diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 7381d4113f617434a5bd306544cb3b86dd1d02ad..0000000000000000000000000000000000000000 --- a/.gitignore +++ /dev/null @@ -1,27 +0,0 @@ -cache/ -web/cache.json -node_modules/ -cloudflare/node_modules/ -/config.json -server/config.json -server/cookie.json -server/logs/ -bundle/clip/config.json -bundle/clip/data/ -bundle/clip/models/ -*.zip -*.7z -*.rar -bundle/python/ -bundle/python-cpu/ -bundle/python-runtime/ -bundle/_delete_pending_python_gpu/ -__pycache__/ -test_*.py -build/ -runtime/*credential* -runtime/cloud-config-state.json -cloudflare/.npm-cache/ -cloudflare/.wrangler/ -cloudflare/.wrangler-config/ -cloudflare/.dev.vars diff --git a/.zipignore b/.zipignore deleted file mode 100644 index c18f6d7a78174bf330ed380f8eb517bbd00e8269..0000000000000000000000000000000000000000 --- a/.zipignore +++ /dev/null @@ -1,56 +0,0 @@ -# Git -.git/ -.gitignore - -# Python -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -*.egg-info/ -/dist/ -/build/ -*.egg - -# Node -npm-debug.log* - -# IDE -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# OS -.DS_Store -Thumbs.db -desktop.ini - -# Logs -*.log -logs/ - -# Temp -tmp/ -temp/ -*.tmp -*.zip -*.7z -*.rar -test_*.py -/build/ - -# Local data and private config -cache/ -server/config.json -bundle/clip/config.json -bundle/python/ -bundle/_delete_pending_python_gpu/ - -# Large runtime files (keep models and data but exclude huge node_modules) -runtime/node/node_modules/ - -# Test outputs -test_output/ diff --git a/README.md b/README.md index 1374a39fc889d9c812d09503c70f3a63561e780c..4a957278c07a26ca82e9846b45b63bb820ab69b2 100644 --- a/README.md +++ b/README.md @@ -1,143 +1,57 @@ --- license: apache-2.0 -language: - - zh - - en +pipeline_tag: image-to-text tags: - - clip - - faiss - - ecommerce - - image-retrieval - - product-search - temu -pipeline_tag: image-feature-extraction + - product-bundling + - clip + - offline-package + - windows --- -
+# 自动组货 -# Bundle CLIP +> 一个面向 Temu / 1688 组货工作流的 Windows 离线整包归档。 -### Temu product retrieval pack for local bundle-building workflows +![Version](https://img.shields.io/badge/package-7z_archive-2f6feb?style=for-the-badge) +![Runtime](https://img.shields.io/badge/runtime-Windows_CPU-22c55e?style=for-the-badge) +![Status](https://img.shields.io/badge/status-warehouse_only-ff7a45?style=for-the-badge) -

- CLIP - FAISS - Local - License -

+## What Is Inside -**Image in. Product ideas out.** +`自动组货.7z` 是完整项目压缩包,用来做仓库存档和迁移分发。它不是 Hugging Face Space,不会在网页上直接运行。 -This repository packages a local CLIP + FAISS retrieval service for ecommerce bundle discovery. +这个包主要包含: -
- ---- +- 自动组货前端工作台 +- 本地后端服务 +- 浏览器扩展 +- CLIP 组货相关模型与索引 +- Windows 启动脚本与离线运行环境 -## Screenshots +## Preview -

- Bundle CLIP local listing search UI -

+### Workbench -

- Local listing search page served by the bundled 9990 runtime. -

+![Workbench overview](docs/images/workbench-overview.png) - - - - - - - - - -
- Auto Bundle smart workflow - - Temu and 1688 workbench overview -
Smart Bundle WorkflowIntegrated Workbench
+### Workflow -## What This Is +![Smart workflow](docs/images/smart-workflow.png) -`bundle-clip` is a self-contained local retrieval bundle used by the Auto Bundle workbench. It combines: +### CLIP Search -| Layer | Role | -| --- | --- | -| Kimi planning | Turns an input product image into bundle-search directions. | -| OpenCLIP encoder | Embeds image and text prompts into the same semantic space. | -| FAISS listing index | Retrieves high-similarity Temu listings from the prepared metadata. | -| Local web UI | Serves a review page at `http://127.0.0.1:9990/`. | +![Bundle CLIP search](docs/images/bundle-clip-search.png) -The pack is designed for fast local review, private experimentation, and offline-ish product matching after the LFS assets are downloaded. +## Download -## Repository Layout +直接在本页下载: ```text -bundle-clip/ -├─ app/ -│ └─ listing_search.html -├─ docs/ -│ └─ images/ -├─ work/ -│ ├─ full_listing_server.py -│ ├─ stdio_listing_worker.py -│ ├─ full_clip_server.py -│ └─ build_full_listing_index.py -├─ models/ -│ └─ open_clip_pytorch_model.bin -├─ data/ -│ ├─ yunqi_clip_training/ -│ │ └─ last_checkpoint.pt -│ ├─ full_listing_index/ -│ │ ├─ products_listing.index -│ │ ├─ products_listing_meta.runtime.json -│ │ ├─ cleaning_report.json -│ │ └─ progress.json -│ └─ full_clip_index/ -│ └─ products_full_prices.json -├─ config.example.json -├─ .gitattributes -└─ README.md -``` - -## Included Assets - -| Asset | Purpose | -| --- | --- | -| `models/open_clip_pytorch_model.bin` | Base OpenCLIP model weights. | -| `data/yunqi_clip_training/last_checkpoint.pt` | Fine-tuned checkpoint for the bundle-search domain. | -| `data/full_listing_index/products_listing.index` | FAISS index for listing retrieval. | -| `data/full_listing_index/products_listing_meta.runtime.json` | Runtime metadata used to render product cards. | -| `data/full_clip_index/products_full_prices.json` | Price metadata used by the local search UI. | - -Large files are tracked with Git LFS. Run `git lfs pull` after cloning. - -## Clone & Update - -Yes, you can clone this repository directly. The only catch is that the model weights and FAISS index are stored with Git LFS, so a complete first-time setup should be: - -```powershell -git lfs install -git clone https://huggingface.co/mikaassa/bundle-clip -cd bundle-clip -git lfs pull +自动组货.7z ``` -If Git LFS is missing, the large assets will look like tiny text pointer files and the server will fail when loading the model or index. - -To update an existing local copy later: - -```powershell -cd bundle-clip -git pull -git lfs pull -``` - -## Quick Start - -### 1. Clone With LFS +或者用 Git LFS 克隆: ```powershell git lfs install @@ -146,83 +60,16 @@ cd bundle-clip git lfs pull ``` -### 2. Create Local Config +## Use On A New Computer -```powershell -Copy-Item config.example.json config.json -``` - -Fill in your private Kimi or Moonshot key: - -```json -{ - "kimi": { - "api_key": "YOUR_KIMI_API_KEY", - "endpoint": "https://api.moonshot.cn/v1/chat/completions", - "model": "kimi-k2.6", - "temperature": 0.6, - "max_completion_tokens": 1200 - } -} -``` - -`config.json` is ignored by Git. Keep real API keys local. - -### 3. Start The Local Service - -```powershell -python .\work\full_listing_server.py -``` - -Open: - -```text -http://127.0.0.1:9990/ -``` - -## Workflow - -```mermaid -flowchart LR - A[Product image] --> B[Kimi bundle directions] - B --> C[English listing prompt] - C --> D[OpenCLIP embedding] - D --> E[FAISS nearest-neighbor search] - E --> F[Temu product cards] -``` - -The local UI supports two review paths: - -| Mode | Use Case | -| --- | --- | -| Image bundle search | Upload a product image, let Kimi produce bundle directions, then retrieve matching listings. | -| Direct CLIP search | Enter a manual listing keyword or prompt and search the index directly. | - -## Runtime Notes - -- Default local port: `9990` -- Main service entry: `work/full_listing_server.py` -- Workbench worker entry: `work/stdio_listing_worker.py` -- Public page: `app/listing_search.html` -- Main metadata image field: `MAINIMAGE` - -The service prefers repository-local `data/` and `models/` paths first. Older absolute-path fallbacks are only used when local assets are missing. - -## Safety - -- Do not commit `config.json`, `.env`, logs, or local cache output. -- API keys should be supplied through `config.json` or environment variables only. -- This repository is a local runtime pack, not a public hosted inference endpoint. -- Product metadata and retrieval quality depend on the bundled index snapshot. - -## Environment Overrides - -```powershell -$env:MOONSHOT_API_KEY="YOUR_KIMI_API_KEY" -$env:KIMI_ENDPOINT="https://api.moonshot.cn/v1/chat/completions" -$env:KIMI_MODEL="kimi-k2.6" -``` +1. 解压 `自动组货.7z` +2. 进入解压后的 `自动组货` 文件夹 +3. 双击 `启动.bat` +4. 如果是第一次部署,按项目内的示例配置补齐本机配置文件 +5. 浏览器扩展需要在 Chrome / Edge 里以开发者模式加载 `extension` 文件夹 -## License +## Notes -Released under the Apache 2.0 license. Check upstream model and data-source terms before redistribution or commercial deployment. +- 这个仓库只作为大文件仓库使用。 +- 配置、Cookie、授权凭据这类本机敏感文件不应该提交到仓库。 +- 如果重新打包发布,建议保持文件名 `自动组货.7z`,这样下载链接和说明都不用改。 diff --git a/bundle/clip/.gitattributes b/bundle/clip/.gitattributes deleted file mode 100644 index a1813275f2c6da8b6d57dc529f6ec2db30115709..0000000000000000000000000000000000000000 --- a/bundle/clip/.gitattributes +++ /dev/null @@ -1,38 +0,0 @@ -data/** filter=lfs diff=lfs merge=lfs -text -models/** filter=lfs diff=lfs merge=lfs -text -*.7z filter=lfs diff=lfs merge=lfs -text -*.arrow filter=lfs diff=lfs merge=lfs -text -*.bin filter=lfs diff=lfs merge=lfs -text -*.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.ftz filter=lfs diff=lfs merge=lfs -text -*.gz filter=lfs diff=lfs merge=lfs -text -*.h5 filter=lfs diff=lfs merge=lfs -text -*.index filter=lfs diff=lfs merge=lfs -text -*.joblib filter=lfs diff=lfs merge=lfs -text -*.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text -*.model filter=lfs diff=lfs merge=lfs -text -*.msgpack filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.npz filter=lfs diff=lfs merge=lfs -text -*.onnx filter=lfs diff=lfs merge=lfs -text -*.ot filter=lfs diff=lfs merge=lfs -text -*.parquet filter=lfs diff=lfs merge=lfs -text -*.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text -*.pth filter=lfs diff=lfs merge=lfs -text -*.rar filter=lfs diff=lfs merge=lfs -text -*.safetensors filter=lfs diff=lfs merge=lfs -text -saved_model/**/* filter=lfs diff=lfs merge=lfs -text -*.tar.* filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text -*.tflite filter=lfs diff=lfs merge=lfs -text -*.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text -*.xz filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.zst filter=lfs diff=lfs merge=lfs -text -*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/bundle/clip/.gitignore b/bundle/clip/.gitignore deleted file mode 100644 index ea12151ce79f4893c8de4af556601184f98b7c6a..0000000000000000000000000000000000000000 --- a/bundle/clip/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -config.json -.env -__pycache__/ -*.pyc -logs/ -*.log -build/ -dist/ -*.spec diff --git a/bundle/clip/README.md b/bundle/clip/README.md deleted file mode 100644 index 1374a39fc889d9c812d09503c70f3a63561e780c..0000000000000000000000000000000000000000 --- a/bundle/clip/README.md +++ /dev/null @@ -1,228 +0,0 @@ ---- -license: apache-2.0 -language: - - zh - - en -tags: - - clip - - faiss - - ecommerce - - image-retrieval - - product-search - - temu -pipeline_tag: image-feature-extraction ---- - -
- -# Bundle CLIP - -### Temu product retrieval pack for local bundle-building workflows - -

- CLIP - FAISS - Local - License -

- -**Image in. Product ideas out.** - -This repository packages a local CLIP + FAISS retrieval service for ecommerce bundle discovery. - -
- ---- - -## Screenshots - -

- Bundle CLIP local listing search UI -

- -

- Local listing search page served by the bundled 9990 runtime. -

- - - - - - - - - - -
- Auto Bundle smart workflow - - Temu and 1688 workbench overview -
Smart Bundle WorkflowIntegrated Workbench
- -## What This Is - -`bundle-clip` is a self-contained local retrieval bundle used by the Auto Bundle workbench. It combines: - -| Layer | Role | -| --- | --- | -| Kimi planning | Turns an input product image into bundle-search directions. | -| OpenCLIP encoder | Embeds image and text prompts into the same semantic space. | -| FAISS listing index | Retrieves high-similarity Temu listings from the prepared metadata. | -| Local web UI | Serves a review page at `http://127.0.0.1:9990/`. | - -The pack is designed for fast local review, private experimentation, and offline-ish product matching after the LFS assets are downloaded. - -## Repository Layout - -```text -bundle-clip/ -├─ app/ -│ └─ listing_search.html -├─ docs/ -│ └─ images/ -├─ work/ -│ ├─ full_listing_server.py -│ ├─ stdio_listing_worker.py -│ ├─ full_clip_server.py -│ └─ build_full_listing_index.py -├─ models/ -│ └─ open_clip_pytorch_model.bin -├─ data/ -│ ├─ yunqi_clip_training/ -│ │ └─ last_checkpoint.pt -│ ├─ full_listing_index/ -│ │ ├─ products_listing.index -│ │ ├─ products_listing_meta.runtime.json -│ │ ├─ cleaning_report.json -│ │ └─ progress.json -│ └─ full_clip_index/ -│ └─ products_full_prices.json -├─ config.example.json -├─ .gitattributes -└─ README.md -``` - -## Included Assets - -| Asset | Purpose | -| --- | --- | -| `models/open_clip_pytorch_model.bin` | Base OpenCLIP model weights. | -| `data/yunqi_clip_training/last_checkpoint.pt` | Fine-tuned checkpoint for the bundle-search domain. | -| `data/full_listing_index/products_listing.index` | FAISS index for listing retrieval. | -| `data/full_listing_index/products_listing_meta.runtime.json` | Runtime metadata used to render product cards. | -| `data/full_clip_index/products_full_prices.json` | Price metadata used by the local search UI. | - -Large files are tracked with Git LFS. Run `git lfs pull` after cloning. - -## Clone & Update - -Yes, you can clone this repository directly. The only catch is that the model weights and FAISS index are stored with Git LFS, so a complete first-time setup should be: - -```powershell -git lfs install -git clone https://huggingface.co/mikaassa/bundle-clip -cd bundle-clip -git lfs pull -``` - -If Git LFS is missing, the large assets will look like tiny text pointer files and the server will fail when loading the model or index. - -To update an existing local copy later: - -```powershell -cd bundle-clip -git pull -git lfs pull -``` - -## Quick Start - -### 1. Clone With LFS - -```powershell -git lfs install -git clone https://huggingface.co/mikaassa/bundle-clip -cd bundle-clip -git lfs pull -``` - -### 2. Create Local Config - -```powershell -Copy-Item config.example.json config.json -``` - -Fill in your private Kimi or Moonshot key: - -```json -{ - "kimi": { - "api_key": "YOUR_KIMI_API_KEY", - "endpoint": "https://api.moonshot.cn/v1/chat/completions", - "model": "kimi-k2.6", - "temperature": 0.6, - "max_completion_tokens": 1200 - } -} -``` - -`config.json` is ignored by Git. Keep real API keys local. - -### 3. Start The Local Service - -```powershell -python .\work\full_listing_server.py -``` - -Open: - -```text -http://127.0.0.1:9990/ -``` - -## Workflow - -```mermaid -flowchart LR - A[Product image] --> B[Kimi bundle directions] - B --> C[English listing prompt] - C --> D[OpenCLIP embedding] - D --> E[FAISS nearest-neighbor search] - E --> F[Temu product cards] -``` - -The local UI supports two review paths: - -| Mode | Use Case | -| --- | --- | -| Image bundle search | Upload a product image, let Kimi produce bundle directions, then retrieve matching listings. | -| Direct CLIP search | Enter a manual listing keyword or prompt and search the index directly. | - -## Runtime Notes - -- Default local port: `9990` -- Main service entry: `work/full_listing_server.py` -- Workbench worker entry: `work/stdio_listing_worker.py` -- Public page: `app/listing_search.html` -- Main metadata image field: `MAINIMAGE` - -The service prefers repository-local `data/` and `models/` paths first. Older absolute-path fallbacks are only used when local assets are missing. - -## Safety - -- Do not commit `config.json`, `.env`, logs, or local cache output. -- API keys should be supplied through `config.json` or environment variables only. -- This repository is a local runtime pack, not a public hosted inference endpoint. -- Product metadata and retrieval quality depend on the bundled index snapshot. - -## Environment Overrides - -```powershell -$env:MOONSHOT_API_KEY="YOUR_KIMI_API_KEY" -$env:KIMI_ENDPOINT="https://api.moonshot.cn/v1/chat/completions" -$env:KIMI_MODEL="kimi-k2.6" -``` - -## License - -Released under the Apache 2.0 license. Check upstream model and data-source terms before redistribution or commercial deployment. diff --git a/bundle/clip/app/listing_search.html b/bundle/clip/app/listing_search.html deleted file mode 100644 index 6e9cd1c2a8dab7687b4e0ea5c889dc2ed437098d..0000000000000000000000000000000000000000 --- a/bundle/clip/app/listing_search.html +++ /dev/null @@ -1,1288 +0,0 @@ - - - - - - 纯 Listing 组货检索 - - - -
-
-
-

IMAGE TO KIMI TO LISTING CLIP

-

纯 Listing 组货检索

-

图片交给 Kimi 生成 10 个可组货商品检索词,再用纯 listing CLIP 索引召回;下方文本框单独用于直接检索 CLIP。

-
-
9990 等待连接
-
- -
-
-
-

匹配结果

- 等待输入 -
-
-

Kimi 组货商品 JSON

-

-
-
- - 改英文 en 最影响 CLIP 召回 -
-
-
-
还没有结果上传图片跑 Kimi,或输入文本直接检索 CLIP
-
-
- - -
-
- - - - diff --git a/bundle/clip/config.example.json b/bundle/clip/config.example.json deleted file mode 100644 index 7b499b0f4bc04042585fce03e584e77b24a71c78..0000000000000000000000000000000000000000 --- a/bundle/clip/config.example.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "kimi": { - "api_key": "", - "endpoint": "https://api.moonshot.cn/v1/chat/completions", - "model": "kimi-k2.6", - "temperature": 0.6, - "max_completion_tokens": 1200 - } -} diff --git a/bundle/clip/data/full_clip_index/products_full_prices.json b/bundle/clip/data/full_clip_index/products_full_prices.json deleted file mode 100644 index 4930dbbc99adc62b2211fc4bf9117fcc8cda887d..0000000000000000000000000000000000000000 --- a/bundle/clip/data/full_clip_index/products_full_prices.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a63876a98e6c4e4341566c90e00b0f4001a62ad69d55c474884e96c5aca141dc -size 37368003 diff --git a/bundle/clip/data/full_listing_index/cleaning_report.json b/bundle/clip/data/full_listing_index/cleaning_report.json deleted file mode 100644 index bb97c12efe6acd8ad7c346b4686ac780c3b87528..0000000000000000000000000000000000000000 --- a/bundle/clip/data/full_listing_index/cleaning_report.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "source": 286548, - "kept": 273454, - "dropped_missing": 0, - "dropped_short": 7554, - "dropped_duplicate": 5540, - "updated_at": "2026-08-21 09:13:05" -} \ No newline at end of file diff --git a/bundle/clip/data/full_listing_index/products_listing.index b/bundle/clip/data/full_listing_index/products_listing.index deleted file mode 100644 index ee179ed232c81a8153ccc4db635c53944aa347c4..0000000000000000000000000000000000000000 --- a/bundle/clip/data/full_listing_index/products_listing.index +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4e50a979e2386e24b862bfbd23fb90658ccc60e406e612144808318bde3f631e -size 560033837 diff --git a/bundle/clip/data/full_listing_index/products_listing_meta.runtime.json b/bundle/clip/data/full_listing_index/products_listing_meta.runtime.json deleted file mode 100644 index 2f9dfeb29d54159e9b1bb1b49a0d6b025f90806b..0000000000000000000000000000000000000000 --- a/bundle/clip/data/full_listing_index/products_listing_meta.runtime.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0c10a2a02aab00a382f9b02cc521e11a3225a28bf334fa987e226acf5832767f -size 139038585 diff --git a/bundle/clip/data/full_listing_index/progress.json b/bundle/clip/data/full_listing_index/progress.json deleted file mode 100644 index 4eedea312b031d39af37d8eaf1cca2c97299394f..0000000000000000000000000000000000000000 --- a/bundle/clip/data/full_listing_index/progress.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "completed": 273454, - "total": 273454, - "status": "complete", - "updated_at": "2026-08-21 09:51:05" -} \ No newline at end of file diff --git a/bundle/clip/data/yunqi_clip_training/last_checkpoint.pt b/bundle/clip/data/yunqi_clip_training/last_checkpoint.pt deleted file mode 100644 index 2b588636a91ed0a284be50e831b4ecee86529ab3..0000000000000000000000000000000000000000 --- a/bundle/clip/data/yunqi_clip_training/last_checkpoint.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:57f6be30140cb9f03fd6d7b7aa83fefc2c4441ef4244608a8391084c0031c35a -size 692418197 diff --git a/bundle/clip/docs/images/smart-workflow.png b/bundle/clip/docs/images/smart-workflow.png deleted file mode 100644 index dbec53b8fffb3c97c80fc9a3dc636562323673af..0000000000000000000000000000000000000000 --- a/bundle/clip/docs/images/smart-workflow.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dbeee62d6d835c400e42189e53a6d350c15045e33a1a204051f9518994790a19 -size 966630 diff --git a/bundle/clip/docs/images/workbench-overview.png b/bundle/clip/docs/images/workbench-overview.png deleted file mode 100644 index cf343ef7a6be6d60a50736d19e077dedc07d971b..0000000000000000000000000000000000000000 --- a/bundle/clip/docs/images/workbench-overview.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f9eda4e9eb24f0f90d25a6c6d154530a7b7d44110334c2ef4c4757de9c310cb6 -size 1254959 diff --git a/bundle/clip/models/open_clip_pytorch_model.bin b/bundle/clip/models/open_clip_pytorch_model.bin deleted file mode 100644 index 39999b31ecfef4b109e67ca40767a72561389a68..0000000000000000000000000000000000000000 --- a/bundle/clip/models/open_clip_pytorch_model.bin +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1bd3c7172de5b207ceac554f5ab5266166f3b9baccc9af5989bc801016d080ad -size 605219813 diff --git a/bundle/clip/work/build_full_listing_index.py b/bundle/clip/work/build_full_listing_index.py deleted file mode 100644 index 6e1a9fb18656c73eb2899711fd5ad78df19da6b9..0000000000000000000000000000000000000000 --- a/bundle/clip/work/build_full_listing_index.py +++ /dev/null @@ -1,395 +0,0 @@ -"""Build a cleaned full-product CLIP text index from listing metadata.""" - -import argparse -import json -import os -import re -import time -from pathlib import Path - -import faiss -import numpy as np -import open_clip -import torch - - -CLIP_DIR = Path(r"F:\Clip") -SOURCE_INDEX_DIR = CLIP_DIR / "data" / "full_clip_index" -SOURCE_METADATA_PATH = SOURCE_INDEX_DIR / "products_full_meta.json" -PRICE_METADATA_PATH = SOURCE_INDEX_DIR / "products_full_prices.json" -MODEL_PATH = CLIP_DIR / "models" / "open_clip_pytorch_model.bin" -CHECKPOINT_PATH = CLIP_DIR / "data" / "yunqi_clip_training" / "last_checkpoint.pt" -OUTPUT_DIR = CLIP_DIR / "data" / "full_listing_index" -METADATA_PATH = OUTPUT_DIR / "products_listing_meta.json" -EMBEDDINGS_PATH = OUTPUT_DIR / "products_listing_embeddings.npy" -INDEX_PATH = OUTPUT_DIR / "products_listing.index" -PROGRESS_PATH = OUTPUT_DIR / "progress.json" -REPORT_PATH = OUTPUT_DIR / "cleaning_report.json" -MODEL_NAME = "ViT-B-32" -DEFAULT_BATCH_SIZE = 256 -DEFAULT_SAVE_EVERY = 2048 - -STOP_WORDS = { - "with", - "for", - "and", - "the", - "set", - "pcs", - "piece", - "pieces", - "pack", - "new", - "hot", - "sale", - "best", - "high", - "quality", - "portable", - "creative", - "fashion", - "women", - "men", - "kids", - "girls", - "boys", - "home", - "office", - "outdoor", - "indoor", -} - - -def parse_arguments(): - """Parse options for a resumable listing-index build.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE) - parser.add_argument("--save-every", type=int, default=DEFAULT_SAVE_EVERY) - parser.add_argument("--force-clean", action="store_true") - return parser.parse_args() - - -def write_json_atomic(path, payload): - """Write JSON through a temporary file and replace the target atomically.""" - path.parent.mkdir(parents=True, exist_ok=True) - temporary_path = path.with_suffix(path.suffix + ".tmp") - temporary_path.write_text( - json.dumps(payload, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - last_error = None - attempt = 0 - while attempt < 10: - try: - os.replace(temporary_path, path) - return - except PermissionError as error: - last_error = error - time.sleep(0.5) - attempt += 1 - raise last_error - - -def clean_spaces(value): - """Collapse noisy whitespace and remove invisible control characters.""" - text = str(value or "").replace("\u0000", " ") - text = re.sub(r"\s+", " ", text) - return text.strip() - - -def looks_mojibake(value): - """Detect obviously broken text so English listing can be preferred.""" - text = str(value or "") - if not text: - return False - bad_count = 0 - for character in text: - if character == "�": - bad_count += 1 - return bad_count >= max(3, len(text) // 12) - - -def choose_listing_text(record): - """Choose the cleanest searchable listing text from one product record.""" - title_en = clean_spaces(record.get("title_en", "")) - title_cn = clean_spaces(record.get("title_cn", "")) - title = clean_spaces(record.get("title", "")) - if title_en: - return title_en - if title and not looks_mojibake(title): - return title - if title_cn and not looks_mojibake(title_cn): - return title_cn - return title_en or title or title_cn - - -def normalize_for_exact_dedupe(value): - """Build a strict title key for exact duplicate removal.""" - text = clean_spaces(value).lower() - text = re.sub(r"[^a-z0-9]+", " ", text) - text = re.sub(r"\s+", " ", text) - return text.strip() - - -def make_family_key(value): - """Build a coarse key used later to avoid same-looking Top results.""" - normalized = normalize_for_exact_dedupe(value) - tokens = normalized.split(" ") - kept_tokens = [] - for token in tokens: - if len(token) <= 2: - continue - if token in STOP_WORDS: - continue - if token.isdigit(): - continue - kept_tokens.append(token) - unique_tokens = [] - for token in kept_tokens: - if token not in unique_tokens: - unique_tokens.append(token) - if len(unique_tokens) <= 2: - return normalized[:80] - return " ".join(unique_tokens[:10]) - - -def load_source_products(): - """Load the existing full product metadata generated by the image index.""" - if not SOURCE_METADATA_PATH.exists(): - raise FileNotFoundError(f"Missing source metadata: {SOURCE_METADATA_PATH}") - return json.loads(SOURCE_METADATA_PATH.read_text(encoding="utf-8")) - - -def clean_products(raw_products): - """Drop unusable listings and exact duplicate listing records.""" - cleaned_products = [] - seen_titles = set() - seen_ids = set() - dropped_short = 0 - dropped_duplicate = 0 - dropped_missing = 0 - for raw_product in raw_products: - product_id = str(raw_product.get("id", "")).strip() - listing_text = choose_listing_text(raw_product) - exact_key = normalize_for_exact_dedupe(listing_text) - if not product_id or not listing_text: - dropped_missing += 1 - continue - if len(exact_key) < 8: - dropped_short += 1 - continue - if product_id in seen_ids or exact_key in seen_titles: - dropped_duplicate += 1 - continue - product = raw_product.copy() - product["title"] = listing_text - product["listing_text"] = listing_text - product["listing_key"] = exact_key - product["family_key"] = make_family_key(listing_text) - product["img_url"] = f"/listing-images/{product_id}.jpg" - cleaned_products.append(product) - seen_ids.add(product_id) - seen_titles.add(exact_key) - report = { - "source": len(raw_products), - "kept": len(cleaned_products), - "dropped_missing": dropped_missing, - "dropped_short": dropped_short, - "dropped_duplicate": dropped_duplicate, - "updated_at": time.strftime("%Y-%m-%d %H:%M:%S"), - } - return cleaned_products, report - - -def load_or_create_clean_products(force_clean): - """Reuse cleaned metadata unless a fresh cleaning pass is requested.""" - if METADATA_PATH.exists() and not force_clean: - return json.loads(METADATA_PATH.read_text(encoding="utf-8")) - raw_products = load_source_products() - cleaned_products, report = clean_products(raw_products) - write_json_atomic(METADATA_PATH, cleaned_products) - write_json_atomic(REPORT_PATH, report) - return cleaned_products - - -def load_model(): - """Load the trained CLIP text tower on CPU.""" - if not MODEL_PATH.exists(): - raise FileNotFoundError(f"Missing base model: {MODEL_PATH}") - if not CHECKPOINT_PATH.exists(): - raise FileNotFoundError(f"Missing trained checkpoint: {CHECKPOINT_PATH}") - device = torch.device("cpu") - model, _, _ = open_clip.create_model_and_transforms( - MODEL_NAME, - pretrained=str(MODEL_PATH), - ) - checkpoint = torch.load(CHECKPOINT_PATH, map_location=device, weights_only=False) - model.load_state_dict(checkpoint["model"]) - model = model.to(device) - model.eval() - tokenizer = open_clip.get_tokenizer(MODEL_NAME) - return model, tokenizer, device - - -def load_progress(total): - """Read the last completed text embedding count for resume.""" - if not PROGRESS_PATH.exists(): - return 0 - progress = json.loads(PROGRESS_PATH.read_text(encoding="utf-8")) - if int(progress.get("total", total)) != total: - raise RuntimeError("Existing listing progress does not match cleaned metadata") - return max(0, min(total, int(progress.get("completed", 0)))) - - -def open_embedding_memmap(total, dimension): - """Create or reopen the text embedding memmap.""" - OUTPUT_DIR.mkdir(parents=True, exist_ok=True) - if EMBEDDINGS_PATH.exists(): - embedding_matrix = np.lib.format.open_memmap(EMBEDDINGS_PATH, mode="r+") - if embedding_matrix.shape != (total, dimension): - raise RuntimeError("Existing listing embedding matrix shape mismatch") - return embedding_matrix - return np.lib.format.open_memmap( - EMBEDDINGS_PATH, - mode="w+", - dtype="float32", - shape=(total, dimension), - ) - - -def restore_faiss_index(embedding_matrix, completed, dimension): - """Rebuild an in-memory FAISS index from completed text vectors.""" - index = faiss.IndexFlatIP(dimension) - chunk_size = 8192 - start_index = 0 - while start_index < completed: - end_index = min(start_index + chunk_size, completed) - index.add(np.asarray(embedding_matrix[start_index:end_index], dtype="float32")) - start_index = end_index - return index - - -def write_faiss_index_file(index, path): - """Write a FAISS index through Python bytes so Windows Unicode paths stay valid.""" - serialized_index = faiss.serialize_index(index) - path.write_bytes(serialized_index.tobytes()) - - -def encode_text_batch(model, tokenizer, texts, device): - """Encode one batch of listing strings with the trained text tower.""" - tokens = tokenizer(texts).to(device) - with torch.inference_mode(): - features = model.encode_text(tokens) - features = features / features.norm(dim=-1, keepdim=True) - return features.cpu().numpy().astype("float32") - - -def encode_products(products, model, tokenizer, device, batch_size, save_every): - """Encode cleaned listings and print live throughput.""" - dimension = int(model.text_projection.shape[1]) - total = len(products) - completed = load_progress(total) - embedding_matrix = open_embedding_memmap(total, dimension) - index = restore_faiss_index(embedding_matrix, completed, dimension) - started_at = time.perf_counter() - while completed < total: - batch_end = min(completed + batch_size, total) - texts = [] - product_index = completed - while product_index < batch_end: - texts.append(products[product_index]["listing_text"]) - product_index += 1 - batch_features = encode_text_batch(model, tokenizer, texts, device) - embedding_matrix[completed:batch_end] = batch_features - index.add(batch_features) - completed = batch_end - if completed % save_every < batch_size or completed >= total: - embedding_matrix.flush() - write_json_atomic( - PROGRESS_PATH, - { - "completed": completed, - "total": total, - "status": "building", - "updated_at": time.strftime("%Y-%m-%d %H:%M:%S"), - }, - ) - elapsed = time.perf_counter() - started_at - rate = completed / max(elapsed, 1e-6) - eta_minutes = (total - completed) / max(rate, 1e-6) / 60 - print( - "listing_embedding", - completed, - "/", - total, - "rate", - f"{rate:.2f}/s", - "eta_minutes", - f"{eta_minutes:.1f}", - flush=True, - ) - return embedding_matrix, index - - -def write_final_outputs(products, embedding_matrix, index): - """Publish the finished listing index without touching the image index.""" - embedding_matrix.flush() - temporary_index_path = INDEX_PATH.with_suffix(INDEX_PATH.suffix + ".tmp") - write_faiss_index_file(index, temporary_index_path) - os.replace(temporary_index_path, INDEX_PATH) - write_json_atomic(METADATA_PATH, products) - write_json_atomic( - PROGRESS_PATH, - { - "completed": len(products), - "total": len(products), - "status": "complete", - "updated_at": time.strftime("%Y-%m-%d %H:%M:%S"), - }, - ) - return index.ntotal - - -def main(): - """Run the full cleaned listing-index build.""" - arguments = parse_arguments() - products = load_or_create_clean_products(arguments.force_clean) - model, tokenizer, device = load_model() - print( - "listing_index_start", - json.dumps( - { - "device": str(device), - "products": len(products), - "batch_size": arguments.batch_size, - "output_dir": str(OUTPUT_DIR), - }, - ensure_ascii=False, - ), - flush=True, - ) - embedding_matrix, index = encode_products( - products, - model, - tokenizer, - device, - max(1, arguments.batch_size), - max(1, arguments.save_every), - ) - vector_count = write_final_outputs(products, embedding_matrix, index) - print( - "listing_index_complete", - json.dumps( - { - "vectors": vector_count, - "dimension": int(embedding_matrix.shape[1]), - "index": str(INDEX_PATH), - }, - ensure_ascii=False, - ), - flush=True, - ) - - -if __name__ == "__main__": - main() diff --git a/bundle/clip/work/full_clip_server.py b/bundle/clip/work/full_clip_server.py deleted file mode 100644 index 1f9b144b91e75330bd4dea1b53ca09e927cf4d82..0000000000000000000000000000000000000000 --- a/bundle/clip/work/full_clip_server.py +++ /dev/null @@ -1,615 +0,0 @@ -"""Serve the full trained CLIP index and Kimi assembly planner.""" - -import base64 -import io -import json -import os -import time -import urllib.error -import urllib.request -from pathlib import Path -from urllib.parse import urlparse - -import faiss -import numpy as np -import open_clip -import torch -from fastapi import FastAPI, File, Form, UploadFile -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import HTMLResponse, JSONResponse -from fastapi.staticfiles import StaticFiles -from PIL import Image -import uvicorn - - -APP_ROOT = Path(__file__).resolve().parent.parent -BUNDLED_FULL_INDEX_DIR = APP_ROOT / "data" / "full_clip_index" -BUNDLED_MODEL_PATH = APP_ROOT / "models" / "open_clip_pytorch_model.bin" -CLIP_DIR = APP_ROOT if BUNDLED_FULL_INDEX_DIR.exists() and BUNDLED_MODEL_PATH.exists() else Path(r"F:\Clip") -FULL_INDEX_DIR = CLIP_DIR / "data" / "full_clip_index" -INDEX_PATH = FULL_INDEX_DIR / "products_full.index" -METADATA_PATH = FULL_INDEX_DIR / "products_full_meta.json" -PRICE_METADATA_PATH = FULL_INDEX_DIR / "products_full_prices.json" -PROGRESS_PATH = FULL_INDEX_DIR / "progress.json" -IMAGE_DIR = APP_ROOT / "images" -BASE_MODEL_PATH = CLIP_DIR / "models" / "open_clip_pytorch_model.bin" -TRAINED_CHECKPOINT_PATH = CLIP_DIR / "data" / "yunqi_clip_training" / "last_checkpoint.pt" -HTML_PATH = APP_ROOT / "app" / "clip_search.html" if (APP_ROOT / "app" / "clip_search.html").exists() else APP_ROOT / "clip_search.html" -MODEL_NAME = "ViT-B-32" -CONFIG_PATH = APP_ROOT / "config.json" -KIMI_API_KEY_ENV = "MOONSHOT_API_KEY" -KIMI_ENDPOINT_ENV = "KIMI_ENDPOINT" -KIMI_MODEL_ENV = "KIMI_MODEL" -APP_CONFIG = {} -KIMI_CONFIG = {} -KIMI_API_URL = "https://api.moonshot.cn/v1/chat/completions" -KIMI_MODEL = "kimi-k2.6" -KIMI_TEMPERATURE = 0.6 -KIMI_MAX_COMPLETION_TOKENS = 1800 - - -def load_app_config(): - """Load local app configuration without committing user secrets.""" - if not CONFIG_PATH.exists(): - return {} - return json.loads(CONFIG_PATH.read_text(encoding="utf-8")) - - -def read_kimi_config(): - """Return Kimi API settings from environment variables, config.json, and defaults.""" - config = APP_CONFIG.get("kimi", {}) if isinstance(APP_CONFIG, dict) else {} - return { - "api_key": os.environ.get(KIMI_API_KEY_ENV, "").strip() or str(config.get("api_key", "")).strip(), - "endpoint": os.environ.get(KIMI_ENDPOINT_ENV, "").strip() or str(config.get("endpoint", KIMI_API_URL)).strip(), - "model": os.environ.get(KIMI_MODEL_ENV, "").strip() or str(config.get("model", KIMI_MODEL)).strip(), - "temperature": float(config.get("temperature", KIMI_TEMPERATURE)), - "max_completion_tokens": int(config.get("max_completion_tokens", KIMI_MAX_COMPLETION_TOKENS)), - } - - -APP_CONFIG = load_app_config() -KIMI_CONFIG = read_kimi_config() -KIMI_API_URL = KIMI_CONFIG["endpoint"] -KIMI_MODEL = KIMI_CONFIG["model"] -KIMI_TEMPERATURE = KIMI_CONFIG["temperature"] -KIMI_MAX_COMPLETION_TOKENS = KIMI_CONFIG["max_completion_tokens"] - - -def read_faiss_index_file(path): - """Read a FAISS index through Python bytes so Windows Unicode paths stay valid.""" - index_bytes = np.frombuffer(path.read_bytes(), dtype="uint8") - return faiss.deserialize_index(index_bytes) - - -def load_runtime(): - """Load the completed full FAISS index, metadata, and trained CLIP model.""" - if not INDEX_PATH.exists() or not METADATA_PATH.exists(): - raise FileNotFoundError("全量索引尚未生成完成") - if PROGRESS_PATH.exists(): - progress = json.loads(PROGRESS_PATH.read_text(encoding="utf-8")) - if progress.get("status") != "complete": - raise RuntimeError( - f"全量索引仍在构建:{progress.get('completed', 0)}/{progress.get('total', 0)}" - ) - if not BASE_MODEL_PATH.exists() or not TRAINED_CHECKPOINT_PATH.exists(): - raise FileNotFoundError("基础模型或最终训练 checkpoint 不存在") - device = torch.device("cpu") - model, _, preprocess = open_clip.create_model_and_transforms( - MODEL_NAME, - pretrained=str(BASE_MODEL_PATH), - ) - checkpoint = torch.load(TRAINED_CHECKPOINT_PATH, map_location=device, weights_only=False) - model.load_state_dict(checkpoint["model"]) - model = model.to(device) - model.eval() - index = read_faiss_index_file(INDEX_PATH) - products = json.loads(METADATA_PATH.read_text(encoding="utf-8")) - price_metadata = {} - if PRICE_METADATA_PATH.exists(): - price_metadata = json.loads(PRICE_METADATA_PATH.read_text(encoding="utf-8")) - if index.ntotal != len(products): - raise RuntimeError(f"索引数量 {index.ntotal} 与元数据数量 {len(products)} 不一致") - tokenizer = open_clip.get_tokenizer(MODEL_NAME) - return model, preprocess, tokenizer, device, index, products, price_metadata - - -def get_rank_window(top_k): - """Clamp the requested result count to a safe full-index range.""" - return max(1, min(int(top_k), 100)) - - -def encode_image(image, model, preprocess, device): - """Encode one uploaded image with the trained CLIP image tower.""" - tensor = preprocess(image.convert("RGB")).unsqueeze(0).to(device) - with torch.inference_mode(): - feature = model.encode_image(tensor) - feature = feature / feature.norm(dim=-1, keepdim=True) - return feature.cpu().numpy().astype("float32") - - -def encode_text(query, model, tokenizer, device): - """Encode one prompt with the trained CLIP text tower.""" - tokens = tokenizer([query]).to(device) - with torch.inference_mode(): - feature = model.encode_text(tokens) - feature = feature / feature.norm(dim=-1, keepdim=True) - return feature.cpu().numpy().astype("float32") - - -def search_index( - query_vector, - top_k, - index, - products, - price_metadata, - min_price=None, - max_price=None, -): - """Search FAISS and apply an optional price constraint before returning Top K.""" - output_count = get_rank_window(top_k) - search_count = index.ntotal if min_price is not None or max_price is not None else output_count - scores, indices = index.search(query_vector, search_count) - results = [] - for result_position in range(len(indices[0])): - product_index = int(indices[0][result_position]) - if product_index < 0 or product_index >= len(products): - continue - product = products[product_index].copy() - product["similarity"] = round(float(scores[0][result_position]) * 100, 2) - product["img_url"] = f"/images/{product['id']}.jpg" - product_id = str(product["id"]) - if product_id in price_metadata: - product.update(price_metadata[product_id]) - if min_price is not None or max_price is not None: - price = product.get("price_usd") - if price is None: - continue - if min_price is not None and float(price) < float(min_price): - continue - if max_price is not None and float(price) > float(max_price): - continue - product["rank"] = len(results) + 1 - results.append(product) - if len(results) >= output_count: - break - return results - - -def resolve_image_from_request(file, img_url): - """Read an uploaded image or an allowed local image URL into PIL.""" - if isinstance(file, (bytes, bytearray)) and file: - return Image.open(io.BytesIO(file)).convert("RGB") - if file and file.filename: - return Image.open(io.BytesIO(file)).convert("RGB") - if img_url: - parsed_url = urlparse(img_url) - local_name = os.path.basename(parsed_url.path) - local_path = IMAGE_DIR / local_name - if parsed_url.path.startswith("/images/") and local_path.exists(): - return Image.open(local_path).convert("RGB") - raise ValueError("没有提供有效图片") - - -def merge_search_results(image_results, text_results, image_weight): - """Merge image and listing search results by product ID and CLIP score.""" - result_by_id = {} - text_weight = 1.0 - image_weight - for product in image_results: - product_id = str(product.get("id", "")) - item = product.copy() - item["image_similarity"] = float(product.get("similarity", 0.0)) - item["text_similarity"] = 0.0 - result_by_id[product_id] = item - for product in text_results: - product_id = str(product.get("id", "")) - if product_id not in result_by_id: - item = product.copy() - item["image_similarity"] = 0.0 - item["text_similarity"] = float(product.get("similarity", 0.0)) - result_by_id[product_id] = item - else: - result_by_id[product_id]["text_similarity"] = float(product.get("similarity", 0.0)) - results = [] - for item in result_by_id.values(): - item["similarity"] = round( - item["image_similarity"] * image_weight - + item["text_similarity"] * text_weight, - 2, - ) - results.append(item) - index = 0 - while index < len(results): - best_index = index - candidate_index = index + 1 - while candidate_index < len(results): - if results[candidate_index]["similarity"] > results[best_index]["similarity"]: - best_index = candidate_index - candidate_index += 1 - if best_index != index: - results[index], results[best_index] = results[best_index], results[index] - results[index]["rank"] = index + 1 - index += 1 - return results - - -def filter_by_price(products, min_price, max_price): - """Keep products inside the requested USD price range.""" - if min_price is None and max_price is None: - return products - filtered = [] - for product in products: - price = product.get("price_usd") - if price is None: - continue - if min_price is not None and float(price) < float(min_price): - continue - if max_price is not None and float(price) > float(max_price): - continue - filtered.append(product) - return filtered - - -def image_to_data_url(image): - """Resize an input image and encode it as a compact Kimi data URL.""" - image_copy = image.copy().convert("RGB") - image_copy.thumbnail((1280, 1280)) - image_buffer = io.BytesIO() - image_copy.save(image_buffer, format="JPEG", quality=85, optimize=True) - encoded = base64.b64encode(image_buffer.getvalue()).decode("ascii") - return f"data:image/jpeg;base64,{encoded}" - - -def build_clip_evidence(image, listing, model, preprocess, tokenizer, device, index, products, price_metadata): - """Collect weak CLIP evidence so Kimi can correct noisy retrieval signals.""" - evidence = [] - if image is not None: - image_vector = encode_image(image, model, preprocess, device) - image_results = search_index(image_vector, 8, index, products, price_metadata) - evidence.append({"source": "image", "results": image_results}) - if listing: - listing_vector = encode_text(listing, model, tokenizer, device) - listing_results = search_index(listing_vector, 8, index, products, price_metadata) - evidence.append({"source": "listing", "results": listing_results}) - return evidence - - -def call_kimi(image, listing, min_price, max_price, clip_evidence): - """Call domestic Kimi K2.6 in JSON and non-thinking mode.""" - api_key = KIMI_CONFIG["api_key"] - if not api_key: - raise RuntimeError(f"未配置 {CONFIG_PATH.name} 里的 kimi.api_key 或 {KIMI_API_KEY_ENV} 环境变量") - price_rule = { - "currency": "USD", - "min": min_price, - "max": max_price, - } - system_prompt = ( - "你是商品组货规划助手,不是单纯的相似商品检索器。" - "请根据输入图片和Listing生成10个用于商品向量检索的中英文Prompt,目标是找出可以一起销售或一起购买的一组商品。" - "CLIP召回结果不够精准,只能作为弱证据,禁止直接照抄CLIP误召回的品类。" - "10个Prompt必须发散到不同组货方向,不能只是同一商品的颜色、材质或包装改写。" - "10个方向依次覆盖:1核心相似品,2功能替代品,3互补配件,4共同使用工具,5配套耗材,6高概率一起购买的关联品,7收纳整理品,8包装展示品,9人群场景关联品,10套装组合方案。" - "互补品必须和主商品的使用场景有明确关系,不要生成无关的氛围用品。" - "例如主商品是扳手,可以发散到锤子、螺丝刀、卷尺、螺丝螺母、工具收纳包,而不是只生成不同颜色的扳手。" - "Listing明确写出的品类优先;图片用于确认外观、颜色、形状和材质。" - "如果图片和Listing明显冲突,内部自行纠偏,并优先保留Listing主品类。" - "只能返回合法JSON对象,且只能有一个字段 prompts。" - "prompts 必须是长度为10的数组,数组元素只能是对象,且只能包含 zh 和 en 两个字段。" - "zh 是简短具体的中文检索词,en 是语义完全一致的英文检索词。" - "不要返回plan_name、summary、role、reason、price_filter、input_conflict或clip_adjustment。" - ) - user_text = ( - "输入Listing:\n" - + (listing or "未提供") - + "\n价格筛选(美元):\n" - + json.dumps(price_rule, ensure_ascii=False) - + "\nCLIP弱证据(可能不准确,只用于发现偏差):\n" - + json.dumps(clip_evidence, ensure_ascii=False) - + "\n请只返回JSON对象,不要Markdown代码围栏。" - ) - content = [{"type": "text", "text": user_text}] - if image is not None: - content.insert( - 0, - { - "type": "image_url", - "image_url": {"url": image_to_data_url(image)}, - }, - ) - payload = { - "model": KIMI_MODEL, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": content}, - ], - "thinking": {"type": "disabled"}, - "temperature": KIMI_TEMPERATURE, - "response_format": {"type": "json_object"}, - "max_completion_tokens": KIMI_MAX_COMPLETION_TOKENS, - } - request = urllib.request.Request( - KIMI_API_URL, - data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), - headers={ - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - }, - method="POST", - ) - response_data = None - attempt = 0 - while attempt < 3: - try: - with urllib.request.urlopen(request, timeout=120) as response: - response_data = json.loads(response.read().decode("utf-8")) - break - except urllib.error.HTTPError as error: - detail = error.read().decode("utf-8", errors="replace") - retryable = error.code in (429, 500, 502, 503, 504) - if not retryable or attempt >= 2: - raise RuntimeError( - f"Kimi API HTTP {error.code}: {detail[:500]}" - ) from error - retry_after = error.headers.get("Retry-After") - try: - delay = float(retry_after) if retry_after else 2.0 + attempt * 2.0 - except (TypeError, ValueError): - delay = 2.0 + attempt * 2.0 - time.sleep(min(max(delay, 1.0), 10.0)) - attempt += 1 - except urllib.error.URLError as error: - if attempt >= 2: - raise RuntimeError(f"Kimi API 网络错误: {error.reason}") from error - time.sleep(2.0 + attempt * 2.0) - attempt += 1 - choices = response_data.get("choices", []) - if not choices: - raise RuntimeError("Kimi API 没有返回 choices") - content_text = choices[0].get("message", {}).get("content", "") - if isinstance(content_text, dict): - return content_text - try: - return json.loads(content_text) - except (TypeError, json.JSONDecodeError) as error: - raise RuntimeError("Kimi 返回内容不是合法 JSON") from error - - -def normalize_kimi_prompts(raw_plan, listing, min_price, max_price): - """Validate Kimi prompts and fill missing prompts without inventing products.""" - raw_prompts = raw_plan.get("prompts", []) if isinstance(raw_plan, dict) else [] - if not isinstance(raw_prompts, list): - raw_prompts = [] - prompts = [] - default_roles = [ - "核心相似品", - "功能替代品", - "互补配件", - "共同使用工具", - "配套耗材", - "关联加购品", - "收纳整理品", - "包装展示品", - "人群场景关联品", - "场景套装/收纳方案", - ] - for raw_prompt in raw_prompts: - if isinstance(raw_prompt, dict): - prompt_text = str( - raw_prompt.get("zh", raw_prompt.get("prompt", "")) - ).strip() - prompt_english = str( - raw_prompt.get("en", raw_prompt.get("prompt_en", "")) - ).strip() - prompt_role = raw_prompt.get( - "role", - default_roles[min(len(prompts), len(default_roles) - 1)], - ) - prompt_reason = raw_prompt.get("reason", "适合图片和Listing检索") - else: - prompt_text = str(raw_prompt).strip() - prompt_english = "" - prompt_role = default_roles[min(len(prompts), len(default_roles) - 1)] - prompt_reason = "适合图片和Listing检索" - if not prompt_text: - continue - prompts.append( - { - "prompt": prompt_text, - "prompt_en": prompt_english, - "role": prompt_role, - "reason": prompt_reason, - } - ) - if len(prompts) >= 10: - break - fallback_text = listing.strip() or "符合输入图片风格的商品" - fallback_roles = [ - "核心相似品", - "功能替代品", - "互补配件", - "共同使用工具", - "配套耗材", - "关联加购品", - "收纳整理品", - "包装展示品", - "人群场景关联品", - "场景套装/收纳方案", - ] - index = 0 - while len(prompts) < 10: - prompts.append( - { - "prompt": f"{fallback_text},{fallback_roles[index]}", - "prompt_en": "", - "role": fallback_roles[index], - "reason": "Kimi未返回足够Prompt,使用Listing补足检索方向", - } - ) - index += 1 - return {"prompts": prompts} - - -def create_app(): - """Load runtime assets and create the FastAPI application.""" - model, preprocess, tokenizer, device, index, products, price_metadata = load_runtime() - application = FastAPI(title="Full CLIP Product Search") - application.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ) - if IMAGE_DIR.exists(): - application.mount("/images", StaticFiles(directory=IMAGE_DIR), name="images") - - @application.post("/api/search/image") - async def search_image( - file: UploadFile = File(None), - img_url: str = Form(None), - top_k: int = Form(12), - ): - """Search full product metadata using an uploaded image.""" - try: - contents = await file.read() if file and file.filename else None - image = resolve_image_from_request(contents, img_url) - query_vector = encode_image(image, model, preprocess, device) - return {"results": search_index(query_vector, top_k, index, products, price_metadata)} - except Exception as error: - return JSONResponse({"error": str(error)}, status_code=400) - - @application.post("/api/search/text") - async def search_text( - query: str = Form(...), - top_k: int = Form(12), - ): - """Search full product metadata using a text prompt.""" - try: - query_vector = encode_text(query, model, tokenizer, device) - return {"results": search_index(query_vector, top_k, index, products, price_metadata)} - except Exception as error: - return JSONResponse({"error": str(error)}, status_code=400) - - @application.post("/api/assemble") - async def assemble_products( - file: UploadFile = File(None), - listing: str = Form(""), - min_price: float = Form(None), - max_price: float = Form(None), - top_k: int = Form(1), - ): - """Generate ten Kimi prompts and retrieve one product for each direction.""" - try: - contents = await file.read() if file and file.filename else None - image = resolve_image_from_request(contents, "") if contents else None - if image is None and not listing.strip(): - raise ValueError("请至少提供图片或 Listing") - clip_evidence = build_clip_evidence( - image, - listing.strip(), - model, - preprocess, - tokenizer, - device, - index, - products, - price_metadata, - ) - raw_plan = call_kimi( - image, - listing.strip(), - min_price, - max_price, - clip_evidence, - ) - plan = normalize_kimi_prompts( - raw_plan, - listing, - min_price, - max_price, - ) - groups = [] - selected_results = [] - public_prompts = [] - prompt_index = 0 - while prompt_index < len(plan["prompts"]): - prompt_item = plan["prompts"][prompt_index] - prompt_text = prompt_item["prompt"] - prompt_english = prompt_item.get("prompt_en", "").strip() - public_prompts.append( - {"zh": prompt_text, "en": prompt_english} - ) - recall_text = prompt_english or prompt_text - prompt_vector = encode_text(recall_text, model, tokenizer, device) - prompt_matches = search_index( - prompt_vector, - 100, - index, - products, - price_metadata, - min_price, - max_price, - ) - price_matches = prompt_matches - group_results = [] - result_index = 0 - while result_index < min(1, len(price_matches)): - selected = price_matches[result_index].copy() - selected["prompt_index"] = prompt_index + 1 - selected["search_prompt"] = prompt_text - selected["search_prompt_en"] = prompt_english - selected["prompt_role"] = prompt_item["role"] - selected["prompt_reason"] = prompt_item["reason"] - selected["prompt_rank"] = result_index + 1 - group_results.append(selected) - selected_results.append(selected) - result_index += 1 - groups.append( - { - "prompt_index": prompt_index + 1, - "prompt": prompt_text, - "prompt_en": prompt_english, - "role": prompt_item["role"], - "reason": prompt_item["reason"], - "results": group_results, - } - ) - prompt_index += 1 - return { - "plan": {"prompts": public_prompts}, - "results": selected_results, - "groups": groups, - "prompts_searched": len(groups), - "results_per_prompt": 1, - "model": KIMI_MODEL, - "thinking": "disabled", - } - except Exception as error: - return JSONResponse({"error": str(error)}, status_code=400) - - @application.get("/api/index/status") - async def index_status(): - """Return the loaded full-index count for a quick browser health check.""" - return { - "vectors": index.ntotal, - "products": len(products), - "price_records": len(price_metadata), - "device": str(device), - "checkpoint": str(TRAINED_CHECKPOINT_PATH), - "kimi_model": KIMI_MODEL, - "kimi_configured": bool(KIMI_CONFIG["api_key"]), - } - - @application.get("/", response_class=HTMLResponse) - async def search_page(): - """Serve the standalone image and prompt search page.""" - return HTMLResponse(HTML_PATH.read_text(encoding="utf-8")) - - return application - - -app = create_app() - - -if __name__ == "__main__": - uvicorn.run(app, host="127.0.0.1", port=8888) diff --git a/bundle/clip/work/full_listing_server.py b/bundle/clip/work/full_listing_server.py deleted file mode 100644 index 48bf658f928f48fea288423474d214b802682568..0000000000000000000000000000000000000000 --- a/bundle/clip/work/full_listing_server.py +++ /dev/null @@ -1,951 +0,0 @@ -"""Provide the cleaned pure-listing CLIP index for the integrated worker.""" - -import base64 -import asyncio -import concurrent.futures -import io -import json -import os -import subprocess -import sys -import time -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - -import faiss -import numpy as np -import open_clip -import torch -from fastapi import Body, FastAPI, File, Form, Request, UploadFile -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, Response -from fastapi.staticfiles import StaticFiles -from PIL import Image -import uvicorn - - -APP_ROOT = Path(__file__).resolve().parent.parent -BUNDLED_INDEX_DIR = APP_ROOT / "data" / "full_listing_index" -BUNDLED_MODEL_PATH = APP_ROOT / "models" / "open_clip_pytorch_model.bin" -CLIP_DIR = APP_ROOT if BUNDLED_INDEX_DIR.exists() and BUNDLED_MODEL_PATH.exists() else Path(r"F:\Clip") -LISTING_INDEX_DIR = CLIP_DIR / "data" / "full_listing_index" -INDEX_PATH = LISTING_INDEX_DIR / "products_listing.index" -METADATA_PATH = LISTING_INDEX_DIR / "products_listing_meta.json" -RUNTIME_METADATA_PATH = LISTING_INDEX_DIR / "products_listing_meta.runtime.json" -PRICE_METADATA_PATH = CLIP_DIR / "data" / "full_clip_index" / "products_full_prices.json" -PROGRESS_PATH = LISTING_INDEX_DIR / "progress.json" -REPORT_PATH = LISTING_INDEX_DIR / "cleaning_report.json" -BUILD_LOG_PATH = LISTING_INDEX_DIR / "build.log" -SERVER_LOG_PATH = APP_ROOT / "logs" / "server.log" -IMAGE_DIR = APP_ROOT / "images" -BASE_MODEL_PATH = CLIP_DIR / "models" / "open_clip_pytorch_model.bin" -TRAINED_CHECKPOINT_PATH = CLIP_DIR / "data" / "yunqi_clip_training" / "last_checkpoint.pt" -HTML_PATH = APP_ROOT / "app" / "listing_search.html" if (APP_ROOT / "app" / "listing_search.html").exists() else APP_ROOT / "listing_search.html" -BUILD_SCRIPT_PATH = APP_ROOT / "work" / "build_full_listing_index.py" -MODEL_NAME = "ViT-B-32" -CONFIG_PATH = APP_ROOT / "config.json" -RUNTIME_METADATA_FIELDS = [ - "id", - "title", - "listing_key", - "family_key", - "image_url", - "price_usd", - "sales_total", -] -KIMI_API_KEY_ENV = "MOONSHOT_API_KEY" -KIMI_ENDPOINT_ENV = "KIMI_ENDPOINT" -KIMI_MODEL_ENV = "KIMI_MODEL" -APP_CONFIG = {} -KIMI_CONFIG = {} -KIMI_API_URL = "https://api.moonshot.cn/v1/chat/completions" -KIMI_MODEL = "kimi-k2.6" -KIMI_TEMPERATURE = 0.6 -KIMI_MAX_COMPLETION_TOKENS = 1200 -KIMI_PROMPT_BATCH_RANGES = [(1, 3), (4, 6), (7, 10)] -KIMI_PROMPT_SLOT_RANGES = KIMI_PROMPT_BATCH_RANGES -DEFAULT_KIMI_SYSTEM_PROMPT = """ -你是跨境电商组货商品检索词生成器。你只根据用户上传的图片生成可一起售卖/一起购买的商品检索词。 - -任务:输出10个“具体可采购商品”,用于后续纯 listing CLIP 检索。 - -生成原则: -1. 不要只找外观相似品;优先覆盖互补品、同场景加购、替代升级、耗材补充、收纳展示、维护清洁、配套工具、礼盒套装里的其他商品。 -2. 每条必须是具体商品,不要写大类、策略、理由或营销词。不要输出“配件、用品、产品、套装、工具”这种过宽泛词,除非前面有清晰具体限定。 -3. 中文 zh 要像能直接给采购看的商品短名:主体品类 + 关键材质/结构/场景/人群/规格,尽量 6-18 个中文字符。 -4. 英文 en 要像英文 listing 标题检索词:6-14 个英文词,必须包含明确 product noun,并尽量包含 material / shape / color / scene / target user / size / function 中的2-4个要素。 -5. 如果图片主体不确定,根据最明显视觉元素推断;不要解释不确定性。 -6. 10条之间要有明显差异,避免同义改写刷数量。 - -输出格式:只返回合法 JSON 对象,且只能包含 prompts 字段。 -prompts 是长度为10的数组,每个元素只能包含 zh 和 en 两个字段。 -""".strip() - -RUNTIME_CACHE = { - "model": None, - "tokenizer": None, - "device": None, - "index": None, - "products": None, - "prices": None, -} -BUILD_PROCESS = {"process": None} - - -def write_bundle_log(message, payload=None): - """Write one compact bundle API log line for the Node diagnostics collector.""" - entry = {"message": str(message or ""), "payload": payload or {}} - sys.stderr.write("[BUNDLE API] " + json.dumps(entry, ensure_ascii=True) + "\n") - sys.stderr.flush() - - -def load_app_config(): - """Load local app configuration without requiring secrets to be committed.""" - if not CONFIG_PATH.exists(): - return {} - return json.loads(CONFIG_PATH.read_text(encoding="utf-8")) - - -def read_kimi_config(): - """Return Kimi settings from environment variables, config.json, and safe defaults.""" - config = APP_CONFIG.get("kimi", {}) if isinstance(APP_CONFIG, dict) else {} - return { - "api_key": os.environ.get(KIMI_API_KEY_ENV, "").strip() or str(config.get("api_key", "")).strip(), - "endpoint": os.environ.get(KIMI_ENDPOINT_ENV, "").strip() or str(config.get("endpoint", KIMI_API_URL)).strip(), - "model": os.environ.get(KIMI_MODEL_ENV, "").strip() or str(config.get("model", KIMI_MODEL)).strip(), - "temperature": float(config.get("temperature", KIMI_TEMPERATURE)), - "max_completion_tokens": int(config.get("max_completion_tokens", KIMI_MAX_COMPLETION_TOKENS)), - } - - -def read_json_file(path, fallback): - """Read a JSON file when it exists, otherwise return the fallback value.""" - if not path.exists(): - return fallback - return json.loads(path.read_text(encoding="utf-8")) - - -def read_faiss_index_file(path): - """Read a FAISS index through Python bytes so Windows Unicode paths stay valid.""" - index_bytes = np.frombuffer(path.read_bytes(), dtype="uint8") - return faiss.deserialize_index(index_bytes) - - -def resolve_listing_metadata_path(): - """Return the slim runtime metadata when bundled, otherwise use the original build metadata.""" - if RUNTIME_METADATA_PATH.exists(): - return RUNTIME_METADATA_PATH - return METADATA_PATH - - -def normalize_product_metadata_rows(rows): - """Convert compact runtime metadata rows back into product dictionaries.""" - if not rows: - return [] - if isinstance(rows[0], dict): - return rows - products = [] - for row in rows: - product = {} - values = row if isinstance(row, list) else [] - for index, field_name in enumerate(RUNTIME_METADATA_FIELDS): - if index < len(values) and values[index] not in (None, ""): - product[field_name] = values[index] - products.append(product) - return products - - -APP_CONFIG = load_app_config() -KIMI_CONFIG = read_kimi_config() -KIMI_API_URL = KIMI_CONFIG["endpoint"] -KIMI_MODEL = KIMI_CONFIG["model"] -KIMI_TEMPERATURE = KIMI_CONFIG["temperature"] -KIMI_MAX_COMPLETION_TOKENS = KIMI_CONFIG["max_completion_tokens"] - - -def append_server_log(message): - """Append one timestamped server log line without recording secrets.""" - SERVER_LOG_PATH.parent.mkdir(parents=True, exist_ok=True) - timestamp = time.strftime("%Y-%m-%d %H:%M:%S") - with SERVER_LOG_PATH.open("a", encoding="utf-8") as log_file: - log_file.write(f"{timestamp} {message}\n") - - -def should_skip_access_log(path): - """Return whether a noisy internal endpoint should be hidden from server logs.""" - return path in {"/api/index/status", "/api/server/log", "/api/cdn/image"} - - -def validate_cdn_image_url(image_url): - """Validate that the proxied image URL is a plain HTTP(S) CDN URL.""" - parsed_url = urllib.parse.urlparse(str(image_url or "").strip()) - if parsed_url.scheme not in {"http", "https"}: - raise ValueError("CDN image URL must be http or https") - if not parsed_url.netloc: - raise ValueError("CDN image URL host is missing") - return parsed_url.geturl() - - -def fetch_cdn_image_bytes(safe_url): - """Fetch one validated CDN image in a worker thread for the async proxy endpoint.""" - request = urllib.request.Request( - safe_url, - headers={ - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", - "Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", - }, - method="GET", - ) - with urllib.request.urlopen(request, timeout=30) as cdn_response: - image_bytes = cdn_response.read() - content_type = cdn_response.headers.get("Content-Type", "image/jpeg") - status_code = getattr(cdn_response, "status", 200) - return image_bytes, content_type, status_code - - -def read_server_log_filtered(max_bytes): - """Read recent server logs while hiding noisy internal heartbeat entries.""" - raw_log = read_tail(SERVER_LOG_PATH, max_bytes) - hidden_patterns = [ - " /api/index/status ", - " /api/server/log ", - " /api/cdn/image ", - ] - visible_lines = [] - for line in raw_log.splitlines(): - if not line.startswith("20"): - continue - if any(pattern in line for pattern in hidden_patterns): - continue - visible_lines.append(line) - return "\n".join(visible_lines) - - -def load_model_runtime(): - """Load the trained CLIP text tower only once per server process.""" - if RUNTIME_CACHE["model"] is not None: - return - if not BASE_MODEL_PATH.exists() or not TRAINED_CHECKPOINT_PATH.exists(): - raise FileNotFoundError("Base model or trained checkpoint is missing") - write_bundle_log("CLIP load progress", {"stage": "model_base", "progress": 78, "message": "正在载入 CLIP 基础模型。", "status": "loading", "error": ""}) - device = torch.device("cpu") - model, _, _ = open_clip.create_model_and_transforms( - MODEL_NAME, - pretrained=str(BASE_MODEL_PATH), - ) - write_bundle_log("CLIP load progress", {"stage": "model_checkpoint", "progress": 87, "message": "正在载入 CLIP 训练权重。", "status": "loading", "error": ""}) - checkpoint = torch.load(TRAINED_CHECKPOINT_PATH, map_location=device, weights_only=False) - model.load_state_dict(checkpoint["model"]) - model = model.to(device) - model.eval() - RUNTIME_CACHE["model"] = model - RUNTIME_CACHE["tokenizer"] = open_clip.get_tokenizer(MODEL_NAME) - RUNTIME_CACHE["device"] = device - write_bundle_log("CLIP load progress", {"stage": "model_ready", "progress": 96, "message": "CLIP 模型已载入,正在完成初始化。", "status": "loading", "error": ""}) - - -def load_index_runtime(): - """Load or reload the completed listing FAISS index and product metadata.""" - metadata_path = resolve_listing_metadata_path() - if not INDEX_PATH.exists() or not metadata_path.exists(): - raise FileNotFoundError("Listing index is not ready; start the build first") - progress = read_json_file(PROGRESS_PATH, {}) - if progress.get("status") != "complete": - raise RuntimeError( - f"Listing index is still building: {progress.get('completed', 0)}/{progress.get('total', 0)}" - ) - index_mtime = INDEX_PATH.stat().st_mtime - cached_mtime = RUNTIME_CACHE.get("index_mtime") - if RUNTIME_CACHE["index"] is not None and cached_mtime == index_mtime: - return - write_bundle_log("CLIP load progress", {"stage": "index", "progress": 46, "message": "正在载入 FAISS 商品索引。", "status": "loading", "error": ""}) - RUNTIME_CACHE["index"] = read_faiss_index_file(INDEX_PATH) - write_bundle_log("CLIP load progress", {"stage": "metadata", "progress": 60, "message": "正在载入商品元数据。", "status": "loading", "error": ""}) - RUNTIME_CACHE["products"] = normalize_product_metadata_rows(read_json_file(metadata_path, [])) - write_bundle_log("CLIP load progress", {"stage": "prices", "progress": 70, "message": "正在载入价格数据。", "status": "loading", "error": ""}) - RUNTIME_CACHE["prices"] = read_json_file(PRICE_METADATA_PATH, {}) - RUNTIME_CACHE["index_mtime"] = index_mtime - if RUNTIME_CACHE["index"].ntotal != len(RUNTIME_CACHE["products"]): - raise RuntimeError("Listing index count does not match metadata count") - - -def encode_text(query): - """Encode one listing query with the trained CLIP text tower.""" - load_model_runtime() - tokenizer = RUNTIME_CACHE["tokenizer"] - model = RUNTIME_CACHE["model"] - device = RUNTIME_CACHE["device"] - tokens = tokenizer([query]).to(device) - with torch.inference_mode(): - feature = model.encode_text(tokens) - feature = feature / feature.norm(dim=-1, keepdim=True) - return feature.cpu().numpy().astype("float32") - - -def get_rank_window(top_k): - """Clamp the requested result count to a practical range.""" - return max(1, min(int(top_k), 100)) - - -def should_keep_price(product, min_price, max_price): - """Return whether a product is inside the optional USD price range.""" - if min_price is None and max_price is None: - return True - price = product.get("price_usd") - if price is None: - return False - if min_price is not None and float(price) < float(min_price): - return False - if max_price is not None and float(price) > float(max_price): - return False - return True - - -def resolve_product_image_url(product): - """Return the MAINIMAGE/CDN URL from metadata, or an empty string when unavailable.""" - image_fields = [ - "MAINIMAGE", - "mainImage", - "main_image", - "mainimage", - "image_url", - "imgUrl", - "img_url", - ] - for field_name in image_fields: - image_value = str(product.get(field_name, "") or "").strip() - if image_value.lower().startswith(("http://", "https://")): - return image_value - return "" - - -def add_sidecar_fields(product): - """Attach price data and the required CDN image URL to one product.""" - product_id = str(product.get("id", "")) - prices = RUNTIME_CACHE["prices"] or {} - if product_id in prices: - product.update(prices[product_id]) - product["img_url"] = resolve_product_image_url(product) - return product - - -def search_listing_index(query_vector, top_k, min_price, max_price): - """Search the text index and dedupe similar listing families before returning.""" - load_index_runtime() - index = RUNTIME_CACHE["index"] - products = RUNTIME_CACHE["products"] - output_count = get_rank_window(top_k) - if min_price is not None or max_price is not None: - search_count = index.ntotal - else: - search_count = min(index.ntotal, max(output_count * 30, 300)) - scores, indices = index.search(query_vector, search_count) - results = [] - seen_families = set() - seen_images = set() - result_position = 0 - while result_position < len(indices[0]): - product_index = int(indices[0][result_position]) - if product_index < 0 or product_index >= len(products): - result_position += 1 - continue - product = products[product_index].copy() - product = add_sidecar_fields(product) - if not product.get("img_url"): - result_position += 1 - continue - if not should_keep_price(product, min_price, max_price): - result_position += 1 - continue - family_key = str(product.get("family_key", product.get("listing_key", ""))) - image_key = str(product.get("local_img", product.get("image_url", ""))) - if family_key in seen_families or image_key in seen_images: - result_position += 1 - continue - product["similarity"] = round(float(scores[0][result_position]) * 100, 2) - product["rank"] = len(results) + 1 - product["source"] = "Listing" - results.append(product) - seen_families.add(family_key) - seen_images.add(image_key) - if len(results) >= output_count: - break - result_position += 1 - return results - - -def image_to_data_url(image): - """Encode an uploaded image as a compact Kimi-compatible data URL.""" - image_copy = image.copy().convert("RGB") - image_copy.thumbnail((1280, 1280)) - image_buffer = io.BytesIO() - image_copy.save(image_buffer, format="JPEG", quality=85, optimize=True) - encoded = base64.b64encode(image_buffer.getvalue()).decode("ascii") - return f"data:image/jpeg;base64,{encoded}" - - -def read_uploaded_image(contents): - """Read uploaded bytes into a normalized PIL image.""" - if not contents: - return None - return Image.open(io.BytesIO(contents)).convert("RGB") - - -def parse_kimi_prompt_json(response_data): - """Extract the JSON prompt object from one Kimi chat completion response.""" - choices = response_data.get("choices", []) if isinstance(response_data, dict) else [] - if not choices: - raise RuntimeError("Kimi API returned no choices") - content_text = choices[0].get("message", {}).get("content", "") - if isinstance(content_text, dict): - return content_text - try: - return json.loads(content_text) - except (TypeError, json.JSONDecodeError) as error: - raise RuntimeError("Kimi response is not valid JSON") from error - - -def collect_kimi_batch_prompts(raw_plan): - """Return valid prompts in the exact order supplied by one completed Kimi batch.""" - raw_prompts = raw_plan.get("prompts", []) if isinstance(raw_plan, dict) else [] - if not isinstance(raw_prompts, list): - return [] - prompts = [] - skipped_count = 0 - for idx, raw_prompt in enumerate(raw_prompts): - if isinstance(raw_prompt, dict): - prompt_item = { - "zh": str(raw_prompt.get("zh", raw_prompt.get("prompt", ""))).strip(), - "en": str(raw_prompt.get("en", raw_prompt.get("prompt_en", ""))).strip(), - } - else: - prompt_item = {"zh": str(raw_prompt).strip(), "en": ""} - if prompt_item["zh"] or prompt_item["en"]: - prompts.append(prompt_item) - else: - skipped_count += 1 - write_bundle_log("Kimi prompt skipped (empty)", { - "index": idx, - "raw": raw_prompt, - }) - if skipped_count > 0: - write_bundle_log("Kimi batch prompts collected", { - "valid": len(prompts), - "skipped": skipped_count, - "total": len(raw_prompts), - }) - return prompts - - -def call_kimi_prompt_batch(image_data_url, min_price, max_price, kimi_prompt, batch_start, batch_end, kimi_user_prompt=""): - """Ask Kimi for one independent batch whose results keep provider return order.""" - api_key = KIMI_CONFIG["api_key"] - if not api_key: - raise RuntimeError(f"Missing Kimi api_key in {CONFIG_PATH.name} or {KIMI_API_KEY_ENV} environment variable") - price_rule = {"currency": "USD", "min": min_price, "max": max_price} - batch_count = max(1, batch_end - batch_start + 1) - system_prompt = (kimi_prompt or DEFAULT_KIMI_SYSTEM_PROMPT).strip() - user_prompt = str(kimi_user_prompt or "").strip() - schema_guard = ( - "\n\n硬性输出约束:只返回合法JSON对象,不能返回Markdown代码围栏。" - f"JSON只能包含prompts字段;prompts必须是长度为{batch_count}的数组;" - "每个元素只能包含zh和en两个字段,不要输出slot。" - ) - user_text = ( - (user_prompt + "\n\n" if user_prompt else "") - + f"请生成本批次的 {batch_count} 个 CLIP 检索方向。" - "这些结果会按照各批次实际返回先后拼接,不要输出编号。\n" - "Price filter:\n" - + json.dumps(price_rule, ensure_ascii=False) - + "\n只返回JSON,不要Markdown代码围栏。" - ) - content = [{"type": "text", "text": user_text}] - if image_data_url: - content.insert(0, {"type": "image_url", "image_url": {"url": image_data_url}}) - payload = { - "model": KIMI_MODEL, - "messages": [ - {"role": "system", "content": system_prompt + schema_guard}, - {"role": "user", "content": content}, - ], - "thinking": {"type": "disabled"}, - "temperature": KIMI_TEMPERATURE, - "response_format": {"type": "json_object"}, - "max_completion_tokens": min(KIMI_MAX_COMPLETION_TOKENS, 650), - } - request = urllib.request.Request( - KIMI_API_URL, - data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), - headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, - method="POST", - ) - response_data = None - attempt = 0 - while attempt < 3: - try: - write_bundle_log("Kimi prompts POST", { - "endpoint": KIMI_API_URL, - "model": KIMI_MODEL, - "attempt": attempt + 1, - "batch_start": batch_start, - "batch_end": batch_end, - "min_price": min_price, - "max_price": max_price, - }) - with urllib.request.urlopen(request, timeout=120) as response: - response_data = json.loads(response.read().decode("utf-8")) - write_bundle_log("Kimi prompts response", { - "status": "ok", - "model": KIMI_MODEL, - "attempt": attempt + 1, - "batch_start": batch_start, - "batch_end": batch_end, - }) - break - except urllib.error.HTTPError as error: - detail = error.read().decode("utf-8", errors="replace") - write_bundle_log("Kimi prompts HTTP error", { - "status": error.code, - "attempt": attempt + 1, - "batch_start": batch_start, - "batch_end": batch_end, - "detail": detail[:300], - }) - if error.code not in (429, 500, 502, 503, 504) or attempt >= 2: - raise RuntimeError(f"Kimi API HTTP {error.code}: {detail[:500]}") from error - retry_after = error.headers.get("Retry-After") - try: - delay = float(retry_after) if retry_after else 2.0 + attempt * 2.0 - except (TypeError, ValueError): - delay = 2.0 + attempt * 2.0 - time.sleep(min(max(delay, 1.0), 10.0)) - attempt += 1 - except urllib.error.URLError as error: - write_bundle_log("Kimi prompts network error", { - "attempt": attempt + 1, - "batch_start": batch_start, - "batch_end": batch_end, - "reason": str(error.reason), - }) - if attempt >= 2: - raise RuntimeError(f"Kimi API network error: {error.reason}") from error - time.sleep(2.0 + attempt * 2.0) - attempt += 1 - raw_plan = parse_kimi_prompt_json(response_data) - prompts = collect_kimi_batch_prompts(raw_plan) - write_bundle_log("Kimi batch complete", { - "batch_start": batch_start, - "batch_end": batch_end, - "expected_count": batch_end - batch_start + 1, - "actual_count": len(prompts), - "raw_response": response_data, - }) - return prompts - - -def call_kimi_prompts(image, min_price, max_price, kimi_prompt, max_workers=3, on_batch_completed=None, kimi_user_prompt=""): - """Ask Kimi concurrently for ten JSON bundle-product prompts, ordered by completion time.""" - image_data_url = image_to_data_url(image) if image is not None else "" - batch_ranges = KIMI_PROMPT_BATCH_RANGES - safe_workers = max(1, min(int(max_workers or 1), len(batch_ranges))) - completed_prompts = [] - errors = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=safe_workers) as executor: - future_map = {} - for batch_start, batch_end in batch_ranges: - future = executor.submit( - call_kimi_prompt_batch, - image_data_url, - min_price, - max_price, - kimi_prompt, - batch_start, - batch_end, - kimi_user_prompt, - ) - future_map[future] = (batch_start, batch_end) - for future in concurrent.futures.as_completed(future_map): - batch_start, batch_end = future_map[future] - try: - batch_prompts = future.result() - completed_prompts.extend(batch_prompts) - if callable(on_batch_completed) and batch_prompts: - on_batch_completed(batch_start, batch_end, batch_prompts) - except Exception as error: - errors.append(f"{batch_start}-{batch_end}: {error}") - write_bundle_log("Kimi prompt batch failed", { - "batch_start": batch_start, - "batch_end": batch_end, - "error": str(error), - }) - if not completed_prompts and errors: - raise RuntimeError("Kimi parallel prompts failed: " + "; ".join(errors)) - write_bundle_log("Kimi all batches complete", { - "total_prompts": len(completed_prompts), - "expected_total": sum(end - start + 1 for start, end in batch_ranges), - "batch_count": len(batch_ranges), - "errors": errors, - }) - return {"prompts": completed_prompts, "batch_errors": errors, "batch_workers": safe_workers} - - -def normalize_kimi_prompts(raw_plan, fill_missing=True): - """Validate Kimi prompts and optionally fill missing directions.""" - raw_prompts = raw_plan.get("prompts", []) if isinstance(raw_plan, dict) else [] - if not isinstance(raw_prompts, list): - raw_prompts = [] - prompts = [] - for raw_prompt in raw_prompts: - if isinstance(raw_prompt, dict): - prompt_zh = str(raw_prompt.get("zh", raw_prompt.get("prompt", ""))).strip() - prompt_en = str(raw_prompt.get("en", raw_prompt.get("prompt_en", ""))).strip() - else: - prompt_zh = str(raw_prompt).strip() - prompt_en = "" - if not prompt_zh and not prompt_en: - continue - prompts.append({"zh": prompt_zh, "en": prompt_en}) - if len(prompts) >= 10: - break - fallback_text = "related product bundle" - if fill_missing: - while len(prompts) < 10: - prompts.append({"zh": fallback_text, "en": fallback_text}) - return {"prompts": prompts} - - -def search_prompt_groups(prompts, min_price, max_price, top_k, prompt_offset=0): - """Run each Kimi prompt through the listing CLIP index and group results.""" - groups = [] - selected_results = [] - prompt_index = 0 - while prompt_index < len(prompts): - prompt_item = prompts[prompt_index] - recall_text = prompt_item.get("en") or prompt_item.get("zh") or "" - query_vector = encode_text(recall_text) - matches = search_listing_index(query_vector, top_k, min_price, max_price) - group_results = [] - result_index = 0 - while result_index < len(matches): - product = matches[result_index].copy() - product["prompt_index"] = prompt_offset + prompt_index + 1 - product["search_prompt"] = prompt_item.get("zh", "") - product["search_prompt_en"] = prompt_item.get("en", "") - product["prompt_rank"] = result_index + 1 - group_results.append(product) - selected_results.append(product) - result_index += 1 - groups.append( - { - "prompt_index": prompt_offset + prompt_index + 1, - "prompt": prompt_item.get("zh", ""), - "prompt_en": prompt_item.get("en", ""), - "results": group_results, - } - ) - prompt_index += 1 - return groups, selected_results - - -def read_tail(path, max_bytes): - """Read the end of a log file without loading the whole file.""" - if not path.exists(): - return "" - with path.open("rb") as log_file: - log_file.seek(0, os.SEEK_END) - size = log_file.tell() - log_file.seek(max(0, size - max_bytes), os.SEEK_SET) - return log_file.read().decode("utf-8", errors="replace") - - -def is_build_running(): - """Return whether the current build subprocess is still active.""" - process = BUILD_PROCESS.get("process") - if process is None: - return False - return process.poll() is None - - -def start_build_process(batch_size, force_clean): - """Start the listing-index build in the background and append logs.""" - if is_build_running(): - return False - LISTING_INDEX_DIR.mkdir(parents=True, exist_ok=True) - command = [ - sys.executable, - str(BUILD_SCRIPT_PATH), - "--batch-size", - str(max(1, min(int(batch_size), 2048))), - ] - if force_clean: - command.append("--force-clean") - log_file = BUILD_LOG_PATH.open("a", encoding="utf-8") - log_file.write(f"\nserver_start_build {command}\n") - log_file.flush() - BUILD_PROCESS["process"] = subprocess.Popen( - command, - stdout=log_file, - stderr=subprocess.STDOUT, - cwd=str(BUILD_SCRIPT_PATH.parent), - ) - return True - - -def build_status_payload(): - """Return progress, cleaning report, and recent build log for the UI.""" - progress = read_json_file(PROGRESS_PATH, {}) - report = read_json_file(REPORT_PATH, {}) - inferred_running = is_build_running() - if not inferred_running and progress.get("status") == "building": - completed = int(progress.get("completed", 0) or 0) - total = int(progress.get("total", 0) or 0) - inferred_running = total > 0 and completed < total - payload = { - "running": inferred_running, - "progress": progress, - "report": report, - "log": read_tail(BUILD_LOG_PATH, 20000), - "index_exists": INDEX_PATH.exists(), - "metadata_exists": resolve_listing_metadata_path().exists(), - } - if INDEX_PATH.exists(): - payload["index_size_mb"] = round(INDEX_PATH.stat().st_size / 1024 / 1024, 2) - return payload - - -def create_app(): - """Create the legacy FastAPI application for pure listing search.""" - application = FastAPI(title="Pure Listing CLIP Search") - application.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ) - if IMAGE_DIR.exists(): - application.mount("/listing-images", StaticFiles(directory=IMAGE_DIR), name="listing-images") - - @application.middleware("http") - async def log_http_request(request: Request, call_next): - """Write one compact access log line for every API/page request.""" - started_at = time.perf_counter() - skip_access_log = should_skip_access_log(request.url.path) - try: - response = await call_next(request) - except Exception as error: - duration_ms = int((time.perf_counter() - started_at) * 1000) - if not skip_access_log: - append_server_log( - f"{request.method} {request.url.path} ERROR {duration_ms}ms {type(error).__name__}: {error}" - ) - raise - duration_ms = int((time.perf_counter() - started_at) * 1000) - if not skip_access_log: - append_server_log( - f"{request.method} {request.url.path} {response.status_code} {duration_ms}ms" - ) - return response - - @application.get("/api/server/log", response_class=PlainTextResponse) - async def server_log(max_bytes: int = 50000): - """Return the recent local server log as plain text.""" - safe_max_bytes = max(1000, min(int(max_bytes), 1000000)) - return PlainTextResponse(read_server_log_filtered(safe_max_bytes)) - - @application.get("/api/cdn/image") - async def proxy_cdn_image(url: str): - """Fetch one remote CDN image through this server so the request is visible in logs.""" - started_at = time.perf_counter() - try: - safe_url = validate_cdn_image_url(url) - image_bytes, content_type, status_code = await asyncio.to_thread(fetch_cdn_image_bytes, safe_url) - duration_ms = int((time.perf_counter() - started_at) * 1000) - append_server_log(f"CDN GET {safe_url} {status_code} {len(image_bytes)}B {duration_ms}ms") - return Response( - content=image_bytes, - media_type=content_type, - headers={"Cache-Control": "public, max-age=86400"}, - ) - except Exception as error: - duration_ms = int((time.perf_counter() - started_at) * 1000) - append_server_log(f"CDN GET {url} ERROR {duration_ms}ms {type(error).__name__}: {error}") - return Response(status_code=502) - - @application.post("/api/index/build/start") - async def start_index_build( - batch_size: int = Form(256), - force_clean: bool = Form(False), - ): - """Start a background cleaned listing-index build.""" - try: - started = start_build_process(batch_size, force_clean) - return {"started": started, "status": build_status_payload()} - except Exception as error: - return JSONResponse({"error": str(error)}, status_code=400) - - @application.get("/api/index/status") - async def index_status(): - """Return listing-index build and load status.""" - status = build_status_payload() - status["kimi_model"] = KIMI_MODEL - status["kimi_configured"] = bool(KIMI_CONFIG["api_key"]) - if INDEX_PATH.exists() and METADATA_PATH.exists(): - try: - load_index_runtime() - status["vectors"] = RUNTIME_CACHE["index"].ntotal - status["products"] = len(RUNTIME_CACHE["products"]) - status["price_records"] = len(RUNTIME_CACHE["prices"]) - except Exception as error: - status["load_error"] = str(error) - return status - - @application.post("/api/search/text") - async def search_text( - query: str = Form(...), - top_k: int = Form(24), - min_price: float = Form(None), - max_price: float = Form(None), - ): - """Search the cleaned pure-listing index.""" - try: - if not query.strip(): - raise ValueError("Query is empty") - query_vector = encode_text(query.strip()) - results = search_listing_index(query_vector, top_k, min_price, max_price) - return {"results": results} - except Exception as error: - return JSONResponse({"error": str(error)}, status_code=400) - - @application.post("/api/search/prompts") - async def search_prompts(payload: dict = Body(...)): - """Search the listing index with manually edited Kimi prompt JSON.""" - try: - plan = normalize_kimi_prompts(payload, fill_missing=False) - if not plan["prompts"]: - raise ValueError("Edited prompts are empty") - min_price = payload.get("min_price") - max_price = payload.get("max_price") - safe_top_k = max(1, min(int(payload.get("top_k", 1)), 10)) - groups, selected_results = search_prompt_groups( - plan["prompts"], - min_price, - max_price, - safe_top_k, - ) - return { - "plan": plan, - "groups": groups, - "results": selected_results, - "prompts_searched": len(groups), - "results_per_prompt": safe_top_k, - "source": "edited_prompts", - } - except Exception as error: - return JSONResponse({"error": str(error)}, status_code=400) - - @application.post("/api/assemble") - async def assemble_products( - file: UploadFile = File(None), - listing: str = Form(""), - kimi_prompt: str = Form(""), - top_k: int = Form(2), - min_price: float = Form(None), - max_price: float = Form(None), - ): - """Run image-only Kimi JSON prompts, then search the listing CLIP index. - 核心流程(按你要求): - - Kimi 生成 10 个方向(3+3+4 并发) - - 哪个 Kimi 批次先返回,哪个批次先进入 CLIP - - 当前接口保持一次性返回,不使用 SSE - - 最终固定返回 10 个商品,按 Kimi 实际返回顺序排序 - """ - try: - contents = await file.read() if file and file.filename else None - image = read_uploaded_image(contents) - if image is None: - raise ValueError("Please upload an image for Kimi bundle generation") - effective_system_prompt = DEFAULT_KIMI_SYSTEM_PROMPT - effective_user_prompt = kimi_prompt.strip() - safe_top_k = max(1, min(int(top_k), 10)) - batch_groups = [] - selected_results = [] - searched_prompt_count = 0 - - def search_completed_kimi_batch(_batch_start, _batch_end, batch_prompts): - """Search one completed Kimi batch before slower Kimi batches finish.""" - nonlocal searched_prompt_count - batch_plan = normalize_kimi_prompts({"prompts": batch_prompts}, fill_missing=False) - prompts = batch_plan["prompts"] - if not prompts: - return - groups, batch_results = search_prompt_groups( - prompts, - min_price, - max_price, - safe_top_k, - searched_prompt_count, - ) - searched_prompt_count += len(prompts) - batch_groups.extend(groups) - selected_results.extend(batch_results) - - # === Kimi 批次返回后立即进入 CLIP,接口最终一次返回 === - raw_plan = call_kimi_prompts( - image, - min_price, - max_price, - effective_system_prompt, - on_batch_completed=search_completed_kimi_batch, - kimi_user_prompt=effective_user_prompt, - ) - plan = normalize_kimi_prompts(raw_plan) - groups = batch_groups - if not groups: - groups, selected_results = search_prompt_groups(plan["prompts"], min_price, max_price, safe_top_k) - - # === 固定返回 10 个商品 === - if len(selected_results) > 10: - selected_results = selected_results[:10] - elif len(selected_results) < 10: - while len(selected_results) < 10: - selected_results.append({ - "id": f"fallback-{len(selected_results)}", - "title": "related product bundle", - "prompt_index": 1, - "search_prompt": "", - "search_prompt_en": "", - "prompt_rank": 1, - "similarity": 50.0, - "source": "fallback" - }) - - return { - "plan": plan, - "groups": groups, - "results": selected_results, - "prompts_searched": len(groups), - "results_per_prompt": safe_top_k, - "model": KIMI_MODEL, - "thinking": "disabled", - "kimi_prompt": effective_user_prompt, - "return_count": len(selected_results) - } - except Exception as error: - return JSONResponse({"error": str(error)}, status_code=400) - - @application.get("/", response_class=HTMLResponse) - async def listing_page(): - """Serve the standalone pure-listing search page.""" - return HTMLResponse(HTML_PATH.read_text(encoding="utf-8")) - - return application - - -app = create_app() - - -if __name__ == "__main__": - raise SystemExit("9990 HTTP service is disabled. Use stdio_listing_worker.py through the 3000 server.") diff --git a/bundle/clip/work/stdio_listing_worker.py b/bundle/clip/work/stdio_listing_worker.py deleted file mode 100644 index 67614675b710d7168d2ae402e0789187af381b07..0000000000000000000000000000000000000000 --- a/bundle/clip/work/stdio_listing_worker.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Serve CLIP listing operations over stdin/stdout without opening an HTTP port.""" - -import base64 -import json -import sys - -from full_listing_server import ( - call_kimi_prompts, - read_uploaded_image, - normalize_kimi_prompts, - search_prompt_groups, - encode_text, - search_listing_index, - build_status_payload, - load_index_runtime, - load_model_runtime, - RUNTIME_CACHE, - KIMI_CONFIG, - KIMI_MODEL, - DEFAULT_KIMI_SYSTEM_PROMPT, -) - - -def configure_stdio_encoding(): - """Force safe UTF-8 stream writes even when data contains surrogate escapes.""" - for stream in (sys.stdout, sys.stderr): - reconfigure = getattr(stream, "reconfigure", None) - if reconfigure: - reconfigure(encoding="utf-8", errors="backslashreplace") - - -def write_json_line(payload): - """Write one JSON response line and flush immediately for the Node parent.""" - sys.stdout.write(json.dumps(payload, ensure_ascii=True) + "\n") - sys.stdout.flush() - - -def read_optional_float(value): - """Convert one optional numeric value from JSON into a float or None.""" - if value is None or value == "": - return None - return float(value) - - -def read_uploaded_image_from_base64(value): - """Decode one base64 image payload into a normalized PIL image.""" - if not value: - return None - return read_uploaded_image(base64.b64decode(str(value))) - - -def handle_index_status(_payload): - """Return listing-index readiness without requiring an HTTP request.""" - status = build_status_payload() - status["kimi_model"] = KIMI_MODEL - status["kimi_configured"] = bool(KIMI_CONFIG["api_key"]) - try: - load_index_runtime() - status["vectors"] = RUNTIME_CACHE["index"].ntotal - status["products"] = len(RUNTIME_CACHE["products"]) - status["price_records"] = len(RUNTIME_CACHE["prices"]) - except Exception as error: - status["load_error"] = str(error) - return status - - -def handle_warmup(_payload): - """Load the listing index and trained text model before the first search.""" - load_index_runtime() - load_model_runtime() - return { - "vectors": RUNTIME_CACHE["index"].ntotal, - "products": len(RUNTIME_CACHE["products"]), - "model_ready": RUNTIME_CACHE["model"] is not None, - } - - -def handle_search_text(payload): - """Search the listing CLIP index with one manual keyword.""" - query = str(payload.get("query", "")).strip() - if not query: - raise ValueError("Query is empty") - top_k = max(1, min(int(payload.get("top_k", 24)), 100)) - min_price = read_optional_float(payload.get("min_price")) - max_price = read_optional_float(payload.get("max_price")) - query_vector = encode_text(query) - return {"results": search_listing_index(query_vector, top_k, min_price, max_price)} - - -def handle_assemble(payload): - """Run image-only Kimi JSON prompts, then search the listing CLIP index.""" - image = read_uploaded_image_from_base64(payload.get("image_base64")) - if image is None: - raise ValueError("Please provide an image for Kimi bundle generation") - min_price = read_optional_float(payload.get("min_price")) - max_price = read_optional_float(payload.get("max_price")) - effective_system_prompt = str(payload.get("kimi_system_prompt", "")).strip() or DEFAULT_KIMI_SYSTEM_PROMPT - effective_user_prompt = str(payload.get("kimi_prompt", "")).strip() - safe_top_k = max(1, min(int(payload.get("top_k", 2)), 10)) - batch_groups = [] - batch_selected_results = [] - searched_prompt_count = 0 - - def search_completed_kimi_batch(_batch_start, _batch_end, batch_prompts): - """Search one returned Kimi batch immediately so later Kimi calls do not block CLIP.""" - nonlocal searched_prompt_count - batch_plan = normalize_kimi_prompts({"prompts": batch_prompts}, fill_missing=False) - prompts = batch_plan["prompts"] - if not prompts: - return - groups, selected_results = search_prompt_groups( - prompts, - min_price, - max_price, - safe_top_k, - searched_prompt_count, - ) - searched_prompt_count += len(prompts) - batch_groups.extend(groups) - batch_selected_results.extend(selected_results) - - raw_plan = call_kimi_prompts( - image, - min_price, - max_price, - effective_system_prompt, - on_batch_completed=search_completed_kimi_batch, - kimi_user_prompt=effective_user_prompt, - ) - plan = normalize_kimi_prompts(raw_plan) - if batch_groups: - groups = batch_groups - selected_results = batch_selected_results - else: - groups, selected_results = search_prompt_groups(plan["prompts"], min_price, max_price, safe_top_k) - return { - "plan": plan, - "groups": groups, - "results": selected_results, - "prompts_searched": len(groups), - "results_per_prompt": safe_top_k, - "model": KIMI_MODEL, - "thinking": "disabled", - "kimi_prompt": effective_user_prompt, - } - - -def dispatch(payload): - """Route one worker JSON command to the matching CLIP operation.""" - action = str(payload.get("action", "")).strip() - if action == "index_status": - return handle_index_status(payload) - if action == "warmup": - return handle_warmup(payload) - if action == "search_text": - return handle_search_text(payload) - if action == "assemble": - return handle_assemble(payload) - raise ValueError("Unknown CLIP worker action: " + action) - - -def main(): - """Read JSON-line requests forever and return JSON-line responses.""" - for line in sys.stdin: - text = line.strip() - if not text: - continue - request = {} - try: - request = json.loads(text) - request_id = request.get("id") - result = dispatch(request) - write_json_line({"id": request_id, "ok": True, "result": result}) - except Exception as error: - write_json_line({ - "id": request.get("id", ""), - "ok": False, - "error": str(error), - }) - - -if __name__ == "__main__": - configure_stdio_encoding() - main() diff --git a/bundle/python-cpu/Lib/site-packages/PIL/AvifImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/AvifImagePlugin.py deleted file mode 100644 index 2d388c8a83aebb14deebfd01ea1021571f780c23..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/AvifImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:80733b71d411630ea63c782333de241a12bd83f0f1cafcd8ccde4516bed4b5a1 -size 9596 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/BdfFontFile.py b/bundle/python-cpu/Lib/site-packages/PIL/BdfFontFile.py deleted file mode 100644 index 12f2706b06d926b34aeda5cedb3853193e4cd318..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/BdfFontFile.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2c54f5932a79e45789cd0b1fbd64694f2f53009f5485e3fe7bd95e2ddf77cf35 -size 3463 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/BlpImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/BlpImagePlugin.py deleted file mode 100644 index 2ceec1d007e3a7576bc90f2fde1228e3f53354fc..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/BlpImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:522b1467bdc20f8edfd8b19346573e4df6f10dda93915b72a3b8efe6f6104c17 -size 17048 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/BmpImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/BmpImagePlugin.py deleted file mode 100644 index 9a492f95b9c4823bd6d6b1f421e55a648b0c0577..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/BmpImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e23760527cf54ebb06b6e56d73be70ff896666c04f55fdd1047bf99434f9da63 -size 20377 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/BufrStubImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/BufrStubImagePlugin.py deleted file mode 100644 index 6d9586aaf279960fbd72f5f7fce570bae0282f1e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/BufrStubImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3b2a82334b9422e153315099108f4fda20377a83785d540f7c57cdb835bfc9e4 -size 1757 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ContainerIO.py b/bundle/python-cpu/Lib/site-packages/PIL/ContainerIO.py deleted file mode 100644 index 8e772eb97cf82df8e69cbfd23ad42cd674c78f7b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ContainerIO.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:23ac8efd814412a30a035724804133176aff224e69fc68ccf91ad63896234a56 -size 4777 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/CurImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/CurImagePlugin.py deleted file mode 100644 index 22b5014bb3df8e9180c6927a83237b9207f7fc38..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/CurImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3aab7fa3ba3b1d758a01e368073544be1c4a14cbf75b751cf390a73a842e8612 -size 2814 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/DcxImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/DcxImagePlugin.py deleted file mode 100644 index 609b3ab475b50ae5ddbd822e267166d29a36d8cc..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/DcxImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2f6d5cde0557e883e6938cee7ff96db779807917dcb8209c5e1902569b9e7765 -size 2264 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/DdsImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/DdsImagePlugin.py deleted file mode 100644 index 87a2960e30f8a47d08a4be4541eba7620ec53a7d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/DdsImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:eae2016bb3476be917ff549086c18b89d8e911d0f6d3f0900486a90fe2808cac -size 19733 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/EpsImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/EpsImagePlugin.py deleted file mode 100644 index 289e6f0dcaa897500631b8e66776775a8ce8b7f3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/EpsImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3582af51b5623fc766008fa1e3df028a918aaf7073f4f6435fc5d1607f3deb2c -size 17253 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ExifTags.py b/bundle/python-cpu/Lib/site-packages/PIL/ExifTags.py deleted file mode 100644 index 0dc2eddcd0d4c53fbf92bf3d2c94d9ab8da31fba..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ExifTags.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:362cafb665044fbf41322a248a3313a7fabc75e596ce9c9db5371305f272f0a1 -size 10339 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/FitsImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/FitsImagePlugin.py deleted file mode 100644 index b87757a9dbdb82fbe151e57c811da70db9b2adf6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/FitsImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2ff9142b7f02394b95f65a13e64f4366a12df74a3600245d64f63696fb1ed0bd -size 4875 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/FliImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/FliImagePlugin.py deleted file mode 100644 index 9eb2c2b6007f88e05b6b58b7b79237d4ba548c94..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/FliImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2f5bdfe108ec68c57479141399fcb5c27a89f65f61195e65bd320b5bbce7cc3f -size 5113 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/FontFile.py b/bundle/python-cpu/Lib/site-packages/PIL/FontFile.py deleted file mode 100644 index 35a91bda351398b5b341ea4af59c59ecf98d9514..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/FontFile.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a424a370fdf10adc8e780110f3e12f9366e0e479ec49c560400e715108ff0554 -size 4419 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/FpxImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/FpxImagePlugin.py deleted file mode 100644 index 337ed14a70946aaf144cebafd77103a92d3ab64e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/FpxImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b3610e701e56459102379804abd517a576a44e2dfc905322133d8ccd1029c7f5 -size 7724 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/FtexImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/FtexImagePlugin.py deleted file mode 100644 index 74814c40ecc945e1717d4345be79031247d571e1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/FtexImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fb32afbe131373351ede8d1fe07a430bdfc32ba94a1d3e4b0017f4af9fb797ee -size 3685 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/GbrImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/GbrImagePlugin.py deleted file mode 100644 index 50121bb38892ca2084bf41dd2e163f8f3470cd84..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/GbrImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:21bff2df967334e98ca86d92b8eca3c518773b1b78d90a5c11c80b6ba0619ad0 -size 3156 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/GdImageFile.py b/bundle/python-cpu/Lib/site-packages/PIL/GdImageFile.py deleted file mode 100644 index 1eb7d5a6ec34165074a7500ea6fb1c74efb34d07..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/GdImageFile.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ace96a8ea2945fc25827a18b1b3a8792cf85a4033bdb43b2a1d1c636222e9283 -size 2951 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/GifImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/GifImagePlugin.py deleted file mode 100644 index 89647883d50ff5ed090c8e1bd226b536b457e9d1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/GifImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0cf9dd837ba23b412c95765aea4422bbf6e7c2210c6eefde91eab43d4308b0d5 -size 43596 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/GimpGradientFile.py b/bundle/python-cpu/Lib/site-packages/PIL/GimpGradientFile.py deleted file mode 100644 index 756de80879b5a921922c3e6d9f74366ab733d706..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/GimpGradientFile.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cec216d4c915ea4a39964e072555de8121d4df250fc16d695329d1b39cf48990 -size 4137 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/GimpPaletteFile.py b/bundle/python-cpu/Lib/site-packages/PIL/GimpPaletteFile.py deleted file mode 100644 index c8c643471cd19f1bd3649c748991a7fbacda1821..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/GimpPaletteFile.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:126047710f44c42e1fe3bb5a0fb4bb1ba55a324afd1e3ca7955f0d7535d5e624 -size 1935 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/GribStubImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/GribStubImagePlugin.py deleted file mode 100644 index e8cb051af959d8eb811f2e8b9b84e1b012517d4e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/GribStubImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:73c88f12a40e48171616a756122b9cb9b8da5f061f6520761a365761568f0313 -size 1786 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/Hdf5StubImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/Hdf5StubImagePlugin.py deleted file mode 100644 index 1f58be5df03f2b1bb1810c1d3e7df7bea4852836..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/Hdf5StubImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ac831d7fbb83b248c9ab07ba0af18bc3fb6325f875828e6f2929e67ed05d08a8 -size 1768 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/IcnsImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/IcnsImagePlugin.py deleted file mode 100644 index 579f0e8b4b09ea6fd13f430265eb274f12ae6ac5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/IcnsImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4e8092cb968527bffa9e266ccc53917c232bf3f5393babe0775447315aff27fa -size 12798 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/IcoImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/IcoImagePlugin.py deleted file mode 100644 index 8d54108b559d10f03f7055b71423eec1eb961928..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/IcoImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0d9a018f6ccabfbe9293de8f2aaf1e97d69900cd89df050b56a3dea27a3fd33d -size 13499 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/ImImagePlugin.py deleted file mode 100644 index d057dba7d97954893f9074d562c63c996aa9d67c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5fd881490e31eeb6aec5b3889fbda3eefe9301481995da580f970073a67530d0 -size 11992 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/Image.py b/bundle/python-cpu/Lib/site-packages/PIL/Image.py deleted file mode 100644 index 4ab8eb8a86b9c1ca5a7939bdc77c9b61124f9d03..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/Image.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:59f1b98ed7bfd7b516b9f13ffd9dd3bc0aa1a495042accdb5ae4609b41c329d5 -size 158171 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageChops.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageChops.py deleted file mode 100644 index ca0f7ed62e24a0ca9e8963419f5e26581c9b1db8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageChops.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:859f043d43e5408cee82c11e755f2dae4297d230420c66fa0acce66ba65e3194 -size 8257 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageCms.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageCms.py deleted file mode 100644 index c5404a625e621c7e60c11ed48fc296c32c7d4591..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageCms.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0effeeecadc71257d6eabcbd4cf43277ff35c6aeda0e7ba4c60ab67676bf49de -size 41773 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageColor.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageColor.py deleted file mode 100644 index 95f9a86a5ce222d7efb8b7bf3e0fb77877f1b5d3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageColor.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:295faeec79d95abacbdf3b8100e2ea7ab23eeef15c5d3c5d2dea1a6158ec9f02 -size 9761 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageDraw.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageDraw.py deleted file mode 100644 index aa9f933c4bf32f47dc971a723cde845f92a6f9a4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageDraw.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:98aa4388d2629866aad9dd734bec67ad3f9981ab1ed4390281b4c17be9027cf6 -size 36421 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageDraw2.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageDraw2.py deleted file mode 100644 index 135964fd4bc23c985a57016fe87c03db7a78f0cd..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageDraw2.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1b4388cfbb0e5bf4e7af5b431ee8d3c8f54476aa216fb33838e3ade19424550c -size 7470 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageEnhance.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageEnhance.py deleted file mode 100644 index c1ac5492a36175af3e7520d02a3485992c727bc1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageEnhance.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ba00d4d2c963691ff369389bb6e9a54ef7fea851f5c755ba97640ca5cdfce975 -size 3740 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageFile.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageFile.py deleted file mode 100644 index c29c5be3a5ae1fc691a7e3a669f7ed66c736720f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageFile.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e972c9d8785ad1fd44f860a375d8719f3fdff8e46da1cc311950699f93e4f539 -size 30828 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageFilter.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageFilter.py deleted file mode 100644 index f99d78109f45823a1d39e69cbbf28a02664ebcbe..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageFilter.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1702a1b8f1c497a99f816a3aa94509b2ba2ce357ec480308c015e406c6bcc15e -size 19714 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageFont.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageFont.py deleted file mode 100644 index dfd2e49f461520d8971bedd94e53bf1426814fce..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageFont.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e9601111f8817e8387fd792b3643e709a5a11febea43cb49de1732e776ba5f60 -size 65080 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageGrab.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageGrab.py deleted file mode 100644 index 53ad47ac6895bf07661bc6e13b9550b64f033443..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageGrab.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b0df963537d02a3f90a20b6385edd5338efc7b6b28ee2248f30de562d1358530 -size 8475 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageMath.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageMath.py deleted file mode 100644 index fb0cc97f045d3070d3f4d51a32310b16262cf69a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageMath.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d39d8fd05608b184178f75263ff0b04a0509de58e8e6945f14dfe4eac85e4496 -size 10659 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageMode.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageMode.py deleted file mode 100644 index 934b016744118db6bc4ebc7830f659242a835f06..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageMode.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:50b900fa3b02806c31563af62cf48fe367439a3ea759a424f4be142a3eca7d28 -size 2480 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageMorph.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageMorph.py deleted file mode 100644 index dad24c0db6cb898180fed8764b0aff97ec12b607..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageMorph.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:56bd50401e6f198a4655f7d11894287781fbfc095098ede5f81961a5a61e0f22 -size 10673 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageOps.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageOps.py deleted file mode 100644 index 6c5f0de3292a980a8477bbf76c6ec02a1ea4e767..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageOps.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ad146c4c218335150d4099e851f1f78c582f5df6c46648893f0ab56dea38a3e4 -size 26446 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImagePalette.py b/bundle/python-cpu/Lib/site-packages/PIL/ImagePalette.py deleted file mode 100644 index 67bdce15fb45890d01f0ffe8aa8a1421a6fb5624..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImagePalette.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f4a973fbb23a253494a7695f64933a5740f270540a926e91c5fc970d3e14fe7c -size 9498 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImagePath.py b/bundle/python-cpu/Lib/site-packages/PIL/ImagePath.py deleted file mode 100644 index b75127e5150aa3946469df714aea7b97afbb83ca..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImagePath.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6679c9baf40db5b2918429abeb54c4984875bd5579ffdd163043cbf0ea72e65f -size 391 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageQt.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageQt.py deleted file mode 100644 index a311ffdd5a2460ca014beaf85152b77aac52e705..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageQt.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4a9c4778f81687981d8d5206342e061b9bfd577b26426fe73e224372b8671d77 -size 6903 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageSequence.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageSequence.py deleted file mode 100644 index c740c56bcce6ddb5262988ea9b10ea5196f17596..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageSequence.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:82e64751da56af39f3664718d0e6290f3b85d2e15794212edbad283feedefdf2 -size 2341 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageShow.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageShow.py deleted file mode 100644 index 81431b17673a9369c0b7065c815c3578a34d6bf9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageShow.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d660d3b5502fb540e2d414a9eeefa85541c368deecd916c99196f07d0eb62082 -size 10684 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageStat.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageStat.py deleted file mode 100644 index 9f2b8622fc5f3ee1a0658ba8bd896ab14cd826f9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageStat.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5b4b946bc60d0babe1a3095ead2cd923013b46ef8aa76ad6baa18d2c114b672a -size 5662 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageText.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageText.py deleted file mode 100644 index 9a10aa3583cdaaa61dee0f656ac31519dfd69917..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageText.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:082c1145853cadd845f42775d7ac896b6cc63d9b2e80572cf579acb6fcc01d54 -size 19087 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageTk.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageTk.py deleted file mode 100644 index fbc781eaa01f3717d35ca217508fbe488eaafd36..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageTk.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4f1d337fd913681b801af66ea342401eb35b9379fa7b0af86677569d6c592b44 -size 8398 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageTransform.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageTransform.py deleted file mode 100644 index e70f16f87cb0b6eea4c3e860eba261ca8eb82edf..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageTransform.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:66eea8c925c436dabbd76fa792a14ba9a7d83ef891020b209337cb586d0add5f -size 4052 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImageWin.py b/bundle/python-cpu/Lib/site-packages/PIL/ImageWin.py deleted file mode 100644 index 325d502a42488ea650d1de6f2f2aa2db08f7ea18..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImageWin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7d018c99917dc076f4f6d33c7697ccb8de5ac75bfe1c3d38389a645ce6b1ddc7 -size 8345 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/ImtImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/ImtImagePlugin.py deleted file mode 100644 index dcd7a200a7cd60590c7a534ced88575b4bc0cfc9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/ImtImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:afe643cf75e38ba10117c7949caea2b8211912d965a74c4b452210d3f875e141 -size 2768 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/IptcImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/IptcImagePlugin.py deleted file mode 100644 index 35b281c9749003a907fdb94f22f9043d8ba6da7f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/IptcImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e05b6e7b88dd6bba546342e1dbbd8e0df84e0488893992b4410d9ed0f610c228 -size 6663 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/Jpeg2KImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/Jpeg2KImagePlugin.py deleted file mode 100644 index 03a6c39a02d6c342a6af3b54301d1022fb4326b4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/Jpeg2KImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:00e20588471f2455caa9018885d958480c3d23e6d8f087eb421450fc0f6b3c18 -size 15088 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/JpegImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/JpegImagePlugin.py deleted file mode 100644 index 1df3fb3ad83893b0964e5952809b5ff0858eeb5f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/JpegImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:049fc12cf436b101a5622456a1b83ba8b28b54cf8106d3953aed751bb0c08f3f -size 32223 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/JpegPresets.py b/bundle/python-cpu/Lib/site-packages/PIL/JpegPresets.py deleted file mode 100644 index 71df9a3c657b245045696d2f8d4aad7ae4a7e1af..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/JpegPresets.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:51422c2b3bf2cdd3f3b2774e0ddf747aefe5b8ca9ab86d4f261d7450eb90be68 -size 12621 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/McIdasImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/McIdasImagePlugin.py deleted file mode 100644 index 5b5cb80822e12d2c9bcb9cf80c4caf7d89d71ea9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/McIdasImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3fd77592be12b5ed10564c5faa2c77d6c7c60fcb169fe7959982b4c20e83bc8c -size 1955 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/MicImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/MicImagePlugin.py deleted file mode 100644 index c28705bf8130db29813307a9d1c0564ecbe3ec09..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/MicImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b82c463389652053e37eed8a8e2fb34a73da727aa2194646aacfc3b5bdc992d2 -size 2702 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/MpegImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/MpegImagePlugin.py deleted file mode 100644 index ec4d11143e8c4de34e2db6cb5b0b051e679a0d94..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/MpegImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:91e9bdce87ec06d22aeb194c25cb0590e5d370908f1080a4f657214df47da432 -size 2094 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/MpoImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/MpoImagePlugin.py deleted file mode 100644 index 169a334684dee3e55c3a168aaa59e7f4e63c53d4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/MpoImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ffc580b1881e62444336ab32d08ff2c38a72c3471a75dadfd4c5940cd2808636 -size 6987 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/MspImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/MspImagePlugin.py deleted file mode 100644 index 9bfb31289d3612a1acdf182fabae61af7e8e144b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/MspImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bb75dc1140d81926046847543a19e3867a64650264a537520781251e9c4e3e58 -size 6072 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PSDraw.py b/bundle/python-cpu/Lib/site-packages/PIL/PSDraw.py deleted file mode 100644 index f873ef22584565c6645cf7c92750c8690011c3c1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PSDraw.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:eec05f0067c19aa9d958c4e445f2e6e28057b519415c0b984fffcd3f71ad1f0d -size 7230 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PaletteFile.py b/bundle/python-cpu/Lib/site-packages/PIL/PaletteFile.py deleted file mode 100644 index 65ad74f37bb37ab8e16ee3bbc00572e436cfa863..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PaletteFile.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:41d11af7d8cb0be3cf844a86cbaff87c3b9af13cca4925339007987029455fb3 -size 1270 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PalmImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/PalmImagePlugin.py deleted file mode 100644 index dba1013745fe5bbceb97aae3dfbf759ca6f8dd58..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PalmImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2247aeefa9db5ba83130696a516c1b579ec12441abd775465c842f9162b9e265 -size 8965 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PcdImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/PcdImagePlugin.py deleted file mode 100644 index 95410b056bf99a048c375e01c6450e0bdf8105b8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PcdImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:819b027170c8a845532b6cfe1962a9b53db9a455350143a0a6bdc9d3a7233c28 -size 1842 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PcfFontFile.py b/bundle/python-cpu/Lib/site-packages/PIL/PcfFontFile.py deleted file mode 100644 index 1a3baef536da4e26cd7728735fd3cc9628fabfbc..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PcfFontFile.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:50f65bca80f8295af7df1b6da0f6a2db5d63ce8f8f0e92386dc9f26cb0eee75e -size 7240 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PcxImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/PcxImagePlugin.py deleted file mode 100644 index ce8deb520afa4a11d20094ddb25248e2a4db7b98..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PcxImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5e6f1423fe7f01006c89693b07c489add784ffccfb0d20405269cda636e79396 -size 6596 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PdfImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/PdfImagePlugin.py deleted file mode 100644 index 71d864a406ab29931d8d9cf38e8798994b264909..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PdfImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3c3f19e0ea596e3f572f479bed0d9b18bc3846552d77c74f2c2e14e86ff60036 -size 9720 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PdfParser.py b/bundle/python-cpu/Lib/site-packages/PIL/PdfParser.py deleted file mode 100644 index 7cff4482ca10cf9f320d974176bf84b23bb5155d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PdfParser.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c5e04f3c748cc2b7f711208a1689b7c3af0e61e11cb454ee06a209fb5217b901 -size 39753 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PixarImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/PixarImagePlugin.py deleted file mode 100644 index 86004882ed3af38377eede3237b535f6255b403a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PixarImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f919ee2521a1185f0cb14b221d9ab757307157e04d9cadf99018c97e1633c17b -size 1830 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PngImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/PngImagePlugin.py deleted file mode 100644 index 92ddbc0dc597e4756d1dc0eedd49dae2a262ac02..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PngImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:26e6495edc516f7589dea911c330d7f16d4ec96eed78af838da00902e77e6040 -size 53977 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PpmImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/PpmImagePlugin.py deleted file mode 100644 index b415dde28026a81343588727ad5bbc7c3eed2603..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PpmImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:22331d42c009691d13c90bb591ddebe28eaad28e49295c2ae21475d0e7804715 -size 12719 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/PsdImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/PsdImagePlugin.py deleted file mode 100644 index 27f9179507981d3fb9054317e05c6005949f091d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/PsdImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b3ce408b7129bc190d12ce0db3ad408e114a048b134669021878a7e72afb372d -size 9249 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/QoiImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/QoiImagePlugin.py deleted file mode 100644 index d279efe36d33c96bf646a8dc2f71d08068af6527..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/QoiImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c25feb0cc7e1c9c5aae865c1157f31ce8235d5728c6567afd9629288ce2df372 -size 8831 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/SgiImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/SgiImagePlugin.py deleted file mode 100644 index eb5c3c21a73824bd2ec5d01129ffd9100651747d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/SgiImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4f0891938baa8e0836d7e3a2154066044d893d63a30c716654f8141fbbce3b75 -size 6602 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/SpiderImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/SpiderImagePlugin.py deleted file mode 100644 index 3540b4c86b1c3f7e60475a6b1ad771bd8bbb8959..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/SpiderImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b0087770827cbc65aee9619de92721c9ca2aec36a338fa25680a31abbea5b68c -size 10626 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/SunImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/SunImagePlugin.py deleted file mode 100644 index 90345594eb607f8b427d2eeaf374304398ab8919..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/SunImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:60a604bee1b84149227acdf838ab5758a4d8499f14deacdc13fbdd4eb3ae46cc -size 4734 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/TarIO.py b/bundle/python-cpu/Lib/site-packages/PIL/TarIO.py deleted file mode 100644 index c0643c5744cc54257ce6f83ece1da79e33baa84e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/TarIO.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2bab4b9050dc69aaf7bdbe84c986fb1b8a15101e35c7f2ccb8625e513f70e434 -size 1503 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/TgaImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/TgaImagePlugin.py deleted file mode 100644 index a71f0167d966997fbfdb029c3dfd2a85fb2a0acf..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/TgaImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:52bbc8363a079783087f2c5fe5b2f75519425af38deee0e92507e6cf7c939588 -size 8017 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/TiffImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/TiffImagePlugin.py deleted file mode 100644 index 4f8dcec64b1398b77c954683e3c5aa13f42a0bfd..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/TiffImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7c7c771b1358c4096239c5973199672870c7e4cc34c4ce33a391695a7e251df7 -size 88301 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/TiffTags.py b/bundle/python-cpu/Lib/site-packages/PIL/TiffTags.py deleted file mode 100644 index a7e70c267bcb129ba25309e1afd61c7e94b396f8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/TiffTags.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:16c76a6c8409388ee2846ecc67f2b8614403678ce8fab6986e28550dee3653d1 -size 17772 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/WalImageFile.py b/bundle/python-cpu/Lib/site-packages/PIL/WalImageFile.py deleted file mode 100644 index 3d9a7a17e65b6adc7a84e85bf6f3192b79dedd96..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/WalImageFile.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:70b442298cac9c4d2fe59fd47745ded180de338988c45197a83231f157ae31c6 -size 5891 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/WebPImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/WebPImagePlugin.py deleted file mode 100644 index b82dc5af343f723f3efb10d06234aca6c5cbb8be..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/WebPImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c7022669e4536810944b2d1a5a69a18dc7176706cbd36363994b0fcc2003d69c -size 10289 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/WmfImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/WmfImagePlugin.py deleted file mode 100644 index b4b51091d1279866a8ed4b4c183fe0ea05fa2a46..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/WmfImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:136d180c964920aad44327ed3a6ac9ac0b97ba60c6662c231f237620a282c9b8 -size 5390 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/XVThumbImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/XVThumbImagePlugin.py deleted file mode 100644 index 5d315452b8589a5dd548b8dc877e3d1d788d72f4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/XVThumbImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:384b448508022cb82fc7c275888f75d533338c9e624e538ea90a4b431d5bcd0e -size 2209 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/XbmImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/XbmImagePlugin.py deleted file mode 100644 index ff0a362f2d380ec72063e7b94b40a580d24b1255..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/XbmImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c1cd0da5f4f3952871f589a2d33c28d1a5598c36b8fc1ecd1221b64e008eeb94 -size 2767 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/XpmImagePlugin.py b/bundle/python-cpu/Lib/site-packages/PIL/XpmImagePlugin.py deleted file mode 100644 index a8c4df0eae552cbf4acf9ae14c7e1c4330434b2e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/XpmImagePlugin.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:232f0ac0e8e7546541c5f4f6c6d0f737050a52dcec0f678753b3aeb48461a9d9 -size 4539 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/__init__.py b/bundle/python-cpu/Lib/site-packages/PIL/__init__.py deleted file mode 100644 index 1d761dc5c136aa918b7f4e2cc62aa7a72e525c65..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3fffff8502f4b59ca61d39cce713d55ffef046954262fce1abac3e97f3803734 -size 2122 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/__main__.py b/bundle/python-cpu/Lib/site-packages/PIL/__main__.py deleted file mode 100644 index d3b7630099832787ba98d42c23f710c65652f9a7..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5fc788a469661df9e9ef36b3a7999d6afdb6f08b5c7f696488c3f4b4b53a5fd7 -size 140 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_avif.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/PIL/_avif.cp310-win_amd64.pyd deleted file mode 100644 index 011f8c16458782b909772151e7addd51b9f988e9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_avif.cp310-win_amd64.pyd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:df0c617422668f1b1b5eece35a0689183942402a16632689d60c3d6488679bc8 -size 7893504 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_avif.pyi b/bundle/python-cpu/Lib/site-packages/PIL/_avif.pyi deleted file mode 100644 index ed6235fb86066e32440b939e0234d2ec70b33d44..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_avif.pyi +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cc3f2f0283c2f1a1085637dc90bb45b24456e6c6a255e977fac254036a476867 -size 66 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_binary.py b/bundle/python-cpu/Lib/site-packages/PIL/_binary.py deleted file mode 100644 index 2e6b5f1d95f7e7911e1bd6219f3a166fd6fc0661..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_binary.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ba6dc47fa221e747297ee81afcbdaef2ea9be664c7a4b51f5382563132e0b20b -size 2664 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_deprecate.py b/bundle/python-cpu/Lib/site-packages/PIL/_deprecate.py deleted file mode 100644 index 2f22c307c872fc45ae4e61f38f50450e900688b9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_deprecate.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:522ac3970b3279feaeeac2ffae51379a72ad45fd666af9a7864b7fd4f03b43c0 -size 2106 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imaging.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/PIL/_imaging.cp310-win_amd64.pyd deleted file mode 100644 index 2887913747c47c3f31a514d988b1b1bd3aafe879..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imaging.cp310-win_amd64.pyd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:16a7c881fd3624c8428078ce24460861e91fc5cc43de1b3614dc2531fe327e2e -size 2632192 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imaging.pyi b/bundle/python-cpu/Lib/site-packages/PIL/_imaging.pyi deleted file mode 100644 index 89ad83407337d0f44e639c4a9da9ee74e9e468a3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imaging.pyi +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8d2446dead5dc2204ca8ec0884bb5305edcb30ac2c8b76244dc0dfb0a577d84d -size 924 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imagingcms.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/PIL/_imagingcms.cp310-win_amd64.pyd deleted file mode 100644 index 71249379a0ddddb43b050114d3d4c34726e54a6a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imagingcms.cp310-win_amd64.pyd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c92c55b43a774b8787a040afdd70c1c086daad5780c73efb35b029c219a03d5f -size 273408 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imagingcms.pyi b/bundle/python-cpu/Lib/site-packages/PIL/_imagingcms.pyi deleted file mode 100644 index 23109a0b6faf37463e289e5d2867db80acaa5d55..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imagingcms.pyi +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e7f48329eecac13f5a6f911e1bec27e85e6cbff7ecd28db5db1a0f066dc7f88a -size 4576 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imagingft.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/PIL/_imagingft.cp310-win_amd64.pyd deleted file mode 100644 index 824e8a52d8d59dee3b3cb772eb2c3941243fb996..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imagingft.cp310-win_amd64.pyd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2c5951d8d3d6f2640545afcdd059f21f65e8571e24789f76790c18d501bf505c -size 2173952 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imagingft.pyi b/bundle/python-cpu/Lib/site-packages/PIL/_imagingft.pyi deleted file mode 100644 index 10fab4e7adbf7bb014179846d16942deee083ee0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imagingft.pyi +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:535ae9c2a417d6150bbd3028f1f634fc8ad79e0f953ee09755a581523da9d73d -size 1903 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imagingmath.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/PIL/_imagingmath.cp310-win_amd64.pyd deleted file mode 100644 index 24f40fe08c8dafd1fa67d48121603102cb43611a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imagingmath.cp310-win_amd64.pyd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:91dbb83922daaa0a601b8a8282be60ebf7633a5b743565c3b579fcafda9b9094 -size 21504 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imagingmath.pyi b/bundle/python-cpu/Lib/site-packages/PIL/_imagingmath.pyi deleted file mode 100644 index ed6235fb86066e32440b939e0234d2ec70b33d44..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imagingmath.pyi +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cc3f2f0283c2f1a1085637dc90bb45b24456e6c6a255e977fac254036a476867 -size 66 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imagingmorph.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/PIL/_imagingmorph.cp310-win_amd64.pyd deleted file mode 100644 index 61a714ff6604b320a8608e0ebd91d127c509d8f4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imagingmorph.cp310-win_amd64.pyd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:25f7f3524b992079f9167861e0f41298cb93f4a028095c02bf1e48d6e49bca16 -size 12288 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imagingmorph.pyi b/bundle/python-cpu/Lib/site-packages/PIL/_imagingmorph.pyi deleted file mode 100644 index ed6235fb86066e32440b939e0234d2ec70b33d44..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imagingmorph.pyi +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cc3f2f0283c2f1a1085637dc90bb45b24456e6c6a255e977fac254036a476867 -size 66 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imagingtk.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/PIL/_imagingtk.cp310-win_amd64.pyd deleted file mode 100644 index 812d27ee4e12dd4d3d0223d752604d0f764a534c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imagingtk.cp310-win_amd64.pyd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:832f6a7f22092fdc7962842170ebadd6ab844dc1f7b4d5582cdf547217ab00be -size 14848 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_imagingtk.pyi b/bundle/python-cpu/Lib/site-packages/PIL/_imagingtk.pyi deleted file mode 100644 index ed6235fb86066e32440b939e0234d2ec70b33d44..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_imagingtk.pyi +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cc3f2f0283c2f1a1085637dc90bb45b24456e6c6a255e977fac254036a476867 -size 66 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_tkinter_finder.py b/bundle/python-cpu/Lib/site-packages/PIL/_tkinter_finder.py deleted file mode 100644 index f382f0f8b56a0c82dbc6a03972eb5441cb893cbb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_tkinter_finder.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1f5530ddd57b8161c88ffa2c527245c56875691b10d8c74aa4d3f7bd0c10716d -size 558 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_typing.py b/bundle/python-cpu/Lib/site-packages/PIL/_typing.py deleted file mode 100644 index f9656e70dd698527b3eb44ecf08d2f7b8990484a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_typing.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b7e3154b97ffa5174bf8cd3bd9866a957141fc3f33d4a3a5f0470ab13fddf6ce -size 964 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_util.py b/bundle/python-cpu/Lib/site-packages/PIL/_util.py deleted file mode 100644 index 332a93718dde0b603e0e17b87781220cc11fbf15..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_util.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:78ff087e658999b13f3ea6d425be2a4ea18d2e2e334f44d369daebaa6ca483b6 -size 713 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_version.py b/bundle/python-cpu/Lib/site-packages/PIL/_version.py deleted file mode 100644 index 49d987d86683f4c2c1c061d46eb6c8f81bbaf7c6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_version.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8367607a728dc206da64e3de6b24f826f3030d7f3c053f25c3b3a1fde94fb9d7 -size 91 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_webp.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/PIL/_webp.cp310-win_amd64.pyd deleted file mode 100644 index 2596559d4bb376dbe52ba693c9b1736e160539f4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_webp.cp310-win_amd64.pyd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b2b263bd00abe0c1c120efca36f8050c161ce6dbbdb20bf940cc874cae01bac2 -size 416256 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/_webp.pyi b/bundle/python-cpu/Lib/site-packages/PIL/_webp.pyi deleted file mode 100644 index ed6235fb86066e32440b939e0234d2ec70b33d44..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/_webp.pyi +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cc3f2f0283c2f1a1085637dc90bb45b24456e6c6a255e977fac254036a476867 -size 66 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/features.py b/bundle/python-cpu/Lib/site-packages/PIL/features.py deleted file mode 100644 index 7ea88592088a5669a19342b7d16eea2a125576ac..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/features.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cbf7a5342ce007a46030ff4031fdc274e3f78dda0de21a38bf1a122b383a7967 -size 11118 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/py.typed b/bundle/python-cpu/Lib/site-packages/PIL/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/PIL/report.py b/bundle/python-cpu/Lib/site-packages/PIL/report.py deleted file mode 100644 index c98008d75ab0366b6db410bd80e6851b072a6226..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/PIL/report.py +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ea6ecd3afd5adb8e7bed9889a315fcf62a7925e3a07f63b517de5febed4ae5a3 -size 105 diff --git a/bundle/python-cpu/Lib/site-packages/_distutils_hack/__init__.py b/bundle/python-cpu/Lib/site-packages/_distutils_hack/__init__.py deleted file mode 100644 index b320f600d2b359ce1fca91a2d5f24fe6be3545e4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/_distutils_hack/__init__.py +++ /dev/null @@ -1,239 +0,0 @@ -# don't import any costly modules -import os -import sys - -report_url = ( - "https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml" -) - - -def warn_distutils_present(): - if 'distutils' not in sys.modules: - return - import warnings - - warnings.warn( - "Distutils was imported before Setuptools, but importing Setuptools " - "also replaces the `distutils` module in `sys.modules`. This may lead " - "to undesirable behaviors or errors. To avoid these issues, avoid " - "using distutils directly, ensure that setuptools is installed in the " - "traditional way (e.g. not an editable install), and/or make sure " - "that setuptools is always imported before distutils." - ) - - -def clear_distutils(): - if 'distutils' not in sys.modules: - return - import warnings - - warnings.warn( - "Setuptools is replacing distutils. Support for replacing " - "an already imported distutils is deprecated. In the future, " - "this condition will fail. " - f"Register concerns at {report_url}" - ) - mods = [ - name - for name in sys.modules - if name == "distutils" or name.startswith("distutils.") - ] - for name in mods: - del sys.modules[name] - - -def enabled(): - """ - Allow selection of distutils by environment variable. - """ - which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local') - if which == 'stdlib': - import warnings - - warnings.warn( - "Reliance on distutils from stdlib is deprecated. Users " - "must rely on setuptools to provide the distutils module. " - "Avoid importing distutils or import setuptools first, " - "and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. " - f"Register concerns at {report_url}" - ) - return which == 'local' - - -def ensure_local_distutils(): - import importlib - - clear_distutils() - - # With the DistutilsMetaFinder in place, - # perform an import to cause distutils to be - # loaded from setuptools._distutils. Ref #2906. - with shim(): - importlib.import_module('distutils') - - # check that submodules load as expected - core = importlib.import_module('distutils.core') - assert '_distutils' in core.__file__, core.__file__ - assert 'setuptools._distutils.log' not in sys.modules - - -def do_override(): - """ - Ensure that the local copy of distutils is preferred over stdlib. - - See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 - for more motivation. - """ - if enabled(): - warn_distutils_present() - ensure_local_distutils() - - -class _TrivialRe: - def __init__(self, *patterns) -> None: - self._patterns = patterns - - def match(self, string): - return all(pat in string for pat in self._patterns) - - -class DistutilsMetaFinder: - def find_spec(self, fullname, path, target=None): - # optimization: only consider top level modules and those - # found in the CPython test suite. - if path is not None and not fullname.startswith('test.'): - return None - - method_name = 'spec_for_{fullname}'.format(**locals()) - method = getattr(self, method_name, lambda: None) - return method() - - def spec_for_distutils(self): - if self.is_cpython(): - return None - - import importlib - import importlib.abc - import importlib.util - - try: - mod = importlib.import_module('setuptools._distutils') - except Exception: # noqa: BLE001 # intentional broad fallback - # There are a couple of cases where setuptools._distutils - # may not be present: - # - An older Setuptools without a local distutils is - # taking precedence. Ref #2957. - # - Path manipulation during sitecustomize removes - # setuptools from the path but only after the hook - # has been loaded. Ref #2980. - # In either case, fall back to stdlib behavior. - return None - - class DistutilsLoader(importlib.abc.Loader): - def create_module(self, spec): - mod.__name__ = 'distutils' - return mod - - def exec_module(self, module): - pass - - return importlib.util.spec_from_loader( - 'distutils', DistutilsLoader(), origin=mod.__file__ - ) - - @staticmethod - def is_cpython(): - """ - Suppress supplying distutils for CPython (build and tests). - Ref #2965 and #3007. - """ - return os.path.isfile('pybuilddir.txt') - - def spec_for_pip(self): - """ - Ensure stdlib distutils when running under pip. - See pypa/pip#8761 for rationale. - """ - if sys.version_info >= (3, 12) or self.pip_imported_during_build(): - return - clear_distutils() - self.spec_for_distutils = lambda: None - - @classmethod - def pip_imported_during_build(cls): - """ - Detect if pip is being imported in a build script. Ref #2355. - """ - import traceback - - return any( - cls.frame_file_is_setup(frame) for frame, line in traceback.walk_stack(None) - ) - - @staticmethod - def frame_file_is_setup(frame): - """ - Return True if the indicated frame suggests a setup.py file. - """ - # some frames may not have __file__ (#2940) - return frame.f_globals.get('__file__', '').endswith('setup.py') - - def spec_for_sensitive_tests(self): - """ - Ensure stdlib distutils when running select tests under CPython. - - python/cpython#91169 - """ - clear_distutils() - self.spec_for_distutils = lambda: None - - sensitive_tests = ( - [ - 'test.test_distutils', - 'test.test_peg_generator', - 'test.test_importlib', - ] - if sys.version_info < (3, 10) - else [ - 'test.test_distutils', - ] - ) - - -for name in DistutilsMetaFinder.sensitive_tests: - setattr( - DistutilsMetaFinder, - f'spec_for_{name}', - DistutilsMetaFinder.spec_for_sensitive_tests, - ) - - -DISTUTILS_FINDER = DistutilsMetaFinder() - - -def add_shim(): - DISTUTILS_FINDER in sys.meta_path or insert_shim() - - -class shim: - def __enter__(self) -> None: - insert_shim() - - def __exit__(self, exc: object, value: object, tb: object) -> None: - _remove_shim() - - -def insert_shim(): - sys.meta_path.insert(0, DISTUTILS_FINDER) - - -def _remove_shim(): - try: - sys.meta_path.remove(DISTUTILS_FINDER) - except ValueError: - pass - - -if sys.version_info < (3, 12): - # DistutilsMetaFinder can only be disabled in Python < 3.12 (PEP 632) - remove_shim = _remove_shim diff --git a/bundle/python-cpu/Lib/site-packages/_distutils_hack/override.py b/bundle/python-cpu/Lib/site-packages/_distutils_hack/override.py deleted file mode 100644 index 2cc433a4a55e3b41fa31089918fb62096092f89f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/_distutils_hack/override.py +++ /dev/null @@ -1 +0,0 @@ -__import__('_distutils_hack').do_override() diff --git a/bundle/python-cpu/Lib/site-packages/_yaml/__init__.py b/bundle/python-cpu/Lib/site-packages/_yaml/__init__.py deleted file mode 100644 index 7baa8c4b68127d5cdf0be9a799429e61347c2694..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/_yaml/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -# This is a stub package designed to roughly emulate the _yaml -# extension module, which previously existed as a standalone module -# and has been moved into the `yaml` package namespace. -# It does not perfectly mimic its old counterpart, but should get -# close enough for anyone who's relying on it even when they shouldn't. -import yaml - -# in some circumstances, the yaml module we imoprted may be from a different version, so we need -# to tread carefully when poking at it here (it may not have the attributes we expect) -if not getattr(yaml, '__with_libyaml__', False): - from sys import version_info - - exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError - raise exc("No module named '_yaml'") -else: - from yaml._yaml import * - import warnings - warnings.warn( - 'The _yaml extension module is now located at yaml._yaml' - ' and its location is subject to change. To use the' - ' LibYAML-based parser and emitter, import from `yaml`:' - ' `from yaml import CLoader as Loader, CDumper as Dumper`.', - DeprecationWarning - ) - del warnings - # Don't `del yaml` here because yaml is actually an existing - # namespace member of _yaml. - -__name__ = '_yaml' -# If the module is top-level (i.e. not a part of any specific package) -# then the attribute should be set to ''. -# https://docs.python.org/3.8/library/types.html -__package__ = '' diff --git a/bundle/python-cpu/Lib/site-packages/ada92cb5d92a588d1b93__mypyc.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/ada92cb5d92a588d1b93__mypyc.cp310-win_amd64.pyd deleted file mode 100644 index f88c28ce228a41f6e18f76501cd3348d9a46d2e9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ada92cb5d92a588d1b93__mypyc.cp310-win_amd64.pyd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:df449f6b7397e3a83f84d08ff38efac2c45527944221d1d6f6e78228a05c8ce7 -size 223232 diff --git a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/METADATA deleted file mode 100644 index d162dc81d27f56c4b7add382e32082f311ba1f09..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/METADATA +++ /dev/null @@ -1,144 +0,0 @@ -Metadata-Version: 2.4 -Name: annotated-doc -Version: 0.0.5 -Summary: Document parameters, class attributes, return types, and variables inline, with Annotated. -Author-Email: =?utf-8?q?Sebasti=C3=A1n_Ram=C3=ADrez?= -License-Expression: MIT -License-File: LICENSE -Classifier: Intended Audience :: Information Technology -Classifier: Intended Audience :: System Administrators -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python -Classifier: Topic :: Internet -Classifier: Topic :: Software Development :: Libraries :: Application Frameworks -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Topic :: Software Development :: Libraries -Classifier: Topic :: Software Development -Classifier: Typing :: Typed -Classifier: Development Status :: 4 - Beta -Classifier: Intended Audience :: Developers -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Project-URL: Homepage, https://github.com/fastapi/annotated-doc -Project-URL: Documentation, https://github.com/fastapi/annotated-doc -Project-URL: Repository, https://github.com/fastapi/annotated-doc -Project-URL: Issues, https://github.com/fastapi/annotated-doc/issues -Project-URL: Changelog, https://github.com/fastapi/annotated-doc/release-notes.md -Requires-Python: >=3.9 -Description-Content-Type: text/markdown - -# Annotated Doc - -Document parameters, class attributes, return types, and variables inline, with `Annotated`. - - - Test - - - Coverage - - - Package version - - - Supported Python versions - - -## Installation - -```bash -pip install annotated-doc -``` - -Or with `uv`: - -```Python -uv add annotated-doc -``` - -## Usage - -Import `Doc` and pass a single literal string with the documentation for the specific parameter, class attribute, return type, or variable. - -For example, to document a parameter `name` in a function `hi` you could do: - -```Python -from typing import Annotated - -from annotated_doc import Doc - -def hi(name: Annotated[str, Doc("Who to say hi to")]) -> None: - print(f"Hi, {name}!") -``` - -You can also use it to document class attributes: - -```Python -from typing import Annotated - -from annotated_doc import Doc - -class User: - name: Annotated[str, Doc("The user's name")] - age: Annotated[int, Doc("The user's age")] -``` - -The same way, you could document return types and variables, or anything that could have a type annotation with `Annotated`. - -## Who Uses This - -`annotated-doc` was made for: - -* [FastAPI](https://fastapi.tiangolo.com/) -* [Typer](https://typer.tiangolo.com/) -* [SQLModel](https://sqlmodel.tiangolo.com/) -* [Asyncer](https://asyncer.tiangolo.com/) - -`annotated-doc` is supported by [griffe-typingdoc](https://github.com/mkdocstrings/griffe-typingdoc), which powers reference documentation like the one in the [FastAPI Reference](https://fastapi.tiangolo.com/reference/). - -## Reasons not to use `annotated-doc` - -You are already comfortable with one of the existing docstring formats, like: - -* Sphinx -* numpydoc -* Google -* Keras - -Your team is already comfortable using them. - -You prefer having the documentation about parameters all together in a docstring, separated from the code defining them. - -You care about a specific set of users, using one specific editor, and that editor already has support for the specific docstring format you use. - -## Reasons to use `annotated-doc` - -* No micro-syntax to learn for newcomers, it’s **just Python** syntax. -* **Editing** would be already fully supported by default by any editor (current or future) supporting Python syntax, including syntax errors, syntax highlighting, etc. -* **Rendering** would be relatively straightforward to implement by static tools (tools that don't need runtime execution), as the information can be extracted from the AST they normally already create. -* **Deduplication of information**: the name of a parameter would be defined in a single place, not duplicated inside of a docstring. -* **Elimination** of the possibility of having **inconsistencies** when removing a parameter or class variable and **forgetting to remove** its documentation. -* **Minimization** of the probability of adding a new parameter or class variable and **forgetting to add its documentation**. -* **Elimination** of the possibility of having **inconsistencies** between the **name** of a parameter in the **signature** and the name in the docstring when it is renamed. -* **Access** to the documentation string for each symbol at **runtime**, including existing (older) Python versions. -* A more formalized way to document other symbols, like type aliases, that could use Annotated. -* **Support** for apps using FastAPI, Typer and others. -* **AI Accessibility**: AI tools will have an easier way understanding each parameter as the distance from documentation to parameter is much closer. - -## History - -I ([@tiangolo](https://github.com/tiangolo)) originally wanted for this to be part of the Python standard library (in [PEP 727](https://peps.python.org/pep-0727/)), but the proposal was withdrawn as there was a fair amount of negative feedback and opposition. - -The conclusion was that this was better done as an external effort, in a third-party library. - -So, here it is, with a simpler approach, as a third-party library, in a way that can be used by others, starting with FastAPI and friends. - -## License - -This project is licensed under the terms of the MIT license. diff --git a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/RECORD deleted file mode 100644 index a3ed2855e5add4a3f38ea07e9289fe985c98b189..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/RECORD +++ /dev/null @@ -1,10 +0,0 @@ -annotated_doc-0.0.5.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -annotated_doc-0.0.5.dist-info/METADATA,sha256=jCTXDF2RU9IhIzU6kGBC_XYD3y0VwhlikLD4qWJVjEY,6516 -annotated_doc-0.0.5.dist-info/RECORD,, -annotated_doc-0.0.5.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -annotated_doc-0.0.5.dist-info/WHEEL,sha256=VP-D4TPS230sME9Z3vb3INXvo1yt0924YRm5AOsk_dE,90 -annotated_doc-0.0.5.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34 -annotated_doc-0.0.5.dist-info/licenses/LICENSE,sha256=__Fwd5pqy_ZavbQFwIfxzuF4ZpHkqWpANFF-SlBKDN8,1086 -annotated_doc/__init__.py,sha256=UqEV5evgeySQ5yvJQwfwTqwvMGfIwbxRPP4gf-o2-LA,52 -annotated_doc/main.py,sha256=5Zfvxv80SwwLqpRW73AZyZyiM4bWma9QWRbp_cgD20s,1075 -annotated_doc/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/WHEEL deleted file mode 100644 index e651d8efb5fb543a06227b0ebfab1007f9e19326..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: pdm-backend (2.4.9) -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/entry_points.txt b/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/entry_points.txt deleted file mode 100644 index c3ad4726d437022e5c606a4206ffb6007347a008..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/entry_points.txt +++ /dev/null @@ -1,4 +0,0 @@ -[console_scripts] - -[gui_scripts] - diff --git a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/licenses/LICENSE b/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/licenses/LICENSE deleted file mode 100644 index 7a254464cc78ccea32b3ded00513c44c4e4da412..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_doc-0.0.5.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2025 Sebastián Ramírez - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/bundle/python-cpu/Lib/site-packages/annotated_doc/__init__.py b/bundle/python-cpu/Lib/site-packages/annotated_doc/__init__.py deleted file mode 100644 index d6f729253cbdcfad8b1aa27513d4de07e443cf05..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_doc/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .main import Doc as Doc - -__version__ = "0.0.5" diff --git a/bundle/python-cpu/Lib/site-packages/annotated_doc/main.py b/bundle/python-cpu/Lib/site-packages/annotated_doc/main.py deleted file mode 100644 index 7063c59e4500a1d02bfc9b41887f9e95f8163507..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_doc/main.py +++ /dev/null @@ -1,36 +0,0 @@ -class Doc: - """Define the documentation of a type annotation using `Annotated`, to be - used in class attributes, function and method parameters, return values, - and variables. - - The value should be a positional-only string literal to allow static tools - like editors and documentation generators to use it. - - This complements docstrings. - - The string value passed is available in the attribute `documentation`. - - Example: - - ```Python - from typing import Annotated - from annotated_doc import Doc - - def hi(name: Annotated[str, Doc("Who to say hi to")]) -> None: - print(f"Hi, {name}!") - ``` - """ - - def __init__(self, documentation: str, /) -> None: - self.documentation = documentation - - def __repr__(self) -> str: - return f"Doc({self.documentation!r})" - - def __hash__(self) -> int: - return hash(self.documentation) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Doc): - return NotImplemented - return self.documentation == other.documentation diff --git a/bundle/python-cpu/Lib/site-packages/annotated_doc/py.typed b/bundle/python-cpu/Lib/site-packages/annotated_doc/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/METADATA deleted file mode 100644 index eb5a08847f8a91e58ad8325a7059715e9be44ed6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/METADATA +++ /dev/null @@ -1,295 +0,0 @@ -Metadata-Version: 2.4 -Name: annotated-types -Version: 0.8.0 -Summary: Reusable constraint types to use with typing.Annotated -Project-URL: Homepage, https://github.com/annotated-types/annotated-types -Project-URL: Source, https://github.com/annotated-types/annotated-types -Project-URL: Changelog, https://github.com/annotated-types/annotated-types/releases -Author-email: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com>, Samuel Colvin , Zac Hatfield-Dodds -License-Expression: MIT -License-File: LICENSE -Classifier: Development Status :: 4 - Beta -Classifier: Environment :: Console -Classifier: Environment :: MacOS X -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Information Technology -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: POSIX :: Linux -Classifier: Operating System :: Unix -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Typing :: Typed -Requires-Python: >=3.10 -Description-Content-Type: text/markdown - -# annotated-types - -[![CI](https://github.com/annotated-types/annotated-types/workflows/CI/badge.svg?event=push)](https://github.com/annotated-types/annotated-types/actions?query=event%3Apush+branch%3Amain+workflow%3ACI) -[![pypi](https://img.shields.io/pypi/v/annotated-types.svg)](https://pypi.python.org/pypi/annotated-types) -[![versions](https://img.shields.io/pypi/pyversions/annotated-types.svg)](https://github.com/annotated-types/annotated-types) -[![license](https://img.shields.io/github/license/annotated-types/annotated-types.svg)](https://github.com/annotated-types/annotated-types/blob/main/LICENSE) - -[PEP-593](https://peps.python.org/pep-0593/) added `typing.Annotated` as a way of -adding context-specific metadata to existing types, and specifies that -`Annotated[T, x]` _should_ be treated as `T` by any tool or library without special -logic for `x`. - -This package provides metadata objects which can be used to represent common -constraints such as upper and lower bounds on scalar values and collection sizes, -a `Predicate` marker for runtime checks, and -descriptions of how we intend these metadata to be interpreted. In some cases, -we also note alternative representations which do not require this package. - -## Install - -```bash -pip install annotated-types -``` - -## Examples - -```python -from typing import Annotated -from annotated_types import Gt, Len, Predicate - -class MyClass: - age: Annotated[int, Gt(18)] # Valid: 19, 20, ... - # Invalid: 17, 18, "19", 19.0, ... - factors: list[Annotated[int, Predicate(is_prime)]] # Valid: 2, 3, 5, 7, 11, ... - # Invalid: 4, 8, -2, 5.0, "prime", ... - - my_list: Annotated[list[int], Len(0, 10)] # Valid: [], [10, 20, 30, 40, 50] - # Invalid: (1, 2), ["abc"], [0] * 20 -``` - -## Documentation - -_While `annotated-types` avoids runtime checks for performance, users should not -construct invalid combinations such as `MultipleOf("non-numeric")` or `Annotated[int, Len(3)]`. -Downstream implementors may choose to raise an error, emit a warning, silently ignore -a metadata item, etc., if the metadata objects described below are used with an -incompatible type - or for any other reason!_ - -### Gt, Ge, Lt, Le - -Express inclusive and/or exclusive bounds on orderable values - which may be numbers, -dates, times, strings, sets, etc. Note that the boundary value need not be of the -same type that was annotated, so long as they can be compared: `Annotated[int, Gt(1.5)]` -is fine, for example, and implies that the value is an integer x such that `x > 1.5`. - -We suggest that implementors may also interpret `functools.partial(operator.le, 1.5)` -as being equivalent to `Gt(1.5)`, for users who wish to avoid a runtime dependency on -the `annotated-types` package. - -To be explicit, these types have the following meanings: - -* `Gt(x)` - value must be "Greater Than" `x` - equivalent to exclusive minimum -* `Ge(x)` - value must be "Greater than or Equal" to `x` - equivalent to inclusive minimum -* `Lt(x)` - value must be "Less Than" `x` - equivalent to exclusive maximum -* `Le(x)` - value must be "Less than or Equal" to `x` - equivalent to inclusive maximum - -### Interval - -`Interval(gt, ge, lt, le)` allows you to specify an upper and lower bound with a single -metadata object. `None` attributes should be ignored, and non-`None` attributes -treated as per the single bounds above. - -### MultipleOf - -`MultipleOf(multiple_of=x)` might be interpreted in two ways: - -1. Python semantics, implying `value % multiple_of == 0`, or -2. [JSONschema semantics](https://json-schema.org/draft/2020-12/json-schema-validation.html#rfc.section.6.2.1), - where `int(value / multiple_of) == value / multiple_of`. - -We encourage users to be aware of these two common interpretations and their -distinct behaviours, especially since very large or non-integer numbers make -it easy to cause silent data corruption due to floating-point imprecision. - -We encourage libraries to carefully document which interpretation they implement. - -### MinLen, MaxLen, Len - -`Len()` implies that `min_length <= len(value) <= max_length` - lower and upper bounds are inclusive. - -As well as `Len()` which can optionally include upper and lower bounds, we also -provide `MinLen(x)` and `MaxLen(y)` which are equivalent to `Len(min_length=x)` -and `Len(max_length=y)` respectively. - -`Len`, `MinLen`, and `MaxLen` may be used with any type which supports `len(value)`. - -Examples of usage: - -* `Annotated[list, MaxLen(10)]` (or `Annotated[list, Len(max_length=10)]`) - list must have a length of 10 or less -* `Annotated[str, MaxLen(10)]` - string must have a length of 10 or less -* `Annotated[list, MinLen(3)]` (or `Annotated[list, Len(min_length=3)]`) - list must have a length of 3 or more -* `Annotated[list, Len(4, 6)]` - list must have a length of 4, 5, or 6 -* `Annotated[list, Len(8, 8)]` - list must have a length of exactly 8 - -#### Changed in v0.4.0 - -* `min_inclusive` has been renamed to `min_length`, no change in meaning -* `max_exclusive` has been renamed to `max_length`, upper bound is now **inclusive** instead of **exclusive** -* The recommendation that slices are interpreted as `Len` has been removed due to ambiguity and different semantic - meaning of the upper bound in slices vs. `Len` - -See [issue #23](https://github.com/annotated-types/annotated-types/issues/23) for discussion. - -### Timezone - -`Timezone` can be used with a `datetime` or a `time` to express which timezones -are allowed. `Annotated[datetime, Timezone(None)]` must be a naive datetime. -`Timezone[...]` ([literal ellipsis](https://docs.python.org/3/library/constants.html#Ellipsis)) -expresses that any timezone-aware datetime is allowed. You may also pass a specific -timezone string or [`tzinfo`](https://docs.python.org/3/library/datetime.html#tzinfo-objects) -object such as `Timezone(timezone.utc)` or `Timezone("Africa/Abidjan")` to express that you only -allow a specific timezone, though we note that this is often a symptom of fragile design. - -#### Changed in v0.x.x - -* `Timezone` accepts [`tzinfo`](https://docs.python.org/3/library/datetime.html#tzinfo-objects) objects instead of - `timezone`, extending compatibility to [`zoneinfo`](https://docs.python.org/3/library/zoneinfo.html) and third party libraries. - -### Unit - -`Unit(unit: str)` expresses that the annotated numeric value is the magnitude of -a quantity with the specified unit. For example, `Annotated[float, Unit("m/s")]` -would be a float representing a velocity in meters per second. - -Please note that `annotated_types` itself makes no attempt to parse or validate -the unit string in any way. That is left entirely to downstream libraries, -such as [`pint`](https://pint.readthedocs.io) or -[`astropy.units`](https://docs.astropy.org/en/stable/units/). - -An example of how a library might use this metadata: - -```python -from annotated_types import Unit -from typing import Annotated, TypeVar, Callable, Any, get_origin, get_args - -# given a type annotated with a unit: -Meters = Annotated[float, Unit("m")] - - -# you can cast the annotation to a specific unit type with any -# callable that accepts a string and returns the desired type -T = TypeVar("T") -def cast_unit(tp: Any, unit_cls: Callable[[str], T]) -> T | None: - if get_origin(tp) is Annotated: - for arg in get_args(tp): - if isinstance(arg, Unit): - return unit_cls(arg.unit) - return None - - -# using `pint` -import pint -pint_unit = cast_unit(Meters, pint.Unit) - - -# using `astropy.units` -import astropy.units as u -astropy_unit = cast_unit(Meters, u.Unit) -``` - -### Predicate - -`Predicate(func: Callable)` expresses that `func(value)` is truthy for valid values. -Users should prefer the statically inspectable metadata above, but if you need -the full power and flexibility of arbitrary runtime predicates... here it is. - -For some common constraints, we provide generic types: - -* `LowerCase = Annotated[T, Predicate(str.islower)]` -* `UpperCase = Annotated[T, Predicate(str.isupper)]` -* `IsDigit = Annotated[T, Predicate(str.isdigit)]` -* `IsFinite = Annotated[T, Predicate(math.isfinite)]` -* `IsNotFinite = Annotated[T, Predicate(Not(math.isfinite))]` -* `IsNan = Annotated[T, Predicate(math.isnan)]` -* `IsNotNan = Annotated[T, Predicate(Not(math.isnan))]` -* `IsInfinite = Annotated[T, Predicate(math.isinf)]` -* `IsNotInfinite = Annotated[T, Predicate(Not(math.isinf))]` - -so that you can write e.g. `x: IsFinite[float] = 2.0` instead of the longer -(but exactly equivalent) `x: Annotated[float, Predicate(math.isfinite)] = 2.0`. - -Some libraries might have special logic to handle known or understandable predicates, -for example by checking for `str.isdigit` and using its presence to both call custom -logic to enforce digit-only strings, and customise some generated external schema. -Users are therefore encouraged to avoid indirection like `lambda s: s.lower()`, in -favor of introspectable methods such as `str.lower` or `re.compile("pattern").search`. - -To enable basic negation of commonly used predicates like `math.isnan` without introducing introspection that makes it impossible for implementers to introspect the predicate we provide a `Not` wrapper that simply negates the predicate in an introspectable manner. Several of the predicates listed above are created in this manner. - -We do not specify what behaviour should be expected for predicates that raise -an exception. For example `Annotated[int, Predicate(str.isdigit)]` might silently -skip invalid constraints, or statically raise an error; or it might try calling it -and then propagate or discard the resulting -`TypeError: descriptor 'isdigit' for 'str' objects doesn't apply to a 'int' object` -exception. We encourage libraries to document the behaviour they choose. - -### Doc - -`doc()` can be used to add documentation information in `Annotated`, for function and method parameters, variables, class attributes, return types, and any place where `Annotated` can be used. - -It expects a value that can be statically analyzed, as the main use case is for static analysis, editors, documentation generators, and similar tools. - -It returns a `DocInfo` class with a single attribute `documentation` containing the value passed to `doc()`. - -This is the early adopter's alternative form of the [`typing-doc` proposal](https://github.com/tiangolo/fastapi/blob/typing-doc/typing_doc.md). - -### Integrating downstream types with `GroupedMetadata` - -Implementers may choose to provide a convenience wrapper that groups multiple pieces of metadata. -This can help reduce verbosity and cognitive overhead for users. -For example, an implementer like Pydantic might provide a `Field` or `Meta` type that accepts keyword arguments and transforms these into low-level metadata: - -```python -from dataclasses import dataclass -from typing import Iterator -from annotated_types import GroupedMetadata, Ge - -@dataclass -class Field(GroupedMetadata): - ge: int | None = None - description: str | None = None - - def __iter__(self) -> Iterator[object]: - # Iterating over a GroupedMetadata object should yield annotated-types - # constraint metadata objects which describe it as fully as possible, - # and may include other unknown objects too. - if self.ge is not None: - yield Ge(self.ge) - if self.description is not None: - yield Description(self.description) -``` - -Libraries consuming annotated-types constraints should check for `GroupedMetadata` and unpack it by iterating over the object and treating the results as if they had been "unpacked" in the `Annotated` type. The same logic should be applied to the [PEP 646 `Unpack` type](https://peps.python.org/pep-0646/), so that `Annotated[T, Field(...)]`, `Annotated[T, Unpack[Field(...)]]` and `Annotated[T, *Field(...)]` are all treated consistently. - -Libraries consuming annotated-types should also ignore any metadata they do not recongize that came from unpacking a `GroupedMetadata`, just like they ignore unrecognized metadata in `Annotated` itself. - -Our own `annotated_types.Interval` class is a `GroupedMetadata` which unpacks itself into `Gt`, `Lt`, etc., so this is not an abstract concern. Similarly, `annotated_types.Len` is a `GroupedMetadata` which unpacks itself into `MinLen` (optionally) and `MaxLen`. - -### Consuming metadata - -We intend to not be prescriptive as to _how_ the metadata and constraints are used, but as an example of how one might parse constraints from types annotations see our [implementation in `test_main.py`](https://github.com/annotated-types/annotated-types/blob/f59cf6d1b5255a0fe359b93896759a180bec30ae/tests/test_main.py#L94-L103). - -It is up to the implementer to determine how this metadata is used. -You could use the metadata for runtime type checking, for generating schemas or to generate example data, amongst other use cases. - -## Design & History - -This package was designed at the PyCon 2022 sprints by the maintainers of Pydantic -and Hypothesis, with the goal of making it as easy as possible for end-users to -provide more informative annotations for use by runtime libraries. - -It is deliberately minimal, and following PEP-593 allows considerable downstream -discretion in what (if anything!) they choose to support. Nonetheless, we expect -that staying simple and covering _only_ the most common use-cases will give users -and maintainers the best experience we can. If you'd like more constraints for your -types - follow our lead, by defining them and documenting them downstream! diff --git a/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/RECORD deleted file mode 100644 index b610b60c47045a9d85d99272bd1fd81452ede51a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/RECORD +++ /dev/null @@ -1,9 +0,0 @@ -annotated_types-0.8.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -annotated_types-0.8.0.dist-info/METADATA,sha256=YUmFsnj2Abjvhj-CeDuZHMOtpLifWS63HHxIxfLvIpg,15009 -annotated_types-0.8.0.dist-info/RECORD,, -annotated_types-0.8.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -annotated_types-0.8.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87 -annotated_types-0.8.0.dist-info/licenses/LICENSE,sha256=_hBJiEsaDZNCkB6I4H8ykl0ksxIdmXK2poBfuYJLCV0,1083 -annotated_types/__init__.py,sha256=pxBKTUObJ6n3T8C-I2ubobeDHmBEAmgCogWrwSmKm8g,13273 -annotated_types/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -annotated_types/test_cases.py,sha256=2GdHKstXuBzpf3iXKCjmvFZpyk2zP9Hm9kXGGDhGBo4,6310 diff --git a/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/WHEEL deleted file mode 100644 index 7401812e19af977cc5088f3b8fb1ef6bc0441c0a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: hatchling 1.31.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/licenses/LICENSE b/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/licenses/LICENSE deleted file mode 100644 index d99323a9965f146d5b0888c4ca1bf0727e12b04f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_types-0.8.0.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2022 the contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/bundle/python-cpu/Lib/site-packages/annotated_types/__init__.py b/bundle/python-cpu/Lib/site-packages/annotated_types/__init__.py deleted file mode 100644 index dcb35c54a56b0729f90278c22b402c73127be4d8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/annotated_types/__init__.py +++ /dev/null @@ -1,416 +0,0 @@ -import math -import types -from collections.abc import Callable, Iterator -from dataclasses import dataclass -from datetime import tzinfo -from types import EllipsisType -from typing import ( - TYPE_CHECKING, - Annotated, - Any, - Literal, - Protocol, - SupportsFloat, - SupportsIndex, - TypeVar, - Union, - runtime_checkable, -) - -__all__ = ( - 'BaseMetadata', - 'GroupedMetadata', - 'Gt', - 'Ge', - 'Lt', - 'Le', - 'Interval', - 'MultipleOf', - 'MinLen', - 'MaxLen', - 'Len', - 'Timezone', - 'Predicate', - 'LowerCase', - 'UpperCase', - 'IsDigits', - 'IsFinite', - 'IsNotFinite', - 'IsNan', - 'IsNotNan', - 'IsInfinite', - 'IsNotInfinite', - 'doc', - 'DocInfo', - '__version__', -) - -__version__ = '0.8.0' - - -T = TypeVar('T') - - -# arguments that start with __ are considered -# positional only -# see https://peps.python.org/pep-0484/#positional-only-arguments - - -class SupportsGt(Protocol): - def __gt__(self: T, __other: T) -> bool: - ... - - -class SupportsGe(Protocol): - def __ge__(self: T, __other: T) -> bool: - ... - - -class SupportsLt(Protocol): - def __lt__(self: T, __other: T) -> bool: - ... - - -class SupportsLe(Protocol): - def __le__(self: T, __other: T) -> bool: - ... - - -class SupportsMod(Protocol): - def __mod__(self: T, __other: T) -> T: - ... - - -class SupportsDiv(Protocol): - def __div__(self: T, __other: T) -> T: - ... - - -class BaseMetadata: - """Base class for all metadata. - - This exists mainly so that implementers - can do `isinstance(..., BaseMetadata)` while traversing field annotations. - """ - - __slots__ = () - - -@dataclass(frozen=True, slots=True) -class Gt(BaseMetadata): - """Gt(gt=x) implies that the value must be greater than x. - - It can be used with any type that supports the ``>`` operator, - including numbers, dates and times, strings, sets, and so on. - """ - - gt: SupportsGt - - -@dataclass(frozen=True, slots=True) -class Ge(BaseMetadata): - """Ge(ge=x) implies that the value must be greater than or equal to x. - - It can be used with any type that supports the ``>=`` operator, - including numbers, dates and times, strings, sets, and so on. - """ - - ge: SupportsGe - - -@dataclass(frozen=True, slots=True) -class Lt(BaseMetadata): - """Lt(lt=x) implies that the value must be less than x. - - It can be used with any type that supports the ``<`` operator, - including numbers, dates and times, strings, sets, and so on. - """ - - lt: SupportsLt - - -@dataclass(frozen=True, slots=True) -class Le(BaseMetadata): - """Le(le=x) implies that the value must be less than or equal to x. - - It can be used with any type that supports the ``<=`` operator, - including numbers, dates and times, strings, sets, and so on. - """ - - le: SupportsLe - - -@runtime_checkable -class GroupedMetadata(Protocol): - """A grouping of multiple objects, like typing.Unpack. - - `GroupedMetadata` on its own is not metadata and has no meaning. - All of the constraints and metadata should be fully expressable - in terms of the `BaseMetadata`'s returned by `GroupedMetadata.__iter__()`. - - Concrete implementations should override `GroupedMetadata.__iter__()` - to add their own metadata. - For example: - - >>> @dataclass - >>> class Field(GroupedMetadata): - >>> gt: float | None = None - >>> description: str | None = None - ... - >>> def __iter__(self) -> Iterable[object]: - >>> if self.gt is not None: - >>> yield Gt(self.gt) - >>> if self.description is not None: - >>> yield Description(self.gt) - - Also see the implementation of `Interval` below for an example. - - Parsers should recognize this and unpack it so that it can be used - both with and without unpacking: - - - `Annotated[int, Field(...)]` (parser must unpack Field) - - `Annotated[int, *Field(...)]` (PEP-646) - """ # noqa: trailing-whitespace - - @property - def __is_annotated_types_grouped_metadata__(self) -> Literal[True]: - return True - - def __iter__(self) -> Iterator[object]: - ... - - if not TYPE_CHECKING: - __slots__ = () # allow subclasses to use slots - - def __init_subclass__(cls, *args: Any, **kwargs: Any) -> None: - # Basic ABC like functionality without the complexity of an ABC - super().__init_subclass__(*args, **kwargs) - if cls.__iter__ is GroupedMetadata.__iter__: - raise TypeError("Can't subclass GroupedMetadata without implementing __iter__") - - def __iter__(self) -> Iterator[object]: # noqa: F811 - raise NotImplementedError # more helpful than "None has no attribute..." type errors - - -@dataclass(frozen=True, kw_only=True, slots=True) -class Interval(GroupedMetadata): - """Interval can express inclusive or exclusive bounds with a single object. - - It accepts keyword arguments ``gt``, ``ge``, ``lt``, and/or ``le``, which - are interpreted the same way as the single-bound constraints. - """ - - gt: SupportsGt | None = None - ge: SupportsGe | None = None - lt: SupportsLt | None = None - le: SupportsLe | None = None - - def __iter__(self) -> Iterator[BaseMetadata]: - """Unpack an Interval into zero or more single-bounds.""" - if self.gt is not None: - yield Gt(self.gt) - if self.ge is not None: - yield Ge(self.ge) - if self.lt is not None: - yield Lt(self.lt) - if self.le is not None: - yield Le(self.le) - - -@dataclass(frozen=True, slots=True) -class MultipleOf(BaseMetadata): - """MultipleOf(multiple_of=x) might be interpreted in two ways: - - 1. Python semantics, implying ``value % multiple_of == 0``, or - 2. JSONschema semantics, where ``int(value / multiple_of) == value / multiple_of`` - - We encourage users to be aware of these two common interpretations, - and libraries to carefully document which they implement. - """ - - multiple_of: SupportsDiv | SupportsMod - - -@dataclass(frozen=True, slots=True) -class MinLen(BaseMetadata): - """ - MinLen() implies minimum inclusive length, - e.g. ``len(value) >= min_length``. - """ - - min_length: Annotated[int, Ge(0)] - - -@dataclass(frozen=True, slots=True) -class MaxLen(BaseMetadata): - """ - MaxLen() implies maximum inclusive length, - e.g. ``len(value) <= max_length``. - """ - - max_length: Annotated[int, Ge(0)] - - -@dataclass(frozen=True, slots=True) -class Len(GroupedMetadata): - """ - Len() implies that ``min_length <= len(value) <= max_length``. - - Upper bound may be omitted or ``None`` to indicate no upper length bound. - """ - - min_length: Annotated[int, Ge(0)] = 0 - max_length: Annotated[int, Ge(0)] | None = None - - def __iter__(self) -> Iterator[BaseMetadata]: - """Unpack a Len into zero or more single-bounds.""" - if self.min_length > 0: - yield MinLen(self.min_length) - if self.max_length is not None: - yield MaxLen(self.max_length) - - -@dataclass(frozen=True, slots=True) -class Timezone(BaseMetadata): - """Timezone(tz=...) requires a datetime to be aware (or ``tz=None``, naive). - - ``Annotated[datetime, Timezone(None)]`` must be a naive datetime. - ``Timezone(...)`` (the ellipsis literal) expresses that the datetime must be - tz-aware but any timezone is allowed. - - You may also pass a specific timezone string or tzinfo object such as - ``Timezone(timezone.utc)`` or ``Timezone("Africa/Abidjan")`` to express that - you only allow a specific timezone, though we note that this is often - a symptom of poor design. - """ - - tz: str | tzinfo | EllipsisType | None - - -@dataclass(frozen=True, slots=True) -class Unit(BaseMetadata): - """Indicates that the value is a physical quantity with the specified unit. - - It is intended for usage with numeric types, where the value represents the - magnitude of the quantity. For example, ``distance: Annotated[float, Unit('m')]`` - or ``speed: Annotated[float, Unit('m/s')]``. - - Interpretation of the unit string is left to the discretion of the consumer. - It is suggested to follow conventions established by python libraries that work - with physical quantities, such as - - - ``pint`` : - - ``astropy.units``: - - For indicating a quantity with a certain dimensionality but without a specific unit - it is recommended to use square brackets, e.g. `Annotated[float, Unit('[time]')]`. - Note, however, ``annotated_types`` itself makes no use of the unit string. - """ - - unit: str - - -@dataclass(frozen=True, slots=True) -class Predicate(BaseMetadata): - """``Predicate(func: Callable)`` implies `func(value)` is truthy for valid values. - - Users should prefer statically inspectable metadata, but if you need the full - power and flexibility of arbitrary runtime predicates... here it is. - - We provide a few predefined predicates for common string constraints: - ``LowerCase = Predicate(str.islower)``, ``UpperCase = Predicate(str.isupper)``, and - ``IsDigits = Predicate(str.isdigit)``. Users are encouraged to use methods which - can be given special handling, and avoid indirection like ``lambda s: s.lower()``. - - Some libraries might have special logic to handle certain predicates, e.g. by - checking for `str.isdigit` and using its presence to both call custom logic to - enforce digit-only strings, and customise some generated external schema. - - We do not specify what behaviour should be expected for predicates that raise - an exception. For example `Annotated[int, Predicate(str.isdigit)]` might silently - skip invalid constraints, or statically raise an error; or it might try calling it - and then propagate or discard the resulting exception. - """ - - func: Callable[[Any], bool] - - def __repr__(self) -> str: - if getattr(self.func, "__name__", "") == "": - return f"{self.__class__.__name__}({self.func!r})" - if isinstance(self.func, (types.MethodType, types.BuiltinMethodType)) and ( - namespace := getattr(self.func.__self__, "__name__", None) - ): - return f"{self.__class__.__name__}({namespace}.{self.func.__name__})" - if isinstance(self.func, type(str.isascii)): # method descriptor - return f"{self.__class__.__name__}({self.func.__qualname__})" - return f"{self.__class__.__name__}({self.func.__name__})" - - -@dataclass -class Not: - func: Callable[[Any], bool] - - def __call__(self, __v: Any) -> bool: - return not self.func(__v) - - -_StrType = TypeVar("_StrType", bound=str) - -LowerCase = Annotated[_StrType, Predicate(str.islower)] -""" -Return True if the string is a lowercase string, False otherwise. - -A string is lowercase if all cased characters in the string are lowercase and there is at least one cased character in the string. -""" # noqa: E501 -UpperCase = Annotated[_StrType, Predicate(str.isupper)] -""" -Return True if the string is an uppercase string, False otherwise. - -A string is uppercase if all cased characters in the string are uppercase and there is at least one cased character in the string. -""" # noqa: E501 -IsDigit = Annotated[_StrType, Predicate(str.isdigit)] -IsDigits = IsDigit # type: ignore # plural for backwards compatibility, see #63 -""" -Return True if the string is a digit string, False otherwise. - -A string is a digit string if all characters in the string are digits and there is at least one character in the string. -""" # noqa: E501 -IsAscii = Annotated[_StrType, Predicate(str.isascii)] -""" -Return True if all characters in the string are ASCII, False otherwise. - -ASCII characters have code points in the range U+0000-U+007F. Empty string is ASCII too. -""" - -_NumericType = TypeVar('_NumericType', bound=Union[SupportsFloat, SupportsIndex]) -IsFinite = Annotated[_NumericType, Predicate(math.isfinite)] -"""Return True if x is neither an infinity nor a NaN, and False otherwise.""" -IsNotFinite = Annotated[_NumericType, Predicate(Not(math.isfinite))] -"""Return True if x is one of infinity or NaN, and False otherwise""" -IsNan = Annotated[_NumericType, Predicate(math.isnan)] -"""Return True if x is a NaN (not a number), and False otherwise.""" -IsNotNan = Annotated[_NumericType, Predicate(Not(math.isnan))] -"""Return True if x is anything but NaN (not a number), and False otherwise.""" -IsInfinite = Annotated[_NumericType, Predicate(math.isinf)] -"""Return True if x is a positive or negative infinity, and False otherwise.""" -IsNotInfinite = Annotated[_NumericType, Predicate(Not(math.isinf))] -"""Return True if x is neither a positive or negative infinity, and False otherwise.""" - -try: - # PEP 727 – Documentation in Annotated Metadata - from typing_extensions import Doc # type: ignore[attr-defined] -except ImportError: - - @dataclass(frozen=True, slots=True) - class Doc: # type: ignore [no-redef] - """ " - The return value of doc(), mainly to be used by tools that want to extract the - Annotated documentation at runtime. - """ - - documentation: str - """The documentation string passed to doc().""" - - -DocInfo = Doc # backwards compatibility -doc = Doc diff --git a/bundle/python-cpu/Lib/site-packages/annotated_types/py.typed b/bundle/python-cpu/Lib/site-packages/annotated_types/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/METADATA deleted file mode 100644 index 126a728ba495267b9d9208a6df9ff46221700e3a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/METADATA +++ /dev/null @@ -1,107 +0,0 @@ -Metadata-Version: 2.4 -Name: anyio -Version: 4.14.2 -Summary: High-level concurrency and networking framework on top of asyncio or Trio -Author-email: Alex Grönholm -License-Expression: MIT -Project-URL: Documentation, https://anyio.readthedocs.io/en/latest/ -Project-URL: Changelog, https://anyio.readthedocs.io/en/stable/versionhistory.html -Project-URL: Source code, https://github.com/agronholm/anyio -Project-URL: Issue tracker, https://github.com/agronholm/anyio/issues -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Framework :: AnyIO -Classifier: Typing :: Typed -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Programming Language :: Python :: 3.15 -Requires-Python: >=3.10 -Description-Content-Type: text/x-rst -License-File: LICENSE -Requires-Dist: exceptiongroup>=1.0.2; python_version < "3.11" -Requires-Dist: idna>=2.8 -Requires-Dist: typing_extensions>=4.5; python_version < "3.13" -Provides-Extra: trio -Requires-Dist: trio>=0.32.0; extra == "trio" -Dynamic: license-file - -.. image:: https://github.com/agronholm/anyio/actions/workflows/test.yml/badge.svg - :target: https://github.com/agronholm/anyio/actions/workflows/test.yml - :alt: Build Status -.. image:: https://coveralls.io/repos/github/agronholm/anyio/badge.svg?branch=master - :target: https://coveralls.io/github/agronholm/anyio?branch=master - :alt: Code Coverage -.. image:: https://readthedocs.org/projects/anyio/badge/?version=latest - :target: https://anyio.readthedocs.io/en/latest/?badge=latest - :alt: Documentation -.. image:: https://badges.gitter.im/gitterHQ/gitter.svg - :target: https://gitter.im/python-trio/AnyIO - :alt: Gitter chat -.. image:: https://tidelift.com/badges/package/pypi/anyio - :target: https://tidelift.com/subscription/pkg/pypi-anyio - :alt: Tidelift - -AnyIO is an asynchronous networking and concurrency library that works on top of either asyncio_ or -Trio_. It implements Trio-like `structured concurrency`_ (SC) on top of asyncio and works in harmony -with the native SC of Trio itself. - -Applications and libraries written against AnyIO's API will run unmodified on either asyncio_ or -Trio_. AnyIO can also be adopted into a library or application incrementally – bit by bit, no full -refactoring necessary. It will blend in with the native libraries of your chosen backend. - -To find out why you might want to use AnyIO's APIs instead of asyncio's, you can read about it -`here `_. - -Documentation -------------- - -View full documentation at: https://anyio.readthedocs.io/ - -Features --------- - -AnyIO offers the following functionality: - -* Task groups (nurseries_ in trio terminology) -* High-level networking (TCP, UDP and UNIX sockets) - - * `Happy eyeballs`_ algorithm for TCP connections (more robust than that of asyncio on Python - 3.8) - * async/await style UDP sockets (unlike asyncio where you still have to use Transports and - Protocols) - -* A versatile API for byte streams and object streams -* Inter-task synchronization and communication (locks, conditions, events, semaphores, object - streams) -* Worker threads -* Subprocesses -* Subinterpreter support for code parallelization (on Python 3.13 and later) -* Asynchronous file I/O (using worker threads) -* Signal handling -* Asynchronous versions of the functools_ and itertools_ modules - -AnyIO also comes with its own pytest_ plugin which also supports asynchronous fixtures. -It even works with the popular Hypothesis_ library. - -.. _asyncio: https://docs.python.org/3/library/asyncio.html -.. _Trio: https://github.com/python-trio/trio -.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency -.. _nurseries: https://trio.readthedocs.io/en/stable/reference-core.html#nurseries-and-spawning -.. _Happy eyeballs: https://en.wikipedia.org/wiki/Happy_Eyeballs -.. _pytest: https://docs.pytest.org/en/latest/ -.. _functools: https://docs.python.org/3/library/functools.html -.. _itertools: https://docs.python.org/3/library/itertools.html -.. _Hypothesis: https://hypothesis.works/ - -Security contact information ----------------------------- - -To report a security vulnerability, please use the `Tidelift security contact`_. -Tidelift will coordinate the fix and disclosure. - -.. _Tidelift security contact: https://tidelift.com/security diff --git a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/RECORD deleted file mode 100644 index ccc50e86a9cf72e7cb60572ca6c9619c2b5564cd..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/RECORD +++ /dev/null @@ -1,54 +0,0 @@ -anyio-4.14.2.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -anyio-4.14.2.dist-info/METADATA,sha256=xeb8Tf2DMxmROyh756MNmTXpIIx--RdjBiGFWqrE1Bk,4645 -anyio-4.14.2.dist-info/RECORD,, -anyio-4.14.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -anyio-4.14.2.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91 -anyio-4.14.2.dist-info/entry_points.txt,sha256=_d6Yu6uiaZmNe0CydowirE9Cmg7zUL2g08tQpoS3Qvc,39 -anyio-4.14.2.dist-info/licenses/LICENSE,sha256=U2GsncWPLvX9LpsJxoKXwX8ElQkJu8gCO9uC6s8iwrA,1081 -anyio-4.14.2.dist-info/scm_file_list.json,sha256=wDSXGv8Ehn5ZW5BhB-RlaAc16zY_OfO27qrlMfMMZy8,3654 -anyio-4.14.2.dist-info/scm_version.json,sha256=KgaUx31SyaqGFQFKpG3FO9kCk-ygOKS7Uf0yk56OzaY,161 -anyio-4.14.2.dist-info/top_level.txt,sha256=QglSMiWX8_5dpoVAEIHdEYzvqFMdSYWmCj6tYw2ITkQ,6 -anyio/__init__.py,sha256=HitUIfzvAojSeaHVmJ9rFn8k_yI63G6s_jUL2QChf4U,6405 -anyio/_backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -anyio/_backends/_asyncio.py,sha256=eK0j8PI0O7y3bX8cUBsafoWP8wsXR-szP6ciyP4Mkkk,104400 -anyio/_backends/_trio.py,sha256=hoTXL8zI81v4UE-YT_cr2fzh_zY1mKdmZD3oQHmeR0E,45583 -anyio/_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -anyio/_core/_asyncio_selector_thread.py,sha256=2PdxFM3cs02Kp6BSppbvmRT7q7asreTW5FgBxEsflBo,5626 -anyio/_core/_contextmanagers.py,sha256=YInBCabiEeS-UaP_Jdxa1CaFC71ETPW8HZTHIM8Rsc8,7215 -anyio/_core/_eventloop.py,sha256=ByZUeJD9alMfcyTseRo5IzTO0IltEul_Gyq9iqSjqDk,6658 -anyio/_core/_exceptions.py,sha256=OfzLO4Z3Hog1TnipbIn72YNtkoYxS4lHW9MqKDeGc88,4936 -anyio/_core/_fileio.py,sha256=hHfyV0bXDL-R2ZNnInwse3nmTAd36AIz1cBxgmAwzAQ,31358 -anyio/_core/_resources.py,sha256=NbmU5O5UX3xEyACnkmYX28Fmwdl-f-ny0tHym26e0w0,435 -anyio/_core/_signals.py,sha256=mjTBB2hTKNPRlU0IhnijeQedpWOGERDiMjSlJQsFrug,1016 -anyio/_core/_sockets.py,sha256=HtjiSH-yzehlqh_LpD3PGIafDuUCwd8gueUuk5MFeNk,35288 -anyio/_core/_streams.py,sha256=FczFwIgDpnkK0bODWJXMpsUJYdvAD04kaUaGzJU8DK0,1806 -anyio/_core/_subprocesses.py,sha256=M2GCc4NKXCbB_GtEskJgndiM2b0VS0NK_ohmFxri7O8,7923 -anyio/_core/_synchronization.py,sha256=kgPk88-eVOmY-pDNs-ReRbcEelY1a7YczuVBPsZwc8A,21591 -anyio/_core/_tasks.py,sha256=y99vRi-AFzEv6kyvIwqrzuskE2NMiNItWnuWGk7eOr4,13126 -anyio/_core/_tempfile.py,sha256=jE2w59FRF3yRo4vjkjfZF2YcqsBZvc66VWRwrJGDYGk,19624 -anyio/_core/_testing.py,sha256=u7MPqGXwpTxqI7hclSdNA30z2GH1Nw258uwKvy_RfBg,2340 -anyio/_core/_typedattr.py,sha256=P4ozZikn3-DbpoYcvyghS_FOYAgbmUxeoU8-L_07pZM,2508 -anyio/abc/__init__.py,sha256=6mWhcl_pGXhrgZVHP_TCfMvIXIOp9mroEFM90fYCU_U,2869 -anyio/abc/_eventloop.py,sha256=OqWYSEj0TmwL_xniCJt3_jHFWsuMk9THk8tCTGsKapI,10681 -anyio/abc/_resources.py,sha256=DrYvkNN1hH6Uvv5_5uKySvDsnknGVDe8FCKfko0VtN8,783 -anyio/abc/_sockets.py,sha256=OmVDrfemVvF9c5K1tpBgQyV6fn5v0XyCExLAqBOGz9o,13124 -anyio/abc/_streams.py,sha256=rnSwRy-6y80TPQtIXit5LjMMiU1CCWS1oMNsmJQHRTg,7582 -anyio/abc/_subprocesses.py,sha256=cumAPJTktOQtw63IqG0lDpyZqu_l1EElvQHMiwJgL08,2067 -anyio/abc/_tasks.py,sha256=m-FtE4phxeNIELSG7A3H7VUz3jA2Ib5J2JIew8-PS6o,6642 -anyio/abc/_testing.py,sha256=9YYM2AXsYFvf4PLjUEr6yRxDiUeB5QbY_gOg0X_C6lY,2034 -anyio/from_thread.py,sha256=JYsbaCaIB_Iit6kNhtXSteJGt4PcQ7ncq0nIpcelIrg,19265 -anyio/functools.py,sha256=T4JS8IXq-x1S0Lbo2owF8l9fza2KypO147QLeyz4cjs,11797 -anyio/itertools.py,sha256=QV-9mnRCr2yBph8g01QFvN-bQ_Yle-8Sl13YSydBlMI,16168 -anyio/lowlevel.py,sha256=-59484Z6K5jF7XJJhHxYHIKnvFH_uAeMhl9a_AfaSHI,6280 -anyio/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -anyio/pytest_plugin.py,sha256=paMpI_VMNQf2bir0LfvgMpXSiYJoHDzWdKUVTyoHmvQ,13609 -anyio/streams/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -anyio/streams/buffered.py,sha256=u7hCD8SNrYHcutG6K5wEiy1F88-pizgjvEFM22Kq2Cw,6746 -anyio/streams/file.py,sha256=6jujI2m-QJITqqKFamrupX_DNsU7y2Fz3omLZxOLuY0,4524 -anyio/streams/memory.py,sha256=ZmKWCLpyItOCXmvCQT-L8IyJHNFaB-OVJbrn88CaMo0,10776 -anyio/streams/stapled.py,sha256=mDNF9Gj4deXfOuKSZmgkEG-QExYAKAGjYBHbrs-rJaQ,4486 -anyio/streams/text.py,sha256=BcVAGJw1VRvtIqnv-o0Rb0pwH7p8vwlvl21xHq522ag,5765 -anyio/streams/tls.py,sha256=Gvs--YOoFxcyn-hakXOiPM8H-aEp8nsU6k9rTrKqJA4,15801 -anyio/to_interpreter.py,sha256=_mLngrMy97TMR6VbW4Y6YzDUk9ZuPcQMPlkuyRh3C9k,7100 -anyio/to_process.py,sha256=jHw7v6XHBNIfYa6DqnqKY0gsRo8ScOCh8fo19fqFvjU,9848 -anyio/to_thread.py,sha256=bYszW0lCDfTmLnXbAYic05HEe_Di_0UWKabgL2vU0T0,2750 diff --git a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/WHEEL deleted file mode 100644 index 1d472b6c22838de58d1c3c0dd2c795a5cde9e415..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/WHEEL +++ /dev/null @@ -1,5 +0,0 @@ -Wheel-Version: 1.0 -Generator: setuptools (83.0.0) -Root-Is-Purelib: true -Tag: py3-none-any - diff --git a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/entry_points.txt b/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/entry_points.txt deleted file mode 100644 index 44dd9bdc3039122cc98014c1439ca254313fd014..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[pytest11] -anyio = anyio.pytest_plugin diff --git a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/licenses/LICENSE b/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/licenses/LICENSE deleted file mode 100644 index 104eebf5a3002fccdaceef3a4cb936173c1c2035..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/licenses/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Alex Grönholm - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/scm_file_list.json b/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/scm_file_list.json deleted file mode 100644 index 72a48145f533714ddb8ec6198464e494fa5c0c13..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/scm_file_list.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "files": [ - ".pre-commit-config.yaml", - "LICENSE", - "pyproject.toml", - "AGENTS.md", - "README.rst", - "CLAUDE.md", - ".readthedocs.yml", - ".gitignore", - "docs/tempfile.rst", - "docs/signals.rst", - "docs/synchronization.rst", - "docs/contextmanagers.rst", - "docs/testing.rst", - "docs/networking.rst", - "docs/contributing.rst", - "docs/index.rst", - "docs/versionhistory.rst", - "docs/threads.rst", - "docs/api.rst", - "docs/typedattrs.rst", - "docs/basics.rst", - "docs/fileio.rst", - "docs/cancellation.rst", - "docs/support.rst", - "docs/streams.rst", - "docs/why.rst", - "docs/tasks.rst", - "docs/migration.rst", - "docs/conf.py", - "docs/subprocesses.rst", - "docs/faq.rst", - "docs/subinterpreters.rst", - "src/anyio/functools.py", - "src/anyio/py.typed", - "src/anyio/__init__.py", - "src/anyio/pytest_plugin.py", - "src/anyio/itertools.py", - "src/anyio/to_interpreter.py", - "src/anyio/from_thread.py", - "src/anyio/to_process.py", - "src/anyio/to_thread.py", - "src/anyio/lowlevel.py", - "src/anyio/_backends/_trio.py", - "src/anyio/_backends/__init__.py", - "src/anyio/_backends/_asyncio.py", - "src/anyio/streams/memory.py", - "src/anyio/streams/__init__.py", - "src/anyio/streams/tls.py", - "src/anyio/streams/file.py", - "src/anyio/streams/text.py", - "src/anyio/streams/stapled.py", - "src/anyio/streams/buffered.py", - "src/anyio/abc/_eventloop.py", - "src/anyio/abc/__init__.py", - "src/anyio/abc/_sockets.py", - "src/anyio/abc/_tasks.py", - "src/anyio/abc/_subprocesses.py", - "src/anyio/abc/_resources.py", - "src/anyio/abc/_streams.py", - "src/anyio/abc/_testing.py", - "src/anyio/_core/_typedattr.py", - "src/anyio/_core/_eventloop.py", - "src/anyio/_core/__init__.py", - "src/anyio/_core/_tempfile.py", - "src/anyio/_core/_sockets.py", - "src/anyio/_core/_tasks.py", - "src/anyio/_core/_fileio.py", - "src/anyio/_core/_synchronization.py", - "src/anyio/_core/_subprocesses.py", - "src/anyio/_core/_resources.py", - "src/anyio/_core/_contextmanagers.py", - "src/anyio/_core/_exceptions.py", - "src/anyio/_core/_streams.py", - "src/anyio/_core/_signals.py", - "src/anyio/_core/_asyncio_selector_thread.py", - "src/anyio/_core/_testing.py", - "tests/test_itertools.py", - "tests/test_functools.py", - "tests/test_eventloop.py", - "tests/__init__.py", - "tests/test_to_thread.py", - "tests/test_from_thread.py", - "tests/test_lowlevel.py", - "tests/test_to_interpreter.py", - "tests/test_sockets.py", - "tests/test_typedattr.py", - "tests/test_to_process.py", - "tests/test_all_attributes.py", - "tests/test_synchronization.py", - "tests/test_debugging.py", - "tests/test_contextmanagers.py", - "tests/test_fileio.py", - "tests/conftest.py", - "tests/test_signals.py", - "tests/test_deprecations.py", - "tests/test_tempfile.py", - "tests/test_taskgroups.py", - "tests/test_pytest_plugin.py", - "tests/test_subprocesses.py", - "tests/streams/test_text.py", - "tests/streams/test_memory.py", - "tests/streams/__init__.py", - "tests/streams/test_file.py", - "tests/streams/test_stapled.py", - "tests/streams/test_tls.py", - "tests/streams/test_buffered.py", - ".github/pull_request_template.md", - ".github/dependabot.yml", - ".github/FUNDING.yml", - ".github/ISSUE_TEMPLATE/features_request.yaml", - ".github/ISSUE_TEMPLATE/bug_report.yaml", - ".github/ISSUE_TEMPLATE/config.yml", - ".github/workflows/test.yml", - ".github/workflows/test-downstream.yml", - ".github/workflows/publish.yml" - ] -} diff --git a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/scm_version.json b/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/scm_version.json deleted file mode 100644 index 13d71062b9fa3d7ce79a838448f646e6da5a3d2d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/scm_version.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "tag": "4.14.2", - "distance": 0, - "node": "gc384f99687c64c59ed8a11c3a0f11a2d57daff71", - "dirty": false, - "branch": "HEAD", - "node_date": "2026-07-12" -} diff --git a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/top_level.txt b/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/top_level.txt deleted file mode 100644 index c77c069ecc9b7f8b1f97dbcfec905725db0253a8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio-4.14.2.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -anyio diff --git a/bundle/python-cpu/Lib/site-packages/anyio/__init__.py b/bundle/python-cpu/Lib/site-packages/anyio/__init__.py deleted file mode 100644 index 2502c760bcc1d640be2de20f620d52fc3364cb55..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/__init__.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -from ._core._contextmanagers import AsyncContextManagerMixin as AsyncContextManagerMixin -from ._core._contextmanagers import ContextManagerMixin as ContextManagerMixin -from ._core._eventloop import current_time as current_time -from ._core._eventloop import get_all_backends as get_all_backends -from ._core._eventloop import get_available_backends as get_available_backends -from ._core._eventloop import get_cancelled_exc_class as get_cancelled_exc_class -from ._core._eventloop import run as run -from ._core._eventloop import sleep as sleep -from ._core._eventloop import sleep_forever as sleep_forever -from ._core._eventloop import sleep_until as sleep_until -from ._core._exceptions import BrokenResourceError as BrokenResourceError -from ._core._exceptions import BrokenWorkerInterpreter as BrokenWorkerInterpreter -from ._core._exceptions import BrokenWorkerProcess as BrokenWorkerProcess -from ._core._exceptions import BusyResourceError as BusyResourceError -from ._core._exceptions import ClosedResourceError as ClosedResourceError -from ._core._exceptions import ConnectionFailed as ConnectionFailed -from ._core._exceptions import DelimiterNotFound as DelimiterNotFound -from ._core._exceptions import EndOfStream as EndOfStream -from ._core._exceptions import IncompleteRead as IncompleteRead -from ._core._exceptions import NoEventLoopError as NoEventLoopError -from ._core._exceptions import RunFinishedError as RunFinishedError -from ._core._exceptions import TaskCancelled as TaskCancelled -from ._core._exceptions import TaskFailed as TaskFailed -from ._core._exceptions import TaskNotFinished as TaskNotFinished -from ._core._exceptions import TypedAttributeLookupError as TypedAttributeLookupError -from ._core._exceptions import WouldBlock as WouldBlock -from ._core._fileio import AsyncFile as AsyncFile -from ._core._fileio import Path as Path -from ._core._fileio import open_file as open_file -from ._core._fileio import wrap_file as wrap_file -from ._core._resources import aclose_forcefully as aclose_forcefully -from ._core._signals import open_signal_receiver as open_signal_receiver -from ._core._sockets import TCPConnectable as TCPConnectable -from ._core._sockets import UNIXConnectable as UNIXConnectable -from ._core._sockets import as_connectable as as_connectable -from ._core._sockets import connect_tcp as connect_tcp -from ._core._sockets import connect_unix as connect_unix -from ._core._sockets import create_connected_udp_socket as create_connected_udp_socket -from ._core._sockets import ( - create_connected_unix_datagram_socket as create_connected_unix_datagram_socket, -) -from ._core._sockets import create_tcp_listener as create_tcp_listener -from ._core._sockets import create_udp_socket as create_udp_socket -from ._core._sockets import create_unix_datagram_socket as create_unix_datagram_socket -from ._core._sockets import create_unix_listener as create_unix_listener -from ._core._sockets import getaddrinfo as getaddrinfo -from ._core._sockets import getnameinfo as getnameinfo -from ._core._sockets import notify_closing as notify_closing -from ._core._sockets import wait_readable as wait_readable -from ._core._sockets import wait_socket_readable as wait_socket_readable -from ._core._sockets import wait_socket_writable as wait_socket_writable -from ._core._sockets import wait_writable as wait_writable -from ._core._streams import create_memory_object_stream as create_memory_object_stream -from ._core._subprocesses import open_process as open_process -from ._core._subprocesses import run_process as run_process -from ._core._synchronization import CapacityLimiter as CapacityLimiter -from ._core._synchronization import ( - CapacityLimiterStatistics as CapacityLimiterStatistics, -) -from ._core._synchronization import Condition as Condition -from ._core._synchronization import ConditionStatistics as ConditionStatistics -from ._core._synchronization import Event as Event -from ._core._synchronization import EventStatistics as EventStatistics -from ._core._synchronization import Lock as Lock -from ._core._synchronization import LockStatistics as LockStatistics -from ._core._synchronization import ResourceGuard as ResourceGuard -from ._core._synchronization import Semaphore as Semaphore -from ._core._synchronization import SemaphoreStatistics as SemaphoreStatistics -from ._core._tasks import TASK_STATUS_IGNORED as TASK_STATUS_IGNORED -from ._core._tasks import CancelScope as CancelScope -from ._core._tasks import TaskHandle as TaskHandle -from ._core._tasks import create_task_group as create_task_group -from ._core._tasks import current_effective_deadline as current_effective_deadline -from ._core._tasks import fail_after as fail_after -from ._core._tasks import move_on_after as move_on_after -from ._core._tempfile import NamedTemporaryFile as NamedTemporaryFile -from ._core._tempfile import SpooledTemporaryFile as SpooledTemporaryFile -from ._core._tempfile import TemporaryDirectory as TemporaryDirectory -from ._core._tempfile import TemporaryFile as TemporaryFile -from ._core._tempfile import gettempdir as gettempdir -from ._core._tempfile import gettempdirb as gettempdirb -from ._core._tempfile import mkdtemp as mkdtemp -from ._core._tempfile import mkstemp as mkstemp -from ._core._testing import TaskInfo as TaskInfo -from ._core._testing import get_current_task as get_current_task -from ._core._testing import get_running_tasks as get_running_tasks -from ._core._testing import wait_all_tasks_blocked as wait_all_tasks_blocked -from ._core._typedattr import TypedAttributeProvider as TypedAttributeProvider -from ._core._typedattr import TypedAttributeSet as TypedAttributeSet -from ._core._typedattr import typed_attribute as typed_attribute - -# Re-export imports so they look like they live directly in this package -for __value in list(locals().values()): - if getattr(__value, "__module__", "").startswith("anyio."): - __value.__module__ = __name__ - - -del __value - - -def __getattr__(attr: str) -> type[BrokenWorkerInterpreter]: - """Support deprecated aliases.""" - if attr == "BrokenWorkerIntepreter": - import warnings - - warnings.warn( - "The 'BrokenWorkerIntepreter' alias is deprecated, use 'BrokenWorkerInterpreter' instead.", - DeprecationWarning, - stacklevel=2, - ) - return BrokenWorkerInterpreter - - raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_backends/__init__.py b/bundle/python-cpu/Lib/site-packages/anyio/_backends/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_backends/_asyncio.py b/bundle/python-cpu/Lib/site-packages/anyio/_backends/_asyncio.py deleted file mode 100644 index c00c2cd9be4d773c1ed38498659021bc45099e0c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_backends/_asyncio.py +++ /dev/null @@ -1,3136 +0,0 @@ -from __future__ import annotations - -import array -import asyncio -import concurrent.futures -import contextvars -import math -import os -import socket -import sys -import threading -import weakref -from asyncio import ( - AbstractEventLoop, - CancelledError, - all_tasks, - create_task, - current_task, - get_running_loop, - sleep, -) -from asyncio.base_events import _run_until_complete_cb # type: ignore[attr-defined] -from collections import OrderedDict, deque -from collections.abc import ( - AsyncGenerator, - AsyncIterator, - Awaitable, - Callable, - Collection, - Coroutine, - Iterable, - Sequence, -) -from concurrent.futures import Future -from contextlib import AbstractContextManager -from contextvars import Context, copy_context -from dataclasses import dataclass, field -from functools import partial, wraps -from inspect import ( - CORO_RUNNING, - CORO_SUSPENDED, - getcoroutinestate, -) -from io import IOBase -from os import PathLike -from queue import Queue -from signal import Signals -from socket import AddressFamily, SocketKind -from threading import Thread -from types import CodeType, TracebackType -from typing import ( - IO, - TYPE_CHECKING, - Any, - Literal, - ParamSpec, - TypeVar, - cast, -) -from weakref import WeakKeyDictionary - -from .. import ( - CapacityLimiterStatistics, - EventStatistics, - LockStatistics, - TaskInfo, - abc, -) -from .._core._eventloop import ( - claim_worker_thread, - set_current_async_library, - threadlocals, -) -from .._core._exceptions import ( - BrokenResourceError, - BusyResourceError, - ClosedResourceError, - EndOfStream, - RunFinishedError, - WouldBlock, -) -from .._core._sockets import convert_ipv6_sockaddr -from .._core._streams import create_memory_object_stream -from .._core._synchronization import ( - CapacityLimiter as BaseCapacityLimiter, -) -from .._core._synchronization import Event as BaseEvent -from .._core._synchronization import Lock as BaseLock -from .._core._synchronization import ( - ResourceGuard, - SemaphoreStatistics, -) -from .._core._synchronization import Semaphore as BaseSemaphore -from .._core._tasks import CancelScope as BaseCancelScope -from .._core._tasks import TaskHandle -from ..abc import ( - AsyncBackend, - IPSockAddrType, - SocketListener, - UDPPacketType, - UNIXDatagramPacketType, -) -from ..abc._eventloop import StrOrBytesPath -from ..abc._tasks import call_for_coroutine, get_callable_name -from ..lowlevel import RunVar -from ..streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike -else: - FileDescriptorLike = object - -if sys.version_info >= (3, 11): - from asyncio import Runner - from typing import TypeVarTuple, Unpack -else: - import contextvars - import enum - import signal - from asyncio import coroutines, events, exceptions, tasks - - from exceptiongroup import BaseExceptionGroup - from typing_extensions import TypeVarTuple, Unpack - - class _State(enum.Enum): - CREATED = "created" - INITIALIZED = "initialized" - CLOSED = "closed" - - class Runner: - # Copied from CPython 3.11 - def __init__( - self, - *, - debug: bool | None = None, - loop_factory: Callable[[], AbstractEventLoop] | None = None, - ): - self._state = _State.CREATED - self._debug = debug - self._loop_factory = loop_factory - self._loop: AbstractEventLoop | None = None - self._context = None - self._interrupt_count = 0 - self._set_event_loop = False - - def __enter__(self) -> Runner: - self._lazy_init() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.close() - - def close(self) -> None: - """Shutdown and close event loop.""" - loop = self._loop - if self._state is not _State.INITIALIZED or loop is None: - return - try: - _cancel_all_tasks(loop) - loop.run_until_complete(loop.shutdown_asyncgens()) - if hasattr(loop, "shutdown_default_executor"): - loop.run_until_complete(loop.shutdown_default_executor()) - else: - loop.run_until_complete(_shutdown_default_executor(loop)) - finally: - if self._set_event_loop: - events.set_event_loop(None) - loop.close() - self._loop = None - self._state = _State.CLOSED - - def get_loop(self) -> AbstractEventLoop: - """Return embedded event loop.""" - self._lazy_init() - return self._loop - - def run(self, coro: Coroutine[T_Retval], *, context=None) -> T_Retval: - """Run a coroutine inside the embedded event loop.""" - if not coroutines.iscoroutine(coro): - raise ValueError(f"a coroutine was expected, got {coro!r}") - - if events._get_running_loop() is not None: - # fail fast with short traceback - raise RuntimeError( - "Runner.run() cannot be called from a running event loop" - ) - - self._lazy_init() - - if context is None: - context = self._context - task = context.run(self._loop.create_task, coro) - - if ( - threading.current_thread() is threading.main_thread() - and signal.getsignal(signal.SIGINT) is signal.default_int_handler - ): - sigint_handler = partial(self._on_sigint, main_task=task) - try: - signal.signal(signal.SIGINT, sigint_handler) - except ValueError: - # `signal.signal` may throw if `threading.main_thread` does - # not support signals (e.g. embedded interpreter with signals - # not registered - see gh-91880) - sigint_handler = None - else: - sigint_handler = None - - self._interrupt_count = 0 - try: - return self._loop.run_until_complete(task) - except exceptions.CancelledError: - if self._interrupt_count > 0: - uncancel = getattr(task, "uncancel", None) - if uncancel is not None and uncancel() == 0: - raise KeyboardInterrupt # noqa: B904 - raise # CancelledError - finally: - if ( - sigint_handler is not None - and signal.getsignal(signal.SIGINT) is sigint_handler - ): - signal.signal(signal.SIGINT, signal.default_int_handler) - - def _lazy_init(self) -> None: - if self._state is _State.CLOSED: - raise RuntimeError("Runner is closed") - if self._state is _State.INITIALIZED: - return - if self._loop_factory is None: - self._loop = events.new_event_loop() - if not self._set_event_loop: - # Call set_event_loop only once to avoid calling - # attach_loop multiple times on child watchers - events.set_event_loop(self._loop) - self._set_event_loop = True - else: - self._loop = self._loop_factory() - if self._debug is not None: - self._loop.set_debug(self._debug) - self._context = contextvars.copy_context() - self._state = _State.INITIALIZED - - def _on_sigint(self, signum, frame, main_task: asyncio.Task) -> None: - self._interrupt_count += 1 - if self._interrupt_count == 1 and not main_task.done(): - main_task.cancel() - # wakeup loop if it is blocked by select() with long timeout - self._loop.call_soon_threadsafe(lambda: None) - return - raise KeyboardInterrupt() - - def _cancel_all_tasks(loop: AbstractEventLoop) -> None: - to_cancel = tasks.all_tasks(loop) - if not to_cancel: - return - - for task in to_cancel: - task.cancel() - - loop.run_until_complete(tasks.gather(*to_cancel, return_exceptions=True)) - - for task in to_cancel: - if task.cancelled(): - continue - if task.exception() is not None: - loop.call_exception_handler( - { - "message": "unhandled exception during asyncio.run() shutdown", - "exception": task.exception(), - "task": task, - } - ) - - async def _shutdown_default_executor(loop: AbstractEventLoop) -> None: - """Schedule the shutdown of the default executor.""" - - def _do_shutdown(future: asyncio.futures.Future) -> None: - try: - loop._default_executor.shutdown(wait=True) # type: ignore[attr-defined] - loop.call_soon_threadsafe(future.set_result, None) - except Exception as ex: - loop.call_soon_threadsafe(future.set_exception, ex) - - loop._executor_shutdown_called = True - if loop._default_executor is None: - return - future = loop.create_future() - thread = threading.Thread(target=_do_shutdown, args=(future,)) - thread.start() - try: - await future - finally: - thread.join() - - -T_Retval = TypeVar("T_Retval") -T_co = TypeVar("T_co", covariant=True) -T_contra = TypeVar("T_contra", contravariant=True) -PosArgsT = TypeVarTuple("PosArgsT") -P = ParamSpec("P") - -_root_task: RunVar[asyncio.Task | None] = RunVar("_root_task") - - -def find_root_task() -> asyncio.Task: - root_task = _root_task.get(None) - if root_task is not None and not root_task.done(): - return root_task - - # Look for a task that has been started via run_until_complete() - for task in all_tasks(): - if task._callbacks and not task.done(): - callbacks = [cb for cb, context in task._callbacks] - for cb in callbacks: - if ( - cb is _run_until_complete_cb - or getattr(cb, "__module__", None) == "uvloop.loop" - ): - _root_task.set(task) - return task - - # Look up the topmost task in the AnyIO task tree, if possible - task = cast(asyncio.Task, current_task()) - state = _task_states.get(task) - if state: - cancel_scope = state.cancel_scope - while cancel_scope and cancel_scope._parent_scope is not None: - cancel_scope = cancel_scope._parent_scope - - if cancel_scope is not None: - return cast(asyncio.Task, cancel_scope._host_task) - - return task - - -# -# Event loop -# - -_run_vars: WeakKeyDictionary[asyncio.AbstractEventLoop, Any] = WeakKeyDictionary() - - -def _task_started(task: asyncio.Task) -> bool: - """Return ``True`` if the task has been started and has not finished.""" - # The task coro should never be None here, as we never add finished tasks to the - # task list - coro = task.get_coro() - assert coro is not None - return getcoroutinestate(coro) in (CORO_RUNNING, CORO_SUSPENDED) - - -# -# Timeouts and cancellation -# - - -def is_anyio_cancellation(exc: CancelledError) -> bool: - # Sometimes third party frameworks catch a CancelledError and raise a new one, so as - # a workaround we have to look at the previous ones in __context__ too for a - # matching cancel message - while True: - if ( - exc.args - and isinstance(exc.args[0], str) - and exc.args[0].startswith("Cancelled via cancel scope ") - ): - return True - - if isinstance(exc.__context__, CancelledError): - exc = exc.__context__ - continue - - return False - - -class CancelScope(BaseCancelScope): - __slots__ = ( - "_active", - "_cancel_called", - "_cancel_handle", - "_cancel_reason", - "_cancelled_caught", - "_child_scopes", - "_deadline", - "_host_task", - "_parent_scope", - "_pending_uncancellations", - "_shield", - "_tasks", - "_timeout_handle", - ) - - def __new__( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - return object.__new__(cls) - - def __init__(self, deadline: float = math.inf, shield: bool = False): - self._deadline = deadline - self._shield = shield - self._parent_scope: CancelScope | None = None - self._child_scopes: set[CancelScope] = set() - self._cancel_called = False - self._cancel_reason: str | None = None - self._cancelled_caught = False - self._active = False - self._timeout_handle: asyncio.TimerHandle | None = None - self._cancel_handle: asyncio.Handle | None = None - self._tasks: set[asyncio.Task] = set() - self._host_task: asyncio.Task | None = None - if sys.version_info >= (3, 11): - self._pending_uncancellations: int | None = 0 - else: - self._pending_uncancellations = None - - def __enter__(self) -> CancelScope: - if self._active: - raise RuntimeError( - "Each CancelScope may only be used for a single 'with' block" - ) - - self._host_task = host_task = cast(asyncio.Task, current_task()) - self._tasks.add(host_task) - try: - task_state = _task_states[host_task] - except KeyError: - task_state = TaskState(None, self) - _task_states[host_task] = task_state - else: - self._parent_scope = task_state.cancel_scope - task_state.cancel_scope = self - if self._parent_scope is not None: - # If using an eager task factory, the parent scope may not even contain - # the host task - self._parent_scope._child_scopes.add(self) - self._parent_scope._tasks.discard(host_task) - - self._timeout() - self._active = True - - # Start cancelling the host task if the scope was cancelled before entering - if self._cancel_called: - self._deliver_cancellation(self) - - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - del exc_tb - - if not self._active: - raise RuntimeError("This cancel scope is not active") - if current_task() is not self._host_task: - raise RuntimeError( - "Attempted to exit cancel scope in a different task than it was " - "entered in" - ) - - assert self._host_task is not None - host_task_state = _task_states.get(self._host_task) - if host_task_state is None or host_task_state.cancel_scope is not self: - raise RuntimeError( - "Attempted to exit a cancel scope that isn't the current tasks's " - "current cancel scope" - ) - - try: - self._active = False - if self._timeout_handle: - self._timeout_handle.cancel() - self._timeout_handle = None - - self._tasks.remove(self._host_task) - if self._parent_scope is not None: - self._parent_scope._child_scopes.remove(self) - self._parent_scope._tasks.add(self._host_task) - - host_task_state.cancel_scope = self._parent_scope - - # Restart the cancellation effort in the closest visible, cancelled parent - # scope if necessary - self._restart_cancellation_in_parent() - - # We only swallow the exception iff it was an AnyIO CancelledError, either - # directly as exc_val or inside an exception group and there are no cancelled - # parent cancel scopes visible to us here - if self._cancel_called and not self._parent_cancellation_is_visible_to_us: - # For each level-cancel() call made on the host task, call uncancel() - while self._pending_uncancellations: - self._host_task.uncancel() - self._pending_uncancellations -= 1 - - # Update cancelled_caught and check for exceptions we must not swallow - if isinstance(exc_val, BaseExceptionGroup): - cancelleds_caught, remaining = exc_val.split( - lambda exc: ( - isinstance(exc, CancelledError) - and is_anyio_cancellation(exc) - ) - ) - - if cancelleds_caught is None: - return False - - self._cancelled_caught = True - - if remaining is None: - return True - - context = remaining.__context__ - try: - # Preserve __cause__ and __suppress_context__ by avoiding `raise - # ... from ...` - raise remaining - finally: - # Preserve __context__ - remaining.__context__ = context - del context - else: - if isinstance(exc_val, CancelledError) and is_anyio_cancellation( - exc_val - ): - self._cancelled_caught = True - return True - else: - return False - else: - if self._pending_uncancellations: - assert self._parent_scope is not None - assert self._parent_scope._pending_uncancellations is not None - self._parent_scope._pending_uncancellations += ( - self._pending_uncancellations - ) - self._pending_uncancellations = 0 - - return False - finally: - self._host_task = None - del exc_val - - @property - def _effectively_cancelled(self) -> bool: - cancel_scope: CancelScope | None = self - while cancel_scope is not None: - if cancel_scope._cancel_called: - return True - - if cancel_scope.shield: - return False - - cancel_scope = cancel_scope._parent_scope - - return False - - @property - def _parent_cancellation_is_visible_to_us(self) -> bool: - return ( - self._parent_scope is not None - and not self.shield - and self._parent_scope._effectively_cancelled - ) - - def _timeout(self) -> None: - if self._deadline != math.inf: - loop = get_running_loop() - if loop.time() >= self._deadline: - self.cancel("deadline exceeded") - else: - self._timeout_handle = loop.call_at(self._deadline, self._timeout) - - def _deliver_cancellation(self, origin: CancelScope) -> bool: - """ - Deliver cancellation to directly contained tasks and nested cancel scopes. - - Schedule another run at the end if we still have tasks eligible for - cancellation. - - :param origin: the cancel scope that originated the cancellation - :return: ``True`` if the delivery needs to be retried on the next cycle - - """ - should_retry = False - current = current_task() - for task in self._tasks: - # Always skip tasks that are already done (see issue #1111) - if task.done(): - continue - - should_retry = True - if task._must_cancel: # type: ignore[attr-defined] - continue - - # The task is eligible for cancellation if it has started - if task is not current and (task is self._host_task or _task_started(task)): - waiter = task._fut_waiter # type: ignore[attr-defined] - if not isinstance(waiter, asyncio.Future) or not waiter.done(): - task.cancel(origin._cancel_reason) - if ( - task is origin._host_task - and origin._pending_uncancellations is not None - ): - origin._pending_uncancellations += 1 - - # Deliver cancellation to child scopes that aren't shielded or running their own - # cancellation callbacks - for scope in self._child_scopes: - if not scope._shield and not scope.cancel_called: - should_retry = scope._deliver_cancellation(origin) or should_retry - - # Schedule another callback if there are still tasks left - if origin is self: - if should_retry: - self._cancel_handle = get_running_loop().call_soon( - self._deliver_cancellation, origin - ) - else: - self._cancel_handle = None - - return should_retry - - def _restart_cancellation_in_parent(self) -> None: - """ - Restart the cancellation effort in the closest directly cancelled parent scope. - - """ - scope = self._parent_scope - while scope is not None: - if scope._cancel_called: - if scope._cancel_handle is None: - scope._deliver_cancellation(scope) - - break - - # No point in looking beyond any shielded scope - if scope._shield: - break - - scope = scope._parent_scope - - def cancel(self, reason: str | None = None) -> None: - if not self._cancel_called: - if self._timeout_handle: - self._timeout_handle.cancel() - self._timeout_handle = None - - self._cancel_called = True - self._cancel_reason = f"Cancelled via cancel scope {id(self):x}" - if task := current_task(): - self._cancel_reason += f" by {task}" - - if reason: - self._cancel_reason += f"; reason: {reason}" - - if self._host_task is not None: - self._deliver_cancellation(self) - - @property - def deadline(self) -> float: - return self._deadline - - @deadline.setter - def deadline(self, value: float) -> None: - self._deadline = float(value) - if self._timeout_handle is not None: - self._timeout_handle.cancel() - self._timeout_handle = None - - if self._active and not self._cancel_called: - self._timeout() - - @property - def cancel_called(self) -> bool: - return self._cancel_called - - @property - def cancelled_caught(self) -> bool: - return self._cancelled_caught - - @property - def shield(self) -> bool: - return self._shield - - @shield.setter - def shield(self, value: bool) -> None: - if self._shield != value: - self._shield = value - if not value: - self._restart_cancellation_in_parent() - - -# -# Task states -# - - -class TaskState: - """ - Encapsulates auxiliary task information that cannot be added to the Task instance - itself because there are no guarantees about its implementation. - """ - - __slots__ = "parent_id", "cancel_scope", "__weakref__" - - def __init__(self, parent_id: int | None, cancel_scope: CancelScope | None): - self.parent_id = parent_id - self.cancel_scope = cancel_scope - - -_task_states: WeakKeyDictionary[asyncio.Task, TaskState] = WeakKeyDictionary() - - -# -# Task groups -# - - -class _AsyncioTaskStatus(abc.TaskStatus): - def __init__(self, future: asyncio.Future, parent_id: int): - self._future = future - self._parent_id = parent_id - - def started(self, value: T_contra | None = None) -> None: - try: - self._future.set_result(value) - except asyncio.InvalidStateError: - if not self._future.cancelled(): - raise RuntimeError( - "called 'started' twice on the same task status" - ) from None - - task = cast(asyncio.Task, current_task()) - _task_states[task].parent_id = self._parent_id - - -if sys.version_info >= (3, 12): - _eager_task_factory_code: CodeType | None = asyncio.eager_task_factory.__code__ -else: - _eager_task_factory_code = None - - -class TaskGroup(abc.TaskGroup): - def __init__(self) -> None: - self.cancel_scope: CancelScope = CancelScope() - self._entered = False - self._exceptions: list[BaseException] = [] - self._tasks: set[asyncio.Task] = set() - self._on_completed_fut: asyncio.Future[None] | None = None - - async def __aenter__(self) -> TaskGroup: - if self._entered: - raise RuntimeError("TaskGroup cannot be entered more than once") - - self._entered = True - - self.cancel_scope.__enter__() - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - try: - if exc_val is not None: - self.cancel_scope.cancel() - if not isinstance(exc_val, CancelledError): - self._exceptions.append(exc_val) - - loop = get_running_loop() - try: - if self._tasks: - with CancelScope() as wait_scope: - while self._tasks: - self._on_completed_fut = loop.create_future() - - try: - await self._on_completed_fut - except CancelledError as exc: - # Shield the scope against further cancellation attempts, - # as they're not productive (#695) - wait_scope.shield = True - self.cancel_scope.cancel() - - # Set exc_val from the cancellation exception if it was - # previously unset. However, we should not replace a native - # cancellation exception with one raise by a cancel scope. - if exc_val is None or ( - isinstance(exc_val, CancelledError) - and not is_anyio_cancellation(exc) - ): - exc_val = exc - - self._on_completed_fut = None - else: - # If there are no child tasks to wait on, run at least one checkpoint - # anyway - await AsyncIOBackend.cancel_shielded_checkpoint() - - if self._exceptions: - # The exception that got us here should already have been - # added to self._exceptions so it's ok to break exception - # chaining and avoid adding a "During handling of above..." - # for each nesting level. - raise BaseExceptionGroup( - "unhandled errors in a TaskGroup", self._exceptions - ) from None - elif exc_val: - raise exc_val - except BaseException as exc: - if self.cancel_scope.__exit__(type(exc), exc, exc.__traceback__): - return True - - raise - - return self.cancel_scope.__exit__(exc_type, exc_val, exc_tb) - finally: - del exc_val, exc_tb, self._exceptions - - def _spawn( - self, - coro: Coroutine[Any, Any, T_co], - name: object, - task_status_future: asyncio.Future | None = None, - ) -> TaskHandle[T_co]: - def task_done(_task: asyncio.Task) -> None: - if sys.version_info >= (3, 14) and self.cancel_scope._host_task is not None: - asyncio.future_discard_from_awaited_by( - _task, self.cancel_scope._host_task - ) - - task_state = _task_states[_task] - assert task_state.cancel_scope is not None - assert _task in task_state.cancel_scope._tasks - task_state.cancel_scope._tasks.remove(_task) - self._tasks.remove(task) - del _task_states[_task] - - if self._on_completed_fut is not None and not self._tasks: - try: - self._on_completed_fut.set_result(None) - except asyncio.InvalidStateError: - pass - - try: - exc = _task.exception() - except CancelledError as e: - while isinstance(e.__context__, CancelledError): - e = e.__context__ - - exc = e - - if exc is not None: - # The future can only be in the cancelled state if the host task was - # cancelled, so return immediately instead of adding one more - # CancelledError to the exceptions list - if task_status_future is not None and task_status_future.cancelled(): - return - - if task_status_future is None or task_status_future.done(): - if not isinstance(exc, CancelledError): - self._exceptions.append(exc) - - if not self.cancel_scope._effectively_cancelled: - self.cancel_scope.cancel() - else: - task_status_future.set_exception(exc) - elif task_status_future is not None and not task_status_future.done(): - task_status_future.set_exception( - RuntimeError("Child exited without calling task_status.started()") - ) - - if task_status_future: - parent_id = id(current_task()) - else: - parent_id = id(self.cancel_scope._host_task) - - handle = TaskHandle(coro, name) - loop = asyncio.get_running_loop() - wrapper_coro = handle._run_coro() - if ( - (factory := loop.get_task_factory()) - and getattr(factory, "__code__", None) is _eager_task_factory_code - and (closure := getattr(factory, "__closure__", None)) - ): - custom_task_constructor = closure[0].cell_contents - task = custom_task_constructor(wrapper_coro, loop=loop, name=handle.name) - else: - task = loop.create_task(wrapper_coro, name=handle.name) - - # Make the spawned task inherit the task group's cancel scope - _task_states[task] = TaskState( - parent_id=parent_id, cancel_scope=self.cancel_scope - ) - self.cancel_scope._tasks.add(task) - self._tasks.add(task) - if sys.version_info >= (3, 14) and self.cancel_scope._host_task is not None: - asyncio.future_add_to_awaited_by(task, self.cancel_scope._host_task) - - task.add_done_callback(task_done) - return handle - - def create_task( - self, - coro: Coroutine[Any, Any, T_co], - *, - name: object = None, - context: Context | None = None, - ) -> TaskHandle[T_co]: - if not isinstance(coro, Coroutine): - raise TypeError(f"expected a coroutine, got {coro.__class__.__qualname__}") - - if not self._entered or not self.cancel_scope._active: - coro.close() - raise RuntimeError( - "This task group is not active; no new tasks can be started." - ) - - if context is not None: - return context.run(self._spawn, coro, name=name) - else: - return self._spawn(coro, name=name) - - async def start( - self, - func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], - *args: Unpack[PosArgsT], - name: object = None, - return_handle: Literal[False] | Literal[True] = False, - ) -> Any: - if not self._entered or not self.cancel_scope._active: - raise RuntimeError( - "This task group is not active; no new tasks can be started." - ) - - future: asyncio.Future = asyncio.Future() - final_name = get_callable_name(func, name) - task_status = _AsyncioTaskStatus(future, id(self.cancel_scope._host_task)) - coro = call_for_coroutine(func, args, task_status=task_status) - handle = self._spawn(coro, final_name, future) - - # If the task raises an exception after sending a start value without a switch - # point between, the task group is cancelled and this method never proceeds to - # process the completed future. That's why we have to have a shielded cancel - # scope here. - try: - await future - except BaseException: - if handle.status is TaskHandle.Status.PENDING: - # Cancel the task and wait for it to exit before returning - handle.cancel() - with CancelScope(shield=True): - await handle.wait() - - raise - - if return_handle: - handle._start_value = future.result() - return handle - else: - return future.result() - - -# -# Threads -# - -_Retval_Queue_Type = tuple[T_Retval | None, BaseException | None] - - -class WorkerThread(Thread): - MAX_IDLE_TIME = 10 # seconds - - def __init__( - self, - root_task: asyncio.Task, - workers: set[WorkerThread], - idle_workers: deque[WorkerThread], - ): - super().__init__(name="AnyIO worker thread") - self.root_task = root_task - self.workers = workers - self.idle_workers = idle_workers - self.loop = root_task._loop - self.queue: Queue[ - tuple[Context, Callable, tuple, asyncio.Future, CancelScope] | None - ] = Queue(2) - self.idle_since = AsyncIOBackend.current_time() - self.stopping = False - - def _report_result( - self, future: asyncio.Future, result: Any, exc: BaseException | None - ) -> None: - self.idle_since = AsyncIOBackend.current_time() - if not self.stopping: - self.idle_workers.append(self) - - if not future.cancelled(): - if exc is not None: - if isinstance(exc, StopIteration): - new_exc = RuntimeError("coroutine raised StopIteration") - new_exc.__cause__ = exc - exc = new_exc - - future.set_exception(exc) - else: - future.set_result(result) - - def run(self) -> None: - with claim_worker_thread(AsyncIOBackend, self.loop): - while True: - item = self.queue.get() - if item is None: - # Shutdown command received - return - - context, func, args, future, cancel_scope = item - if not future.cancelled(): - result = None - exception: BaseException | None = None - threadlocals.current_cancel_scope = cancel_scope - try: - result = context.run(func, *args) - except BaseException as exc: - exception = exc - finally: - del threadlocals.current_cancel_scope - - if not self.loop.is_closed(): - self.loop.call_soon_threadsafe( - self._report_result, future, result, exception - ) - - del result, exception - - self.queue.task_done() - del item, context, func, args, future, cancel_scope - - def stop(self, f: asyncio.Task | None = None) -> None: - self.stopping = True - self.queue.put_nowait(None) - self.workers.discard(self) - try: - self.idle_workers.remove(self) - except ValueError: - pass - - -_threadpool_idle_workers: RunVar[deque[WorkerThread]] = RunVar( - "_threadpool_idle_workers" -) -_threadpool_workers: RunVar[set[WorkerThread]] = RunVar("_threadpool_workers") - - -# -# Subprocesses -# - - -@dataclass(eq=False) -class StreamReaderWrapper(abc.ByteReceiveStream): - _stream: asyncio.StreamReader - - async def receive(self, max_bytes: int = 65536) -> bytes: - if max_bytes < 1: - raise ValueError("max_bytes must be a positive integer") - - data = await self._stream.read(max_bytes) - if data: - return data - else: - raise EndOfStream - - async def aclose(self) -> None: - self._stream.set_exception(ClosedResourceError()) - await AsyncIOBackend.checkpoint() - - -@dataclass(eq=False) -class StreamWriterWrapper(abc.ByteSendStream): - _stream: asyncio.StreamWriter - _closed: bool = field(init=False, default=False) - - async def send(self, item: bytes) -> None: - await AsyncIOBackend.checkpoint_if_cancelled() - stream_paused = self._stream._protocol._paused # type: ignore[attr-defined] - try: - self._stream.write(item) - await self._stream.drain() - except (ConnectionResetError, BrokenPipeError, RuntimeError) as exc: - # If closed by us and/or the peer: - # * on stdlib, drain() raises ConnectionResetError or BrokenPipeError - # * on uvloop and Winloop, write() eventually starts raising RuntimeError - if self._closed: - raise ClosedResourceError from exc - elif self._stream.is_closing(): - raise BrokenResourceError from exc - - raise - - if not stream_paused: - await AsyncIOBackend.cancel_shielded_checkpoint() - - async def aclose(self) -> None: - self._closed = True - self._stream.close() - await AsyncIOBackend.checkpoint() - - -@dataclass(eq=False) -class Process(abc.Process): - _process: asyncio.subprocess.Process - _stdin: StreamWriterWrapper | None - _stdout: StreamReaderWrapper | None - _stderr: StreamReaderWrapper | None - _exited: asyncio.Event - _transport: asyncio.SubprocessTransport - - async def aclose(self) -> None: - with CancelScope(shield=True) as scope: - # We need to close the underlying pipe_transports as well to allow a - # process blocking on full buffers to receive SIGPIPE and exit. - if self._stdin: - await self._stdin.aclose() - if pipe := self._transport.get_pipe_transport(0): - pipe.close() - if self._stdout: - await self._stdout.aclose() - if pipe := self._transport.get_pipe_transport(1): - pipe.close() - if self._stderr: - await self._stderr.aclose() - if pipe := self._transport.get_pipe_transport(2): - pipe.close() - - scope.shield = False - try: - await self.wait() - except BaseException: - scope.shield = True - # Closing the transport on asyncio also handles sending kill - self._transport.close() - await self.wait() - raise - - async def wait(self) -> int: - await self._exited.wait() - assert self._process.returncode is not None - return self._process.returncode - - def terminate(self) -> None: - self._process.terminate() - - def kill(self) -> None: - self._process.kill() - - def send_signal(self, signal: int) -> None: - self._process.send_signal(signal) - - @property - def pid(self) -> int: - return self._process.pid - - @property - def returncode(self) -> int | None: - return self._process.returncode - - @property - def stdin(self) -> abc.ByteSendStream | None: - return self._stdin - - @property - def stdout(self) -> abc.ByteReceiveStream | None: - return self._stdout - - @property - def stderr(self) -> abc.ByteReceiveStream | None: - return self._stderr - - -def _forcibly_shutdown_process_pool_on_exit( - workers: set[Process], _task: object -) -> None: - """ - Forcibly shuts down worker processes belonging to this event loop.""" - child_watcher: asyncio.AbstractChildWatcher | None = None # type: ignore[name-defined] - if sys.version_info < (3, 12): - try: - child_watcher = asyncio.get_event_loop_policy().get_child_watcher() - except NotImplementedError: - pass - - # Close as much as possible (w/o async/await) to avoid warnings - for process in workers.copy(): - if process.returncode is not None: - continue - - process._stdin._stream._transport.close() # type: ignore[union-attr] - process._stdout._stream._transport.close() # type: ignore[union-attr] - process._stderr._stream._transport.close() # type: ignore[union-attr] - process.kill() - if child_watcher: - child_watcher.remove_child_handler(process.pid) - - -async def _shutdown_process_pool_on_exit(workers: set[abc.Process]) -> None: - """ - Shuts down worker processes belonging to this event loop. - - NOTE: this only works when the event loop was started using asyncio.run() or - anyio.run(). - - """ - process: abc.Process - try: - await sleep(math.inf) - except asyncio.CancelledError: - workers = workers.copy() - for process in workers: - if process.returncode is None: - process.kill() - - for process in workers: - await process.aclose() - - -# -# Sockets and networking -# - - -class StreamProtocol(asyncio.Protocol): - read_queue: deque[bytes] - read_event: asyncio.Event - write_event: asyncio.Event - exception: Exception | None = None - is_at_eof: bool = False - - def connection_made(self, transport: asyncio.BaseTransport) -> None: - self.read_queue = deque() - self.read_event = asyncio.Event() - self.write_event = asyncio.Event() - self.write_event.set() - cast(asyncio.Transport, transport).set_write_buffer_limits(0) - - def connection_lost(self, exc: Exception | None) -> None: - if exc: - self.exception = exc - - self.read_event.set() - self.write_event.set() - - def data_received(self, data: bytes) -> None: - # ProactorEventloop sometimes sends bytearray instead of bytes - self.read_queue.append(bytes(data)) - self.read_event.set() - - def eof_received(self) -> bool | None: - self.is_at_eof = True - self.read_event.set() - return True - - def pause_writing(self) -> None: - self.write_event = asyncio.Event() - - def resume_writing(self) -> None: - self.write_event.set() - - -class DatagramProtocol(asyncio.DatagramProtocol): - read_queue: deque[tuple[bytes, IPSockAddrType]] - read_event: asyncio.Event - write_event: asyncio.Event - closed_event: asyncio.Event - exception: Exception | None = None - - def connection_made(self, transport: asyncio.BaseTransport) -> None: - self.read_queue = deque(maxlen=100) # arbitrary value - self.read_event = asyncio.Event() - self.write_event = asyncio.Event() - self.closed_event = asyncio.Event() - self.write_event.set() - - def connection_lost(self, exc: Exception | None) -> None: - self.read_event.set() - self.write_event.set() - self.closed_event.set() - - def datagram_received(self, data: bytes, addr: IPSockAddrType) -> None: - addr = convert_ipv6_sockaddr(addr) - self.read_queue.append((data, addr)) - self.read_event.set() - - def error_received(self, exc: Exception) -> None: - self.exception = exc - - def pause_writing(self) -> None: - self.write_event.clear() - - def resume_writing(self) -> None: - self.write_event.set() - - -class SocketStream(abc.SocketStream): - def __init__(self, transport: asyncio.Transport, protocol: StreamProtocol): - self._transport = transport - self._protocol = protocol - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - self._closed = False - - @property - def _raw_socket(self) -> socket.socket: - return self._transport.get_extra_info("socket") - - async def receive(self, max_bytes: int = 65536) -> bytes: - if max_bytes < 1: - raise ValueError("max_bytes must be a positive integer") - - with self._receive_guard: - if ( - not self._protocol.read_event.is_set() - and not self._transport.is_closing() - and not self._protocol.is_at_eof - ): - self._transport.resume_reading() - await self._protocol.read_event.wait() - self._transport.pause_reading() - else: - await AsyncIOBackend.checkpoint() - - try: - chunk = self._protocol.read_queue.popleft() - except IndexError: - if self._closed: - raise ClosedResourceError from None - elif self._protocol.exception: - raise BrokenResourceError from self._protocol.exception - else: - raise EndOfStream from None - - if len(chunk) > max_bytes: - # Split the oversized chunk - chunk, leftover = chunk[:max_bytes], chunk[max_bytes:] - self._protocol.read_queue.appendleft(leftover) - - # If the read queue is empty, clear the flag so that the next call will - # block until data is available - if not self._protocol.read_queue: - self._protocol.read_event.clear() - - return chunk - - async def send(self, item: bytes) -> None: - with self._send_guard: - await AsyncIOBackend.checkpoint() - - if self._closed: - raise ClosedResourceError - elif self._protocol.exception is not None: - raise BrokenResourceError from self._protocol.exception - - try: - self._transport.write(item) - except RuntimeError as exc: - if self._transport.is_closing(): - raise BrokenResourceError from exc - else: - raise - - await self._protocol.write_event.wait() - - async def send_eof(self) -> None: - try: - self._transport.write_eof() - except OSError: - pass - - async def aclose(self) -> None: - self._closed = True - if not self._transport.is_closing(): - try: - self._transport.write_eof() - except OSError: - pass - - self._transport.close() - await sleep(0) - self._transport.abort() - - -class _RawSocketMixin: - _receive_future: asyncio.Future | None = None - _send_future: asyncio.Future | None = None - _closing = False - - def __init__(self, raw_socket: socket.socket): - self.__raw_socket = raw_socket - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - @property - def _raw_socket(self) -> socket.socket: - return self.__raw_socket - - def _wait_until_readable(self, loop: asyncio.AbstractEventLoop) -> asyncio.Future: - def callback(f: object) -> None: - del self._receive_future - loop.remove_reader(self.__raw_socket) - - f = self._receive_future = asyncio.Future() - loop.add_reader(self.__raw_socket, f.set_result, None) - f.add_done_callback(callback) - return f - - def _wait_until_writable(self, loop: asyncio.AbstractEventLoop) -> asyncio.Future: - def callback(f: object) -> None: - del self._send_future - loop.remove_writer(self.__raw_socket) - - f = self._send_future = asyncio.Future() - loop.add_writer(self.__raw_socket, f.set_result, None) - f.add_done_callback(callback) - return f - - async def aclose(self) -> None: - if not self._closing: - self._closing = True - if self.__raw_socket.fileno() != -1: - self.__raw_socket.close() - - if self._receive_future: - self._receive_future.set_result(None) - if self._send_future: - self._send_future.set_result(None) - - -class UNIXSocketStream(_RawSocketMixin, abc.UNIXSocketStream): - async def send_eof(self) -> None: - with self._send_guard: - self._raw_socket.shutdown(socket.SHUT_WR) - - async def receive(self, max_bytes: int = 65536) -> bytes: - if max_bytes < 1: - raise ValueError("max_bytes must be a positive integer") - - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - data = self._raw_socket.recv(max_bytes) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - if not data: - raise EndOfStream - - return data - - async def send(self, item: bytes) -> None: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._send_guard: - view = memoryview(item) - while view: - try: - bytes_sent = self._raw_socket.send(view) - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - view = view[bytes_sent:] - - async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: - if not isinstance(msglen, int) or msglen < 0: - raise ValueError("msglen must be a non-negative integer") - if not isinstance(maxfds, int) or maxfds < 1: - raise ValueError("maxfds must be a positive integer") - - loop = get_running_loop() - fds = array.array("i") - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - message, ancdata, flags, addr = self._raw_socket.recvmsg( - msglen, socket.CMSG_LEN(maxfds * fds.itemsize) - ) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - if not message and not ancdata: - raise EndOfStream - - break - - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level != socket.SOL_SOCKET or cmsg_type != socket.SCM_RIGHTS: - raise RuntimeError( - f"Received unexpected ancillary data; message = {message!r}, " - f"cmsg_level = {cmsg_level}, cmsg_type = {cmsg_type}" - ) - - fds.frombytes(cmsg_data[: len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - - return message, list(fds) - - async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: - if not message: - raise ValueError("message must not be empty") - if not fds: - raise ValueError("fds must not be empty") - - loop = get_running_loop() - filenos: list[int] = [] - for fd in fds: - if isinstance(fd, int): - filenos.append(fd) - elif isinstance(fd, IOBase): - filenos.append(fd.fileno()) - - fdarray = array.array("i", filenos) - await AsyncIOBackend.checkpoint() - with self._send_guard: - while True: - try: - # The ignore can be removed after mypy picks up - # https://github.com/python/typeshed/pull/5545 - self._raw_socket.sendmsg( - [message], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fdarray)] - ) - break - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - - -class TCPSocketListener(abc.SocketListener): - _accept_scope: CancelScope | None = None - _closed = False - - def __init__(self, raw_socket: socket.socket): - self.__raw_socket = raw_socket - self._loop = cast(asyncio.BaseEventLoop, get_running_loop()) - self._accept_guard = ResourceGuard("accepting connections from") - - @property - def _raw_socket(self) -> socket.socket: - return self.__raw_socket - - async def accept(self) -> abc.SocketStream: - if self._closed: - raise ClosedResourceError - - with self._accept_guard: - await AsyncIOBackend.checkpoint() - with CancelScope() as self._accept_scope: - try: - client_sock, _addr = await self._loop.sock_accept(self._raw_socket) - except asyncio.CancelledError: - # Workaround for https://bugs.python.org/issue41317 - try: - self._loop.remove_reader(self._raw_socket) - except (ValueError, NotImplementedError): - pass - - if self._closed: - raise ClosedResourceError from None - - raise - finally: - self._accept_scope = None - - client_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - transport, protocol = await self._loop.connect_accepted_socket( - StreamProtocol, client_sock - ) - return SocketStream(transport, protocol) - - async def aclose(self) -> None: - if self._closed: - return - - self._closed = True - if self._accept_scope: - # Workaround for https://bugs.python.org/issue41317 - try: - self._loop.remove_reader(self._raw_socket) - except (ValueError, NotImplementedError): - pass - - self._accept_scope.cancel() - await sleep(0) - - self._raw_socket.close() - - -class UNIXSocketListener(abc.SocketListener): - def __init__(self, raw_socket: socket.socket): - self.__raw_socket = raw_socket - self._loop = get_running_loop() - self._accept_guard = ResourceGuard("accepting connections from") - self._closed = False - - async def accept(self) -> abc.SocketStream: - await AsyncIOBackend.checkpoint() - with self._accept_guard: - while True: - try: - client_sock, _ = self.__raw_socket.accept() - client_sock.setblocking(False) - return UNIXSocketStream(client_sock) - except BlockingIOError: - f: asyncio.Future = asyncio.Future() - self._loop.add_reader(self.__raw_socket, f.set_result, None) - f.add_done_callback( - lambda _: self._loop.remove_reader(self.__raw_socket) - ) - await f - except OSError as exc: - if self._closed: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - - async def aclose(self) -> None: - self._closed = True - self.__raw_socket.close() - - @property - def _raw_socket(self) -> socket.socket: - return self.__raw_socket - - -class UDPSocket(abc.UDPSocket): - def __init__( - self, transport: asyncio.DatagramTransport, protocol: DatagramProtocol - ): - self._transport = transport - self._protocol = protocol - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - self._closed = False - - @property - def _raw_socket(self) -> socket.socket: - return self._transport.get_extra_info("socket") - - async def aclose(self) -> None: - self._closed = True - if not self._transport.is_closing(): - self._transport.close() - - await self._protocol.closed_event.wait() - - async def receive(self) -> tuple[bytes, IPSockAddrType]: - with self._receive_guard: - await AsyncIOBackend.checkpoint() - - # If the buffer is empty, ask for more data - if not self._protocol.read_queue and not self._transport.is_closing(): - self._protocol.read_event.clear() - await self._protocol.read_event.wait() - - try: - return self._protocol.read_queue.popleft() - except IndexError: - if self._closed: - raise ClosedResourceError from None - else: - raise BrokenResourceError from None - - async def send(self, item: UDPPacketType) -> None: - with self._send_guard: - await AsyncIOBackend.checkpoint() - await self._protocol.write_event.wait() - if self._closed: - raise ClosedResourceError - elif self._transport.is_closing(): - raise BrokenResourceError - else: - self._transport.sendto(*item) - - -class ConnectedUDPSocket(abc.ConnectedUDPSocket): - def __init__( - self, transport: asyncio.DatagramTransport, protocol: DatagramProtocol - ): - self._transport = transport - self._protocol = protocol - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - self._closed = False - - @property - def _raw_socket(self) -> socket.socket: - return self._transport.get_extra_info("socket") - - async def aclose(self) -> None: - self._closed = True - if not self._transport.is_closing(): - self._transport.close() - - await self._protocol.closed_event.wait() - - async def receive(self) -> bytes: - with self._receive_guard: - await AsyncIOBackend.checkpoint() - - # If the buffer is empty, ask for more data - if not self._protocol.read_queue and not self._transport.is_closing(): - self._protocol.read_event.clear() - await self._protocol.read_event.wait() - - try: - packet = self._protocol.read_queue.popleft() - except IndexError: - if self._closed: - raise ClosedResourceError from None - else: - raise BrokenResourceError from None - - return packet[0] - - async def send(self, item: bytes) -> None: - with self._send_guard: - await AsyncIOBackend.checkpoint() - await self._protocol.write_event.wait() - if self._closed: - raise ClosedResourceError - elif self._transport.is_closing(): - raise BrokenResourceError - else: - self._transport.sendto(item) - - -class UNIXDatagramSocket(_RawSocketMixin, abc.UNIXDatagramSocket): - async def receive(self) -> UNIXDatagramPacketType: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - data = self._raw_socket.recvfrom(65536) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return data - - async def send(self, item: UNIXDatagramPacketType) -> None: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._send_guard: - while True: - try: - self._raw_socket.sendto(*item) - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return - - -class ConnectedUNIXDatagramSocket(_RawSocketMixin, abc.ConnectedUNIXDatagramSocket): - async def receive(self) -> bytes: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - data = self._raw_socket.recv(65536) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return data - - async def send(self, item: bytes) -> None: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._send_guard: - while True: - try: - self._raw_socket.send(item) - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return - - -_read_events: RunVar[dict[int, asyncio.Future[bool]]] = RunVar("read_events") -_write_events: RunVar[dict[int, asyncio.Future[bool]]] = RunVar("write_events") - - -# -# Synchronization -# - - -class Event(BaseEvent): - __slots__ = ("_event",) - - def __new__(cls) -> Event: - return object.__new__(cls) - - def __init__(self) -> None: - self._event = asyncio.Event() - - def set(self) -> None: - self._event.set() - - def is_set(self) -> bool: - return self._event.is_set() - - async def wait(self) -> None: - if self.is_set(): - await AsyncIOBackend.checkpoint() - else: - await self._event.wait() - - def statistics(self) -> EventStatistics: - return EventStatistics(len(self._event._waiters)) - - -class Lock(BaseLock): - __slots__ = "_fast_acquire", "_owner_task", "_waiters" - - def __new__(cls, *, fast_acquire: bool = False) -> Lock: - return object.__new__(cls) - - def __init__(self, *, fast_acquire: bool = False) -> None: - self._fast_acquire = fast_acquire - self._owner_task: asyncio.Task | None = None - self._waiters: deque[tuple[asyncio.Task, asyncio.Future]] = deque() - - async def acquire(self) -> None: - task = cast(asyncio.Task, current_task()) - if self._owner_task is None and not self._waiters: - await AsyncIOBackend.checkpoint_if_cancelled() - self._owner_task = task - - # Unless on the "fast path", yield control of the event loop so that other - # tasks can run too - if not self._fast_acquire: - try: - await AsyncIOBackend.cancel_shielded_checkpoint() - except CancelledError: - self.release() - raise - - return - - if self._owner_task == task: - raise RuntimeError("Attempted to acquire an already held Lock") - - fut: asyncio.Future[None] = asyncio.Future() - item = task, fut - self._waiters.append(item) - try: - await fut - except CancelledError: - if fut.cancelled(): - try: - self._waiters.remove(item) - except ValueError: - pass - else: - self.release() - - raise - - def acquire_nowait(self) -> None: - task = cast(asyncio.Task, current_task()) - if self._owner_task is None and not self._waiters: - self._owner_task = task - return - - if self._owner_task is task: - raise RuntimeError("Attempted to acquire an already held Lock") - - raise WouldBlock - - def locked(self) -> bool: - return self._owner_task is not None - - def release(self) -> None: - if self._owner_task != current_task(): - raise RuntimeError("The current task is not holding this lock") - - # A cancelled waiter that already received ownership removes itself from - # _waiters before calling release(); any cancelled waiter still queued here - # was cancelled before being woken, so drop it. - while self._waiters: - task, fut = self._waiters.popleft() - if fut.cancelled(): - continue - - self._owner_task = task - fut.set_result(None) - return - - self._owner_task = None - - def statistics(self) -> LockStatistics: - task_info = AsyncIOTaskInfo(self._owner_task) if self._owner_task else None - return LockStatistics(self.locked(), task_info, len(self._waiters)) - - -class Semaphore(BaseSemaphore): - __slots__ = "_value", "_max_value", "_fast_acquire", "_waiters" - - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - return object.__new__(cls) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ): - super().__init__(initial_value, max_value=max_value) - self._value = initial_value - self._max_value = max_value - self._fast_acquire = fast_acquire - self._waiters: deque[asyncio.Future[None]] = deque() - - async def acquire(self) -> None: - if self._value > 0 and not self._waiters: - await AsyncIOBackend.checkpoint_if_cancelled() - self._value -= 1 - - # Unless on the "fast path", yield control of the event loop so that other - # tasks can run too - if not self._fast_acquire: - try: - await AsyncIOBackend.cancel_shielded_checkpoint() - except CancelledError: - self.release() - raise - - return - - fut: asyncio.Future[None] = asyncio.Future() - self._waiters.append(fut) - try: - await fut - except CancelledError: - if fut.cancelled(): - try: - self._waiters.remove(fut) - except ValueError: - pass - else: - self.release() - - raise - - def acquire_nowait(self) -> None: - if self._value == 0: - raise WouldBlock - - self._value -= 1 - - def release(self) -> None: - if self._max_value is not None and self._value == self._max_value: - raise ValueError("semaphore released too many times") - - while self._waiters: - fut = self._waiters.popleft() - if fut.cancelled(): - continue - - fut.set_result(None) - return - - self._value += 1 - - @property - def value(self) -> int: - return self._value - - @property - def max_value(self) -> int | None: - return self._max_value - - def statistics(self) -> SemaphoreStatistics: - return SemaphoreStatistics(len(self._waiters)) - - -class CapacityLimiter(BaseCapacityLimiter): - __slots__ = "_total_tokens", "_borrowers", "_wait_queue" - - def __new__(cls, total_tokens: float) -> CapacityLimiter: - return object.__new__(cls) - - def __init__(self, total_tokens: float): - self._total_tokens: float = 0 - self._borrowers: set[Any] = set() - self._wait_queue: OrderedDict[Any, asyncio.Event] = OrderedDict() - self.total_tokens = total_tokens - - async def __aenter__(self) -> None: - await self.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - @property - def total_tokens(self) -> float: - return self._total_tokens - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - if not isinstance(value, int) and not math.isinf(value): - raise TypeError("total_tokens must be an int or math.inf") - - if value < 0: - raise ValueError("total_tokens must be >= 0") - - waiters_to_notify = max(value - self._total_tokens, 0) - self._total_tokens = value - - # Notify waiting tasks that they have acquired the limiter - while self._wait_queue and waiters_to_notify: - borrower, event = self._wait_queue.popitem(last=False) - self._borrowers.add(borrower) - event.set() - waiters_to_notify -= 1 - - @property - def borrowed_tokens(self) -> int: - return len(self._borrowers) - - @property - def available_tokens(self) -> float: - return self._total_tokens - len(self._borrowers) - - def _notify_next_waiter(self) -> None: - """Hand a free token to the next task in line, if any.""" - if self._wait_queue and len(self._borrowers) < self._total_tokens: - borrower, event = self._wait_queue.popitem(last=False) - self._borrowers.add(borrower) - event.set() - - def acquire_nowait(self) -> None: - self.acquire_on_behalf_of_nowait(current_task()) - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - if borrower in self._borrowers: - raise RuntimeError( - "this borrower is already holding one of this CapacityLimiter's tokens" - ) - - if self._wait_queue or len(self._borrowers) >= self._total_tokens: - raise WouldBlock - - self._borrowers.add(borrower) - - async def acquire(self) -> None: - return await self.acquire_on_behalf_of(current_task()) - - async def acquire_on_behalf_of(self, borrower: object) -> None: - await AsyncIOBackend.checkpoint_if_cancelled() - try: - self.acquire_on_behalf_of_nowait(borrower) - except WouldBlock: - event = asyncio.Event() - self._wait_queue[borrower] = event - try: - await event.wait() - except BaseException: - self._wait_queue.pop(borrower, None) - if event.is_set(): - self._borrowers.discard(borrower) - self._notify_next_waiter() - - raise - else: - try: - await AsyncIOBackend.cancel_shielded_checkpoint() - except BaseException: - self.release() - raise - - def release(self) -> None: - self.release_on_behalf_of(current_task()) - - def release_on_behalf_of(self, borrower: object) -> None: - try: - self._borrowers.remove(borrower) - except KeyError: - raise RuntimeError( - "this borrower isn't holding any of this CapacityLimiter's tokens" - ) from None - - self._notify_next_waiter() - - def statistics(self) -> CapacityLimiterStatistics: - return CapacityLimiterStatistics( - self.borrowed_tokens, - self.total_tokens, - tuple(self._borrowers), - len(self._wait_queue), - ) - - -_default_thread_limiter: RunVar[CapacityLimiter] = RunVar("_default_thread_limiter") - - -# -# Operating system signals -# - - -class _SignalReceiver: - def __init__(self, signals: tuple[Signals, ...]): - self._signals = signals - self._loop = get_running_loop() - self._signal_queue: deque[Signals] = deque() - self._future: asyncio.Future = asyncio.Future() - self._handled_signals: set[Signals] = set() - - def _deliver(self, signum: Signals) -> None: - self._signal_queue.append(signum) - if not self._future.done(): - self._future.set_result(None) - - def __enter__(self) -> _SignalReceiver: - for sig in set(self._signals): - self._loop.add_signal_handler(sig, self._deliver, sig) - self._handled_signals.add(sig) - - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - for sig in self._handled_signals: - self._loop.remove_signal_handler(sig) - - def __aiter__(self) -> _SignalReceiver: - return self - - async def __anext__(self) -> Signals: - await AsyncIOBackend.checkpoint() - if not self._signal_queue: - self._future = asyncio.Future() - await self._future - - return self._signal_queue.popleft() - - -# -# Testing and debugging -# - - -class AsyncIOTaskInfo(TaskInfo): - def __init__(self, task: asyncio.Task): - task_state = _task_states.get(task) - if task_state is None: - parent_id = None - else: - parent_id = task_state.parent_id - - coro = task.get_coro() - assert coro is not None, "created TaskInfo from a completed Task" - super().__init__(id(task), parent_id, task.get_name(), coro) - self._task = weakref.ref(task) - - def has_pending_cancellation(self) -> bool: - if not (task := self._task()): - # If the task isn't around anymore, it won't have a pending cancellation - return False - - if task._must_cancel: # type: ignore[attr-defined] - return True - elif ( - isinstance(task._fut_waiter, asyncio.Future) # type: ignore[attr-defined] - and task._fut_waiter.cancelled() # type: ignore[attr-defined] - ): - return True - - if task_state := _task_states.get(task): - if cancel_scope := task_state.cancel_scope: - return cancel_scope._effectively_cancelled - - return False - - -class TestRunner(abc.TestRunner): - _send_stream: MemoryObjectSendStream[tuple[Awaitable[Any], asyncio.Future[Any]]] - - def __init__( - self, - *, - debug: bool | None = None, - use_uvloop: bool = False, - loop_factory: Callable[[], AbstractEventLoop] | None = None, - ) -> None: - if use_uvloop and loop_factory is None: - if sys.platform != "win32": - import uvloop - - loop_factory = uvloop.new_event_loop - else: - import winloop - - loop_factory = winloop.new_event_loop - - self._runner = Runner(debug=debug, loop_factory=loop_factory) - self._exceptions: list[BaseException] = [] - self._runner_task: asyncio.Task | None = None - - def __enter__(self) -> TestRunner: - self._runner.__enter__() - self.get_loop().set_exception_handler(self._exception_handler) - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self._runner.__exit__(exc_type, exc_val, exc_tb) - - def get_loop(self) -> AbstractEventLoop: - return self._runner.get_loop() - - def is_running(self) -> bool: - try: - asyncio.get_running_loop() - return True - except RuntimeError: - return False - - def _exception_handler( - self, loop: asyncio.AbstractEventLoop, context: dict[str, Any] - ) -> None: - if isinstance(context.get("exception"), Exception): - self._exceptions.append(context["exception"]) - else: - loop.default_exception_handler(context) - - def _raise_async_exceptions(self) -> None: - # Re-raise any exceptions raised in asynchronous callbacks - if self._exceptions: - exceptions, self._exceptions = self._exceptions, [] - if len(exceptions) == 1: - raise exceptions[0] - elif exceptions: - raise BaseExceptionGroup( - "Multiple exceptions occurred in asynchronous callbacks", exceptions - ) - - async def _run_tests_and_fixtures( - self, - receive_stream: MemoryObjectReceiveStream[ - tuple[Awaitable[T_Retval], asyncio.Future[T_Retval]] - ], - ) -> None: - from _pytest.outcomes import OutcomeException - - with receive_stream, self._send_stream: - async for coro, future in receive_stream: - try: - retval = await coro - except CancelledError as exc: - if not future.cancelled(): - future.cancel(*exc.args) - - raise - except BaseException as exc: - if not future.cancelled(): - future.set_exception(exc) - - if not isinstance(exc, (Exception, OutcomeException)): - raise - else: - if not future.cancelled(): - future.set_result(retval) - - async def _call_in_runner_task( - self, - func: Callable[P, Awaitable[T_Retval]], - /, - *args: P.args, - **kwargs: P.kwargs, - ) -> T_Retval: - if not self._runner_task: - self._send_stream, receive_stream = create_memory_object_stream[ - tuple[Awaitable[Any], asyncio.Future] - ](1) - self._runner_task = self.get_loop().create_task( - self._run_tests_and_fixtures(receive_stream) - ) - - coro = func(*args, **kwargs) - future: asyncio.Future[T_Retval] = self.get_loop().create_future() - self._send_stream.send_nowait((coro, future)) - return await future - - def run_asyncgen_fixture( - self, - fixture_func: Callable[..., AsyncGenerator[T_Retval, Any]], - kwargs: dict[str, Any], - ) -> Iterable[T_Retval]: - asyncgen = fixture_func(**kwargs) - fixturevalue: T_Retval = self.get_loop().run_until_complete( - self._call_in_runner_task(asyncgen.asend, None) - ) - self._raise_async_exceptions() - - yield fixturevalue - - try: - self.get_loop().run_until_complete( - self._call_in_runner_task(asyncgen.asend, None) - ) - except StopAsyncIteration: - self._raise_async_exceptions() - else: - self.get_loop().run_until_complete(asyncgen.aclose()) - raise RuntimeError("Async generator fixture did not stop") - - def run_fixture( - self, - fixture_func: Callable[..., Coroutine[Any, Any, T_Retval]], - kwargs: dict[str, Any], - ) -> T_Retval: - retval = self.get_loop().run_until_complete( - self._call_in_runner_task(fixture_func, **kwargs) - ) - self._raise_async_exceptions() - return retval - - def run_test( - self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] - ) -> None: - from _pytest.outcomes import OutcomeException - - try: - self.get_loop().run_until_complete( - self._call_in_runner_task(test_func, **kwargs) - ) - except Exception as exc: - self._exceptions.append(exc) - except OutcomeException: - raise - except BaseException: - # A BaseException (e.g. KeyboardInterrupt, SystemExit) interrupted the event loop before - # the test completed. Cancel _runner_task so it does not resume when the event - # loop is re-entered during async generator fixture teardown. - if self._runner_task is not None and not self._runner_task.done(): - self._runner_task.cancel() - self._send_stream.close() - try: - self.get_loop().run_until_complete(self._runner_task) - except CancelledError: - pass - finally: - self._runner_task = None - raise - self._raise_async_exceptions() - - -class _ProcessStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol): - """ - A subprocess protocol that allows us to be notified of ``process_exited`` - - asyncio's own ``Process.wait()`` only resolves once every pipe transport has - disconnected so to get same semantics as on trio and uvloop we need this. - """ - - def __init__(self) -> None: - # Match the standard factory for asyncio.create_process - super().__init__(limit=2**16, loop=asyncio.get_running_loop()) - self.exited = asyncio.Event() - - def process_exited(self) -> None: - super().process_exited() - self.exited.set() - - -class AsyncIOBackend(AsyncBackend): - @classmethod - def run( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - options: dict[str, Any], - ) -> T_Retval: - @wraps(func) - async def wrapper() -> T_Retval: - task = cast(asyncio.Task, current_task()) - task.set_name(get_callable_name(func)) - _task_states[task] = TaskState(None, None) - - try: - return await func(*args) - finally: - del _task_states[task] - - debug = options.get("debug", None) - loop_factory = options.get("loop_factory", None) - if loop_factory is None and options.get("use_uvloop", False): - if sys.platform != "win32": - import uvloop - - loop_factory = uvloop.new_event_loop - else: - import winloop - - loop_factory = winloop.new_event_loop - - with Runner(debug=debug, loop_factory=loop_factory) as runner: - return runner.run(wrapper()) - - @classmethod - def current_token(cls) -> object: - return get_running_loop() - - @classmethod - def current_time(cls) -> float: - return get_running_loop().time() - - @classmethod - def cancelled_exception_class(cls) -> type[BaseException]: - return CancelledError - - @classmethod - async def checkpoint(cls) -> None: - await sleep(0) - - @classmethod - async def checkpoint_if_cancelled(cls) -> None: - task = current_task() - if task is None: - return - - try: - cancel_scope = _task_states[task].cancel_scope - except KeyError: - return - - while cancel_scope: - if cancel_scope.cancel_called: - await sleep(0) - elif cancel_scope.shield: - break - else: - cancel_scope = cancel_scope._parent_scope - - @classmethod - async def cancel_shielded_checkpoint(cls) -> None: - with CancelScope(shield=True): - await sleep(0) - - @classmethod - async def sleep(cls, delay: float) -> None: - await sleep(delay) - - @classmethod - def create_cancel_scope( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - return CancelScope(deadline=deadline, shield=shield) - - @classmethod - def current_effective_deadline(cls) -> float: - if (task := current_task()) is None: - return math.inf - - try: - cancel_scope = _task_states[task].cancel_scope - except KeyError: - return math.inf - - deadline = math.inf - while cancel_scope: - deadline = min(deadline, cancel_scope.deadline) - if cancel_scope._cancel_called: - deadline = -math.inf - break - elif cancel_scope.shield: - break - else: - cancel_scope = cancel_scope._parent_scope - - return deadline - - @classmethod - def create_task_group(cls) -> abc.TaskGroup: - return TaskGroup() - - @classmethod - def create_event(cls) -> abc.Event: - return Event() - - @classmethod - def create_lock(cls, *, fast_acquire: bool) -> abc.Lock: - return Lock(fast_acquire=fast_acquire) - - @classmethod - def create_semaphore( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> abc.Semaphore: - return Semaphore(initial_value, max_value=max_value, fast_acquire=fast_acquire) - - @classmethod - def create_capacity_limiter(cls, total_tokens: float) -> abc.CapacityLimiter: - return CapacityLimiter(total_tokens) - - @classmethod - async def run_sync_in_worker_thread( # type: ignore[return] - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - abandon_on_cancel: bool = False, - limiter: abc.CapacityLimiter | None = None, - ) -> T_Retval: - await cls.checkpoint() - - # If this is the first run in this event loop thread, set up the necessary - # variables - try: - idle_workers = _threadpool_idle_workers.get() - workers = _threadpool_workers.get() - except LookupError: - idle_workers = deque() - workers = set() - _threadpool_idle_workers.set(idle_workers) - _threadpool_workers.set(workers) - - async with limiter or cls.current_default_thread_limiter(): - with CancelScope(shield=not abandon_on_cancel) as scope: - future = asyncio.Future[T_Retval]() - root_task = find_root_task() - if not idle_workers: - worker = WorkerThread(root_task, workers, idle_workers) - worker.start() - workers.add(worker) - root_task.add_done_callback( - worker.stop, context=contextvars.Context() - ) - else: - worker = idle_workers.pop() - - # Prune any other workers that have been idle for MAX_IDLE_TIME - # seconds or longer - now = cls.current_time() - while idle_workers: - if ( - now - idle_workers[0].idle_since - < WorkerThread.MAX_IDLE_TIME - ): - break - - expired_worker = idle_workers.popleft() - expired_worker.root_task.remove_done_callback( - expired_worker.stop - ) - expired_worker.stop() - - context = copy_context() - context.run(set_current_async_library, None) - if abandon_on_cancel or scope._parent_scope is None: - worker_scope = scope - else: - worker_scope = scope._parent_scope - - worker.queue.put_nowait((context, func, args, future, worker_scope)) - return await future - - @classmethod - def check_cancelled(cls) -> None: - scope: CancelScope | None = threadlocals.current_cancel_scope - while scope is not None: - if scope.cancel_called: - raise CancelledError(f"Cancelled via cancel scope {id(scope):x}") - - if scope.shield: - return - - scope = scope._parent_scope - - @classmethod - def run_async_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_co: - async def task_wrapper() -> T_co: - __tracebackhide__ = True - if scope is not None: - task = cast(asyncio.Task, current_task()) - _task_states[task] = TaskState(None, scope) - scope._tasks.add(task) - try: - return await func(*args) - except CancelledError as exc: - raise concurrent.futures.CancelledError(str(exc)) from None - finally: - if scope is not None: - scope._tasks.discard(task) - - loop = cast( - "AbstractEventLoop", token or threadlocals.current_token.native_token - ) - if loop.is_closed(): - raise RunFinishedError - - context = copy_context() - context.run(set_current_async_library, "asyncio") - scope = getattr(threadlocals, "current_cancel_scope", None) - f: concurrent.futures.Future[T_co] = context.run( - asyncio.run_coroutine_threadsafe, task_wrapper(), loop=loop - ) - return f.result() - - @classmethod - def run_sync_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - @wraps(func) - def wrapper() -> None: - try: - set_current_async_library("asyncio") - f.set_result(func(*args)) - except BaseException as exc: - f.set_exception(exc) - if not isinstance(exc, Exception): - raise - - loop = cast( - "AbstractEventLoop", token or threadlocals.current_token.native_token - ) - if loop.is_closed(): - raise RunFinishedError - - f: concurrent.futures.Future[T_Retval] = Future() - loop.call_soon_threadsafe(wrapper) - return f.result() - - @classmethod - async def open_process( - cls, - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None, - stdout: int | IO[Any] | None, - stderr: int | IO[Any] | None, - **kwargs: Any, - ) -> Process: - await cls.checkpoint() - if isinstance(command, PathLike): - command = os.fspath(command) - - # Use loop.subprocess_shell()/subprocess_exec() rather than their - # asyncio.create_subprocess_*() counterparts to get access to - # transport/protocol. - loop = asyncio.get_running_loop() - if isinstance(command, (str, bytes)): - transport, protocol = await loop.subprocess_shell( - _ProcessStreamProtocol, - command, - stdin=stdin, - stdout=stdout, - stderr=stderr, - **kwargs, - ) - else: - transport, protocol = await loop.subprocess_exec( - _ProcessStreamProtocol, - *command, - stdin=stdin, - stdout=stdout, - stderr=stderr, - **kwargs, - ) - - process = asyncio.subprocess.Process(transport, protocol, loop) - stdin_stream = StreamWriterWrapper(process.stdin) if process.stdin else None - stdout_stream = StreamReaderWrapper(process.stdout) if process.stdout else None - stderr_stream = StreamReaderWrapper(process.stderr) if process.stderr else None - return Process( - process, - stdin_stream, - stdout_stream, - stderr_stream, - protocol.exited, - transport, - ) - - @classmethod - def setup_process_pool_exit_at_shutdown(cls, workers: set[abc.Process]) -> None: - create_task( - _shutdown_process_pool_on_exit(workers), - name="AnyIO process pool shutdown task", - ) - find_root_task().add_done_callback( - partial(_forcibly_shutdown_process_pool_on_exit, workers) # type:ignore[arg-type] - ) - - @classmethod - async def connect_tcp( - cls, host: str, port: int, local_address: IPSockAddrType | None = None - ) -> abc.SocketStream: - transport, protocol = cast( - tuple[asyncio.Transport, StreamProtocol], - await get_running_loop().create_connection( - StreamProtocol, host, port, local_addr=local_address - ), - ) - transport.pause_reading() - return SocketStream(transport, protocol) - - @classmethod - async def connect_unix(cls, path: str | bytes) -> abc.UNIXSocketStream: - await cls.checkpoint() - loop = get_running_loop() - raw_socket = socket.socket(socket.AF_UNIX) - raw_socket.setblocking(False) - while True: - try: - raw_socket.connect(path) - except BlockingIOError: - f: asyncio.Future = asyncio.Future() - loop.add_writer(raw_socket, f.set_result, None) - f.add_done_callback(lambda _: loop.remove_writer(raw_socket)) - await f - except BaseException: - raw_socket.close() - raise - else: - return UNIXSocketStream(raw_socket) - - @classmethod - def create_tcp_listener(cls, sock: socket.socket) -> SocketListener: - return TCPSocketListener(sock) - - @classmethod - def create_unix_listener(cls, sock: socket.socket) -> SocketListener: - return UNIXSocketListener(sock) - - @classmethod - async def create_udp_socket( - cls, - family: AddressFamily, - local_address: IPSockAddrType | None, - remote_address: IPSockAddrType | None, - reuse_port: bool, - ) -> UDPSocket | ConnectedUDPSocket: - transport, protocol = await get_running_loop().create_datagram_endpoint( - DatagramProtocol, - local_addr=local_address, - remote_addr=remote_address, - family=family, - reuse_port=reuse_port, - ) - if protocol.exception: - transport.close() - raise protocol.exception - - if not remote_address: - return UDPSocket(transport, protocol) - else: - return ConnectedUDPSocket(transport, protocol) - - @classmethod - async def create_unix_datagram_socket( # type: ignore[override] - cls, raw_socket: socket.socket, remote_path: str | bytes | None - ) -> abc.UNIXDatagramSocket | abc.ConnectedUNIXDatagramSocket: - await cls.checkpoint() - loop = get_running_loop() - - if remote_path: - while True: - try: - raw_socket.connect(remote_path) - except BlockingIOError: - f: asyncio.Future = asyncio.Future() - loop.add_writer(raw_socket, f.set_result, None) - f.add_done_callback(lambda _: loop.remove_writer(raw_socket)) - await f - except BaseException: - raw_socket.close() - raise - else: - return ConnectedUNIXDatagramSocket(raw_socket) - else: - return UNIXDatagramSocket(raw_socket) - - @classmethod - async def getaddrinfo( - cls, - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, - ) -> Sequence[ - tuple[ - AddressFamily, - SocketKind, - int, - str, - tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes], - ] - ]: - return await get_running_loop().getaddrinfo( - host, port, family=family, type=type, proto=proto, flags=flags - ) - - @classmethod - async def getnameinfo( - cls, sockaddr: IPSockAddrType, flags: int = 0 - ) -> tuple[str, str]: - return await get_running_loop().getnameinfo(sockaddr, flags) - - @classmethod - async def wait_readable(cls, obj: FileDescriptorLike) -> None: - try: - read_events = _read_events.get() - except LookupError: - read_events = {} - _read_events.set(read_events) - - fd = obj if isinstance(obj, int) else obj.fileno() - if read_events.get(fd): - raise BusyResourceError("reading from") - - loop = get_running_loop() - fut: asyncio.Future[bool] = loop.create_future() - - def cb() -> None: - try: - del read_events[fd] - except KeyError: - pass - else: - remove_reader(fd) - - try: - fut.set_result(True) - except asyncio.InvalidStateError: - pass - - try: - loop.add_reader(fd, cb) - except NotImplementedError: - from anyio._core._asyncio_selector_thread import get_selector - - selector = get_selector() - selector.add_reader(fd, cb) - remove_reader = selector.remove_reader - else: - remove_reader = loop.remove_reader - - read_events[fd] = fut - try: - success = await fut - finally: - try: - del read_events[fd] - except KeyError: - pass - else: - remove_reader(fd) - - if not success: - raise ClosedResourceError - - @classmethod - async def wait_writable(cls, obj: FileDescriptorLike) -> None: - try: - write_events = _write_events.get() - except LookupError: - write_events = {} - _write_events.set(write_events) - - fd = obj if isinstance(obj, int) else obj.fileno() - if write_events.get(fd): - raise BusyResourceError("writing to") - - loop = get_running_loop() - fut: asyncio.Future[bool] = loop.create_future() - - def cb() -> None: - try: - del write_events[fd] - except KeyError: - pass - else: - remove_writer(fd) - - try: - fut.set_result(True) - except asyncio.InvalidStateError: - pass - - try: - loop.add_writer(fd, cb) - except NotImplementedError: - from anyio._core._asyncio_selector_thread import get_selector - - selector = get_selector() - selector.add_writer(fd, cb) - remove_writer = selector.remove_writer - else: - remove_writer = loop.remove_writer - - write_events[fd] = fut - try: - success = await fut - finally: - try: - del write_events[fd] - except KeyError: - pass - else: - remove_writer(fd) - - if not success: - raise ClosedResourceError - - @classmethod - def notify_closing(cls, obj: FileDescriptorLike) -> None: - fd = obj if isinstance(obj, int) else obj.fileno() - loop = get_running_loop() - - try: - write_events = _write_events.get() - except LookupError: - pass - else: - try: - fut = write_events.pop(fd) - except KeyError: - pass - else: - try: - fut.set_result(False) - except asyncio.InvalidStateError: - pass - - try: - loop.remove_writer(fd) - except NotImplementedError: - from anyio._core._asyncio_selector_thread import get_selector - - get_selector().remove_writer(fd) - - try: - read_events = _read_events.get() - except LookupError: - pass - else: - try: - fut = read_events.pop(fd) - except KeyError: - pass - else: - try: - fut.set_result(False) - except asyncio.InvalidStateError: - pass - - try: - loop.remove_reader(fd) - except NotImplementedError: - from anyio._core._asyncio_selector_thread import get_selector - - get_selector().remove_reader(fd) - - @classmethod - async def wrap_listener_socket(cls, sock: socket.socket) -> SocketListener: - if hasattr(socket, "AF_UNIX") and sock.family == socket.AF_UNIX: - return UNIXSocketListener(sock) - - return TCPSocketListener(sock) - - @classmethod - async def wrap_stream_socket(cls, sock: socket.socket) -> SocketStream: - transport, protocol = await get_running_loop().create_connection( - StreamProtocol, sock=sock - ) - return SocketStream(transport, protocol) - - @classmethod - async def wrap_unix_stream_socket(cls, sock: socket.socket) -> UNIXSocketStream: - return UNIXSocketStream(sock) - - @classmethod - async def wrap_udp_socket(cls, sock: socket.socket) -> UDPSocket: - transport, protocol = await get_running_loop().create_datagram_endpoint( - DatagramProtocol, sock=sock - ) - return UDPSocket(transport, protocol) - - @classmethod - async def wrap_connected_udp_socket(cls, sock: socket.socket) -> ConnectedUDPSocket: - transport, protocol = await get_running_loop().create_datagram_endpoint( - DatagramProtocol, sock=sock - ) - return ConnectedUDPSocket(transport, protocol) - - @classmethod - async def wrap_unix_datagram_socket(cls, sock: socket.socket) -> UNIXDatagramSocket: - return UNIXDatagramSocket(sock) - - @classmethod - async def wrap_connected_unix_datagram_socket( - cls, sock: socket.socket - ) -> ConnectedUNIXDatagramSocket: - return ConnectedUNIXDatagramSocket(sock) - - @classmethod - def current_default_thread_limiter(cls) -> CapacityLimiter: - try: - return _default_thread_limiter.get() - except LookupError: - limiter = CapacityLimiter(40) - _default_thread_limiter.set(limiter) - return limiter - - @classmethod - def open_signal_receiver( - cls, *signals: Signals - ) -> AbstractContextManager[AsyncIterator[Signals]]: - return _SignalReceiver(signals) - - @classmethod - def get_current_task(cls) -> TaskInfo: - return AsyncIOTaskInfo(current_task()) # type: ignore[arg-type] - - @classmethod - def get_running_tasks(cls) -> Sequence[TaskInfo]: - return [AsyncIOTaskInfo(task) for task in all_tasks() if not task.done()] - - @classmethod - async def wait_all_tasks_blocked(cls) -> None: - await cls.checkpoint() - this_task = current_task() - while True: - for task in all_tasks(): - if task is this_task: - continue - - waiter = task._fut_waiter # type: ignore[attr-defined] - if waiter is None or waiter.done(): - await sleep(0.1) - break - else: - return - - @classmethod - def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: - return TestRunner(**options) - - -backend_class = AsyncIOBackend diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_backends/_trio.py b/bundle/python-cpu/Lib/site-packages/anyio/_backends/_trio.py deleted file mode 100644 index 43d24d23315322eef2df539d5d959b66fc1e2dbb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_backends/_trio.py +++ /dev/null @@ -1,1468 +0,0 @@ -from __future__ import annotations - -import array -import math -import os -import socket -import sys -import types -import weakref -from collections.abc import ( - AsyncGenerator, - AsyncIterator, - Awaitable, - Callable, - Collection, - Coroutine, - Iterable, - Sequence, -) -from contextlib import AbstractContextManager -from contextvars import Context -from dataclasses import dataclass -from functools import partial, wraps -from io import IOBase -from os import PathLike -from signal import Signals -from socket import AddressFamily, SocketKind -from types import TracebackType -from typing import ( - IO, - TYPE_CHECKING, - Any, - Generic, - Literal, - NoReturn, - ParamSpec, - TypeVar, - cast, - overload, -) - -import trio.from_thread -import trio.lowlevel -from outcome import Error, Outcome, Value -from trio.lowlevel import ( - current_root_task, - current_task, - notify_closing, - wait_readable, - wait_writable, -) -from trio.socket import SocketType as TrioSocketType -from trio.to_thread import run_sync - -from .. import ( - CapacityLimiterStatistics, - EventStatistics, - LockStatistics, - RunFinishedError, - TaskInfo, - WouldBlock, - abc, -) -from .._core._eventloop import claim_worker_thread -from .._core._exceptions import ( - BrokenResourceError, - BusyResourceError, - ClosedResourceError, - EndOfStream, -) -from .._core._sockets import convert_ipv6_sockaddr -from .._core._streams import create_memory_object_stream -from .._core._synchronization import ( - CapacityLimiter as BaseCapacityLimiter, -) -from .._core._synchronization import Event as BaseEvent -from .._core._synchronization import Lock as BaseLock -from .._core._synchronization import ( - ResourceGuard, - SemaphoreStatistics, -) -from .._core._synchronization import Semaphore as BaseSemaphore -from .._core._tasks import CancelScope as BaseCancelScope -from .._core._tasks import TaskHandle -from ..abc import IPSockAddrType, UDPPacketType, UNIXDatagramPacketType -from ..abc._eventloop import AsyncBackend, StrOrBytesPath -from ..abc._tasks import T_contra, call_for_coroutine, get_callable_name -from ..streams.memory import MemoryObjectSendStream - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from exceptiongroup import BaseExceptionGroup - from typing_extensions import TypeVarTuple, Unpack - -T = TypeVar("T") -T_Retval = TypeVar("T_Retval") -T_co = TypeVar("T_co", covariant=True) -T_SockAddr = TypeVar("T_SockAddr", str, IPSockAddrType) -PosArgsT = TypeVarTuple("PosArgsT") -P = ParamSpec("P") - - -def ensure_returns_coro( - func: Callable[P, Awaitable[T_Retval]], -) -> Callable[P, Coroutine[Any, Any, T_Retval]]: - @wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> Coroutine[Any, Any, T_Retval]: - awaitable = func(*args, **kwargs) - # Check the common case first. - if isinstance(awaitable, Coroutine): - return awaitable - elif not isinstance(awaitable, Awaitable): - # The user violated the type annotations. Still, we should pass this on to - # Trio so it can raise with an appropriate message. - return awaitable - else: - - @wraps(func) - async def inner_wrapper() -> T_Retval: - return await awaitable - - return inner_wrapper() - - return wrapper - - -# -# Event loop -# - -RunVar = trio.lowlevel.RunVar - - -# -# Timeouts and cancellation -# - - -class CancelScope(BaseCancelScope): - __slots__ = ("__original",) - - def __new__( - cls, original: trio.CancelScope | None = None, **kwargs: object - ) -> CancelScope: - return object.__new__(cls) - - def __init__(self, original: trio.CancelScope | None = None, **kwargs: Any) -> None: - self.__original = original or trio.CancelScope(**kwargs) - - def __enter__(self) -> CancelScope: - self.__original.__enter__() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - return self.__original.__exit__(exc_type, exc_val, exc_tb) - - def cancel(self, reason: str | None = None) -> None: - self.__original.cancel(reason) - - @property - def deadline(self) -> float: - return self.__original.deadline - - @deadline.setter - def deadline(self, value: float) -> None: - self.__original.deadline = value - - @property - def cancel_called(self) -> bool: - return self.__original.cancel_called - - @property - def cancelled_caught(self) -> bool: - return self.__original.cancelled_caught - - @property - def shield(self) -> bool: - return self.__original.shield - - @shield.setter - def shield(self, value: bool) -> None: - self.__original.shield = value - - -# -# Task groups -# - -empty_start_value = object() - - -class _TrioTaskStatus(Generic[T_contra], abc.TaskStatus[T_contra]): - early_start_value: T_contra | object = empty_start_value - real_task_status: trio.TaskStatus[T_contra | None] | None = None - - def started(self, value: T_contra | None = None) -> None: - if self.real_task_status is None: - if self.early_start_value is not empty_start_value: - raise RuntimeError("called 'started' twice on the same task status") - - self.early_start_value = value - else: - self.real_task_status.started(value) - - -class TaskGroup(abc.TaskGroup): - def __init__(self) -> None: - self._entered = False - self._active = False - self._nursery_manager = trio.open_nursery(strict_exception_groups=True) - self.cancel_scope = None # type: ignore[assignment] - - async def __aenter__(self) -> TaskGroup: - if self._entered: - raise RuntimeError("TaskGroup cannot be entered more than once") - - self._entered = True - self._active = True - self._nursery = await self._nursery_manager.__aenter__() - self.cancel_scope = CancelScope(self._nursery.cancel_scope) - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - try: - # trio.Nursery.__exit__ returns bool; .open_nursery has wrong type - return await self._nursery_manager.__aexit__(exc_type, exc_val, exc_tb) # type: ignore[return-value] - except BaseExceptionGroup as exc: - if not exc.split(trio.Cancelled)[1]: - raise trio.Cancelled._create() from exc - - raise - finally: - del exc_val, exc_tb - self._active = False - - def _check_active(self, coro: Coroutine | None = None) -> None: - if not self._active: - if coro is not None: - coro.close() - - raise RuntimeError( - "This task group is not active; no new tasks can be started." - ) - - def create_task( - self, - coro: Coroutine[Any, Any, T_co], - *, - name: object = None, - context: Context | None = None, - ) -> TaskHandle[T_co]: - if not isinstance(coro, Coroutine): - raise TypeError(f"expected a coroutine, got {coro.__class__.__qualname__}") - - self._check_active(coro) - handle = TaskHandle(coro, name) - if context is not None: - context.run( - partial(self._nursery.start_soon, handle._run_coro, name=handle.name) - ) - else: - self._nursery.start_soon(handle._run_coro, name=handle.name) - - return handle - - async def start( - self, - func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], - *args: Unpack[PosArgsT], - name: object = None, - return_handle: Literal[False] | Literal[True] = False, - ) -> Any: - handle: TaskHandle[T_co] - - async def run_coro_with_task_status( - *, task_status: trio.TaskStatus[Any] - ) -> None: - nonlocal handle - wrapper_task_status = _TrioTaskStatus() - coro = call_for_coroutine(func, args, task_status=wrapper_task_status) - if wrapper_task_status.early_start_value is not empty_start_value: - task_status.started(wrapper_task_status.early_start_value) - else: - wrapper_task_status.real_task_status = task_status - - handle = TaskHandle(coro, name) - await handle._run_coro() - - self._check_active() - final_name = get_callable_name(func, name) - start_value = await self._nursery.start( - run_coro_with_task_status, name=final_name - ) - if return_handle: - handle._start_value = start_value - return handle - else: - return start_value - - -# -# Subprocesses -# - - -@dataclass(eq=False) -class ReceiveStreamWrapper(abc.ByteReceiveStream): - _stream: trio.abc.ReceiveStream - - async def receive(self, max_bytes: int | None = None) -> bytes: - if max_bytes is not None and max_bytes < 1: - raise ValueError("max_bytes must be a positive integer") - - try: - data = await self._stream.receive_some(max_bytes) - except trio.ClosedResourceError as exc: - raise ClosedResourceError from exc.__cause__ - except trio.BrokenResourceError as exc: - raise BrokenResourceError from exc.__cause__ - - if data: - return bytes(data) - else: - raise EndOfStream - - async def aclose(self) -> None: - await self._stream.aclose() - - -@dataclass(eq=False) -class SendStreamWrapper(abc.ByteSendStream): - _stream: trio.abc.SendStream - - async def send(self, item: bytes) -> None: - try: - await self._stream.send_all(item) - except trio.ClosedResourceError as exc: - raise ClosedResourceError from exc.__cause__ - except trio.BrokenResourceError as exc: - raise BrokenResourceError from exc.__cause__ - - async def aclose(self) -> None: - await self._stream.aclose() - - -@dataclass(eq=False) -class Process(abc.Process): - _process: trio.Process - _stdin: abc.ByteSendStream | None - _stdout: abc.ByteReceiveStream | None - _stderr: abc.ByteReceiveStream | None - - async def aclose(self) -> None: - with CancelScope(shield=True): - if self._stdin: - await self._stdin.aclose() - if self._stdout: - await self._stdout.aclose() - if self._stderr: - await self._stderr.aclose() - - try: - await self.wait() - except BaseException: - self.kill() - with CancelScope(shield=True): - await self.wait() - raise - - async def wait(self) -> int: - return await self._process.wait() - - def terminate(self) -> None: - self._process.terminate() - - def kill(self) -> None: - self._process.kill() - - def send_signal(self, signal: Signals) -> None: - self._process.send_signal(signal) - - @property - def pid(self) -> int: - return self._process.pid - - @property - def returncode(self) -> int | None: - return self._process.returncode - - @property - def stdin(self) -> abc.ByteSendStream | None: - return self._stdin - - @property - def stdout(self) -> abc.ByteReceiveStream | None: - return self._stdout - - @property - def stderr(self) -> abc.ByteReceiveStream | None: - return self._stderr - - -class _ProcessPoolShutdownInstrument(trio.abc.Instrument): - def after_run(self) -> None: - super().after_run() - - -current_default_worker_process_limiter: trio.lowlevel.RunVar = RunVar( - "current_default_worker_process_limiter" -) - - -async def _shutdown_process_pool(workers: set[abc.Process]) -> None: - try: - await trio.sleep(math.inf) - except trio.Cancelled: - for process in workers: - if process.returncode is None: - process.kill() - - with CancelScope(shield=True): - for process in workers: - await process.aclose() - - -# -# Sockets and networking -# - - -class _TrioSocketMixin(Generic[T_SockAddr]): - def __init__(self, trio_socket: TrioSocketType) -> None: - self._trio_socket = trio_socket - self._closed = False - - def _check_closed(self) -> None: - if self._closed: - raise ClosedResourceError - if self._trio_socket.fileno() < 0: - raise BrokenResourceError - - @property - def _raw_socket(self) -> socket.socket: - return self._trio_socket._sock # type: ignore[attr-defined] - - async def aclose(self) -> None: - if self._trio_socket.fileno() >= 0: - self._closed = True - self._trio_socket.close() - - def _convert_socket_error(self, exc: BaseException) -> NoReturn: - if isinstance(exc, trio.ClosedResourceError): - raise ClosedResourceError from exc - elif self._trio_socket.fileno() < 0 and self._closed: - raise ClosedResourceError from None - elif isinstance(exc, OSError): - raise BrokenResourceError from exc - else: - raise exc - - -class SocketStream(_TrioSocketMixin, abc.SocketStream): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self, max_bytes: int = 65536) -> bytes: - if max_bytes < 1: - raise ValueError("max_bytes must be a positive integer") - - with self._receive_guard: - try: - data = await self._trio_socket.recv(max_bytes) - except BaseException as exc: - self._convert_socket_error(exc) - - if data: - return data - else: - raise EndOfStream - - async def send(self, item: bytes) -> None: - with self._send_guard: - view = memoryview(item) - while view: - try: - bytes_sent = await self._trio_socket.send(view) - except BaseException as exc: - self._convert_socket_error(exc) - - view = view[bytes_sent:] - - async def send_eof(self) -> None: - self._trio_socket.shutdown(socket.SHUT_WR) - - -class UNIXSocketStream(SocketStream, abc.UNIXSocketStream): - async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: - if not isinstance(msglen, int) or msglen < 0: - raise ValueError("msglen must be a non-negative integer") - if not isinstance(maxfds, int) or maxfds < 1: - raise ValueError("maxfds must be a positive integer") - - fds = array.array("i") - await trio.lowlevel.checkpoint() - with self._receive_guard: - while True: - try: - message, ancdata, flags, addr = await self._trio_socket.recvmsg( - msglen, socket.CMSG_LEN(maxfds * fds.itemsize) - ) - except BaseException as exc: - self._convert_socket_error(exc) - else: - if not message and not ancdata: - raise EndOfStream - - break - - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level != socket.SOL_SOCKET or cmsg_type != socket.SCM_RIGHTS: - raise RuntimeError( - f"Received unexpected ancillary data; message = {message!r}, " - f"cmsg_level = {cmsg_level}, cmsg_type = {cmsg_type}" - ) - - fds.frombytes(cmsg_data[: len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - - return message, list(fds) - - async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: - if not message: - raise ValueError("message must not be empty") - if not fds: - raise ValueError("fds must not be empty") - - filenos: list[int] = [] - for fd in fds: - if isinstance(fd, int): - filenos.append(fd) - elif isinstance(fd, IOBase): - filenos.append(fd.fileno()) - - fdarray = array.array("i", filenos) - await trio.lowlevel.checkpoint() - with self._send_guard: - while True: - try: - await self._trio_socket.sendmsg( - [message], - [ - ( - socket.SOL_SOCKET, - socket.SCM_RIGHTS, - fdarray, - ) - ], - ) - break - except BaseException as exc: - self._convert_socket_error(exc) - - -class TCPSocketListener(_TrioSocketMixin, abc.SocketListener): - def __init__(self, raw_socket: socket.socket): - super().__init__(trio.socket.from_stdlib_socket(raw_socket)) - self._accept_guard = ResourceGuard("accepting connections from") - - async def accept(self) -> SocketStream: - with self._accept_guard: - try: - trio_socket, _addr = await self._trio_socket.accept() - except BaseException as exc: - self._convert_socket_error(exc) - - trio_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - return SocketStream(trio_socket) - - -class UNIXSocketListener(_TrioSocketMixin, abc.SocketListener): - def __init__(self, raw_socket: socket.socket): - super().__init__(trio.socket.from_stdlib_socket(raw_socket)) - self._accept_guard = ResourceGuard("accepting connections from") - - async def accept(self) -> UNIXSocketStream: - with self._accept_guard: - try: - trio_socket, _addr = await self._trio_socket.accept() - except BaseException as exc: - self._convert_socket_error(exc) - - return UNIXSocketStream(trio_socket) - - -class UDPSocket(_TrioSocketMixin[IPSockAddrType], abc.UDPSocket): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> tuple[bytes, IPSockAddrType]: - with self._receive_guard: - try: - data, addr = await self._trio_socket.recvfrom(65536) - return data, convert_ipv6_sockaddr(addr) - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: UDPPacketType) -> None: - with self._send_guard: - try: - await self._trio_socket.sendto(*item) - except BaseException as exc: - self._convert_socket_error(exc) - - -class ConnectedUDPSocket(_TrioSocketMixin[IPSockAddrType], abc.ConnectedUDPSocket): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> bytes: - with self._receive_guard: - try: - return await self._trio_socket.recv(65536) - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: bytes) -> None: - with self._send_guard: - try: - await self._trio_socket.send(item) - except BaseException as exc: - self._convert_socket_error(exc) - - -class UNIXDatagramSocket(_TrioSocketMixin[str], abc.UNIXDatagramSocket): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> UNIXDatagramPacketType: - with self._receive_guard: - try: - data, addr = await self._trio_socket.recvfrom(65536) - return data, addr - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: UNIXDatagramPacketType) -> None: - with self._send_guard: - try: - await self._trio_socket.sendto(*item) - except BaseException as exc: - self._convert_socket_error(exc) - - -class ConnectedUNIXDatagramSocket( - _TrioSocketMixin[str], abc.ConnectedUNIXDatagramSocket -): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> bytes: - with self._receive_guard: - try: - return await self._trio_socket.recv(65536) - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: bytes) -> None: - with self._send_guard: - try: - await self._trio_socket.send(item) - except BaseException as exc: - self._convert_socket_error(exc) - - -# -# Synchronization -# - - -class Event(BaseEvent): - __slots__ = ("__original",) - - def __new__(cls) -> Event: - return object.__new__(cls) - - def __init__(self) -> None: - self.__original = trio.Event() - - def is_set(self) -> bool: - return self.__original.is_set() - - async def wait(self) -> None: - return await self.__original.wait() - - def statistics(self) -> EventStatistics: - orig_statistics = self.__original.statistics() - return EventStatistics(tasks_waiting=orig_statistics.tasks_waiting) - - def set(self) -> None: - self.__original.set() - - -class Lock(BaseLock): - __slots__ = "_fast_acquire", "__original" - - def __new__(cls, *, fast_acquire: bool = False) -> Lock: - return object.__new__(cls) - - def __init__(self, *, fast_acquire: bool = False) -> None: - self._fast_acquire = fast_acquire - self.__original = trio.Lock() - - @staticmethod - def _convert_runtime_error_msg(exc: RuntimeError) -> None: - if exc.args == ("attempt to re-acquire an already held Lock",): - exc.args = ("Attempted to acquire an already held Lock",) - - async def acquire(self) -> None: - if not self._fast_acquire: - try: - await self.__original.acquire() - except RuntimeError as exc: - self._convert_runtime_error_msg(exc) - raise - - return - - # This is the "fast path" where we don't let other tasks run - await trio.lowlevel.checkpoint_if_cancelled() - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - await self.__original._lot.park() - except RuntimeError as exc: - self._convert_runtime_error_msg(exc) - raise - - def acquire_nowait(self) -> None: - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - raise WouldBlock from None - except RuntimeError as exc: - self._convert_runtime_error_msg(exc) - raise - - def locked(self) -> bool: - return self.__original.locked() - - def release(self) -> None: - self.__original.release() - - def statistics(self) -> LockStatistics: - orig_statistics = self.__original.statistics() - owner = TrioTaskInfo(orig_statistics.owner) if orig_statistics.owner else None - return LockStatistics( - orig_statistics.locked, owner, orig_statistics.tasks_waiting - ) - - -class Semaphore(BaseSemaphore): - __slots__ = ("__original",) - - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - return object.__new__(cls) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> None: - super().__init__(initial_value, max_value=max_value, fast_acquire=fast_acquire) - self.__original = trio.Semaphore(initial_value, max_value=max_value) - - async def acquire(self) -> None: - if not self._fast_acquire: - await self.__original.acquire() - return - - # This is the "fast path" where we don't let other tasks run - await trio.lowlevel.checkpoint_if_cancelled() - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - await self.__original._lot.park() - - def acquire_nowait(self) -> None: - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - raise WouldBlock from None - - @property - def max_value(self) -> int | None: - return self.__original.max_value - - @property - def value(self) -> int: - return self.__original.value - - def release(self) -> None: - self.__original.release() - - def statistics(self) -> SemaphoreStatistics: - orig_statistics = self.__original.statistics() - return SemaphoreStatistics(orig_statistics.tasks_waiting) - - -class CapacityLimiter(BaseCapacityLimiter): - __slots__ = ("__original",) - - def __new__( - cls, - total_tokens: float | None = None, - *, - original: trio.CapacityLimiter | None = None, - ) -> CapacityLimiter: - return object.__new__(cls) - - def __init__( - self, - total_tokens: float | None = None, - *, - original: trio.CapacityLimiter | None = None, - ) -> None: - if original is not None: - self.__original = original - else: - assert total_tokens is not None - self.__original = trio.CapacityLimiter(total_tokens) - - async def __aenter__(self) -> None: - return await self.__original.__aenter__() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - await self.__original.__aexit__(exc_type, exc_val, exc_tb) - - @property - def total_tokens(self) -> float: - return self.__original.total_tokens - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - self.__original.total_tokens = value - - @property - def borrowed_tokens(self) -> int: - return self.__original.borrowed_tokens - - @property - def available_tokens(self) -> float: - return self.__original.available_tokens - - def acquire_nowait(self) -> None: - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - raise WouldBlock from None - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - try: - self.__original.acquire_on_behalf_of_nowait(borrower) - except trio.WouldBlock: - raise WouldBlock from None - - async def acquire(self) -> None: - await self.__original.acquire() - - async def acquire_on_behalf_of(self, borrower: object) -> None: - await self.__original.acquire_on_behalf_of(borrower) - - def release(self) -> None: - return self.__original.release() - - def release_on_behalf_of(self, borrower: object) -> None: - return self.__original.release_on_behalf_of(borrower) - - def statistics(self) -> CapacityLimiterStatistics: - orig = self.__original.statistics() - return CapacityLimiterStatistics( - borrowed_tokens=orig.borrowed_tokens, - total_tokens=orig.total_tokens, - borrowers=tuple(orig.borrowers), - tasks_waiting=orig.tasks_waiting, - ) - - -_capacity_limiter_wrapper: trio.lowlevel.RunVar = RunVar("_capacity_limiter_wrapper") - - -# -# Signal handling -# - - -class _SignalReceiver: - _iterator: AsyncIterator[int] - - def __init__(self, signals: tuple[Signals, ...]): - self._signals = signals - - def __enter__(self) -> _SignalReceiver: - self._cm = trio.open_signal_receiver(*self._signals) - self._iterator = self._cm.__enter__() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool | None: - return self._cm.__exit__(exc_type, exc_val, exc_tb) - - def __aiter__(self) -> _SignalReceiver: - return self - - async def __anext__(self) -> Signals: - signum = await self._iterator.__anext__() - return Signals(signum) - - -# -# Testing and debugging -# - - -class TestRunner(abc.TestRunner): - def __init__(self, **options: Any) -> None: - from queue import Queue - - self._call_queue: Queue[Callable[[], object]] = Queue() - self._send_stream: ( - MemoryObjectSendStream[tuple[Awaitable[Any], list[Outcome]]] | None - ) = None - self._options = options - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: types.TracebackType | None, - ) -> None: - if self._send_stream: - self._send_stream.close() - while self._send_stream is not None: - self._call_queue.get()() - - def is_running(self) -> bool: - return trio.lowlevel.in_trio_task() - - async def _run_tests_and_fixtures(self) -> None: - self._send_stream, receive_stream = create_memory_object_stream[ - tuple[Awaitable[Any], list[Outcome]] - ](1) - with receive_stream: - async for awaitable, outcome_holder in receive_stream: - try: - retval = await awaitable - except BaseException as exc: - outcome_holder.append(Error(exc)) - else: - outcome_holder.append(Value(retval)) - - def _main_task_finished(self, outcome: object) -> None: - self._send_stream = None - - def _call_in_runner_task( - self, - func: Callable[P, Awaitable[T_Retval]], - /, - *args: P.args, - **kwargs: P.kwargs, - ) -> T_Retval: - if self._send_stream is None: - trio.lowlevel.start_guest_run( - self._run_tests_and_fixtures, - run_sync_soon_threadsafe=self._call_queue.put, - done_callback=self._main_task_finished, - **self._options, - ) - while self._send_stream is None: - self._call_queue.get()() - - outcome_holder: list[Outcome] = [] - self._send_stream.send_nowait((func(*args, **kwargs), outcome_holder)) - while not outcome_holder: - self._call_queue.get()() - - return outcome_holder[0].unwrap() - - def run_asyncgen_fixture( - self, - fixture_func: Callable[..., AsyncGenerator[T_Retval, Any]], - kwargs: dict[str, Any], - ) -> Iterable[T_Retval]: - asyncgen = fixture_func(**kwargs) - fixturevalue: T_Retval = self._call_in_runner_task(asyncgen.asend, None) - - yield fixturevalue - - try: - self._call_in_runner_task(asyncgen.asend, None) - except StopAsyncIteration: - pass - else: - self._call_in_runner_task(asyncgen.aclose) - raise RuntimeError("Async generator fixture did not stop") - - def run_fixture( - self, - fixture_func: Callable[..., Coroutine[Any, Any, T_Retval]], - kwargs: dict[str, Any], - ) -> T_Retval: - return self._call_in_runner_task(fixture_func, **kwargs) - - def run_test( - self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] - ) -> None: - self._call_in_runner_task(test_func, **kwargs) - - -class TrioTaskInfo(TaskInfo): - def __init__(self, task: trio.lowlevel.Task): - parent_id = None - if task.parent_nursery and task.parent_nursery.parent_task: - parent_id = id(task.parent_nursery.parent_task) - - super().__init__(id(task), parent_id, task.name, task.coro) - self._task = weakref.proxy(task) - - def has_pending_cancellation(self) -> bool: - try: - return self._task._cancel_status.effectively_cancelled - except ReferenceError: - # If the task is no longer around, it surely doesn't have a cancellation - # pending - return False - - -class TrioBackend(AsyncBackend): - @classmethod - def run( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - options: dict[str, Any], - ) -> T_Retval: - assert not kwargs, "unreachable, and not supported by Trio" - return trio.run(ensure_returns_coro(func), *args, **options) - - @classmethod - def current_token(cls) -> object: - return trio.lowlevel.current_trio_token() - - @classmethod - def current_time(cls) -> float: - return trio.current_time() - - @classmethod - def cancelled_exception_class(cls) -> type[BaseException]: - return trio.Cancelled - - @classmethod - async def checkpoint(cls) -> None: - await trio.lowlevel.checkpoint() - - @classmethod - async def checkpoint_if_cancelled(cls) -> None: - await trio.lowlevel.checkpoint_if_cancelled() - - @classmethod - async def cancel_shielded_checkpoint(cls) -> None: - await trio.lowlevel.cancel_shielded_checkpoint() - - @classmethod - async def sleep(cls, delay: float) -> None: - await trio.sleep(delay) - - @classmethod - def create_cancel_scope( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> abc.CancelScope: - return CancelScope(deadline=deadline, shield=shield) - - @classmethod - def current_effective_deadline(cls) -> float: - return trio.current_effective_deadline() - - @classmethod - def create_task_group(cls) -> abc.TaskGroup: - return TaskGroup() - - @classmethod - def create_event(cls) -> abc.Event: - return Event() - - @classmethod - def create_lock(cls, *, fast_acquire: bool) -> Lock: - return Lock(fast_acquire=fast_acquire) - - @classmethod - def create_semaphore( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> abc.Semaphore: - return Semaphore(initial_value, max_value=max_value, fast_acquire=fast_acquire) - - @classmethod - def create_capacity_limiter(cls, total_tokens: float) -> CapacityLimiter: - return CapacityLimiter(total_tokens) - - @classmethod - async def run_sync_in_worker_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - abandon_on_cancel: bool = False, - limiter: abc.CapacityLimiter | None = None, - ) -> T_Retval: - def wrapper() -> T_Retval: - with claim_worker_thread(TrioBackend, token): - return func(*args) - - token = TrioBackend.current_token() - return await run_sync( - wrapper, - abandon_on_cancel=abandon_on_cancel, - limiter=cast(trio.CapacityLimiter, limiter), - ) - - @classmethod - def check_cancelled(cls) -> None: - trio.from_thread.check_cancelled() - - @classmethod - def run_async_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_co: - trio_token = cast("trio.lowlevel.TrioToken | None", token) - try: - return trio.from_thread.run(func, *args, trio_token=trio_token) - except trio.RunFinishedError: - raise RunFinishedError from None - - @classmethod - def run_sync_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - trio_token = cast("trio.lowlevel.TrioToken | None", token) - try: - return trio.from_thread.run_sync(func, *args, trio_token=trio_token) - except trio.RunFinishedError: - raise RunFinishedError from None - - @classmethod - async def open_process( - cls, - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None, - stdout: int | IO[Any] | None, - stderr: int | IO[Any] | None, - **kwargs: Any, - ) -> Process: - def convert_item(item: StrOrBytesPath) -> str: - str_or_bytes = os.fspath(item) - if isinstance(str_or_bytes, str): - return str_or_bytes - else: - return os.fsdecode(str_or_bytes) - - if isinstance(command, (str, bytes, PathLike)): - process = await trio.lowlevel.open_process( - convert_item(command), - stdin=stdin, - stdout=stdout, - stderr=stderr, - shell=True, - **kwargs, - ) - else: - process = await trio.lowlevel.open_process( - [convert_item(item) for item in command], - stdin=stdin, - stdout=stdout, - stderr=stderr, - shell=False, - **kwargs, - ) - - stdin_stream = SendStreamWrapper(process.stdin) if process.stdin else None - stdout_stream = ReceiveStreamWrapper(process.stdout) if process.stdout else None - stderr_stream = ReceiveStreamWrapper(process.stderr) if process.stderr else None - return Process(process, stdin_stream, stdout_stream, stderr_stream) - - @classmethod - def setup_process_pool_exit_at_shutdown(cls, workers: set[abc.Process]) -> None: - trio.lowlevel.spawn_system_task(_shutdown_process_pool, workers) - - @classmethod - async def connect_tcp( - cls, host: str, port: int, local_address: IPSockAddrType | None = None - ) -> SocketStream: - family = socket.AF_INET6 if ":" in host else socket.AF_INET - trio_socket = trio.socket.socket(family) - trio_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - if local_address: - await trio_socket.bind(local_address) - - try: - await trio_socket.connect((host, port)) - except BaseException: - trio_socket.close() - raise - - return SocketStream(trio_socket) - - @classmethod - async def connect_unix(cls, path: str | bytes) -> abc.UNIXSocketStream: - trio_socket = trio.socket.socket(socket.AF_UNIX) - try: - await trio_socket.connect(path) - except BaseException: - trio_socket.close() - raise - - return UNIXSocketStream(trio_socket) - - @classmethod - def create_tcp_listener(cls, sock: socket.socket) -> abc.SocketListener: - return TCPSocketListener(sock) - - @classmethod - def create_unix_listener(cls, sock: socket.socket) -> abc.SocketListener: - return UNIXSocketListener(sock) - - @classmethod - async def create_udp_socket( - cls, - family: socket.AddressFamily, - local_address: IPSockAddrType | None, - remote_address: IPSockAddrType | None, - reuse_port: bool, - ) -> UDPSocket | ConnectedUDPSocket: - trio_socket = trio.socket.socket(family=family, type=socket.SOCK_DGRAM) - - if reuse_port: - trio_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - - if local_address: - await trio_socket.bind(local_address) - - if remote_address: - await trio_socket.connect(remote_address) - return ConnectedUDPSocket(trio_socket) - else: - return UDPSocket(trio_socket) - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket.socket, remote_path: None - ) -> abc.UNIXDatagramSocket: ... - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket.socket, remote_path: str | bytes - ) -> abc.ConnectedUNIXDatagramSocket: ... - - @classmethod - async def create_unix_datagram_socket( - cls, raw_socket: socket.socket, remote_path: str | bytes | None - ) -> abc.UNIXDatagramSocket | abc.ConnectedUNIXDatagramSocket: - trio_socket = trio.socket.from_stdlib_socket(raw_socket) - - if remote_path: - await trio_socket.connect(remote_path) - return ConnectedUNIXDatagramSocket(trio_socket) - else: - return UNIXDatagramSocket(trio_socket) - - @classmethod - async def getaddrinfo( - cls, - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, - ) -> Sequence[ - tuple[ - AddressFamily, - SocketKind, - int, - str, - tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes], - ] - ]: - return await trio.socket.getaddrinfo(host, port, family, type, proto, flags) - - @classmethod - async def getnameinfo( - cls, sockaddr: IPSockAddrType, flags: int = 0 - ) -> tuple[str, str]: - return await trio.socket.getnameinfo(sockaddr, flags) - - @classmethod - async def wait_readable(cls, obj: FileDescriptorLike) -> None: - try: - await wait_readable(obj) - except trio.ClosedResourceError as exc: - raise ClosedResourceError().with_traceback(exc.__traceback__) from None - except trio.BusyResourceError: - raise BusyResourceError("reading from") from None - - @classmethod - async def wait_writable(cls, obj: FileDescriptorLike) -> None: - try: - await wait_writable(obj) - except trio.ClosedResourceError as exc: - raise ClosedResourceError().with_traceback(exc.__traceback__) from None - except trio.BusyResourceError: - raise BusyResourceError("writing to") from None - - @classmethod - def notify_closing(cls, obj: FileDescriptorLike) -> None: - notify_closing(obj) - - @classmethod - async def wrap_listener_socket(cls, sock: socket.socket) -> abc.SocketListener: - if hasattr(socket, "AF_UNIX") and sock.family == socket.AF_UNIX: - return UNIXSocketListener(sock) - - return TCPSocketListener(sock) - - @classmethod - async def wrap_stream_socket(cls, sock: socket.socket) -> SocketStream: - trio_sock = trio.socket.from_stdlib_socket(sock) - return SocketStream(trio_sock) - - @classmethod - async def wrap_unix_stream_socket(cls, sock: socket.socket) -> UNIXSocketStream: - trio_sock = trio.socket.from_stdlib_socket(sock) - return UNIXSocketStream(trio_sock) - - @classmethod - async def wrap_udp_socket(cls, sock: socket.socket) -> UDPSocket: - trio_sock = trio.socket.from_stdlib_socket(sock) - return UDPSocket(trio_sock) - - @classmethod - async def wrap_connected_udp_socket(cls, sock: socket.socket) -> ConnectedUDPSocket: - trio_sock = trio.socket.from_stdlib_socket(sock) - return ConnectedUDPSocket(trio_sock) - - @classmethod - async def wrap_unix_datagram_socket(cls, sock: socket.socket) -> UNIXDatagramSocket: - trio_sock = trio.socket.from_stdlib_socket(sock) - return UNIXDatagramSocket(trio_sock) - - @classmethod - async def wrap_connected_unix_datagram_socket( - cls, sock: socket.socket - ) -> ConnectedUNIXDatagramSocket: - trio_sock = trio.socket.from_stdlib_socket(sock) - return ConnectedUNIXDatagramSocket(trio_sock) - - @classmethod - def current_default_thread_limiter(cls) -> CapacityLimiter: - try: - return _capacity_limiter_wrapper.get() - except LookupError: - limiter = CapacityLimiter( - original=trio.to_thread.current_default_thread_limiter() - ) - _capacity_limiter_wrapper.set(limiter) - return limiter - - @classmethod - def open_signal_receiver( - cls, *signals: Signals - ) -> AbstractContextManager[AsyncIterator[Signals]]: - return _SignalReceiver(signals) - - @classmethod - def get_current_task(cls) -> TaskInfo: - task = current_task() - return TrioTaskInfo(task) - - @classmethod - def get_running_tasks(cls) -> Sequence[TaskInfo]: - root_task = current_root_task() - assert root_task - task_infos = [TrioTaskInfo(root_task)] - nurseries = root_task.child_nurseries - while nurseries: - new_nurseries: list[trio.Nursery] = [] - for nursery in nurseries: - for task in nursery.child_tasks: - task_infos.append(TrioTaskInfo(task)) - new_nurseries.extend(task.child_nurseries) - - nurseries = new_nurseries - - return task_infos - - @classmethod - async def wait_all_tasks_blocked(cls) -> None: - from trio.testing import wait_all_tasks_blocked - - await wait_all_tasks_blocked() - - @classmethod - def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: - return TestRunner(**options) - - -backend_class = TrioBackend diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/__init__.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_asyncio_selector_thread.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_asyncio_selector_thread.py deleted file mode 100644 index 9f35bae568e33e6a9e1219761c83cc8350fa0532..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_asyncio_selector_thread.py +++ /dev/null @@ -1,167 +0,0 @@ -from __future__ import annotations - -import asyncio -import socket -import threading -from collections.abc import Callable -from selectors import EVENT_READ, EVENT_WRITE, DefaultSelector -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike - -_selector_lock = threading.Lock() -_selector: Selector | None = None - - -class Selector: - def __init__(self) -> None: - self._thread = threading.Thread(target=self.run, name="AnyIO socket selector") - self._selector = DefaultSelector() - self._send, self._receive = socket.socketpair() - self._send.setblocking(False) - self._receive.setblocking(False) - # This somewhat reduces the amount of memory wasted queueing up data - # for wakeups. With these settings, maximum number of 1-byte sends - # before getting BlockingIOError: - # Linux 4.8: 6 - # macOS (darwin 15.5): 1 - # Windows 10: 525347 - # Windows you're weird. (And on Windows setting SNDBUF to 0 makes send - # blocking, even on non-blocking sockets, so don't do that.) - self._receive.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1) - self._send.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1) - # On Windows this is a TCP socket so this might matter. On other - # platforms this fails b/c AF_UNIX sockets aren't actually TCP. - try: - self._send.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - except OSError: - pass - - self._selector.register(self._receive, EVENT_READ) - self._closed = False - - def start(self) -> None: - self._thread.start() - threading._register_atexit(self._stop) # type: ignore[attr-defined] - - def _stop(self) -> None: - global _selector - self._closed = True - self._notify_self() - self._send.close() - self._thread.join() - self._selector.unregister(self._receive) - self._receive.close() - self._selector.close() - _selector = None - assert not self._selector.get_map(), ( - "selector still has registered file descriptors after shutdown" - ) - - def _notify_self(self) -> None: - try: - self._send.send(b"\x00") - except BlockingIOError: - pass - - def add_reader(self, fd: FileDescriptorLike, callback: Callable[[], Any]) -> None: - loop = asyncio.get_running_loop() - try: - key = self._selector.get_key(fd) - except KeyError: - self._selector.register(fd, EVENT_READ, {EVENT_READ: (loop, callback)}) - else: - if EVENT_READ in key.data: - raise ValueError( - "this file descriptor is already registered for reading" - ) - - key.data[EVENT_READ] = loop, callback - self._selector.modify(fd, key.events | EVENT_READ, key.data) - - self._notify_self() - - def add_writer(self, fd: FileDescriptorLike, callback: Callable[[], Any]) -> None: - loop = asyncio.get_running_loop() - try: - key = self._selector.get_key(fd) - except KeyError: - self._selector.register(fd, EVENT_WRITE, {EVENT_WRITE: (loop, callback)}) - else: - if EVENT_WRITE in key.data: - raise ValueError( - "this file descriptor is already registered for writing" - ) - - key.data[EVENT_WRITE] = loop, callback - self._selector.modify(fd, key.events | EVENT_WRITE, key.data) - - self._notify_self() - - def remove_reader(self, fd: FileDescriptorLike) -> bool: - try: - key = self._selector.get_key(fd) - except KeyError: - return False - - if new_events := key.events ^ EVENT_READ: - del key.data[EVENT_READ] - self._selector.modify(fd, new_events, key.data) - else: - self._selector.unregister(fd) - - return True - - def remove_writer(self, fd: FileDescriptorLike) -> bool: - try: - key = self._selector.get_key(fd) - except KeyError: - return False - - if new_events := key.events ^ EVENT_WRITE: - del key.data[EVENT_WRITE] - self._selector.modify(fd, new_events, key.data) - else: - self._selector.unregister(fd) - - return True - - def run(self) -> None: - while not self._closed: - for key, events in self._selector.select(): - if key.fileobj is self._receive: - try: - while self._receive.recv(4096): - pass - except BlockingIOError: - pass - - continue - - if events & EVENT_READ: - loop, callback = key.data[EVENT_READ] - self.remove_reader(key.fd) - try: - loop.call_soon_threadsafe(callback) - except RuntimeError: - pass # the loop was already closed - - if events & EVENT_WRITE: - loop, callback = key.data[EVENT_WRITE] - self.remove_writer(key.fd) - try: - loop.call_soon_threadsafe(callback) - except RuntimeError: - pass # the loop was already closed - - -def get_selector() -> Selector: - global _selector - - with _selector_lock: - if _selector is None: - _selector = Selector() - _selector.start() - - return _selector diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_contextmanagers.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_contextmanagers.py deleted file mode 100644 index 302f32b0c78a7071605b195c55054cfdb0b55f37..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_contextmanagers.py +++ /dev/null @@ -1,200 +0,0 @@ -from __future__ import annotations - -from abc import abstractmethod -from contextlib import AbstractAsyncContextManager, AbstractContextManager -from inspect import isasyncgen, iscoroutine, isgenerator -from types import TracebackType -from typing import Protocol, TypeVar, cast, final - -_T_co = TypeVar("_T_co", covariant=True) -_ExitT_co = TypeVar("_ExitT_co", covariant=True, bound="bool | None") - - -class _SupportsCtxMgr(Protocol[_T_co, _ExitT_co]): - def __contextmanager__(self) -> AbstractContextManager[_T_co, _ExitT_co]: ... - - -class _SupportsAsyncCtxMgr(Protocol[_T_co, _ExitT_co]): - def __asynccontextmanager__( - self, - ) -> AbstractAsyncContextManager[_T_co, _ExitT_co]: ... - - -class ContextManagerMixin: - """ - Mixin class providing context manager functionality via a generator-based - implementation. - - This class allows you to implement a context manager via :meth:`__contextmanager__` - which should return a generator. The mechanics are meant to mirror those of - :func:`@contextmanager `. - - .. note:: Classes using this mix-in are not reentrant as context managers, meaning - that once you enter it, you can't re-enter before first exiting it. - - .. seealso:: :doc:`contextmanagers` - """ - - __cm: AbstractContextManager[object, bool | None] | None = None - - @final - def __enter__(self: _SupportsCtxMgr[_T_co, bool | None]) -> _T_co: - # Needed for mypy to assume self still has the __cm member - assert isinstance(self, ContextManagerMixin) - if self.__cm is not None: - raise RuntimeError( - f"this {self.__class__.__qualname__} has already been entered" - ) - - cm = self.__contextmanager__() - if not isinstance(cm, AbstractContextManager): - if isgenerator(cm): - raise TypeError( - "__contextmanager__() returned a generator object instead of " - "a context manager. Did you forget to add the @contextmanager " - "decorator?" - ) - - raise TypeError( - f"__contextmanager__() did not return a context manager object, " - f"but {cm.__class__!r}" - ) - - if cm is self: - raise TypeError( - f"{self.__class__.__qualname__}.__contextmanager__() returned " - f"self. Did you forget to add the @contextmanager decorator and a " - f"'yield' statement?" - ) - - value = cm.__enter__() - self.__cm = cm - return value - - @final - def __exit__( - self: _SupportsCtxMgr[object, _ExitT_co], - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> _ExitT_co: - # Needed for mypy to assume self still has the __cm member - assert isinstance(self, ContextManagerMixin) - if self.__cm is None: - raise RuntimeError( - f"this {self.__class__.__qualname__} has not been entered yet" - ) - - # Prevent circular references - cm = self.__cm - del self.__cm - - return cast(_ExitT_co, cm.__exit__(exc_type, exc_val, exc_tb)) - - @abstractmethod - def __contextmanager__(self) -> AbstractContextManager[object, bool | None]: - """ - Implement your context manager logic here. - - This method **must** be decorated with - :func:`@contextmanager `. - - .. note:: Remember that the ``yield`` will raise any exception raised in the - enclosed context block, so use a ``finally:`` block to clean up resources! - - :return: a context manager object - """ - - -class AsyncContextManagerMixin: - """ - Mixin class providing async context manager functionality via a generator-based - implementation. - - This class allows you to implement a context manager via - :meth:`__asynccontextmanager__`. The mechanics are meant to mirror those of - :func:`@asynccontextmanager `. - - .. note:: Classes using this mix-in are not reentrant as context managers, meaning - that once you enter it, you can't re-enter before first exiting it. - - .. seealso:: :doc:`contextmanagers` - """ - - __cm: AbstractAsyncContextManager[object, bool | None] | None = None - - @final - async def __aenter__(self: _SupportsAsyncCtxMgr[_T_co, bool | None]) -> _T_co: - # Needed for mypy to assume self still has the __cm member - assert isinstance(self, AsyncContextManagerMixin) - if self.__cm is not None: - raise RuntimeError( - f"this {self.__class__.__qualname__} has already been entered" - ) - - cm = self.__asynccontextmanager__() - if not isinstance(cm, AbstractAsyncContextManager): - if isasyncgen(cm): - raise TypeError( - "__asynccontextmanager__() returned an async generator instead of " - "an async context manager. Did you forget to add the " - "@asynccontextmanager decorator?" - ) - elif iscoroutine(cm): - cm.close() - raise TypeError( - "__asynccontextmanager__() returned a coroutine object instead of " - "an async context manager. Did you forget to add the " - "@asynccontextmanager decorator and a 'yield' statement?" - ) - - raise TypeError( - f"__asynccontextmanager__() did not return an async context manager, " - f"but {cm.__class__!r}" - ) - - if cm is self: - raise TypeError( - f"{self.__class__.__qualname__}.__asynccontextmanager__() returned " - f"self. Did you forget to add the @asynccontextmanager decorator and a " - f"'yield' statement?" - ) - - value = await cm.__aenter__() - self.__cm = cm - return value - - @final - async def __aexit__( - self: _SupportsAsyncCtxMgr[object, _ExitT_co], - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> _ExitT_co: - assert isinstance(self, AsyncContextManagerMixin) - if self.__cm is None: - raise RuntimeError( - f"this {self.__class__.__qualname__} has not been entered yet" - ) - - # Prevent circular references - cm = self.__cm - del self.__cm - - return cast(_ExitT_co, await cm.__aexit__(exc_type, exc_val, exc_tb)) - - @abstractmethod - def __asynccontextmanager__( - self, - ) -> AbstractAsyncContextManager[object, bool | None]: - """ - Implement your async context manager logic here. - - This method **must** be decorated with - :func:`@asynccontextmanager `. - - .. note:: Remember that the ``yield`` will raise any exception raised in the - enclosed context block, so use a ``finally:`` block to clean up resources! - - :return: an async context manager object - """ diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_eventloop.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_eventloop.py deleted file mode 100644 index a3e2ab1cc172f2fbd94f244ab110cb16aa7eb8eb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_eventloop.py +++ /dev/null @@ -1,240 +0,0 @@ -from __future__ import annotations - -import math -import sys -import threading -from collections.abc import Awaitable, Callable, Generator -from contextlib import contextmanager -from contextvars import Token -from importlib import import_module -from typing import TYPE_CHECKING, Any, TypeVar - -from ._exceptions import NoEventLoopError - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -sniffio: Any -try: - import sniffio -except ModuleNotFoundError: - sniffio = None - -if TYPE_CHECKING: - from ..abc import AsyncBackend - -# This must be updated when new backends are introduced -BACKENDS = "asyncio", "trio" - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - -threadlocals = threading.local() -loaded_backends: dict[str, type[AsyncBackend]] = {} - - -def run( - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - *args: Unpack[PosArgsT], - backend: str = "asyncio", - backend_options: dict[str, Any] | None = None, -) -> T_Retval: - """ - Run the given coroutine function in an asynchronous event loop. - - The current thread must not be already running an event loop. - - :param func: a coroutine function - :param args: positional arguments to ``func`` - :param backend: name of the asynchronous event loop implementation – currently - either ``asyncio`` or ``trio`` - :param backend_options: keyword arguments to call the backend ``run()`` - implementation with (documented :ref:`here `) - :return: the return value of the coroutine function - :raises RuntimeError: if an asynchronous event loop is already running in this - thread - :raises LookupError: if the named backend is not found - - """ - if asynclib_name := current_async_library(): - raise RuntimeError(f"Already running {asynclib_name} in this thread") - - try: - async_backend = get_async_backend(backend) - except ImportError as exc: - if backend in BACKENDS: - raise LookupError( - f"Backend {backend!r} is not available. " - f"Install it with: pip install anyio[{backend}]" - ) from exc - - raise LookupError(f"No such backend: {backend}") from exc - - token = None - if asynclib_name is None: - # Since we're in control of the event loop, we can cache the name of the async - # library - token = set_current_async_library(backend) - - try: - backend_options = backend_options or {} - return async_backend.run(func, args, {}, backend_options) - finally: - reset_current_async_library(token) - - -async def sleep(delay: float) -> None: - """ - Pause the current task for the specified duration. - - :param delay: the duration, in seconds - - """ - return await get_async_backend().sleep(delay) - - -async def sleep_forever() -> None: - """ - Pause the current task until it's cancelled. - - This is a shortcut for ``sleep(math.inf)``. - - .. versionadded:: 3.1 - - """ - await sleep(math.inf) - - -async def sleep_until(deadline: float) -> None: - """ - Pause the current task until the given time. - - :param deadline: the absolute time to wake up at (according to the internal - monotonic clock of the event loop) - - .. versionadded:: 3.1 - - """ - now = current_time() - await sleep(max(deadline - now, 0)) - - -def current_time() -> float: - """ - Return the current value of the event loop's internal clock. - - :return: the clock value (seconds) - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().current_time() - - -def get_all_backends() -> tuple[str, ...]: - """Return a tuple of the names of all built-in backends.""" - return BACKENDS - - -def get_available_backends() -> tuple[str, ...]: - """ - Test for the availability of built-in backends. - - :return a tuple of the built-in backend names that were successfully imported - - .. versionadded:: 4.12 - - """ - available_backends: list[str] = [] - for backend_name in get_all_backends(): - try: - get_async_backend(backend_name) - except ImportError: - continue - - available_backends.append(backend_name) - - return tuple(available_backends) - - -def get_cancelled_exc_class() -> type[BaseException]: - """ - Return the current async library's cancellation exception class. - - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().cancelled_exception_class() - - -# -# Private API -# - - -@contextmanager -def claim_worker_thread( - backend_class: type[AsyncBackend], token: object -) -> Generator[Any, None, None]: - from ..lowlevel import EventLoopToken - - threadlocals.current_token = EventLoopToken(backend_class, token) - try: - yield - finally: - del threadlocals.current_token - - -def get_async_backend(asynclib_name: str | None = None) -> type[AsyncBackend]: - if asynclib_name is None: - asynclib_name = current_async_library() - if not asynclib_name: - raise NoEventLoopError( - f"Not currently running on any asynchronous event loop. " - f"Available async backends: {', '.join(get_all_backends())}" - ) - - # We use our own dict instead of sys.modules to get the already imported back-end - # class because the appropriate modules in sys.modules could potentially be only - # partially initialized - try: - return loaded_backends[asynclib_name] - except KeyError: - module = import_module(f"anyio._backends._{asynclib_name}") - loaded_backends[asynclib_name] = module.backend_class - return module.backend_class - - -def current_async_library() -> str | None: - if sniffio is None: - # If sniffio is not installed, we assume we're either running asyncio or nothing - import asyncio - - try: - asyncio.get_running_loop() - return "asyncio" - except RuntimeError: - pass - else: - try: - return sniffio.current_async_library() - except sniffio.AsyncLibraryNotFoundError: - pass - - return None - - -def set_current_async_library(asynclib_name: str | None) -> Token | None: - # no-op if sniffio is not installed - if sniffio is None: - return None - - return sniffio.current_async_library_cvar.set(asynclib_name) - - -def reset_current_async_library(token: Token | None) -> None: - if token is not None: - sniffio.current_async_library_cvar.reset(token) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_exceptions.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_exceptions.py deleted file mode 100644 index cd6eb9b5ca82c70014fc610670a34cd076b872fe..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_exceptions.py +++ /dev/null @@ -1,177 +0,0 @@ -from __future__ import annotations - -import sys -from collections.abc import Generator -from textwrap import dedent -from typing import Any - -if sys.version_info < (3, 11): - from exceptiongroup import BaseExceptionGroup - - -class BrokenResourceError(Exception): - """ - Raised when trying to use a resource that has been rendered unusable due to external - causes (e.g. a send stream whose peer has disconnected). - """ - - -class BrokenWorkerProcess(Exception): - """ - Raised by :meth:`~anyio.to_process.run_sync` if the worker process terminates abruptly or - otherwise misbehaves. - """ - - -class BrokenWorkerInterpreter(Exception): - """ - Raised by :meth:`~anyio.to_interpreter.run_sync` if an unexpected exception is - raised in the subinterpreter. - """ - - def __init__(self, excinfo: Any): - # This was adapted from concurrent.futures.interpreter.ExecutionFailed - msg = excinfo.formatted - if not msg: - if excinfo.type and excinfo.msg: - msg = f"{excinfo.type.__name__}: {excinfo.msg}" - else: - msg = excinfo.type.__name__ or excinfo.msg - - super().__init__(msg) - self.excinfo = excinfo - - def __str__(self) -> str: - try: - formatted = self.excinfo.errdisplay - except Exception: - return super().__str__() - else: - return dedent( - f""" - {super().__str__()} - - Uncaught in the interpreter: - - {formatted} - """.strip() - ) - - -class BusyResourceError(Exception): - """ - Raised when two tasks are trying to read from or write to the same resource - concurrently. - """ - - def __init__(self, action: str): - super().__init__(f"Another task is already {action} this resource") - - -class ClosedResourceError(Exception): - """Raised when trying to use a resource that has been closed.""" - - -class ConnectionFailed(OSError): - """ - Raised when a connection attempt fails. - - .. note:: This class inherits from :exc:`OSError` for backwards compatibility. - """ - - -def iterate_exceptions( - exception: BaseException, -) -> Generator[BaseException, None, None]: - if isinstance(exception, BaseExceptionGroup): - for exc in exception.exceptions: - yield from iterate_exceptions(exc) - else: - yield exception - - -class DelimiterNotFound(Exception): - """ - Raised during - :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_until` if the - maximum number of bytes has been read without the delimiter being found. - """ - - def __init__(self, max_bytes: int) -> None: - super().__init__( - f"The delimiter was not found among the first {max_bytes} bytes" - ) - - -class EndOfStream(Exception): - """ - Raised when trying to read from a stream that has been closed from the other end. - """ - - -class IncompleteRead(Exception): - """ - Raised during - :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_exactly` or - :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_until` if the - connection is closed before the requested amount of bytes has been read. - """ - - def __init__(self) -> None: - super().__init__( - "The stream was closed before the read operation could be completed" - ) - - -class TypedAttributeLookupError(LookupError): - """ - Raised by :meth:`~anyio.TypedAttributeProvider.extra` when the given typed attribute - is not found and no default value has been given. - """ - - -class WouldBlock(Exception): - """Raised by ``X_nowait`` functions if ``X()`` would block.""" - - -class NoEventLoopError(RuntimeError): - """ - Raised by several functions that require an event loop to be running in the current - thread when there is no running event loop. - - This is also raised by :func:`.from_thread.run` and :func:`.from_thread.run_sync` - if not calling from an AnyIO worker thread, and no ``token`` was passed. - """ - - -class RunFinishedError(RuntimeError): - """ - Raised by :func:`.from_thread.run` and :func:`.from_thread.run_sync` if the event - loop associated with the explicitly passed token has already finished. - """ - - def __init__(self) -> None: - super().__init__( - "The event loop associated with the given token has already finished" - ) - - -class TaskFailed(Exception): - """ - Raised when awaiting on, or attempting to access the return value of, a - :class:`.TaskHandle` that raised an exception. - """ - - -class TaskCancelled(TaskFailed): - """ - Raised when awaiting on, or attempting to access the return value of, a - :class:`.TaskHandle` that was cancelled. - """ - - -class TaskNotFinished(Exception): - """ - Raised when attempting to access the return value or exception of a - :class:`.TaskHandle` that is still pending completion. - """ diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_fileio.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_fileio.py deleted file mode 100644 index 692c754b3fd5b8a2e795d7c981b2785423161216..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_fileio.py +++ /dev/null @@ -1,960 +0,0 @@ -from __future__ import annotations - -import os -import pathlib -import sys -from collections.abc import ( - AsyncIterator, - Callable, - Iterable, - Iterator, - Sequence, -) -from dataclasses import dataclass -from functools import partial -from os import PathLike -from typing import ( - IO, - TYPE_CHECKING, - Any, - AnyStr, - ClassVar, - Final, - Generic, - TypeVar, - overload, -) - -from .. import to_thread -from ..abc import AsyncResource -from ._synchronization import CapacityLimiter - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self - -if sys.version_info >= (3, 14): - from pathlib.types import PathInfo - -if TYPE_CHECKING: - from types import ModuleType - - from _typeshed import OpenBinaryMode, OpenTextMode, ReadableBuffer, WriteableBuffer -else: - ReadableBuffer = OpenBinaryMode = OpenTextMode = WriteableBuffer = object - - -T = TypeVar("T", bound="Path") - - -class AsyncFile(AsyncResource, Generic[AnyStr]): - """ - An asynchronous file object. - - This class wraps a standard file object and provides async friendly versions of the - following blocking methods (where available on the original file object): - - * read - * read1 - * readline - * readlines - * readinto - * readinto1 - * write - * writelines - * truncate - * seek - * tell - * flush - - All other methods are directly passed through. - - This class supports the asynchronous context manager protocol which closes the - underlying file at the end of the context block. - - This class also supports asynchronous iteration:: - - async with await open_file(...) as f: - async for line in f: - print(line) - """ - - def __init__( - self, fp: IO[AnyStr], *, limiter: CapacityLimiter | None = None - ) -> None: - if limiter is not None and not isinstance(limiter, CapacityLimiter): - raise TypeError( - f"limiter must be a CapacityLimiter or None, not " - f"{limiter.__class__.__name__}" - ) - - self._fp: Any = fp - self._limiter = limiter - - def __getattr__(self, name: str) -> object: - return getattr(self._fp, name) - - @property - def limiter(self) -> CapacityLimiter | None: - """The capacity limiter used by this file object, if not the global limiter.""" - return self._limiter - - @property - def wrapped(self) -> IO[AnyStr]: - """The wrapped file object.""" - return self._fp - - async def __aiter__(self) -> AsyncIterator[AnyStr]: - while True: - line = await self.readline() - if line: - yield line - else: - break - - async def aclose(self) -> None: - return await to_thread.run_sync(self._fp.close, limiter=self._limiter) - - async def read(self, size: int = -1) -> AnyStr: - return await to_thread.run_sync(self._fp.read, size, limiter=self._limiter) - - async def read1(self: AsyncFile[bytes], size: int = -1) -> bytes: - return await to_thread.run_sync(self._fp.read1, size, limiter=self._limiter) - - async def readline(self) -> AnyStr: - return await to_thread.run_sync(self._fp.readline, limiter=self._limiter) - - async def readlines(self) -> list[AnyStr]: - return await to_thread.run_sync(self._fp.readlines, limiter=self._limiter) - - async def readinto(self: AsyncFile[bytes], b: WriteableBuffer) -> int: - return await to_thread.run_sync(self._fp.readinto, b, limiter=self._limiter) - - async def readinto1(self: AsyncFile[bytes], b: WriteableBuffer) -> int: - return await to_thread.run_sync(self._fp.readinto1, b, limiter=self._limiter) - - @overload - async def write(self: AsyncFile[bytes], b: ReadableBuffer) -> int: ... - - @overload - async def write(self: AsyncFile[str], b: str) -> int: ... - - async def write(self, b: ReadableBuffer | str) -> int: - return await to_thread.run_sync(self._fp.write, b, limiter=self._limiter) - - @overload - async def writelines( - self: AsyncFile[bytes], lines: Iterable[ReadableBuffer] - ) -> None: ... - - @overload - async def writelines(self: AsyncFile[str], lines: Iterable[str]) -> None: ... - - async def writelines(self, lines: Iterable[ReadableBuffer] | Iterable[str]) -> None: - return await to_thread.run_sync( - self._fp.writelines, lines, limiter=self._limiter - ) - - async def truncate(self, size: int | None = None) -> int: - return await to_thread.run_sync(self._fp.truncate, size, limiter=self._limiter) - - async def seek(self, offset: int, whence: int | None = os.SEEK_SET) -> int: - return await to_thread.run_sync( - self._fp.seek, offset, whence, limiter=self._limiter - ) - - async def tell(self) -> int: - return await to_thread.run_sync(self._fp.tell, limiter=self._limiter) - - async def flush(self) -> None: - return await to_thread.run_sync(self._fp.flush, limiter=self._limiter) - - -@overload -async def open_file( - file: str | PathLike[str] | int, - mode: OpenBinaryMode, - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - closefd: bool = ..., - opener: Callable[[str, int], int] | None = ..., - *, - limiter: CapacityLimiter | None = ..., -) -> AsyncFile[bytes]: ... - - -@overload -async def open_file( - file: str | PathLike[str] | int, - mode: OpenTextMode = ..., - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - closefd: bool = ..., - opener: Callable[[str, int], int] | None = ..., - *, - limiter: CapacityLimiter | None = ..., -) -> AsyncFile[str]: ... - - -async def open_file( - file: str | PathLike[str] | int, - mode: str = "r", - buffering: int = -1, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - closefd: bool = True, - opener: Callable[[str, int], int] | None = None, - *, - limiter: CapacityLimiter | None = None, -) -> AsyncFile[Any]: - """ - Open a file asynchronously. - - Except for ``limiter``, the arguments are exactly the same as for the builtin :func:`open`. - - :param limiter: an optional capacity limiter to use with the file - instead of the default one - :return: an asynchronous file object - - .. versionchanged:: 4.14.0 - Added the ``limiter`` keyword argument. - - """ - fp = await to_thread.run_sync( - open, - file, - mode, - buffering, - encoding, - errors, - newline, - closefd, - opener, - limiter=limiter, - ) - return AsyncFile(fp, limiter=limiter) - - -def wrap_file( - file: IO[AnyStr], *, limiter: CapacityLimiter | None = None -) -> AsyncFile[AnyStr]: - """ - Wrap an existing file as an asynchronous file. - - :param file: an existing file-like object - :param limiter: an optional capacity limiter to use with the file - instead of the default one - :return: an asynchronous file object - - .. versionchanged:: 4.14.0 - Added the ``limiter`` keyword argument. - - """ - return AsyncFile(file, limiter=limiter) - - -@dataclass(eq=False) -class _PathIterator(AsyncIterator[T]): - iterator: Iterator[PathLike[str]] - limiter: CapacityLimiter | None - # This was added to ensure that iterating over a subclass of Path yields instances - # of that subclass rather than the base Path class. - path_cls: type[T] - - async def __anext__(self) -> T: - nextval = await to_thread.run_sync( - next, self.iterator, None, abandon_on_cancel=True, limiter=self.limiter - ) - if nextval is None: - raise StopAsyncIteration from None - - return self.path_cls(nextval, limiter=self.limiter) - - -class Path: - """ - An asynchronous version of :class:`pathlib.Path`. - - This class cannot be substituted for :class:`pathlib.Path` or - :class:`pathlib.PurePath`, but it is compatible with the :class:`os.PathLike` - interface. - - It implements the Python 3.10 version of :class:`pathlib.Path` interface, except for - the deprecated :meth:`~pathlib.Path.link_to` method. - - Some methods may be unavailable or have limited functionality, based on the Python - version: - - * :meth:`~pathlib.Path.copy` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.copy_into` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.from_uri` (available on Python 3.13 or later) - * :meth:`~pathlib.PurePath.full_match` (available on Python 3.13 or later) - * :attr:`~pathlib.Path.info` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.is_junction` (available on Python 3.12 or later) - * :meth:`~pathlib.PurePath.match` (the ``case_sensitive`` parameter is only - available on Python 3.13 or later) - * :meth:`~pathlib.Path.move` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.move_into` (available on Python 3.14 or later) - * :meth:`~pathlib.PurePath.relative_to` (the ``walk_up`` parameter is only available - on Python 3.12 or later) - * :meth:`~pathlib.Path.walk` (available on Python 3.12 or later) - - Any methods that do disk I/O need to be awaited on. These methods are: - - * :meth:`~pathlib.Path.absolute` - * :meth:`~pathlib.Path.chmod` - * :meth:`~pathlib.Path.cwd` - * :meth:`~pathlib.Path.exists` - * :meth:`~pathlib.Path.expanduser` - * :meth:`~pathlib.Path.group` - * :meth:`~pathlib.Path.hardlink_to` - * :meth:`~pathlib.Path.home` - * :meth:`~pathlib.Path.is_block_device` - * :meth:`~pathlib.Path.is_char_device` - * :meth:`~pathlib.Path.is_dir` - * :meth:`~pathlib.Path.is_fifo` - * :meth:`~pathlib.Path.is_file` - * :meth:`~pathlib.Path.is_junction` - * :meth:`~pathlib.Path.is_mount` - * :meth:`~pathlib.Path.is_socket` - * :meth:`~pathlib.Path.is_symlink` - * :meth:`~pathlib.Path.lchmod` - * :meth:`~pathlib.Path.lstat` - * :meth:`~pathlib.Path.mkdir` - * :meth:`~pathlib.Path.open` - * :meth:`~pathlib.Path.owner` - * :meth:`~pathlib.Path.read_bytes` - * :meth:`~pathlib.Path.read_text` - * :meth:`~pathlib.Path.readlink` - * :meth:`~pathlib.Path.rename` - * :meth:`~pathlib.Path.replace` - * :meth:`~pathlib.Path.resolve` - * :meth:`~pathlib.Path.rmdir` - * :meth:`~pathlib.Path.samefile` - * :meth:`~pathlib.Path.stat` - * :meth:`~pathlib.Path.symlink_to` - * :meth:`~pathlib.Path.touch` - * :meth:`~pathlib.Path.unlink` - * :meth:`~pathlib.Path.walk` - * :meth:`~pathlib.Path.write_bytes` - * :meth:`~pathlib.Path.write_text` - - Additionally, the following methods return an async iterator yielding - :class:`~.Path` objects: - - * :meth:`~pathlib.Path.glob` - * :meth:`~pathlib.Path.iterdir` - * :meth:`~pathlib.Path.rglob` - - .. versionchanged:: 4.14.0 - Added the ``limiter`` keyword argument. - """ - - __slots__ = "_path", "_limiter", "__weakref__" - - __weakref__: Any - - def __init__( - self, *args: str | PathLike[str], limiter: CapacityLimiter | None = None - ) -> None: - if limiter is not None and not isinstance(limiter, CapacityLimiter): - raise TypeError( - f"limiter must be a CapacityLimiter or None, not " - f"{limiter.__class__.__name__}" - ) - - self._path: Final[pathlib.Path] = pathlib.Path(*args) - self._limiter = limiter - - def __fspath__(self) -> str: - return self._path.__fspath__() - - if sys.version_info >= (3, 15): - - def __vfspath__(self) -> str: - return self._path.__vfspath__() - - def __str__(self) -> str: - return self._path.__str__() - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.as_posix()!r})" - - def __bytes__(self) -> bytes: - return self._path.__bytes__() - - def __hash__(self) -> int: - return self._path.__hash__() - - def __eq__(self, other: object) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__eq__(target) - - def __lt__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__lt__(target) - - def __le__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__le__(target) - - def __gt__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__gt__(target) - - def __ge__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__ge__(target) - - def __truediv__(self, other: str | PathLike[str]) -> Self: - return type(self)(self._path / other, limiter=self._limiter) - - def __rtruediv__(self, other: str | PathLike[str]) -> Self: - return type(self)(other, limiter=self._limiter) / self - - @property - def limiter(self) -> CapacityLimiter | None: - """The capacity limiter used by this path, if not the global limiter.""" - return self._limiter - - @property - def parts(self) -> tuple[str, ...]: - return self._path.parts - - @property - def drive(self) -> str: - return self._path.drive - - @property - def root(self) -> str: - return self._path.root - - @property - def anchor(self) -> str: - return self._path.anchor - - @property - def parents(self) -> Sequence[Self]: - return tuple(type(self)(p, limiter=self._limiter) for p in self._path.parents) - - @property - def parent(self) -> Self: - return type(self)(self._path.parent, limiter=self._limiter) - - @property - def name(self) -> str: - return self._path.name - - @property - def suffix(self) -> str: - return self._path.suffix - - @property - def suffixes(self) -> list[str]: - return self._path.suffixes - - @property - def stem(self) -> str: - return self._path.stem - - async def absolute(self) -> Self: - path = await to_thread.run_sync(self._path.absolute, limiter=self._limiter) - return type(self)(path, limiter=self._limiter) - - def as_posix(self) -> str: - return self._path.as_posix() - - def as_uri(self) -> str: - return self._path.as_uri() - - if sys.version_info >= (3, 13): - parser: ClassVar[ModuleType] = pathlib.Path.parser - - @classmethod - def from_uri(cls, uri: str, *, limiter: CapacityLimiter | None = None) -> Self: - return cls(pathlib.Path.from_uri(uri), limiter=limiter) - - def full_match( - self, path_pattern: str, *, case_sensitive: bool | None = None - ) -> bool: - return self._path.full_match(path_pattern, case_sensitive=case_sensitive) - - def match( - self, path_pattern: str, *, case_sensitive: bool | None = None - ) -> bool: - return self._path.match(path_pattern, case_sensitive=case_sensitive) - else: - - def match(self, path_pattern: str) -> bool: - return self._path.match(path_pattern) - - if sys.version_info >= (3, 14): - - @property - def info(self) -> PathInfo: - return self._path.info - - async def copy( - self, - target: str | os.PathLike[str], - *, - follow_symlinks: bool = True, - preserve_metadata: bool = False, - ) -> Self: - func = partial( - self._path.copy, - follow_symlinks=follow_symlinks, - preserve_metadata=preserve_metadata, - ) - return type(self)( - await to_thread.run_sync( - func, pathlib.Path(target), limiter=self._limiter - ), - limiter=self._limiter, - ) - - async def copy_into( - self, - target_dir: str | os.PathLike[str], - *, - follow_symlinks: bool = True, - preserve_metadata: bool = False, - ) -> Self: - func = partial( - self._path.copy_into, - follow_symlinks=follow_symlinks, - preserve_metadata=preserve_metadata, - ) - return type(self)( - await to_thread.run_sync( - func, pathlib.Path(target_dir), limiter=self._limiter - ), - limiter=self._limiter, - ) - - async def move(self, target: str | os.PathLike[str]) -> Self: - # Upstream does not handle anyio.Path properly as a PathLike - target = pathlib.Path(target) - return type(self)( - await to_thread.run_sync( - self._path.move, target, limiter=self._limiter - ), - limiter=self._limiter, - ) - - async def move_into( - self, - target_dir: str | os.PathLike[str], - ) -> Self: - return type(self)( - await to_thread.run_sync( - self._path.move_into, target_dir, limiter=self._limiter - ), - limiter=self._limiter, - ) - - def is_relative_to(self, other: str | PathLike[str]) -> bool: - try: - self.relative_to(other) - return True - except ValueError: - return False - - async def chmod(self, mode: int, *, follow_symlinks: bool = True) -> None: - func = partial(os.chmod, follow_symlinks=follow_symlinks) - return await to_thread.run_sync(func, self._path, mode, limiter=self._limiter) - - @classmethod - async def cwd(cls, *, limiter: CapacityLimiter | None = None) -> Self: - path = await to_thread.run_sync(pathlib.Path.cwd, limiter=limiter) - return cls(path, limiter=limiter) - - async def exists(self) -> bool: - return await to_thread.run_sync( - self._path.exists, abandon_on_cancel=True, limiter=self._limiter - ) - - async def expanduser(self) -> Self: - return type(self)( - await to_thread.run_sync( - self._path.expanduser, abandon_on_cancel=True, limiter=self._limiter - ), - limiter=self._limiter, - ) - - if sys.version_info < (3, 12): - # Python 3.11 and earlier - def glob(self, pattern: str) -> AsyncIterator[Self]: - gen = self._path.glob(pattern) - return _PathIterator(gen, self._limiter, type(self)) - elif (3, 12) <= sys.version_info < (3, 13): - # changed in Python 3.12: - # - The case_sensitive parameter was added. - def glob( - self, - pattern: str, - *, - case_sensitive: bool | None = None, - ) -> AsyncIterator[Self]: - gen = self._path.glob(pattern, case_sensitive=case_sensitive) - return _PathIterator(gen, self._limiter, type(self)) - elif sys.version_info >= (3, 13): - # Changed in Python 3.13: - # - The recurse_symlinks parameter was added. - # - The pattern parameter accepts a path-like object. - def glob( # type: ignore[misc] # mypy doesn't allow for differing signatures in a conditional block - self, - pattern: str | PathLike[str], - *, - case_sensitive: bool | None = None, - recurse_symlinks: bool = False, - ) -> AsyncIterator[Self]: - gen = self._path.glob( - pattern, # type: ignore[arg-type] - case_sensitive=case_sensitive, - recurse_symlinks=recurse_symlinks, - ) - return _PathIterator(gen, self._limiter, type(self)) - - async def group(self) -> str: - return await to_thread.run_sync( - self._path.group, abandon_on_cancel=True, limiter=self._limiter - ) - - async def hardlink_to( - self, target: str | bytes | PathLike[str] | PathLike[bytes] - ) -> None: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync(os.link, target, self, limiter=self._limiter) - - @classmethod - async def home(cls, *, limiter: CapacityLimiter | None = None) -> Self: - home_path = await to_thread.run_sync(pathlib.Path.home, limiter=limiter) - return cls(home_path, limiter=limiter) - - def is_absolute(self) -> bool: - return self._path.is_absolute() - - async def is_block_device(self) -> bool: - return await to_thread.run_sync( - self._path.is_block_device, abandon_on_cancel=True, limiter=self._limiter - ) - - async def is_char_device(self) -> bool: - return await to_thread.run_sync( - self._path.is_char_device, abandon_on_cancel=True, limiter=self._limiter - ) - - async def is_dir(self) -> bool: - return await to_thread.run_sync( - self._path.is_dir, abandon_on_cancel=True, limiter=self._limiter - ) - - async def is_fifo(self) -> bool: - return await to_thread.run_sync( - self._path.is_fifo, abandon_on_cancel=True, limiter=self._limiter - ) - - async def is_file(self) -> bool: - return await to_thread.run_sync( - self._path.is_file, abandon_on_cancel=True, limiter=self._limiter - ) - - if sys.version_info >= (3, 12): - - async def is_junction(self) -> bool: - return await to_thread.run_sync( - self._path.is_junction, limiter=self._limiter - ) - - async def is_mount(self) -> bool: - return await to_thread.run_sync( - os.path.ismount, self._path, abandon_on_cancel=True, limiter=self._limiter - ) - - if sys.version_info < (3, 15): - - def is_reserved(self) -> bool: - return self._path.is_reserved() - - async def is_socket(self) -> bool: - return await to_thread.run_sync( - self._path.is_socket, abandon_on_cancel=True, limiter=self._limiter - ) - - async def is_symlink(self) -> bool: - return await to_thread.run_sync( - self._path.is_symlink, abandon_on_cancel=True, limiter=self._limiter - ) - - async def iterdir(self) -> AsyncIterator[Self]: - gen = ( - self._path.iterdir() - if sys.version_info < (3, 13) - else await to_thread.run_sync( - self._path.iterdir, abandon_on_cancel=True, limiter=self._limiter - ) - ) - async for path in _PathIterator(gen, self._limiter, type(self)): - yield path - - def joinpath(self, *args: str | PathLike[str]) -> Self: - return type(self)(self._path.joinpath(*args), limiter=self._limiter) - - async def lchmod(self, mode: int) -> None: - await to_thread.run_sync(self._path.lchmod, mode, limiter=self._limiter) - - async def lstat(self) -> os.stat_result: - return await to_thread.run_sync( - self._path.lstat, abandon_on_cancel=True, limiter=self._limiter - ) - - async def mkdir( - self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False - ) -> None: - await to_thread.run_sync( - self._path.mkdir, mode, parents, exist_ok, limiter=self._limiter - ) - - @overload - async def open( - self, - mode: OpenBinaryMode, - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - ) -> AsyncFile[bytes]: ... - - @overload - async def open( - self, - mode: OpenTextMode = ..., - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - ) -> AsyncFile[str]: ... - - async def open( - self, - mode: str = "r", - buffering: int = -1, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - ) -> AsyncFile[Any]: - fp = await to_thread.run_sync( - self._path.open, - mode, - buffering, - encoding, - errors, - newline, - limiter=self._limiter, - ) - return AsyncFile(fp, limiter=self._limiter) - - async def owner(self) -> str: - return await to_thread.run_sync( - self._path.owner, abandon_on_cancel=True, limiter=self._limiter - ) - - async def read_bytes(self) -> bytes: - return await to_thread.run_sync(self._path.read_bytes, limiter=self._limiter) - - async def read_text( - self, encoding: str | None = None, errors: str | None = None - ) -> str: - return await to_thread.run_sync( - self._path.read_text, encoding, errors, limiter=self._limiter - ) - - if sys.version_info >= (3, 12): - - def relative_to( - self, *other: str | PathLike[str], walk_up: bool = False - ) -> Self: - # relative_to() should work with any PathLike but it doesn't - others = [pathlib.Path(other) for other in other] - return type(self)( - self._path.relative_to(*others, walk_up=walk_up), limiter=self._limiter - ) - - else: - - def relative_to(self, *other: str | PathLike[str]) -> Self: - return type(self)(self._path.relative_to(*other), limiter=self._limiter) - - async def readlink(self) -> Self: - target = await to_thread.run_sync( - os.readlink, self._path, limiter=self._limiter - ) - return type(self)(target, limiter=self._limiter) - - async def rename(self, target: str | pathlib.PurePath | Path) -> Self: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync(self._path.rename, target, limiter=self._limiter) - return type(self)(target, limiter=self._limiter) - - async def replace(self, target: str | pathlib.PurePath | Path) -> Self: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync(self._path.replace, target, limiter=self._limiter) - return type(self)(target, limiter=self._limiter) - - async def resolve(self, strict: bool = False) -> Self: - func = partial(self._path.resolve, strict=strict) - return type(self)( - await to_thread.run_sync( - func, abandon_on_cancel=True, limiter=self._limiter - ), - limiter=self._limiter, - ) - - if sys.version_info < (3, 12): - # Pre Python 3.12 - def rglob(self, pattern: str) -> AsyncIterator[Self]: - gen = self._path.rglob(pattern) - return _PathIterator(gen, self._limiter, type(self)) - elif (3, 12) <= sys.version_info < (3, 13): - # Changed in Python 3.12: - # - The case_sensitive parameter was added. - def rglob( - self, pattern: str, *, case_sensitive: bool | None = None - ) -> AsyncIterator[Self]: - gen = self._path.rglob(pattern, case_sensitive=case_sensitive) - return _PathIterator(gen, self._limiter, type(self)) - elif sys.version_info >= (3, 13): - # Changed in Python 3.13: - # - The recurse_symlinks parameter was added. - # - The pattern parameter accepts a path-like object. - def rglob( # type: ignore[misc] # mypy doesn't allow for differing signatures in a conditional block - self, - pattern: str | PathLike[str], - *, - case_sensitive: bool | None = None, - recurse_symlinks: bool = False, - ) -> AsyncIterator[Self]: - gen = self._path.rglob( - pattern, # type: ignore[arg-type] - case_sensitive=case_sensitive, - recurse_symlinks=recurse_symlinks, - ) - return _PathIterator(gen, self._limiter, type(self)) - - async def rmdir(self) -> None: - await to_thread.run_sync(self._path.rmdir, limiter=self._limiter) - - async def samefile(self, other_path: str | PathLike[str]) -> bool: - if isinstance(other_path, Path): - other_path = other_path._path - - return await to_thread.run_sync( - self._path.samefile, - other_path, - abandon_on_cancel=True, - limiter=self._limiter, - ) - - async def stat(self, *, follow_symlinks: bool = True) -> os.stat_result: - func = partial(os.stat, follow_symlinks=follow_symlinks) - return await to_thread.run_sync( - func, self._path, abandon_on_cancel=True, limiter=self._limiter - ) - - async def symlink_to( - self, - target: str | bytes | PathLike[str] | PathLike[bytes], - target_is_directory: bool = False, - ) -> None: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync( - self._path.symlink_to, target, target_is_directory, limiter=self._limiter - ) - - async def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: - await to_thread.run_sync( - self._path.touch, mode, exist_ok, limiter=self._limiter - ) - - async def unlink(self, missing_ok: bool = False) -> None: - try: - await to_thread.run_sync(self._path.unlink, limiter=self._limiter) - except FileNotFoundError: - if not missing_ok: - raise - - if sys.version_info >= (3, 12): - - async def walk( - self, - top_down: bool = True, - on_error: Callable[[OSError], object] | None = None, - follow_symlinks: bool = False, - ) -> AsyncIterator[tuple[Self, list[str], list[str]]]: - def get_next_value() -> tuple[pathlib.Path, list[str], list[str]] | None: - try: - return next(gen) - except StopIteration: - return None - - gen = self._path.walk(top_down, on_error, follow_symlinks) - while True: - value = await to_thread.run_sync(get_next_value, limiter=self._limiter) - if value is None: - return - - root, dirs, paths = value - yield type(self)(root, limiter=self._limiter), dirs, paths - - def with_name(self, name: str) -> Self: - return type(self)(self._path.with_name(name), limiter=self._limiter) - - def with_stem(self, stem: str) -> Self: - return type(self)( - self._path.with_name(stem + self._path.suffix), limiter=self._limiter - ) - - def with_suffix(self, suffix: str) -> Self: - return type(self)(self._path.with_suffix(suffix), limiter=self._limiter) - - def with_segments(self, *pathsegments: str | PathLike[str]) -> Self: - return type(self)(*pathsegments, limiter=self._limiter) - - async def write_bytes(self, data: ReadableBuffer) -> int: - return await to_thread.run_sync( - self._path.write_bytes, data, limiter=self._limiter - ) - - async def write_text( - self, - data: str, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - ) -> int: - return await to_thread.run_sync( - self._path.write_text, - data, - encoding, - errors, - newline, - limiter=self._limiter, - ) - - -PathLike.register(Path) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_resources.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_resources.py deleted file mode 100644 index b9a5344aef2962670f9b305a02cd0b11f2087d2f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_resources.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -from ..abc import AsyncResource -from ._tasks import CancelScope - - -async def aclose_forcefully(resource: AsyncResource) -> None: - """ - Close an asynchronous resource in a cancelled scope. - - Doing this closes the resource without waiting on anything. - - :param resource: the resource to close - - """ - with CancelScope() as scope: - scope.cancel() - await resource.aclose() diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_signals.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_signals.py deleted file mode 100644 index e24c79e10d4b76775679f7dd0dbe3f5860150451..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_signals.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from collections.abc import AsyncIterator -from contextlib import AbstractContextManager -from signal import Signals - -from ._eventloop import get_async_backend - - -def open_signal_receiver( - *signals: Signals, -) -> AbstractContextManager[AsyncIterator[Signals]]: - """ - Start receiving operating system signals. - - :param signals: signals to receive (e.g. ``signal.SIGINT``) - :return: an asynchronous context manager for an asynchronous iterator which yields - signal numbers - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - .. warning:: Windows does not support signals natively so it is best to avoid - relying on this in cross-platform applications. - - .. warning:: On asyncio, this permanently replaces any previous signal handler for - the given signals, as set via :meth:`~asyncio.loop.add_signal_handler`. - - """ - return get_async_backend().open_signal_receiver(*signals) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_sockets.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_sockets.py deleted file mode 100644 index c75791b85b5549e7b56ac501da2fbe33bc58f99e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_sockets.py +++ /dev/null @@ -1,1011 +0,0 @@ -from __future__ import annotations - -import errno -import os -import socket -import ssl -import stat -import sys -from collections.abc import Awaitable -from dataclasses import dataclass -from ipaddress import IPv4Address, IPv6Address, ip_address -from os import PathLike, chmod -from socket import AddressFamily, SocketKind -from typing import TYPE_CHECKING, Any, Literal, cast, overload - -from .. import ConnectionFailed, to_thread -from ..abc import ( - ByteStreamConnectable, - ConnectedUDPSocket, - ConnectedUNIXDatagramSocket, - IPAddressType, - IPSockAddrType, - SocketListener, - SocketStream, - UDPSocket, - UNIXDatagramSocket, - UNIXSocketStream, -) -from ..streams.stapled import MultiListener -from ..streams.tls import TLSConnectable, TLSStream -from ._eventloop import get_async_backend -from ._resources import aclose_forcefully -from ._synchronization import Event -from ._tasks import create_task_group, move_on_after - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike -else: - FileDescriptorLike = object - -if sys.version_info < (3, 11): - from exceptiongroup import ExceptionGroup - -if sys.version_info >= (3, 12): - from typing import override -else: - from typing_extensions import override - -if sys.version_info < (3, 13): - from typing_extensions import deprecated -else: - from warnings import deprecated - -IPPROTO_IPV6 = getattr(socket, "IPPROTO_IPV6", 41) # https://bugs.python.org/issue29515 - -AnyIPAddressFamily = Literal[ - AddressFamily.AF_UNSPEC, AddressFamily.AF_INET, AddressFamily.AF_INET6 -] -IPAddressFamily = Literal[AddressFamily.AF_INET, AddressFamily.AF_INET6] - - -def idna2008_resolve(host: str) -> bytes: - try: - return host.encode("ascii") - except UnicodeEncodeError: - import idna - - return idna.encode(host, uts46=True) - - -# tls_hostname given -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - local_port: int | None = ..., - ssl_context: ssl.SSLContext | None = ..., - tls_standard_compatible: bool = ..., - tls_hostname: str, - happy_eyeballs_delay: float = ..., -) -> TLSStream: ... - - -# ssl_context given -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - local_port: int | None = ..., - ssl_context: ssl.SSLContext, - tls_standard_compatible: bool = ..., - tls_hostname: str | None = ..., - happy_eyeballs_delay: float = ..., -) -> TLSStream: ... - - -# tls=True -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - local_port: int | None = ..., - tls: Literal[True], - ssl_context: ssl.SSLContext | None = ..., - tls_standard_compatible: bool = ..., - tls_hostname: str | None = ..., - happy_eyeballs_delay: float = ..., -) -> TLSStream: ... - - -# tls=False -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - local_port: int | None = ..., - tls: Literal[False], - ssl_context: ssl.SSLContext | None = ..., - tls_standard_compatible: bool = ..., - tls_hostname: str | None = ..., - happy_eyeballs_delay: float = ..., -) -> SocketStream: ... - - -# No TLS arguments -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - local_port: int | None = ..., - happy_eyeballs_delay: float = ..., -) -> SocketStream: ... - - -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = None, - local_port: int | None = None, - tls: bool = False, - ssl_context: ssl.SSLContext | None = None, - tls_standard_compatible: bool = True, - tls_hostname: str | None = None, - happy_eyeballs_delay: float = 0.25, -) -> SocketStream | TLSStream: - """ - Connect to a host using the TCP protocol. - - This function implements the stateless version of the Happy Eyeballs algorithm (RFC - 6555). If ``remote_host`` is a host name that resolves to multiple IP addresses, - each one is tried until one connection attempt succeeds. If the first attempt does - not connected within 250 milliseconds, a second attempt is started using the next - address in the list, and so on. On IPv6 enabled systems, an IPv6 address (if - available) is tried first. - - When the connection has been established, a TLS handshake will be done if either - ``ssl_context`` or ``tls_hostname`` is not ``None``, or if ``tls`` is ``True``. - - :param remote_host: the IP address or host name to connect to - :param remote_port: port on the target host to connect to - :param local_host: the interface address or name to bind the socket to before - connecting - :param local_port: the local port to bind to (requires ``local_host`` to also be - set) - :param tls: ``True`` to do a TLS handshake with the connected stream and return a - :class:`~anyio.streams.tls.TLSStream` instead - :param ssl_context: the SSL context object to use (if omitted, a default context is - created) - :param tls_standard_compatible: If ``True``, performs the TLS shutdown handshake - before closing the stream and requires that the server does this as well. - Otherwise, :exc:`~ssl.SSLEOFError` may be raised during reads from the stream. - Some protocols, such as HTTP, require this option to be ``False``. - See :meth:`~ssl.SSLContext.wrap_socket` for details. - :param tls_hostname: host name to check the server certificate against (defaults to - the value of ``remote_host``) - :param happy_eyeballs_delay: delay (in seconds) before starting the next connection - attempt - :return: a socket stream object if no TLS handshake was done, otherwise a TLS stream - :raises ConnectionFailed: if the connection fails - - """ - # Placed here due to https://github.com/python/mypy/issues/7057 - connected_stream: SocketStream | None = None - - async def try_connect(remote_host: str, event: Event) -> None: - nonlocal connected_stream - try: - stream = await asynclib.connect_tcp(remote_host, remote_port, local_address) - except OSError as exc: - oserrors.append(exc) - return - else: - if connected_stream is None: - connected_stream = stream - tg.cancel_scope.cancel() - else: - await stream.aclose() - finally: - event.set() - - asynclib = get_async_backend() - local_address: IPSockAddrType | None = None - family = socket.AF_UNSPEC - if local_host: - gai_res = await getaddrinfo(str(local_host), local_port) - family, *_, local_address = gai_res[0] - - target_host = str(remote_host) - try: - addr_obj = ip_address(remote_host) - except ValueError: - addr_obj = None - - if addr_obj is not None: - if isinstance(addr_obj, IPv6Address): - target_addrs = [(socket.AF_INET6, addr_obj.compressed)] - else: - target_addrs = [(socket.AF_INET, addr_obj.compressed)] - else: - # getaddrinfo() will raise an exception if name resolution fails - gai_res = await getaddrinfo( - target_host, remote_port, family=family, type=socket.SOCK_STREAM - ) - - # Organize the list so that the first address is an IPv6 address (if available) - # and the second one is an IPv4 addresses. The rest can be in whatever order. - v6_found = v4_found = False - target_addrs = [] - for af, *_, sa in gai_res: - if af == socket.AF_INET6 and not v6_found: - v6_found = True - target_addrs.insert(0, (af, sa[0])) - elif af == socket.AF_INET and not v4_found and v6_found: - v4_found = True - target_addrs.insert(1, (af, sa[0])) - else: - target_addrs.append((af, sa[0])) - - oserrors: list[OSError] = [] - try: - async with create_task_group() as tg: - for _af, addr in target_addrs: - event = Event() - tg.start_soon(try_connect, addr, event) - with move_on_after(happy_eyeballs_delay): - await event.wait() - - if connected_stream is None: - cause = ( - oserrors[0] - if len(oserrors) == 1 - else ExceptionGroup("multiple connection attempts failed", oserrors) - ) - raise OSError("All connection attempts failed") from cause - finally: - oserrors.clear() - - if tls or tls_hostname or ssl_context: - try: - return await TLSStream.wrap( - connected_stream, - server_side=False, - hostname=tls_hostname or str(remote_host), - ssl_context=ssl_context, - standard_compatible=tls_standard_compatible, - ) - except BaseException: - await aclose_forcefully(connected_stream) - raise - - return connected_stream - - -async def connect_unix(path: str | bytes | PathLike[Any]) -> UNIXSocketStream: - """ - Connect to the given UNIX socket. - - Not available on Windows. - - :param path: path to the socket - :return: a socket stream object - :raises ConnectionFailed: if the connection fails - - """ - path = os.fspath(path) - return await get_async_backend().connect_unix(path) - - -async def create_tcp_listener( - *, - local_host: IPAddressType | None = None, - local_port: int = 0, - family: AnyIPAddressFamily = socket.AddressFamily.AF_UNSPEC, - backlog: int = 65536, - reuse_port: bool = False, -) -> MultiListener[SocketStream]: - """ - Create a TCP socket listener. - - :param local_port: port number to listen on - :param local_host: IP address of the interface to listen on. If omitted, listen on - all IPv4 and IPv6 interfaces. To listen on all interfaces on a specific address - family, use ``0.0.0.0`` for IPv4 or ``::`` for IPv6. - :param family: address family (used if ``local_host`` was omitted) - :param backlog: maximum number of queued incoming connections (up to a maximum of - 2**16, or 65536) - :param reuse_port: ``True`` to allow multiple sockets to bind to the same - address/port (not supported on Windows) - :return: a multi-listener object containing one or more socket listeners - :raises OSError: if there's an error creating a socket, or binding to one or more - interfaces failed - - """ - asynclib = get_async_backend() - backlog = min(backlog, 65536) - local_host = str(local_host) if local_host is not None else None - - def setup_raw_socket( - fam: AddressFamily, - bind_addr: tuple[str, int] | tuple[str, int, int, int], - *, - v6only: bool = True, - ) -> socket.socket: - sock = socket.socket(fam) - try: - sock.setblocking(False) - - if fam == AddressFamily.AF_INET6: - sock.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, v6only) - - # For Windows, enable exclusive address use. For others, enable address - # reuse. - if sys.platform == "win32": - sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) - else: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - - if reuse_port: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - - # Workaround for #554 - if fam == socket.AF_INET6 and "%" in bind_addr[0]: - addr, scope_id = bind_addr[0].split("%", 1) - bind_addr = (addr, bind_addr[1], 0, int(scope_id)) - - sock.bind(bind_addr) - sock.listen(backlog) - except BaseException: - sock.close() - raise - - return sock - - # We passing type=0 on non-Windows platforms as a workaround for a uvloop bug - # where we don't get the correct scope ID for IPv6 link-local addresses when passing - # type=socket.SOCK_STREAM to getaddrinfo(): - # https://github.com/MagicStack/uvloop/issues/539 - gai_res = await getaddrinfo( - local_host, - local_port, - family=family, - type=socket.SOCK_STREAM if sys.platform == "win32" else 0, - flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, - ) - - # The set comprehension is here to work around a glibc bug: - # https://sourceware.org/bugzilla/show_bug.cgi?id=14969 - sockaddrs = sorted({res for res in gai_res if res[1] == SocketKind.SOCK_STREAM}) - - # Special case for dual-stack binding on the "any" interface - if ( - local_host is None - and family == AddressFamily.AF_UNSPEC - and socket.has_dualstack_ipv6() - and any(fam == AddressFamily.AF_INET6 for fam, *_ in gai_res) - ): - raw_socket = setup_raw_socket( - AddressFamily.AF_INET6, ("::", local_port), v6only=False - ) - listener = asynclib.create_tcp_listener(raw_socket) - return MultiListener([listener]) - - errors: list[OSError] = [] - try: - for _ in range(len(sockaddrs)): - listeners: list[SocketListener] = [] - bound_ephemeral_port = local_port - try: - for fam, *_, sockaddr in sockaddrs: - sockaddr = sockaddr[0], bound_ephemeral_port, *sockaddr[2:] - raw_socket = setup_raw_socket(fam, sockaddr) - - # Store the assigned port if an ephemeral port was requested, so - # we'll bind to the same port on all interfaces - if local_port == 0 and len(gai_res) > 1: - bound_ephemeral_port = raw_socket.getsockname()[1] - - listeners.append(asynclib.create_tcp_listener(raw_socket)) - except BaseException as exc: - for listener in listeners: - await listener.aclose() - - # If an ephemeral port was requested but binding the assigned port - # failed for another interface, rotate the address list and try again - if ( - isinstance(exc, OSError) - and exc.errno == errno.EADDRINUSE - and local_port == 0 - and bound_ephemeral_port - ): - errors.append(exc) - sockaddrs.append(sockaddrs.pop(0)) - continue - - raise - - return MultiListener(listeners) - - raise OSError( - f"Could not create {len(sockaddrs)} listeners with a consistent port" - ) from ExceptionGroup("Several bind attempts failed", errors) - finally: - del errors # Prevent reference cycles - - -async def create_unix_listener( - path: str | bytes | PathLike[Any], - *, - mode: int | None = None, - backlog: int = 65536, -) -> SocketListener: - """ - Create a UNIX socket listener. - - Not available on Windows. - - :param path: path of the socket - :param mode: permissions to set on the socket - :param backlog: maximum number of queued incoming connections (up to a maximum of - 2**16, or 65536) - :return: a listener object - - .. versionchanged:: 3.0 - If a socket already exists on the file system in the given path, it will be - removed first. - - """ - backlog = min(backlog, 65536) - raw_socket = await setup_unix_local_socket(path, mode, socket.SOCK_STREAM) - try: - raw_socket.listen(backlog) - return get_async_backend().create_unix_listener(raw_socket) - except BaseException: - raw_socket.close() - raise - - -async def create_udp_socket( - family: AnyIPAddressFamily = AddressFamily.AF_UNSPEC, - *, - local_host: IPAddressType | None = None, - local_port: int = 0, - reuse_port: bool = False, -) -> UDPSocket: - """ - Create a UDP socket. - - If ``port`` has been given, the socket will be bound to this port on the local - machine, making this socket suitable for providing UDP based services. - - :param family: address family (``AF_INET`` or ``AF_INET6``) – automatically - determined from ``local_host`` if omitted - :param local_host: IP address or host name of the local interface to bind to - :param local_port: local port to bind to - :param reuse_port: ``True`` to allow multiple sockets to bind to the same - address/port (not supported on Windows) - :return: a UDP socket - - """ - if family is AddressFamily.AF_UNSPEC and not local_host: - raise ValueError('Either "family" or "local_host" must be given') - - if local_host: - gai_res = await getaddrinfo( - str(local_host), - local_port, - family=family, - type=socket.SOCK_DGRAM, - flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, - ) - family = cast(AnyIPAddressFamily, gai_res[0][0]) - local_address = gai_res[0][-1] - elif family is AddressFamily.AF_INET6: - local_address = ("::", 0) - else: - local_address = ("0.0.0.0", 0) - - sock = await get_async_backend().create_udp_socket( - family, local_address, None, reuse_port - ) - return cast(UDPSocket, sock) - - -async def create_connected_udp_socket( - remote_host: IPAddressType, - remote_port: int, - *, - family: AnyIPAddressFamily = AddressFamily.AF_UNSPEC, - local_host: IPAddressType | None = None, - local_port: int = 0, - reuse_port: bool = False, -) -> ConnectedUDPSocket: - """ - Create a connected UDP socket. - - Connected UDP sockets can only communicate with the specified remote host/port, an - any packets sent from other sources are dropped. - - :param remote_host: remote host to set as the default target - :param remote_port: port on the remote host to set as the default target - :param family: address family (``AF_INET`` or ``AF_INET6``) – automatically - determined from ``local_host`` or ``remote_host`` if omitted - :param local_host: IP address or host name of the local interface to bind to - :param local_port: local port to bind to - :param reuse_port: ``True`` to allow multiple sockets to bind to the same - address/port (not supported on Windows) - :return: a connected UDP socket - - """ - local_address = None - if local_host: - gai_res = await getaddrinfo( - str(local_host), - local_port, - family=family, - type=socket.SOCK_DGRAM, - flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, - ) - family = cast(AnyIPAddressFamily, gai_res[0][0]) - local_address = gai_res[0][-1] - - gai_res = await getaddrinfo( - str(remote_host), remote_port, family=family, type=socket.SOCK_DGRAM - ) - family = cast(AnyIPAddressFamily, gai_res[0][0]) - remote_address = gai_res[0][-1] - - sock = await get_async_backend().create_udp_socket( - family, local_address, remote_address, reuse_port - ) - return cast(ConnectedUDPSocket, sock) - - -async def create_unix_datagram_socket( - *, - local_path: None | str | bytes | PathLike[Any] = None, - local_mode: int | None = None, -) -> UNIXDatagramSocket: - """ - Create a UNIX datagram socket. - - Not available on Windows. - - If ``local_path`` has been given, the socket will be bound to this path, making this - socket suitable for receiving datagrams from other processes. Other processes can - send datagrams to this socket only if ``local_path`` is set. - - If a socket already exists on the file system in the ``local_path``, it will be - removed first. - - :param local_path: the path on which to bind to - :param local_mode: permissions to set on the local socket - :return: a UNIX datagram socket - - """ - raw_socket = await setup_unix_local_socket( - local_path, local_mode, socket.SOCK_DGRAM - ) - return await get_async_backend().create_unix_datagram_socket(raw_socket, None) - - -async def create_connected_unix_datagram_socket( - remote_path: str | bytes | PathLike[Any], - *, - local_path: None | str | bytes | PathLike[Any] = None, - local_mode: int | None = None, -) -> ConnectedUNIXDatagramSocket: - """ - Create a connected UNIX datagram socket. - - Connected datagram sockets can only communicate with the specified remote path. - - If ``local_path`` has been given, the socket will be bound to this path, making - this socket suitable for receiving datagrams from other processes. Other processes - can send datagrams to this socket only if ``local_path`` is set. - - If a socket already exists on the file system in the ``local_path``, it will be - removed first. - - :param remote_path: the path to set as the default target - :param local_path: the path on which to bind to - :param local_mode: permissions to set on the local socket - :return: a connected UNIX datagram socket - - """ - remote_path = os.fspath(remote_path) - raw_socket = await setup_unix_local_socket( - local_path, local_mode, socket.SOCK_DGRAM - ) - return await get_async_backend().create_unix_datagram_socket( - raw_socket, remote_path - ) - - -async def getaddrinfo( - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, -) -> list[tuple[AddressFamily, SocketKind, int, str, tuple[str, int]]]: - """ - Look up a numeric IP address given a host name. - - Internationalized domain names are translated according to the (non-transitional) - IDNA 2008 standard. - - .. note:: 4-tuple IPv6 socket addresses are automatically converted to 2-tuples of - (host, port), unlike what :func:`socket.getaddrinfo` does. - - :param host: host name - :param port: port number - :param family: socket family (`'AF_INET``, ...) - :param type: socket type (``SOCK_STREAM``, ...) - :param proto: protocol number - :param flags: flags to pass to upstream ``getaddrinfo()`` - :return: list of tuples containing (family, type, proto, canonname, sockaddr) - - .. seealso:: :func:`socket.getaddrinfo` - - """ - # Handle unicode hostnames - encoded_host = idna2008_resolve(host) if isinstance(host, str) else host - gai_res = await get_async_backend().getaddrinfo( - encoded_host, port, family=family, type=type, proto=proto, flags=flags - ) - return [ - (family, type, proto, canonname, convert_ipv6_sockaddr(sockaddr)) - for family, type, proto, canonname, sockaddr in gai_res - # filter out IPv6 results when IPv6 is disabled - if not isinstance(sockaddr[0], int) - ] - - -def getnameinfo(sockaddr: IPSockAddrType, flags: int = 0) -> Awaitable[tuple[str, str]]: - """ - Look up the host name of an IP address. - - :param sockaddr: socket address (e.g. (ipaddress, port) for IPv4) - :param flags: flags to pass to upstream ``getnameinfo()`` - :return: a tuple of (host name, service name) - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - .. seealso:: :func:`socket.getnameinfo` - - """ - return get_async_backend().getnameinfo(sockaddr, flags) - - -@deprecated("This function is deprecated; use `wait_readable` instead") -def wait_socket_readable(sock: socket.socket) -> Awaitable[None]: - """ - .. deprecated:: 4.7.0 - Use :func:`wait_readable` instead. - - Wait until the given socket has data to be read. - - .. warning:: Only use this on raw sockets that have not been wrapped by any higher - level constructs like socket streams! - - :param sock: a socket object - :raises ~anyio.ClosedResourceError: if the socket was closed while waiting for the - socket to become readable - :raises ~anyio.BusyResourceError: if another task is already waiting for the socket - to become readable - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().wait_readable(sock.fileno()) - - -@deprecated("This function is deprecated; use `wait_writable` instead") -def wait_socket_writable(sock: socket.socket) -> Awaitable[None]: - """ - .. deprecated:: 4.7.0 - Use :func:`wait_writable` instead. - - Wait until the given socket can be written to. - - This does **NOT** work on Windows when using the asyncio backend with a proactor - event loop (default on py3.8+). - - .. warning:: Only use this on raw sockets that have not been wrapped by any higher - level constructs like socket streams! - - :param sock: a socket object - :raises ~anyio.ClosedResourceError: if the socket was closed while waiting for the - socket to become writable - :raises ~anyio.BusyResourceError: if another task is already waiting for the socket - to become writable - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().wait_writable(sock.fileno()) - - -def wait_readable(obj: FileDescriptorLike) -> Awaitable[None]: - """ - Wait until the given object has data to be read. - - On Unix systems, ``obj`` must either be an integer file descriptor, or else an - object with a ``.fileno()`` method which returns an integer file descriptor. Any - kind of file descriptor can be passed, though the exact semantics will depend on - your kernel. For example, this probably won't do anything useful for on-disk files. - - On Windows systems, ``obj`` must either be an integer ``SOCKET`` handle, or else an - object with a ``.fileno()`` method which returns an integer ``SOCKET`` handle. File - descriptors aren't supported, and neither are handles that refer to anything besides - a ``SOCKET``. - - On backends where this functionality is not natively provided (asyncio - ``ProactorEventLoop`` on Windows), it is provided using a separate selector thread - which is set to shut down when the interpreter shuts down. - - .. warning:: Don't use this on raw sockets that have been wrapped by any higher - level constructs like socket streams! - - :param obj: an object with a ``.fileno()`` method or an integer handle - :raises ~anyio.ClosedResourceError: if the object was closed while waiting for the - object to become readable - :raises ~anyio.BusyResourceError: if another task is already waiting for the object - to become readable - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().wait_readable(obj) - - -def wait_writable(obj: FileDescriptorLike) -> Awaitable[None]: - """ - Wait until the given object can be written to. - - :param obj: an object with a ``.fileno()`` method or an integer handle - :raises ~anyio.ClosedResourceError: if the object was closed while waiting for the - object to become writable - :raises ~anyio.BusyResourceError: if another task is already waiting for the object - to become writable - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - .. seealso:: See the documentation of :func:`wait_readable` for the definition of - ``obj`` and notes on backend compatibility. - - .. warning:: Don't use this on raw sockets that have been wrapped by any higher - level constructs like socket streams! - - """ - return get_async_backend().wait_writable(obj) - - -def notify_closing(obj: FileDescriptorLike) -> None: - """ - Call this before closing a file descriptor (on Unix) or socket (on - Windows). This will cause any `wait_readable` or `wait_writable` - calls on the given object to immediately wake up and raise - `~anyio.ClosedResourceError`. - - This doesn't actually close the object – you still have to do that - yourself afterwards. Also, you want to be careful to make sure no - new tasks start waiting on the object in between when you call this - and when it's actually closed. So to close something properly, you - usually want to do these steps in order: - - 1. Explicitly mark the object as closed, so that any new attempts - to use it will abort before they start. - 2. Call `notify_closing` to wake up any already-existing users. - 3. Actually close the object. - - It's also possible to do them in a different order if that's more - convenient, *but only if* you make sure not to have any checkpoints in - between the steps. This way they all happen in a single atomic - step, so other tasks won't be able to tell what order they happened - in anyway. - - :param obj: an object with a ``.fileno()`` method or an integer handle - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - get_async_backend().notify_closing(obj) - - -# -# Private API -# - - -def convert_ipv6_sockaddr( - sockaddr: tuple[str, int, int, int] | tuple[str, int], -) -> tuple[str, int]: - """ - Convert a 4-tuple IPv6 socket address to a 2-tuple (address, port) format. - - If the scope ID is nonzero, it is added to the address, separated with ``%``. - Otherwise the flow id and scope id are simply cut off from the tuple. - Any other kinds of socket addresses are returned as-is. - - :param sockaddr: the result of :meth:`~socket.socket.getsockname` - :return: the converted socket address - - """ - # This is more complicated than it should be because of MyPy - if isinstance(sockaddr, tuple) and len(sockaddr) == 4: - host, port, flowinfo, scope_id = sockaddr - if scope_id: - # PyPy (as of v7.3.11) leaves the interface name in the result, so - # we discard it and only get the scope ID from the end - # (https://foss.heptapod.net/pypy/pypy/-/issues/3938) - host = host.split("%")[0] - - # Add scope_id to the address - return f"{host}%{scope_id}", port - else: - return host, port - else: - return sockaddr - - -async def setup_unix_local_socket( - path: None | str | bytes | PathLike[Any], - mode: int | None, - socktype: int, -) -> socket.socket: - """ - Create a UNIX local socket object, deleting the socket at the given path if it - exists. - - Not available on Windows. - - :param path: path of the socket - :param mode: permissions to set on the socket - :param socktype: socket.SOCK_STREAM or socket.SOCK_DGRAM - - """ - path_str: str | None - if path is not None: - path_str = os.fsdecode(path) - - # Linux abstract namespace sockets aren't backed by a concrete file so skip stat call - if not path_str.startswith("\0"): - # Copied from pathlib... - try: - stat_result = os.stat(path) - except OSError as e: - if e.errno not in ( - errno.ENOENT, - errno.ENOTDIR, - errno.EBADF, - errno.ELOOP, - ): - raise - else: - if stat.S_ISSOCK(stat_result.st_mode): - os.unlink(path) - else: - path_str = None - - raw_socket = socket.socket(socket.AF_UNIX, socktype) - raw_socket.setblocking(False) - - if path_str is not None: - try: - await to_thread.run_sync(raw_socket.bind, path_str, abandon_on_cancel=True) - if mode is not None: - await to_thread.run_sync(chmod, path_str, mode, abandon_on_cancel=True) - except BaseException: - raw_socket.close() - raise - - return raw_socket - - -@dataclass -class TCPConnectable(ByteStreamConnectable): - """ - Connects to a TCP server at the given host and port. - - :param host: host name or IP address of the server - :param port: TCP port number of the server - """ - - host: str | IPv4Address | IPv6Address - port: int - - def __post_init__(self) -> None: - if self.port < 1 or self.port > 65535: - raise ValueError("TCP port number out of range") - - @override - async def connect(self) -> SocketStream: - try: - return await connect_tcp(self.host, self.port) - except OSError as exc: - raise ConnectionFailed( - f"error connecting to {self.host}:{self.port}: {exc}" - ) from exc - - -@dataclass -class UNIXConnectable(ByteStreamConnectable): - """ - Connects to a UNIX domain socket at the given path. - - :param path: the file system path of the socket - """ - - path: str | bytes | PathLike[str] | PathLike[bytes] - - @override - async def connect(self) -> UNIXSocketStream: - try: - return await connect_unix(self.path) - except OSError as exc: - raise ConnectionFailed(f"error connecting to {self.path!r}: {exc}") from exc - - -def as_connectable( - remote: ByteStreamConnectable - | tuple[str | IPv4Address | IPv6Address, int] - | str - | bytes - | PathLike[str], - /, - *, - tls: bool = False, - ssl_context: ssl.SSLContext | None = None, - tls_hostname: str | None = None, - tls_standard_compatible: bool = True, -) -> ByteStreamConnectable: - """ - Return a byte stream connectable from the given object. - - If a bytestream connectable is given, it is returned unchanged. - If a tuple of (host, port) is given, a TCP connectable is returned. - If a string or bytes path is given, a UNIX connectable is returned. - - If ``tls=True``, the connectable will be wrapped in a - :class:`~.streams.tls.TLSConnectable`. - - :param remote: a connectable, a tuple of (host, port) or a path to a UNIX socket - :param tls: if ``True``, wrap the plaintext connectable in a - :class:`~.streams.tls.TLSConnectable`, using the provided TLS settings) - :param ssl_context: if ``tls=True``, the SSLContext object to use (if not provided, - a secure default will be created) - :param tls_hostname: if ``tls=True``, host name of the server to use for checking - the server certificate (defaults to the host portion of the address for TCP - connectables) - :param tls_standard_compatible: if ``False`` and ``tls=True``, makes the TLS stream - skip the closing handshake when closing the connection, so it won't raise an - exception if the server does the same - - """ - connectable: TCPConnectable | UNIXConnectable | TLSConnectable - if isinstance(remote, ByteStreamConnectable): - return remote - elif isinstance(remote, tuple) and len(remote) == 2: - connectable = TCPConnectable(*remote) - elif isinstance(remote, (str, bytes, PathLike)): - connectable = UNIXConnectable(remote) - else: - raise TypeError(f"cannot convert {remote!r} to a connectable") - - if tls: - if not tls_hostname and isinstance(connectable, TCPConnectable): - tls_hostname = str(connectable.host) - - connectable = TLSConnectable( - connectable, - ssl_context=ssl_context, - hostname=tls_hostname, - standard_compatible=tls_standard_compatible, - ) - - return connectable diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_streams.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_streams.py deleted file mode 100644 index 2b9c7df200f9520357503c754bcdea1c047bdda3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_streams.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -import math -from typing import TypeVar -from warnings import warn - -from ..streams.memory import ( - MemoryObjectReceiveStream, - MemoryObjectSendStream, - _MemoryObjectStreamState, -) - -T_Item = TypeVar("T_Item") - - -class create_memory_object_stream( - tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]], -): - """ - Create a memory object stream. - - The stream's item type can be annotated like - :func:`create_memory_object_stream[T_Item]`. - - :param max_buffer_size: number of items held in the buffer until ``send()`` starts - blocking - :param item_type: old way of marking the streams with the right generic type for - static typing (does nothing on AnyIO 4) - - .. deprecated:: 4.0 - Use ``create_memory_object_stream[YourItemType](...)`` instead. - :return: a tuple of (send stream, receive stream) - - """ - - def __new__( # type: ignore[misc] - cls, max_buffer_size: float = 0, item_type: object = None - ) -> tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]]: - if max_buffer_size != math.inf and not isinstance(max_buffer_size, int): - raise ValueError("max_buffer_size must be either an integer or math.inf") - if max_buffer_size < 0: - raise ValueError("max_buffer_size cannot be negative") - if item_type is not None: - warn( - "The item_type argument has been deprecated in AnyIO 4.0. " - "Use create_memory_object_stream[YourItemType](...) instead.", - DeprecationWarning, - stacklevel=2, - ) - - state = _MemoryObjectStreamState[T_Item](max_buffer_size) - return (MemoryObjectSendStream(state), MemoryObjectReceiveStream(state)) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_subprocesses.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_subprocesses.py deleted file mode 100644 index a6590ca623f5dd3a1c0fa7a2a155b2f7637fd82b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_subprocesses.py +++ /dev/null @@ -1,196 +0,0 @@ -from __future__ import annotations - -from collections.abc import AsyncIterable, Iterable, Mapping, Sequence -from io import BytesIO -from os import PathLike -from subprocess import PIPE, CalledProcessError, CompletedProcess -from typing import IO, Any, TypeAlias, cast - -from ..abc import Process -from ._eventloop import get_async_backend -from ._tasks import create_task_group - -StrOrBytesPath: TypeAlias = str | bytes | PathLike[str] | PathLike[bytes] - - -async def run_process( - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - input: bytes | None = None, - stdin: int | IO[Any] | None = None, - stdout: int | IO[Any] | None = PIPE, - stderr: int | IO[Any] | None = PIPE, - check: bool = True, - cwd: StrOrBytesPath | None = None, - env: Mapping[str, str] | None = None, - startupinfo: Any = None, - creationflags: int = 0, - start_new_session: bool = False, - pass_fds: Sequence[int] = (), - user: str | int | None = None, - group: str | int | None = None, - extra_groups: Iterable[str | int] | None = None, - umask: int = -1, -) -> CompletedProcess[bytes]: - """ - Run an external command in a subprocess and wait until it completes. - - .. seealso:: :func:`subprocess.run` - - :param command: either a string to pass to the shell, or an iterable of strings - containing the executable name or path and its arguments - :param input: bytes passed to the standard input of the subprocess - :param stdin: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - a file-like object, or `None`; ``input`` overrides this - :param stdout: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - a file-like object, or `None` - :param stderr: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - :data:`subprocess.STDOUT`, a file-like object, or `None` - :param check: if ``True``, raise :exc:`~subprocess.CalledProcessError` if the - process terminates with a return code other than 0 - :param cwd: If not ``None``, change the working directory to this before running the - command - :param env: if not ``None``, this mapping replaces the inherited environment - variables from the parent process - :param startupinfo: an instance of :class:`subprocess.STARTUPINFO` that can be used - to specify process startup parameters (Windows only) - :param creationflags: flags that can be used to control the creation of the - subprocess (see :class:`subprocess.Popen` for the specifics) - :param start_new_session: if ``true`` the setsid() system call will be made in the - child process prior to the execution of the subprocess. (POSIX only) - :param pass_fds: sequence of file descriptors to keep open between the parent and - child processes. (POSIX only) - :param user: effective user to run the process as (Python >= 3.9, POSIX only) - :param group: effective group to run the process as (Python >= 3.9, POSIX only) - :param extra_groups: supplementary groups to set in the subprocess (Python >= 3.9, - POSIX only) - :param umask: if not negative, this umask is applied in the child process before - running the given command (Python >= 3.9, POSIX only) - :return: an object representing the completed process - :raises ~subprocess.CalledProcessError: if ``check`` is ``True`` and the process - exits with a nonzero return code - - """ - - async def drain_stream(stream: AsyncIterable[bytes], index: int) -> None: - buffer = BytesIO() - async for chunk in stream: - buffer.write(chunk) - - stream_contents[index] = buffer.getvalue() - - if stdin is not None and input is not None: - raise ValueError("only one of stdin and input is allowed") - - async with await open_process( - command, - stdin=PIPE if input else stdin, - stdout=stdout, - stderr=stderr, - cwd=cwd, - env=env, - startupinfo=startupinfo, - creationflags=creationflags, - start_new_session=start_new_session, - pass_fds=pass_fds, - user=user, - group=group, - extra_groups=extra_groups, - umask=umask, - ) as process: - stream_contents: list[bytes | None] = [None, None] - async with create_task_group() as tg: - if process.stdout: - tg.start_soon(drain_stream, process.stdout, 0) - - if process.stderr: - tg.start_soon(drain_stream, process.stderr, 1) - - if process.stdin and input: - await process.stdin.send(input) - await process.stdin.aclose() - - await process.wait() - - output, errors = stream_contents - if check and process.returncode != 0: - raise CalledProcessError(cast(int, process.returncode), command, output, errors) - - return CompletedProcess(command, cast(int, process.returncode), output, errors) - - -async def open_process( - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None = PIPE, - stdout: int | IO[Any] | None = PIPE, - stderr: int | IO[Any] | None = PIPE, - cwd: StrOrBytesPath | None = None, - env: Mapping[str, str] | None = None, - startupinfo: Any = None, - creationflags: int = 0, - start_new_session: bool = False, - pass_fds: Sequence[int] = (), - user: str | int | None = None, - group: str | int | None = None, - extra_groups: Iterable[str | int] | None = None, - umask: int = -1, -) -> Process: - """ - Start an external command in a subprocess. - - .. seealso:: :class:`subprocess.Popen` - - :param command: either a string to pass to the shell, or an iterable of strings - containing the executable name or path and its arguments - :param stdin: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, a - file-like object, or ``None`` - :param stdout: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - a file-like object, or ``None`` - :param stderr: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - :data:`subprocess.STDOUT`, a file-like object, or ``None`` - :param cwd: If not ``None``, the working directory is changed before executing - :param env: If env is not ``None``, it must be a mapping that defines the - environment variables for the new process - :param creationflags: flags that can be used to control the creation of the - subprocess (see :class:`subprocess.Popen` for the specifics) - :param startupinfo: an instance of :class:`subprocess.STARTUPINFO` that can be used - to specify process startup parameters (Windows only) - :param start_new_session: if ``true`` the setsid() system call will be made in the - child process prior to the execution of the subprocess. (POSIX only) - :param pass_fds: sequence of file descriptors to keep open between the parent and - child processes. (POSIX only) - :param user: effective user to run the process as (POSIX only) - :param group: effective group to run the process as (POSIX only) - :param extra_groups: supplementary groups to set in the subprocess (POSIX only) - :param umask: if not negative, this umask is applied in the child process before - running the given command (POSIX only) - :return: an asynchronous process object - - """ - kwargs: dict[str, Any] = {} - if user is not None: - kwargs["user"] = user - - if group is not None: - kwargs["group"] = group - - if extra_groups is not None: - kwargs["extra_groups"] = extra_groups - - if umask >= 0: - kwargs["umask"] = umask - - return await get_async_backend().open_process( - command, - stdin=stdin, - stdout=stdout, - stderr=stderr, - cwd=cwd, - env=env, - startupinfo=startupinfo, - creationflags=creationflags, - start_new_session=start_new_session, - pass_fds=pass_fds, - **kwargs, - ) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_synchronization.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_synchronization.py deleted file mode 100644 index f1990a5003a6b99cbcb3c72ae8fb01e6e299a9aa..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_synchronization.py +++ /dev/null @@ -1,772 +0,0 @@ -from __future__ import annotations - -import math -from collections import deque -from collections.abc import Callable -from dataclasses import dataclass -from types import TracebackType -from typing import TypeVar - -from ..lowlevel import checkpoint_if_cancelled -from ._eventloop import get_async_backend -from ._exceptions import BusyResourceError, NoEventLoopError -from ._tasks import CancelScope -from ._testing import TaskInfo, get_current_task - -T = TypeVar("T") - - -@dataclass(frozen=True) -class EventStatistics: - """ - :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Event.wait` - """ - - tasks_waiting: int - - -@dataclass(frozen=True) -class CapacityLimiterStatistics: - """ - :ivar int borrowed_tokens: number of tokens currently borrowed by tasks - :ivar float total_tokens: total number of available tokens - :ivar tuple borrowers: tasks or other objects currently holding tokens borrowed from - this limiter - :ivar int tasks_waiting: number of tasks waiting on - :meth:`~.CapacityLimiter.acquire` or - :meth:`~.CapacityLimiter.acquire_on_behalf_of` - """ - - borrowed_tokens: int - total_tokens: float - borrowers: tuple[object, ...] - tasks_waiting: int - - -@dataclass(frozen=True) -class LockStatistics: - """ - :ivar bool locked: flag indicating if this lock is locked or not - :ivar ~anyio.TaskInfo owner: task currently holding the lock (or ``None`` if the - lock is not held by any task) - :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Lock.acquire` - """ - - locked: bool - owner: TaskInfo | None - tasks_waiting: int - - -@dataclass(frozen=True) -class ConditionStatistics: - """ - :ivar int tasks_waiting: number of tasks blocked on :meth:`~.Condition.wait` - :ivar ~anyio.LockStatistics lock_statistics: statistics of the underlying - :class:`~.Lock` - """ - - tasks_waiting: int - lock_statistics: LockStatistics - - -@dataclass(frozen=True) -class SemaphoreStatistics: - """ - :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Semaphore.acquire` - - """ - - tasks_waiting: int - - -class Event: - __slots__ = ("__weakref__",) - - def __new__(cls) -> Event: - try: - return get_async_backend().create_event() - except NoEventLoopError: - return EventAdapter() - - def set(self) -> None: - """Set the flag, notifying all listeners.""" - raise NotImplementedError - - def is_set(self) -> bool: - """Return ``True`` if the flag is set, ``False`` if not.""" - raise NotImplementedError - - async def wait(self) -> None: - """ - Wait until the flag has been set. - - If the flag has already been set when this method is called, it returns - immediately. - - """ - raise NotImplementedError - - def statistics(self) -> EventStatistics: - """Return statistics about the current state of this event.""" - raise NotImplementedError - - -class EventAdapter(Event): - __slots__ = "_internal_event", "_is_set" - - def __new__(cls) -> EventAdapter: - return object.__new__(cls) - - def __init__(self) -> None: - self._internal_event: Event | None = None - self._is_set = False - - @property - def _event(self) -> Event: - if self._internal_event is None: - self._internal_event = get_async_backend().create_event() - if self._is_set: - self._internal_event.set() - - return self._internal_event - - def set(self) -> None: - if self._internal_event is None: - self._is_set = True - else: - self._event.set() - - def is_set(self) -> bool: - if self._internal_event is None: - return self._is_set - - return self._internal_event.is_set() - - async def wait(self) -> None: - await self._event.wait() - - def statistics(self) -> EventStatistics: - if self._internal_event is None: - return EventStatistics(tasks_waiting=0) - - return self._internal_event.statistics() - - -class Lock: - __slots__ = ("__weakref__",) - - def __new__(cls, *, fast_acquire: bool = False) -> Lock: - try: - return get_async_backend().create_lock(fast_acquire=fast_acquire) - except NoEventLoopError: - return LockAdapter(fast_acquire=fast_acquire) - - async def __aenter__(self) -> None: - await self.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - async def acquire(self) -> None: - """Acquire the lock.""" - raise NotImplementedError - - def acquire_nowait(self) -> None: - """ - Acquire the lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - raise NotImplementedError - - def release(self) -> None: - """Release the lock.""" - raise NotImplementedError - - def locked(self) -> bool: - """Return True if the lock is currently held.""" - raise NotImplementedError - - def statistics(self) -> LockStatistics: - """ - Return statistics about the current state of this lock. - - .. versionadded:: 3.0 - """ - raise NotImplementedError - - -class LockAdapter(Lock): - __slots__ = "_internal_lock", "_fast_acquire" - - def __new__(cls, *, fast_acquire: bool = False) -> LockAdapter: - return object.__new__(cls) - - def __init__(self, *, fast_acquire: bool = False): - self._internal_lock: Lock | None = None - self._fast_acquire = fast_acquire - - @property - def _lock(self) -> Lock: - if self._internal_lock is None: - self._internal_lock = get_async_backend().create_lock( - fast_acquire=self._fast_acquire - ) - - return self._internal_lock - - async def __aenter__(self) -> None: - await self._lock.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - if self._internal_lock is not None: - self._internal_lock.release() - - async def acquire(self) -> None: - """Acquire the lock.""" - await self._lock.acquire() - - def acquire_nowait(self) -> None: - """ - Acquire the lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - self._lock.acquire_nowait() - - def release(self) -> None: - """Release the lock.""" - self._lock.release() - - def locked(self) -> bool: - """Return True if the lock is currently held.""" - return self._lock.locked() - - def statistics(self) -> LockStatistics: - """ - Return statistics about the current state of this lock. - - .. versionadded:: 3.0 - - """ - if self._internal_lock is None: - return LockStatistics(False, None, 0) - - return self._internal_lock.statistics() - - -class Condition: - __slots__ = "__weakref__", "_owner_task", "_lock", "_waiters" - - def __init__(self, lock: Lock | None = None): - self._owner_task: TaskInfo | None = None - self._lock = lock or Lock() - self._waiters: deque[Event] = deque() - - async def __aenter__(self) -> None: - await self.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - def _check_acquired(self) -> None: - if self._owner_task != get_current_task(): - raise RuntimeError("The current task is not holding the underlying lock") - - async def acquire(self) -> None: - """Acquire the underlying lock.""" - await self._lock.acquire() - self._owner_task = get_current_task() - - def acquire_nowait(self) -> None: - """ - Acquire the underlying lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - self._lock.acquire_nowait() - self._owner_task = get_current_task() - - def release(self) -> None: - """Release the underlying lock.""" - self._lock.release() - - def locked(self) -> bool: - """Return True if the lock is set.""" - return self._lock.locked() - - def notify(self, n: int = 1) -> None: - """Notify exactly n listeners.""" - self._check_acquired() - for _ in range(n): - try: - event = self._waiters.popleft() - except IndexError: - break - - event.set() - - def notify_all(self) -> None: - """Notify all the listeners.""" - self._check_acquired() - for event in self._waiters: - event.set() - - self._waiters.clear() - - async def wait(self) -> None: - """Wait for a notification.""" - await checkpoint_if_cancelled() - self._check_acquired() - event = Event() - self._waiters.append(event) - self.release() - try: - await event.wait() - except BaseException: - if not event.is_set(): - self._waiters.remove(event) - elif self._waiters: - # This task was notified by could not act on it, so pass - # it on to the next task - self._waiters.popleft().set() - - raise - finally: - with CancelScope(shield=True): - await self.acquire() - - async def wait_for(self, predicate: Callable[[], T]) -> T: - """ - Wait until a predicate becomes true. - - :param predicate: a callable that returns a truthy value when the condition is - met - :return: the result of the predicate - - .. versionadded:: 4.11.0 - - """ - while not (result := predicate()): - await self.wait() - - return result - - def statistics(self) -> ConditionStatistics: - """ - Return statistics about the current state of this condition. - - .. versionadded:: 3.0 - """ - return ConditionStatistics(len(self._waiters), self._lock.statistics()) - - -class Semaphore: - __slots__ = "__weakref__", "_fast_acquire" - - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - try: - return get_async_backend().create_semaphore( - initial_value, max_value=max_value, fast_acquire=fast_acquire - ) - except NoEventLoopError: - return SemaphoreAdapter(initial_value, max_value=max_value) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ): - if not isinstance(initial_value, int): - raise TypeError("initial_value must be an integer") - if initial_value < 0: - raise ValueError("initial_value must be >= 0") - if max_value is not None: - if not isinstance(max_value, int): - raise TypeError("max_value must be an integer or None") - if max_value < initial_value: - raise ValueError( - "max_value must be equal to or higher than initial_value" - ) - - self._fast_acquire = fast_acquire - - async def __aenter__(self) -> Semaphore: - await self.acquire() - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - async def acquire(self) -> None: - """Decrement the semaphore value, blocking if necessary.""" - raise NotImplementedError - - def acquire_nowait(self) -> None: - """ - Acquire the underlying lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - raise NotImplementedError - - def release(self) -> None: - """Increment the semaphore value.""" - raise NotImplementedError - - @property - def value(self) -> int: - """The current value of the semaphore.""" - raise NotImplementedError - - @property - def max_value(self) -> int | None: - """The maximum value of the semaphore.""" - raise NotImplementedError - - def statistics(self) -> SemaphoreStatistics: - """ - Return statistics about the current state of this semaphore. - - .. versionadded:: 3.0 - """ - raise NotImplementedError - - -class SemaphoreAdapter(Semaphore): - __slots__ = "_internal_semaphore", "_initial_value", "_max_value" - - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> SemaphoreAdapter: - return object.__new__(cls) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> None: - super().__init__(initial_value, max_value=max_value, fast_acquire=fast_acquire) - self._internal_semaphore: Semaphore | None = None - self._initial_value = initial_value - self._max_value = max_value - - @property - def _semaphore(self) -> Semaphore: - if self._internal_semaphore is None: - self._internal_semaphore = get_async_backend().create_semaphore( - self._initial_value, max_value=self._max_value - ) - - return self._internal_semaphore - - async def acquire(self) -> None: - await self._semaphore.acquire() - - def acquire_nowait(self) -> None: - self._semaphore.acquire_nowait() - - def release(self) -> None: - self._semaphore.release() - - @property - def value(self) -> int: - if self._internal_semaphore is None: - return self._initial_value - - return self._semaphore.value - - @property - def max_value(self) -> int | None: - return self._max_value - - def statistics(self) -> SemaphoreStatistics: - if self._internal_semaphore is None: - return SemaphoreStatistics(tasks_waiting=0) - - return self._semaphore.statistics() - - -class CapacityLimiter: - __slots__ = ("__weakref__",) - - def __new__(cls, total_tokens: float) -> CapacityLimiter: - try: - return get_async_backend().create_capacity_limiter(total_tokens) - except NoEventLoopError: - return CapacityLimiterAdapter(total_tokens) - - async def __aenter__(self) -> None: - raise NotImplementedError - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - raise NotImplementedError - - @property - def total_tokens(self) -> float: - """ - The total number of tokens available for borrowing. - - This is a read-write property. If the total number of tokens is increased, the - proportionate number of tasks waiting on this limiter will be granted their - tokens. - - .. versionchanged:: 3.0 - The property is now writable. - .. versionchanged:: 4.12 - The value can now be set to 0. - - """ - raise NotImplementedError - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - raise NotImplementedError - - @property - def borrowed_tokens(self) -> int: - """The number of tokens that have currently been borrowed.""" - raise NotImplementedError - - @property - def available_tokens(self) -> float: - """The number of tokens currently available to be borrowed""" - raise NotImplementedError - - def acquire_nowait(self) -> None: - """ - Acquire a token for the current task without waiting for one to become - available. - - :raises ~anyio.WouldBlock: if there are no tokens available for borrowing - - """ - raise NotImplementedError - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - """ - Acquire a token without waiting for one to become available. - - :param borrower: the entity borrowing a token - :raises ~anyio.WouldBlock: if there are no tokens available for borrowing - - """ - raise NotImplementedError - - async def acquire(self) -> None: - """ - Acquire a token for the current task, waiting if necessary for one to become - available. - - """ - raise NotImplementedError - - async def acquire_on_behalf_of(self, borrower: object) -> None: - """ - Acquire a token, waiting if necessary for one to become available. - - :param borrower: the entity borrowing a token - - """ - raise NotImplementedError - - def release(self) -> None: - """ - Release the token held by the current task. - - :raises RuntimeError: if the current task has not borrowed a token from this - limiter. - - """ - raise NotImplementedError - - def release_on_behalf_of(self, borrower: object) -> None: - """ - Release the token held by the given borrower. - - :raises RuntimeError: if the borrower has not borrowed a token from this - limiter. - - """ - raise NotImplementedError - - def statistics(self) -> CapacityLimiterStatistics: - """ - Return statistics about the current state of this limiter. - - .. versionadded:: 3.0 - - """ - raise NotImplementedError - - -class CapacityLimiterAdapter(CapacityLimiter): - __slots__ = "_internal_limiter", "_total_tokens" - - def __new__(cls, total_tokens: float) -> CapacityLimiterAdapter: - return object.__new__(cls) - - def __init__(self, total_tokens: float) -> None: - self._internal_limiter: CapacityLimiter | None = None - self.total_tokens = total_tokens - - @property - def _limiter(self) -> CapacityLimiter: - if self._internal_limiter is None: - self._internal_limiter = get_async_backend().create_capacity_limiter( - self._total_tokens - ) - - return self._internal_limiter - - async def __aenter__(self) -> None: - await self._limiter.__aenter__() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - return await self._limiter.__aexit__(exc_type, exc_val, exc_tb) - - @property - def total_tokens(self) -> float: - if self._internal_limiter is None: - return self._total_tokens - - return self._internal_limiter.total_tokens - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - if not isinstance(value, int) and not math.isinf(value): - raise TypeError("total_tokens must be an int or math.inf") - elif value < 0: - raise ValueError("total_tokens must be >= 0") - - if self._internal_limiter is None: - self._total_tokens = value - return - - self._limiter.total_tokens = value - - @property - def borrowed_tokens(self) -> int: - if self._internal_limiter is None: - return 0 - - return self._internal_limiter.borrowed_tokens - - @property - def available_tokens(self) -> float: - if self._internal_limiter is None: - return self._total_tokens - - return self._internal_limiter.available_tokens - - def acquire_nowait(self) -> None: - self._limiter.acquire_nowait() - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - self._limiter.acquire_on_behalf_of_nowait(borrower) - - async def acquire(self) -> None: - await self._limiter.acquire() - - async def acquire_on_behalf_of(self, borrower: object) -> None: - await self._limiter.acquire_on_behalf_of(borrower) - - def release(self) -> None: - self._limiter.release() - - def release_on_behalf_of(self, borrower: object) -> None: - self._limiter.release_on_behalf_of(borrower) - - def statistics(self) -> CapacityLimiterStatistics: - if self._internal_limiter is None: - return CapacityLimiterStatistics( - borrowed_tokens=0, - total_tokens=self.total_tokens, - borrowers=(), - tasks_waiting=0, - ) - - return self._internal_limiter.statistics() - - -class ResourceGuard: - """ - A context manager for ensuring that a resource is only used by a single task at a - time. - - Entering this context manager while the previous has not exited it yet will trigger - :exc:`BusyResourceError`. - - :param action: the action to guard against (visible in the :exc:`BusyResourceError` - when triggered, e.g. "Another task is already {action} this resource") - - .. versionadded:: 4.1 - """ - - __slots__ = "__weakref__", "action", "_guarded" - - def __init__(self, action: str = "using"): - self.action: str = action - self._guarded = False - - def __enter__(self) -> None: - if self._guarded: - raise BusyResourceError(self.action) - - self._guarded = True - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self._guarded = False diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_tasks.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_tasks.py deleted file mode 100644 index ced54b2e20f7dea2f238600a1a5833fe0d64d70d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_tasks.py +++ /dev/null @@ -1,412 +0,0 @@ -from __future__ import annotations - -import math -import sys -from collections.abc import ( - Coroutine, - Generator, -) -from contextlib import ( - contextmanager, -) -from enum import Enum, auto -from inspect import iscoroutine -from types import TracebackType -from typing import Any, Generic, final - -from ..abc import TaskGroup, TaskStatus -from ._eventloop import get_async_backend, get_cancelled_exc_class -from ._exceptions import TaskCancelled, TaskFailed, TaskNotFinished - -if sys.version_info >= (3, 13): - from typing import TypeVar -else: - from typing_extensions import TypeVar - -if sys.version_info >= (3, 11): - from typing import Never, TypeVarTuple -else: - from typing_extensions import Never, TypeVarTuple - -T = TypeVar("T") -T_co = TypeVar("T_co", covariant=True) -T_startval = TypeVar("T_startval", covariant=True, default=Never) -PosArgsT = TypeVarTuple("PosArgsT") - - -class _IgnoredTaskStatus(TaskStatus[object]): - def started(self, value: object = None) -> None: - pass - - -TASK_STATUS_IGNORED = _IgnoredTaskStatus() - - -class CancelScope: - """ - Wraps a unit of work that can be made separately cancellable. - - :param deadline: The time (clock value) when this scope is cancelled automatically - :param shield: ``True`` to shield the cancel scope from external cancellation - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - """ - - __slots__ = ("__weakref__",) - - def __new__( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - return get_async_backend().create_cancel_scope(shield=shield, deadline=deadline) - - def cancel(self, reason: str | None = None) -> None: - """ - Cancel this scope immediately. - - :param reason: a message describing the reason for the cancellation - - """ - raise NotImplementedError - - @property - def deadline(self) -> float: - """ - The time (clock value) when this scope is cancelled automatically. - - Will be ``float('inf')`` if no timeout has been set. - - """ - raise NotImplementedError - - @deadline.setter - def deadline(self, value: float) -> None: - raise NotImplementedError - - @property - def cancel_called(self) -> bool: - """``True`` if :meth:`cancel` has been called.""" - raise NotImplementedError - - @property - def cancelled_caught(self) -> bool: - """ - ``True`` if this scope suppressed a cancellation exception it itself raised. - - This is typically used to check if any work was interrupted, or to see if the - scope was cancelled due to its deadline being reached. The value will, however, - only be ``True`` if the cancellation was triggered by the scope itself (and not - an outer scope). - - """ - raise NotImplementedError - - @property - def shield(self) -> bool: - """ - ``True`` if this scope is shielded from external cancellation. - - While a scope is shielded, it will not receive cancellations from outside. - - """ - raise NotImplementedError - - @shield.setter - def shield(self, value: bool) -> None: - raise NotImplementedError - - def __enter__(self) -> CancelScope: - raise NotImplementedError - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - raise NotImplementedError - - -@contextmanager -def fail_after( - delay: float | None, shield: bool = False -) -> Generator[CancelScope, None, None]: - """ - Create a context manager which raises a :class:`TimeoutError` if does not finish in - time. - - :param delay: maximum allowed time (in seconds) before raising the exception, or - ``None`` to disable the timeout - :param shield: ``True`` to shield the cancel scope from external cancellation - :return: a context manager that yields a cancel scope - :rtype: :class:`~typing.ContextManager`\\[:class:`~anyio.CancelScope`\\] - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - current_time = get_async_backend().current_time - deadline = (current_time() + delay) if delay is not None else math.inf - with get_async_backend().create_cancel_scope( - deadline=deadline, shield=shield - ) as cancel_scope: - yield cancel_scope - - if cancel_scope.cancelled_caught and current_time() >= cancel_scope.deadline: - raise TimeoutError - - -def move_on_after(delay: float | None, shield: bool = False) -> CancelScope: - """ - Create a cancel scope with a deadline that expires after the given delay. - - :param delay: maximum allowed time (in seconds) before exiting the context block, or - ``None`` to disable the timeout - :param shield: ``True`` to shield the cancel scope from external cancellation - :return: a cancel scope - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - deadline = ( - (get_async_backend().current_time() + delay) if delay is not None else math.inf - ) - return get_async_backend().create_cancel_scope(deadline=deadline, shield=shield) - - -def current_effective_deadline() -> float: - """ - Return the nearest deadline among all the cancel scopes effective for the current - task. - - :return: a clock value from the event loop's internal clock (or ``float('inf')`` if - there is no deadline in effect, or ``float('-inf')`` if the current scope has - been cancelled) - :rtype: float - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().current_effective_deadline() - - -def create_task_group() -> TaskGroup: - """ - Create a task group. - - :return: a task group - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().create_task_group() - - -@final -class TaskHandle(Generic[T_co, T_startval]): - """ - Returned from the task-spawning methods of :class:`TaskGroup`. Can be awaited on to - get the return value of the task (or the raised exception). If the task was - terminated by a :exc:`BaseException`, :exc:`TaskFailed` will be raised (or its - subclass :exc:`TaskCancelled` if the task was cancelled). - - .. versionadded:: 4.14.0 - """ - - class Status(Enum): - """ - The status of a task handle. - - .. attribute:: PENDING - - The task has not finished yet. - .. attribute:: FINISHED - - The task has finished with a return value. - .. attribute:: CANCELLING - - The task has been cancelled but has not finished yet. - .. attribute:: CANCELLED - - The task was cancelled and has finished since. - .. attribute:: FAILED - - The task raised an exception. - """ - - PENDING = auto() - FINISHED = auto() - CANCELLING = auto() - CANCELLED = auto() - FAILED = auto() - - __slots__ = ( - "__weakref__", - "_coro", - "_name", - "_cancel_scope", - "_finished_event", - "_return_value", - "_start_value", - "_exception", - ) - - _return_value: T_co - _start_value: T_startval - - def __init__(self, coro: Coroutine[Any, Any, T_co], name: object) -> None: - from ._synchronization import Event - - self._coro = coro - self._cancel_scope = CancelScope() - self._finished_event = Event() - self._exception: BaseException | None = None - - if name is not None: - self._name = str(name) - elif iscoroutine(coro): - self._name = coro.__qualname__ - else: - self._name = str(coro) # coroutine-like object (e.g. asend() objects) - - async def _run_coro(self) -> None: - __tracebackhide__ = True - - with self._cancel_scope: - try: - retval = await self._coro - except BaseException as exc: - self._exception = exc - raise - else: - self._return_value = retval - finally: - self._finished_event.set() - del self # Break the reference cycle - - def cancel(self) -> None: - """ - Set the task to a cancelled state. - - This will interrupt any interruptible asynchronous operation, and will cause - any further awaits on this task to get immediately cancelled, unless done in - a shielded cancel scope. - - If the task has already finished, this method has no effect. - """ - if not self._finished_event.is_set(): - self._cancel_scope.cancel() - - @property - def coro(self) -> Coroutine[Any, Any, T_co]: - """ - The coroutine object that was passed to one of the task-spawning methods in - :class:`TaskGroup`. - """ - return self._coro - - @property - def status(self) -> TaskHandle.Status: - """ - The current status of the task. - - Every task starts in the :attr:`~TaskHandle.Status.PENDING` state. - If a task is cancelled while in this state, it will transition to the - :attr:`~TaskHandle.Status.CANCELLING` state. When the task finishes, it will - transition to one of the three final states ( - :attr:`~TaskHandle.Status.FINISHED`, :attr:`~TaskHandle.Status.FAILED`, or - :attr:`~TaskHandle.Status.CANCELLING`) depending on the exception the task - raised, if any. No other status transitions will happen. - """ - if not self._finished_event.is_set(): - if self._cancel_scope.cancel_called: - return TaskHandle.Status.CANCELLING - else: - return TaskHandle.Status.PENDING - elif self._exception is not None: - if isinstance(self._exception, get_cancelled_exc_class()): - return TaskHandle.Status.CANCELLED - else: - return TaskHandle.Status.FAILED - else: - return TaskHandle.Status.FINISHED - - @property - def name(self) -> str: - """The name of the task.""" - return self._name - - @property - def exception(self) -> BaseException | None: - """ - The exception raised by the task, or ``None`` if it finished without raising. - - :raises TaskNotFinished: if the task has not finished yet - :raises TaskCancelled: if the task was cancelled - - """ - match self.status: - case TaskHandle.Status.PENDING: - raise TaskNotFinished("the task has not finished yet") - case TaskHandle.Status.FINISHED: - return None - case TaskHandle.Status.CANCELLING: - raise TaskCancelled("the task was cancelled") - case TaskHandle.Status.CANCELLED: - raise TaskCancelled("the task was cancelled") from self._exception - case TaskHandle.Status.FAILED: - return self._exception - - @property - def return_value(self) -> T_co: - """ - The return value of the task. - - :raises TaskNotFinished: if the task has not finished yet - :raises TaskCancelled: if the task was cancelled - :raises TaskFailed: if the task raised an exception - - """ - match self.status: - case TaskHandle.Status.PENDING: - raise TaskNotFinished("the task has not finished yet") - case TaskHandle.Status.FINISHED: - return self._return_value - case TaskHandle.Status.CANCELLING: - raise TaskCancelled("the task was cancelled") - case TaskHandle.Status.CANCELLED: - raise TaskCancelled("the task was cancelled") from self._exception - case TaskHandle.Status.FAILED: - raise TaskFailed("the task raised an exception") from self._exception - - @property - def start_value(self) -> T_startval: - """ - The value passed to :meth:`task_status.started() <.abc.TaskStatus.started>`, - - :raises RuntimeError: if the task was not started with :meth:`TaskGroup.start() - <.abc.TaskGroup.start>` - """ - try: - return self._start_value - except AttributeError: - raise RuntimeError( - "the task was not started with TaskGroup.start()" - ) from None - - async def wait(self) -> None: - """ - Wait for the task to finish. - - This method will return as soon as the task has finished, no matter how it - happened. - """ - await self._finished_event.wait() - - def __await__(self) -> Generator[Any, Any, T_co]: - yield from self._finished_event.wait().__await__() - return self.return_value - - def __repr__(self) -> str: - return ( - f"<{self.__class__.__name__} {self.status.name.lower()} " - f"name={self._name!r} coro={self._coro!r}>" - ) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_tempfile.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_tempfile.py deleted file mode 100644 index 75a09f793744b8e60375ce2efab98307d077bc21..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_tempfile.py +++ /dev/null @@ -1,613 +0,0 @@ -from __future__ import annotations - -import os -import sys -import tempfile -from collections.abc import Iterable -from io import BytesIO, TextIOWrapper -from types import TracebackType -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Generic, - overload, -) - -from .. import to_thread -from .._core._fileio import AsyncFile -from ..lowlevel import checkpoint_if_cancelled - -if TYPE_CHECKING: - from _typeshed import OpenBinaryMode, OpenTextMode, ReadableBuffer, WriteableBuffer - - -class TemporaryFile(Generic[AnyStr]): - """ - An asynchronous temporary file that is automatically created and cleaned up. - - This class provides an asynchronous context manager interface to a temporary file. - The file is created using Python's standard `tempfile.TemporaryFile` function in a - background thread, and is wrapped as an asynchronous file using `AsyncFile`. - - :param mode: The mode in which the file is opened. Defaults to "w+b". - :param buffering: The buffering policy (-1 means the default buffering). - :param encoding: The encoding used to decode or encode the file. Only applicable in - text mode. - :param newline: Controls how universal newlines mode works (only applicable in text - mode). - :param suffix: The suffix for the temporary file name. - :param prefix: The prefix for the temporary file name. - :param dir: The directory in which the temporary file is created. - :param errors: The error handling scheme used for encoding/decoding errors. - """ - - _async_file: AsyncFile[AnyStr] - - @overload - def __init__( - self: TemporaryFile[bytes], - mode: OpenBinaryMode = ..., - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - *, - errors: str | None = ..., - ): ... - @overload - def __init__( - self: TemporaryFile[str], - mode: OpenTextMode, - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - *, - errors: str | None = ..., - ): ... - - def __init__( - self, - mode: OpenTextMode | OpenBinaryMode = "w+b", - buffering: int = -1, - encoding: str | None = None, - newline: str | None = None, - suffix: str | None = None, - prefix: str | None = None, - dir: str | None = None, - *, - errors: str | None = None, - ) -> None: - self.mode = mode - self.buffering = buffering - self.encoding = encoding - self.newline = newline - self.suffix: str | None = suffix - self.prefix: str | None = prefix - self.dir: str | None = dir - self.errors = errors - - async def __aenter__(self) -> AsyncFile[AnyStr]: - fp = await to_thread.run_sync( - lambda: tempfile.TemporaryFile( - self.mode, - self.buffering, - self.encoding, - self.newline, - self.suffix, - self.prefix, - self.dir, - errors=self.errors, - ) - ) - self._async_file = AsyncFile(fp) - return self._async_file - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - await self._async_file.aclose() - - -class NamedTemporaryFile(Generic[AnyStr]): - """ - An asynchronous named temporary file that is automatically created and cleaned up. - - This class provides an asynchronous context manager for a temporary file with a - visible name in the file system. It uses Python's standard - :func:`~tempfile.NamedTemporaryFile` function and wraps the file object with - :class:`AsyncFile` for asynchronous operations. - - :param mode: The mode in which the file is opened. Defaults to "w+b". - :param buffering: The buffering policy (-1 means the default buffering). - :param encoding: The encoding used to decode or encode the file. Only applicable in - text mode. - :param newline: Controls how universal newlines mode works (only applicable in text - mode). - :param suffix: The suffix for the temporary file name. - :param prefix: The prefix for the temporary file name. - :param dir: The directory in which the temporary file is created. - :param delete: Whether to delete the file when it is closed. - :param errors: The error handling scheme used for encoding/decoding errors. - :param delete_on_close: (Python 3.12+) Whether to delete the file on close. - """ - - _async_file: AsyncFile[AnyStr] - - @overload - def __init__( - self: NamedTemporaryFile[bytes], - mode: OpenBinaryMode = ..., - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - delete: bool = ..., - *, - errors: str | None = ..., - delete_on_close: bool = ..., - ): ... - @overload - def __init__( - self: NamedTemporaryFile[str], - mode: OpenTextMode, - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - delete: bool = ..., - *, - errors: str | None = ..., - delete_on_close: bool = ..., - ): ... - - def __init__( - self, - mode: OpenBinaryMode | OpenTextMode = "w+b", - buffering: int = -1, - encoding: str | None = None, - newline: str | None = None, - suffix: str | None = None, - prefix: str | None = None, - dir: str | None = None, - delete: bool = True, - *, - errors: str | None = None, - delete_on_close: bool = True, - ) -> None: - self._params: dict[str, Any] = { - "mode": mode, - "buffering": buffering, - "encoding": encoding, - "newline": newline, - "suffix": suffix, - "prefix": prefix, - "dir": dir, - "delete": delete, - "errors": errors, - } - if sys.version_info >= (3, 12): - self._params["delete_on_close"] = delete_on_close - - async def __aenter__(self) -> AsyncFile[AnyStr]: - fp = await to_thread.run_sync( - lambda: tempfile.NamedTemporaryFile(**self._params) - ) - self._async_file = AsyncFile(fp) - return self._async_file - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - await self._async_file.aclose() - - -class SpooledTemporaryFile(AsyncFile[AnyStr]): - """ - An asynchronous spooled temporary file that starts in memory and is spooled to disk. - - This class provides an asynchronous interface to a spooled temporary file, much like - Python's standard :class:`~tempfile.SpooledTemporaryFile`. It supports asynchronous - write operations and provides a method to force a rollover to disk. - - :param max_size: Maximum size in bytes before the file is rolled over to disk. - :param mode: The mode in which the file is opened. Defaults to "w+b". - :param buffering: The buffering policy (-1 means the default buffering). - :param encoding: The encoding used to decode or encode the file (text mode only). - :param newline: Controls how universal newlines mode works (text mode only). - :param suffix: The suffix for the temporary file name. - :param prefix: The prefix for the temporary file name. - :param dir: The directory in which the temporary file is created. - :param errors: The error handling scheme used for encoding/decoding errors. - """ - - _rolled: bool = False - - @overload - def __init__( - self: SpooledTemporaryFile[bytes], - max_size: int = ..., - mode: OpenBinaryMode = ..., - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - *, - errors: str | None = ..., - ): ... - @overload - def __init__( - self: SpooledTemporaryFile[str], - max_size: int = ..., - mode: OpenTextMode = ..., - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - *, - errors: str | None = ..., - ): ... - - def __init__( - self, - max_size: int = 0, - mode: OpenBinaryMode | OpenTextMode = "w+b", - buffering: int = -1, - encoding: str | None = None, - newline: str | None = None, - suffix: str | None = None, - prefix: str | None = None, - dir: str | None = None, - *, - errors: str | None = None, - ) -> None: - self._tempfile_params: dict[str, Any] = { - "mode": mode, - "buffering": buffering, - "encoding": encoding, - "newline": newline, - "suffix": suffix, - "prefix": prefix, - "dir": dir, - "errors": errors, - } - self._max_size = max_size - if "b" in mode: - super().__init__(BytesIO()) # type: ignore[arg-type] - else: - super().__init__( - TextIOWrapper( # type: ignore[arg-type] - BytesIO(), - encoding=encoding, - errors=errors, - newline=newline, - write_through=True, - ) - ) - - async def aclose(self) -> None: - if not self._rolled: - self._fp.close() - return - - await super().aclose() - - async def _check(self) -> None: - if self._rolled or self._fp.tell() <= self._max_size: - return - - await self.rollover() - - async def rollover(self) -> None: - if self._rolled: - return - - self._rolled = True - buffer = self._fp - buffer.seek(0) - self._fp = await to_thread.run_sync( - lambda: tempfile.TemporaryFile(**self._tempfile_params) - ) - await self.write(buffer.read()) - buffer.close() - - @property - def closed(self) -> bool: - return self._fp.closed - - async def read(self, size: int = -1) -> AnyStr: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.read(size) - - return await super().read(size) # type: ignore[return-value] - - async def read1(self: SpooledTemporaryFile[bytes], size: int = -1) -> bytes: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.read1(size) - - return await super().read1(size) - - async def readline(self) -> AnyStr: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.readline() - - return await super().readline() # type: ignore[return-value] - - async def readlines(self) -> list[AnyStr]: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.readlines() - - return await super().readlines() # type: ignore[return-value] - - async def readinto(self: SpooledTemporaryFile[bytes], b: WriteableBuffer) -> int: - if not self._rolled: - await checkpoint_if_cancelled() - self._fp.readinto(b) - - return await super().readinto(b) - - async def readinto1(self: SpooledTemporaryFile[bytes], b: WriteableBuffer) -> int: - if not self._rolled: - await checkpoint_if_cancelled() - self._fp.readinto(b) - - return await super().readinto1(b) - - async def seek(self, offset: int, whence: int | None = os.SEEK_SET) -> int: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.seek(offset, whence) - - return await super().seek(offset, whence) - - async def tell(self) -> int: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.tell() - - return await super().tell() - - async def truncate(self, size: int | None = None) -> int: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.truncate(size) - - return await super().truncate(size) - - @overload - async def write(self: SpooledTemporaryFile[bytes], b: ReadableBuffer) -> int: ... - @overload - async def write(self: SpooledTemporaryFile[str], b: str) -> int: ... - - async def write(self, b: ReadableBuffer | str) -> int: - """ - Asynchronously write data to the spooled temporary file. - - If the file has not yet been rolled over, the data is written synchronously, - and a rollover is triggered if the size exceeds the maximum size. - - :param s: The data to write. - :return: The number of bytes written. - :raises RuntimeError: If the underlying file is not initialized. - - """ - if not self._rolled: - await checkpoint_if_cancelled() - result = self._fp.write(b) - await self._check() - return result - - return await super().write(b) # type: ignore[misc] - - @overload - async def writelines( - self: SpooledTemporaryFile[bytes], lines: Iterable[ReadableBuffer] - ) -> None: ... - @overload - async def writelines( - self: SpooledTemporaryFile[str], lines: Iterable[str] - ) -> None: ... - - async def writelines(self, lines: Iterable[str] | Iterable[ReadableBuffer]) -> None: - """ - Asynchronously write a list of lines to the spooled temporary file. - - If the file has not yet been rolled over, the lines are written synchronously, - and a rollover is triggered if the size exceeds the maximum size. - - :param lines: An iterable of lines to write. - :raises RuntimeError: If the underlying file is not initialized. - - """ - if not self._rolled: - await checkpoint_if_cancelled() - result = self._fp.writelines(lines) - await self._check() - return result - - return await super().writelines(lines) # type: ignore[misc] - - -class TemporaryDirectory(Generic[AnyStr]): - """ - An asynchronous temporary directory that is created and cleaned up automatically. - - This class provides an asynchronous context manager for creating a temporary - directory. It wraps Python's standard :class:`~tempfile.TemporaryDirectory` to - perform directory creation and cleanup operations in a background thread. - - :param suffix: Suffix to be added to the temporary directory name. - :param prefix: Prefix to be added to the temporary directory name. - :param dir: The parent directory where the temporary directory is created. - :param ignore_cleanup_errors: Whether to ignore errors during cleanup - :param delete: Whether to delete the directory upon closing (Python 3.12+). - """ - - def __init__( - self, - suffix: AnyStr | None = None, - prefix: AnyStr | None = None, - dir: AnyStr | None = None, - *, - ignore_cleanup_errors: bool = False, - delete: bool = True, - ) -> None: - self.suffix: AnyStr | None = suffix - self.prefix: AnyStr | None = prefix - self.dir: AnyStr | None = dir - self.ignore_cleanup_errors = ignore_cleanup_errors - self.delete = delete - - self._tempdir: tempfile.TemporaryDirectory | None = None - - async def __aenter__(self) -> str: - params: dict[str, Any] = { - "suffix": self.suffix, - "prefix": self.prefix, - "dir": self.dir, - "ignore_cleanup_errors": self.ignore_cleanup_errors, - } - if sys.version_info >= (3, 12): - params["delete"] = self.delete - - self._tempdir = await to_thread.run_sync( - lambda: tempfile.TemporaryDirectory(**params) - ) - return await to_thread.run_sync(self._tempdir.__enter__) - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - if self._tempdir is not None: - await to_thread.run_sync( - self._tempdir.__exit__, exc_type, exc_value, traceback - ) - - async def cleanup(self) -> None: - if self._tempdir is not None: - await to_thread.run_sync(self._tempdir.cleanup) - - -@overload -async def mkstemp( - suffix: str | None = None, - prefix: str | None = None, - dir: str | None = None, - text: bool = False, -) -> tuple[int, str]: ... - - -@overload -async def mkstemp( - suffix: bytes | None = None, - prefix: bytes | None = None, - dir: bytes | None = None, - text: bool = False, -) -> tuple[int, bytes]: ... - - -async def mkstemp( - suffix: AnyStr | None = None, - prefix: AnyStr | None = None, - dir: AnyStr | None = None, - text: bool = False, -) -> tuple[int, str | bytes]: - """ - Asynchronously create a temporary file and return an OS-level handle and the file - name. - - This function wraps `tempfile.mkstemp` and executes it in a background thread. - - :param suffix: Suffix to be added to the file name. - :param prefix: Prefix to be added to the file name. - :param dir: Directory in which the temporary file is created. - :param text: Whether the file is opened in text mode. - :return: A tuple containing the file descriptor and the file name. - - """ - return await to_thread.run_sync(tempfile.mkstemp, suffix, prefix, dir, text) - - -@overload -async def mkdtemp( - suffix: str | None = None, - prefix: str | None = None, - dir: str | None = None, -) -> str: ... - - -@overload -async def mkdtemp( - suffix: bytes | None = None, - prefix: bytes | None = None, - dir: bytes | None = None, -) -> bytes: ... - - -async def mkdtemp( - suffix: AnyStr | None = None, - prefix: AnyStr | None = None, - dir: AnyStr | None = None, -) -> str | bytes: - """ - Asynchronously create a temporary directory and return its path. - - This function wraps `tempfile.mkdtemp` and executes it in a background thread. - - :param suffix: Suffix to be added to the directory name. - :param prefix: Prefix to be added to the directory name. - :param dir: Parent directory where the temporary directory is created. - :return: The path of the created temporary directory. - - """ - return await to_thread.run_sync(tempfile.mkdtemp, suffix, prefix, dir) - - -async def gettempdir() -> str: - """ - Asynchronously return the name of the directory used for temporary files. - - This function wraps `tempfile.gettempdir` and executes it in a background thread. - - :return: The path of the temporary directory as a string. - - """ - return await to_thread.run_sync(tempfile.gettempdir) - - -async def gettempdirb() -> bytes: - """ - Asynchronously return the name of the directory used for temporary files in bytes. - - This function wraps `tempfile.gettempdirb` and executes it in a background thread. - - :return: The path of the temporary directory as bytes. - - """ - return await to_thread.run_sync(tempfile.gettempdirb) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_testing.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_testing.py deleted file mode 100644 index 369e65c068a426e99b7e8571209e80ce35b71f47..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_testing.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -from collections.abc import Awaitable, Generator -from typing import Any, cast - -from ._eventloop import get_async_backend - - -class TaskInfo: - """ - Represents an asynchronous task. - - :ivar int id: the unique identifier of the task - :ivar parent_id: the identifier of the parent task, if any - :vartype parent_id: Optional[int] - :ivar str name: the description of the task (if any) - :ivar ~collections.abc.Coroutine coro: the coroutine object of the task - """ - - __slots__ = "_name", "id", "parent_id", "name", "coro" - - def __init__( - self, - id: int, - parent_id: int | None, - name: str | None, - coro: Generator[Any, Any, Any] | Awaitable[Any], - ): - func = get_current_task - self._name = f"{func.__module__}.{func.__qualname__}" - self.id: int = id - self.parent_id: int | None = parent_id - self.name: str | None = name - self.coro: Generator[Any, Any, Any] | Awaitable[Any] = coro - - def __eq__(self, other: object) -> bool: - if isinstance(other, TaskInfo): - return self.id == other.id - - return NotImplemented - - def __hash__(self) -> int: - return hash(self.id) - - def __repr__(self) -> str: - return f"{self.__class__.__name__}(id={self.id!r}, name={self.name!r})" - - def has_pending_cancellation(self) -> bool: - """ - Return ``True`` if the task has a cancellation pending, ``False`` otherwise. - - """ - return False - - -def get_current_task() -> TaskInfo: - """ - Return the current task. - - :return: a representation of the current task - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().get_current_task() - - -def get_running_tasks() -> list[TaskInfo]: - """ - Return a list of running tasks in the current event loop. - - :return: a list of task info objects - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return cast("list[TaskInfo]", get_async_backend().get_running_tasks()) - - -async def wait_all_tasks_blocked() -> None: - """Wait until all other tasks are waiting for something.""" - await get_async_backend().wait_all_tasks_blocked() diff --git a/bundle/python-cpu/Lib/site-packages/anyio/_core/_typedattr.py b/bundle/python-cpu/Lib/site-packages/anyio/_core/_typedattr.py deleted file mode 100644 index f358a448cb12739fd4eda4f4859d3a24ddd1de63..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/_core/_typedattr.py +++ /dev/null @@ -1,81 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable, Mapping -from typing import Any, TypeVar, final, overload - -from ._exceptions import TypedAttributeLookupError - -T_Attr = TypeVar("T_Attr") -T_Default = TypeVar("T_Default") -undefined = object() - - -def typed_attribute() -> Any: - """Return a unique object, used to mark typed attributes.""" - return object() - - -class TypedAttributeSet: - """ - Superclass for typed attribute collections. - - Checks that every public attribute of every subclass has a type annotation. - """ - - def __init_subclass__(cls) -> None: - annotations: dict[str, Any] = getattr(cls, "__annotations__", {}) - for attrname in dir(cls): - if not attrname.startswith("_") and attrname not in annotations: - raise TypeError( - f"Attribute {attrname!r} is missing its type annotation" - ) - - super().__init_subclass__() - - -class TypedAttributeProvider: - """Base class for classes that wish to provide typed extra attributes.""" - - @property - def extra_attributes(self) -> Mapping[T_Attr, Callable[[], T_Attr]]: - """ - A mapping of the extra attributes to callables that return the corresponding - values. - - If the provider wraps another provider, the attributes from that wrapper should - also be included in the returned mapping (but the wrapper may override the - callables from the wrapped instance). - - """ - return {} - - @overload - def extra(self, attribute: T_Attr) -> T_Attr: ... - - @overload - def extra(self, attribute: T_Attr, default: T_Default) -> T_Attr | T_Default: ... - - @final - def extra(self, attribute: Any, default: object = undefined) -> object: - """ - extra(attribute, default=undefined) - - Return the value of the given typed extra attribute. - - :param attribute: the attribute (member of a :class:`~TypedAttributeSet`) to - look for - :param default: the value that should be returned if no value is found for the - attribute - :raises ~anyio.TypedAttributeLookupError: if the search failed and no default - value was given - - """ - try: - getter = self.extra_attributes[attribute] - except KeyError: - if default is undefined: - raise TypedAttributeLookupError("Attribute not found") from None - else: - return default - - return getter() diff --git a/bundle/python-cpu/Lib/site-packages/anyio/abc/__init__.py b/bundle/python-cpu/Lib/site-packages/anyio/abc/__init__.py deleted file mode 100644 index d560ce3f1fa45a7ee4a3bc8958aa59702caa9d0c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/abc/__init__.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -from ._eventloop import AsyncBackend as AsyncBackend -from ._resources import AsyncResource as AsyncResource -from ._sockets import ConnectedUDPSocket as ConnectedUDPSocket -from ._sockets import ConnectedUNIXDatagramSocket as ConnectedUNIXDatagramSocket -from ._sockets import IPAddressType as IPAddressType -from ._sockets import IPSockAddrType as IPSockAddrType -from ._sockets import SocketAttribute as SocketAttribute -from ._sockets import SocketListener as SocketListener -from ._sockets import SocketStream as SocketStream -from ._sockets import UDPPacketType as UDPPacketType -from ._sockets import UDPSocket as UDPSocket -from ._sockets import UNIXDatagramPacketType as UNIXDatagramPacketType -from ._sockets import UNIXDatagramSocket as UNIXDatagramSocket -from ._sockets import UNIXSocketStream as UNIXSocketStream -from ._streams import AnyByteReceiveStream as AnyByteReceiveStream -from ._streams import AnyByteSendStream as AnyByteSendStream -from ._streams import AnyByteStream as AnyByteStream -from ._streams import AnyByteStreamConnectable as AnyByteStreamConnectable -from ._streams import AnyUnreliableByteReceiveStream as AnyUnreliableByteReceiveStream -from ._streams import AnyUnreliableByteSendStream as AnyUnreliableByteSendStream -from ._streams import AnyUnreliableByteStream as AnyUnreliableByteStream -from ._streams import ByteReceiveStream as ByteReceiveStream -from ._streams import ByteSendStream as ByteSendStream -from ._streams import ByteStream as ByteStream -from ._streams import ByteStreamConnectable as ByteStreamConnectable -from ._streams import Listener as Listener -from ._streams import ObjectReceiveStream as ObjectReceiveStream -from ._streams import ObjectSendStream as ObjectSendStream -from ._streams import ObjectStream as ObjectStream -from ._streams import ObjectStreamConnectable as ObjectStreamConnectable -from ._streams import UnreliableObjectReceiveStream as UnreliableObjectReceiveStream -from ._streams import UnreliableObjectSendStream as UnreliableObjectSendStream -from ._streams import UnreliableObjectStream as UnreliableObjectStream -from ._subprocesses import Process as Process -from ._tasks import TaskGroup as TaskGroup -from ._tasks import TaskStatus as TaskStatus -from ._testing import TestRunner as TestRunner - -# Re-exported here, for backwards compatibility -# isort: off -from .._core._synchronization import ( - CapacityLimiter as CapacityLimiter, - Condition as Condition, - Event as Event, - Lock as Lock, - Semaphore as Semaphore, -) -from .._core._tasks import CancelScope as CancelScope -from ..from_thread import BlockingPortal as BlockingPortal - -# Re-export imports so they look like they live directly in this package -for __value in list(locals().values()): - if getattr(__value, "__module__", "").startswith("anyio.abc."): - __value.__module__ = __name__ - -del __value diff --git a/bundle/python-cpu/Lib/site-packages/anyio/abc/_eventloop.py b/bundle/python-cpu/Lib/site-packages/anyio/abc/_eventloop.py deleted file mode 100644 index cad3fa76370ff573469c1c52fd82d2eff0bc83eb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/abc/_eventloop.py +++ /dev/null @@ -1,410 +0,0 @@ -from __future__ import annotations - -import math -import sys -from abc import ABCMeta, abstractmethod -from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Sequence -from contextlib import AbstractContextManager -from os import PathLike -from signal import Signals -from socket import AddressFamily, SocketKind, socket -from typing import ( - IO, - TYPE_CHECKING, - Any, - TypeAlias, - TypeVar, - overload, -) - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike - - from .._core._synchronization import CapacityLimiter, Event, Lock, Semaphore - from .._core._tasks import CancelScope - from .._core._testing import TaskInfo - from ._sockets import ( - ConnectedUDPSocket, - ConnectedUNIXDatagramSocket, - IPSockAddrType, - SocketListener, - SocketStream, - UDPSocket, - UNIXDatagramSocket, - UNIXSocketStream, - ) - from ._subprocesses import Process - from ._tasks import TaskGroup - from ._testing import TestRunner - -T_Retval = TypeVar("T_Retval") -T_co = TypeVar("T_co", covariant=True) -PosArgsT = TypeVarTuple("PosArgsT") -StrOrBytesPath: TypeAlias = str | bytes | PathLike[str] | PathLike[bytes] - - -class AsyncBackend(metaclass=ABCMeta): - @classmethod - @abstractmethod - def run( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - options: dict[str, Any], - ) -> T_Retval: - """ - Run the given coroutine function in an asynchronous event loop. - - The current thread must not be already running an event loop. - - :param func: a coroutine function - :param args: positional arguments to ``func`` - :param kwargs: positional arguments to ``func`` - :param options: keyword arguments to call the backend ``run()`` implementation - with - :return: the return value of the coroutine function - """ - - @classmethod - @abstractmethod - def current_token(cls) -> object: - """ - Return an object that allows other threads to run code inside the event loop. - - :return: a token object, specific to the event loop running in the current - thread - """ - - @classmethod - @abstractmethod - def current_time(cls) -> float: - """ - Return the current value of the event loop's internal clock. - - :return: the clock value (seconds) - """ - - @classmethod - @abstractmethod - def cancelled_exception_class(cls) -> type[BaseException]: - """Return the exception class that is raised in a task if it's cancelled.""" - - @classmethod - @abstractmethod - async def checkpoint(cls) -> None: - """ - Check if the task has been cancelled, and allow rescheduling of other tasks. - - This is effectively the same as running :meth:`checkpoint_if_cancelled` and then - :meth:`cancel_shielded_checkpoint`. - """ - - @classmethod - async def checkpoint_if_cancelled(cls) -> None: - """ - Check if the current task group has been cancelled. - - This will check if the task has been cancelled, but will not allow other tasks - to be scheduled if not. - - """ - if cls.current_effective_deadline() == -math.inf: - await cls.checkpoint() - - @classmethod - async def cancel_shielded_checkpoint(cls) -> None: - """ - Allow the rescheduling of other tasks. - - This will give other tasks the opportunity to run, but without checking if the - current task group has been cancelled, unlike with :meth:`checkpoint`. - - """ - with cls.create_cancel_scope(shield=True): - await cls.sleep(0) - - @classmethod - @abstractmethod - async def sleep(cls, delay: float) -> None: - """ - Pause the current task for the specified duration. - - :param delay: the duration, in seconds - """ - - @classmethod - @abstractmethod - def create_cancel_scope( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - pass - - @classmethod - @abstractmethod - def current_effective_deadline(cls) -> float: - """ - Return the nearest deadline among all the cancel scopes effective for the - current task. - - :return: - - a clock value from the event loop's internal clock - - ``inf`` if there is no deadline in effect - - ``-inf`` if the current scope has been cancelled - :rtype: float - """ - - @classmethod - @abstractmethod - def create_task_group(cls) -> TaskGroup: - pass - - @classmethod - @abstractmethod - def create_event(cls) -> Event: - pass - - @classmethod - @abstractmethod - def create_lock(cls, *, fast_acquire: bool) -> Lock: - pass - - @classmethod - @abstractmethod - def create_semaphore( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - pass - - @classmethod - @abstractmethod - def create_capacity_limiter(cls, total_tokens: float) -> CapacityLimiter: - pass - - @classmethod - @abstractmethod - async def run_sync_in_worker_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - abandon_on_cancel: bool = False, - limiter: CapacityLimiter | None = None, - ) -> T_Retval: - pass - - @classmethod - @abstractmethod - def check_cancelled(cls) -> None: - pass - - @classmethod - @abstractmethod - def run_async_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_co: - pass - - @classmethod - @abstractmethod - def run_sync_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - pass - - @classmethod - @abstractmethod - async def open_process( - cls, - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None, - stdout: int | IO[Any] | None, - stderr: int | IO[Any] | None, - **kwargs: Any, - ) -> Process: - pass - - @classmethod - @abstractmethod - def setup_process_pool_exit_at_shutdown(cls, workers: set[Process]) -> None: - pass - - @classmethod - @abstractmethod - async def connect_tcp( - cls, host: str, port: int, local_address: IPSockAddrType | None = None - ) -> SocketStream: - pass - - @classmethod - @abstractmethod - async def connect_unix(cls, path: str | bytes) -> UNIXSocketStream: - pass - - @classmethod - @abstractmethod - def create_tcp_listener(cls, sock: socket) -> SocketListener: - pass - - @classmethod - @abstractmethod - def create_unix_listener(cls, sock: socket) -> SocketListener: - pass - - @classmethod - @abstractmethod - async def create_udp_socket( - cls, - family: AddressFamily, - local_address: IPSockAddrType | None, - remote_address: IPSockAddrType | None, - reuse_port: bool, - ) -> UDPSocket | ConnectedUDPSocket: - pass - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket, remote_path: None - ) -> UNIXDatagramSocket: ... - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket, remote_path: str | bytes - ) -> ConnectedUNIXDatagramSocket: ... - - @classmethod - @abstractmethod - async def create_unix_datagram_socket( - cls, raw_socket: socket, remote_path: str | bytes | None - ) -> UNIXDatagramSocket | ConnectedUNIXDatagramSocket: - pass - - @classmethod - @abstractmethod - async def getaddrinfo( - cls, - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, - ) -> Sequence[ - tuple[ - AddressFamily, - SocketKind, - int, - str, - tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes], - ] - ]: - pass - - @classmethod - @abstractmethod - async def getnameinfo( - cls, sockaddr: IPSockAddrType, flags: int = 0 - ) -> tuple[str, str]: - pass - - @classmethod - @abstractmethod - async def wait_readable(cls, obj: FileDescriptorLike) -> None: - pass - - @classmethod - @abstractmethod - async def wait_writable(cls, obj: FileDescriptorLike) -> None: - pass - - @classmethod - @abstractmethod - def notify_closing(cls, obj: FileDescriptorLike) -> None: - pass - - @classmethod - @abstractmethod - async def wrap_listener_socket(cls, sock: socket) -> SocketListener: - pass - - @classmethod - @abstractmethod - async def wrap_stream_socket(cls, sock: socket) -> SocketStream: - pass - - @classmethod - @abstractmethod - async def wrap_unix_stream_socket(cls, sock: socket) -> UNIXSocketStream: - pass - - @classmethod - @abstractmethod - async def wrap_udp_socket(cls, sock: socket) -> UDPSocket: - pass - - @classmethod - @abstractmethod - async def wrap_connected_udp_socket(cls, sock: socket) -> ConnectedUDPSocket: - pass - - @classmethod - @abstractmethod - async def wrap_unix_datagram_socket(cls, sock: socket) -> UNIXDatagramSocket: - pass - - @classmethod - @abstractmethod - async def wrap_connected_unix_datagram_socket( - cls, sock: socket - ) -> ConnectedUNIXDatagramSocket: - pass - - @classmethod - @abstractmethod - def current_default_thread_limiter(cls) -> CapacityLimiter: - pass - - @classmethod - @abstractmethod - def open_signal_receiver( - cls, *signals: Signals - ) -> AbstractContextManager[AsyncIterator[Signals]]: - pass - - @classmethod - @abstractmethod - def get_current_task(cls) -> TaskInfo: - pass - - @classmethod - @abstractmethod - def get_running_tasks(cls) -> Sequence[TaskInfo]: - pass - - @classmethod - @abstractmethod - async def wait_all_tasks_blocked(cls) -> None: - pass - - @classmethod - @abstractmethod - def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: - pass diff --git a/bundle/python-cpu/Lib/site-packages/anyio/abc/_resources.py b/bundle/python-cpu/Lib/site-packages/anyio/abc/_resources.py deleted file mode 100644 index 10df115a7b9f975493476da763cc1e26dbd822e5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/abc/_resources.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import annotations - -from abc import ABCMeta, abstractmethod -from types import TracebackType -from typing import TypeVar - -T = TypeVar("T") - - -class AsyncResource(metaclass=ABCMeta): - """ - Abstract base class for all closeable asynchronous resources. - - Works as an asynchronous context manager which returns the instance itself on enter, - and calls :meth:`aclose` on exit. - """ - - __slots__ = () - - async def __aenter__(self: T) -> T: - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - await self.aclose() - - @abstractmethod - async def aclose(self) -> None: - """Close the resource.""" diff --git a/bundle/python-cpu/Lib/site-packages/anyio/abc/_sockets.py b/bundle/python-cpu/Lib/site-packages/anyio/abc/_sockets.py deleted file mode 100644 index feb26bd44a240acb20fd0f2498dff5631b8e2fb3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/abc/_sockets.py +++ /dev/null @@ -1,399 +0,0 @@ -from __future__ import annotations - -import errno -import socket -from abc import abstractmethod -from collections.abc import Callable, Collection, Mapping -from contextlib import AsyncExitStack -from io import IOBase -from ipaddress import IPv4Address, IPv6Address -from socket import AddressFamily -from typing import Any, TypeAlias, TypeVar - -from .._core._eventloop import get_async_backend -from .._core._typedattr import ( - TypedAttributeProvider, - TypedAttributeSet, - typed_attribute, -) -from ._streams import ByteStream, Listener, UnreliableObjectStream -from ._tasks import TaskGroup - -IPAddressType: TypeAlias = str | IPv4Address | IPv6Address -IPSockAddrType: TypeAlias = tuple[str, int] -SockAddrType: TypeAlias = IPSockAddrType | str -UDPPacketType: TypeAlias = tuple[bytes, IPSockAddrType] -UNIXDatagramPacketType: TypeAlias = tuple[bytes, str] -T_Retval = TypeVar("T_Retval") - - -def _validate_socket( - sock_or_fd: socket.socket | int, - sock_type: socket.SocketKind, - addr_family: socket.AddressFamily = socket.AF_UNSPEC, - *, - require_connected: bool = False, - require_bound: bool = False, -) -> socket.socket: - if isinstance(sock_or_fd, int): - try: - sock = socket.socket(fileno=sock_or_fd) - except OSError as exc: - if exc.errno == errno.ENOTSOCK: - raise ValueError( - "the file descriptor does not refer to a socket" - ) from exc - elif require_connected: - raise ValueError("the socket must be connected") from exc - elif require_bound: - raise ValueError("the socket must be bound to a local address") from exc - else: - raise - elif isinstance(sock_or_fd, socket.socket): - sock = sock_or_fd - else: - raise TypeError( - f"expected an int or socket, got {type(sock_or_fd).__qualname__} instead" - ) - - try: - if require_connected: - try: - sock.getpeername() - except OSError as exc: - raise ValueError("the socket must be connected") from exc - - if require_bound: - try: - if sock.family in (socket.AF_INET, socket.AF_INET6): - bound_addr = sock.getsockname()[1] - else: - bound_addr = sock.getsockname() - except OSError: - bound_addr = None - - if not bound_addr: - raise ValueError("the socket must be bound to a local address") - - if addr_family != socket.AF_UNSPEC and sock.family != addr_family: - raise ValueError( - f"address family mismatch: expected {addr_family.name}, got " - f"{sock.family.name}" - ) - - if sock.type != sock_type: - raise ValueError( - f"socket type mismatch: expected {sock_type.name}, got {sock.type.name}" - ) - except BaseException: - # Avoid ResourceWarning from the locally constructed socket object - if isinstance(sock_or_fd, int): - sock.detach() - - raise - - sock.setblocking(False) - return sock - - -class SocketAttribute(TypedAttributeSet): - """ - .. attribute:: family - :type: socket.AddressFamily - - the address family of the underlying socket - - .. attribute:: local_address - :type: tuple[str, int] | str - - the local address the underlying socket is connected to - - .. attribute:: local_port - :type: int - - for IP based sockets, the local port the underlying socket is bound to - - .. attribute:: raw_socket - :type: socket.socket - - the underlying stdlib socket object - - .. attribute:: remote_address - :type: tuple[str, int] | str - - the remote address the underlying socket is connected to - - .. attribute:: remote_port - :type: int - - for IP based sockets, the remote port the underlying socket is connected to - """ - - family: AddressFamily = typed_attribute() - local_address: SockAddrType = typed_attribute() - local_port: int = typed_attribute() - raw_socket: socket.socket = typed_attribute() - remote_address: SockAddrType = typed_attribute() - remote_port: int = typed_attribute() - - -class _SocketProvider(TypedAttributeProvider): - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - from .._core._sockets import convert_ipv6_sockaddr as convert - - attributes: dict[Any, Callable[[], Any]] = { - SocketAttribute.family: lambda: self._raw_socket.family, - SocketAttribute.local_address: lambda: convert( - self._raw_socket.getsockname() - ), - SocketAttribute.raw_socket: lambda: self._raw_socket, - } - try: - peername: tuple[str, int] | None = convert(self._raw_socket.getpeername()) - except OSError: - peername = None - - # Provide the remote address for connected sockets - if peername is not None: - attributes[SocketAttribute.remote_address] = lambda: peername - - # Provide local and remote ports for IP based sockets - if self._raw_socket.family in (AddressFamily.AF_INET, AddressFamily.AF_INET6): - attributes[SocketAttribute.local_port] = lambda: ( - self._raw_socket.getsockname()[1] - ) - if peername is not None: - remote_port = peername[1] - attributes[SocketAttribute.remote_port] = lambda: remote_port - - return attributes - - @property - @abstractmethod - def _raw_socket(self) -> socket.socket: - pass - - -class SocketStream(ByteStream, _SocketProvider): - """ - Transports bytes over a socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket(cls, sock_or_fd: socket.socket | int) -> SocketStream: - """ - Wrap an existing socket object or file descriptor as a socket stream. - - The newly created socket wrapper takes ownership of the socket being passed in. - The existing socket must already be connected. - - :param sock_or_fd: a socket object or file descriptor - :return: a socket stream - - """ - sock = _validate_socket(sock_or_fd, socket.SOCK_STREAM, require_connected=True) - return await get_async_backend().wrap_stream_socket(sock) - - -class UNIXSocketStream(SocketStream): - @classmethod - async def from_socket(cls, sock_or_fd: socket.socket | int) -> UNIXSocketStream: - """ - Wrap an existing socket object or file descriptor as a UNIX socket stream. - - The newly created socket wrapper takes ownership of the socket being passed in. - The existing socket must already be connected. - - :param sock_or_fd: a socket object or file descriptor - :return: a UNIX socket stream - - """ - sock = _validate_socket( - sock_or_fd, socket.SOCK_STREAM, socket.AF_UNIX, require_connected=True - ) - return await get_async_backend().wrap_unix_stream_socket(sock) - - @abstractmethod - async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: - """ - Send file descriptors along with a message to the peer. - - :param message: a non-empty bytestring - :param fds: a collection of files (either numeric file descriptors or open file - or socket objects) - """ - - @abstractmethod - async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: - """ - Receive file descriptors along with a message from the peer. - - :param msglen: length of the message to expect from the peer - :param maxfds: maximum number of file descriptors to expect from the peer - :return: a tuple of (message, file descriptors) - """ - - -class SocketListener(Listener[SocketStream], _SocketProvider): - """ - Listens to incoming socket connections. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket( - cls, - sock_or_fd: socket.socket | int, - ) -> SocketListener: - """ - Wrap an existing socket object or file descriptor as a socket listener. - - The newly created listener takes ownership of the socket being passed in. - - :param sock_or_fd: a socket object or file descriptor - :return: a socket listener - - """ - sock = _validate_socket(sock_or_fd, socket.SOCK_STREAM, require_bound=True) - return await get_async_backend().wrap_listener_socket(sock) - - @abstractmethod - async def accept(self) -> SocketStream: - """Accept an incoming connection.""" - - async def serve( - self, - handler: Callable[[SocketStream], Any], - task_group: TaskGroup | None = None, - ) -> None: - from .. import create_task_group - - async with AsyncExitStack() as stack: - if task_group is None: - task_group = await stack.enter_async_context(create_task_group()) - - while True: - stream = await self.accept() - task_group.start_soon(handler, stream) - - -class UDPSocket(UnreliableObjectStream[UDPPacketType], _SocketProvider): - """ - Represents an unconnected UDP socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket(cls, sock_or_fd: socket.socket | int) -> UDPSocket: - """ - Wrap an existing socket object or file descriptor as a UDP socket. - - The newly created socket wrapper takes ownership of the socket being passed in. - The existing socket must be bound to a local address. - - :param sock_or_fd: a socket object or file descriptor - :return: a UDP socket - - """ - sock = _validate_socket(sock_or_fd, socket.SOCK_DGRAM, require_bound=True) - return await get_async_backend().wrap_udp_socket(sock) - - async def sendto(self, data: bytes, host: str, port: int) -> None: - """ - Alias for :meth:`~.UnreliableObjectSendStream.send` ((data, (host, port))). - - """ - return await self.send((data, (host, port))) - - -class ConnectedUDPSocket(UnreliableObjectStream[bytes], _SocketProvider): - """ - Represents an connected UDP socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket(cls, sock_or_fd: socket.socket | int) -> ConnectedUDPSocket: - """ - Wrap an existing socket object or file descriptor as a connected UDP socket. - - The newly created socket wrapper takes ownership of the socket being passed in. - The existing socket must already be connected. - - :param sock_or_fd: a socket object or file descriptor - :return: a connected UDP socket - - """ - sock = _validate_socket( - sock_or_fd, - socket.SOCK_DGRAM, - require_connected=True, - ) - return await get_async_backend().wrap_connected_udp_socket(sock) - - -class UNIXDatagramSocket( - UnreliableObjectStream[UNIXDatagramPacketType], _SocketProvider -): - """ - Represents an unconnected Unix datagram socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket( - cls, - sock_or_fd: socket.socket | int, - ) -> UNIXDatagramSocket: - """ - Wrap an existing socket object or file descriptor as a UNIX datagram - socket. - - The newly created socket wrapper takes ownership of the socket being passed in. - - :param sock_or_fd: a socket object or file descriptor - :return: a UNIX datagram socket - - """ - sock = _validate_socket(sock_or_fd, socket.SOCK_DGRAM, socket.AF_UNIX) - return await get_async_backend().wrap_unix_datagram_socket(sock) - - async def sendto(self, data: bytes, path: str) -> None: - """Alias for :meth:`~.UnreliableObjectSendStream.send` ((data, path)).""" - return await self.send((data, path)) - - -class ConnectedUNIXDatagramSocket(UnreliableObjectStream[bytes], _SocketProvider): - """ - Represents a connected Unix datagram socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket( - cls, - sock_or_fd: socket.socket | int, - ) -> ConnectedUNIXDatagramSocket: - """ - Wrap an existing socket object or file descriptor as a connected UNIX datagram - socket. - - The newly created socket wrapper takes ownership of the socket being passed in. - The existing socket must already be connected. - - :param sock_or_fd: a socket object or file descriptor - :return: a connected UNIX datagram socket - - """ - sock = _validate_socket( - sock_or_fd, socket.SOCK_DGRAM, socket.AF_UNIX, require_connected=True - ) - return await get_async_backend().wrap_connected_unix_datagram_socket(sock) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/abc/_streams.py b/bundle/python-cpu/Lib/site-packages/anyio/abc/_streams.py deleted file mode 100644 index 34ebfc1c0fa6f533e0621ac25659e13ffd5d7508..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/abc/_streams.py +++ /dev/null @@ -1,235 +0,0 @@ -from __future__ import annotations - -from abc import ABCMeta, abstractmethod -from collections.abc import Callable -from typing import Any, Generic, TypeAlias, TypeVar - -from .._core._exceptions import EndOfStream -from .._core._typedattr import TypedAttributeProvider -from ._resources import AsyncResource -from ._tasks import TaskGroup - -T_Item = TypeVar("T_Item") -T_co = TypeVar("T_co", covariant=True) -T_contra = TypeVar("T_contra", contravariant=True) - - -class UnreliableObjectReceiveStream( - Generic[T_co], AsyncResource, TypedAttributeProvider -): - """ - An interface for receiving objects. - - This interface makes no guarantees that the received messages arrive in the order in - which they were sent, or that no messages are missed. - - Asynchronously iterating over objects of this type will yield objects matching the - given type parameter. - """ - - def __aiter__(self) -> UnreliableObjectReceiveStream[T_co]: - return self - - async def __anext__(self) -> T_co: - try: - return await self.receive() - except EndOfStream: - raise StopAsyncIteration from None - - @abstractmethod - async def receive(self) -> T_co: - """ - Receive the next item. - - :raises ~anyio.ClosedResourceError: if the receive stream has been explicitly - closed - :raises ~anyio.EndOfStream: if this stream has been closed from the other end - :raises ~anyio.BrokenResourceError: if this stream has been rendered unusable - due to external causes - """ - - -class UnreliableObjectSendStream( - Generic[T_contra], AsyncResource, TypedAttributeProvider -): - """ - An interface for sending objects. - - This interface makes no guarantees that the messages sent will reach the - recipient(s) in the same order in which they were sent, or at all. - """ - - @abstractmethod - async def send(self, item: T_contra) -> None: - """ - Send an item to the peer(s). - - :param item: the item to send - :raises ~anyio.ClosedResourceError: if the send stream has been explicitly - closed - :raises ~anyio.BrokenResourceError: if this stream has been rendered unusable - due to external causes - """ - - -class UnreliableObjectStream( - UnreliableObjectReceiveStream[T_Item], UnreliableObjectSendStream[T_Item] -): - """ - A bidirectional message stream which does not guarantee the order or reliability of - message delivery. - """ - - -class ObjectReceiveStream(UnreliableObjectReceiveStream[T_co]): - """ - A receive message stream which guarantees that messages are received in the same - order in which they were sent, and that no messages are missed. - """ - - -class ObjectSendStream(UnreliableObjectSendStream[T_contra]): - """ - A send message stream which guarantees that messages are delivered in the same order - in which they were sent, without missing any messages in the middle. - """ - - -class ObjectStream( - ObjectReceiveStream[T_Item], - ObjectSendStream[T_Item], - UnreliableObjectStream[T_Item], -): - """ - A bidirectional message stream which guarantees the order and reliability of message - delivery. - """ - - @abstractmethod - async def send_eof(self) -> None: - """ - Send an end-of-file indication to the peer. - - You should not try to send any further data to this stream after calling this - method. This method is idempotent (does nothing on successive calls). - """ - - -class ByteReceiveStream(AsyncResource, TypedAttributeProvider): - """ - An interface for receiving bytes from a single peer. - - Iterating this byte stream will yield a byte string of arbitrary length, but no more - than 65536 bytes. - """ - - def __aiter__(self) -> ByteReceiveStream: - return self - - async def __anext__(self) -> bytes: - try: - return await self.receive() - except EndOfStream: - raise StopAsyncIteration from None - - @abstractmethod - async def receive(self, max_bytes: int = 65536) -> bytes: - """ - Receive at most ``max_bytes`` bytes from the peer. - - .. note:: Implementers of this interface should not return an empty - :class:`bytes` object, and users should ignore them. - - :param max_bytes: maximum number of bytes to receive (must be a positive - integer) - :return: the received bytes - :raises ValueError: if ``max_bytes`` is less than 1 - :raises ~anyio.EndOfStream: if this stream has been closed from the other end - """ - - -class ByteSendStream(AsyncResource, TypedAttributeProvider): - """An interface for sending bytes to a single peer.""" - - @abstractmethod - async def send(self, item: bytes) -> None: - """ - Send the given bytes to the peer. - - :param item: the bytes to send - """ - - -class ByteStream(ByteReceiveStream, ByteSendStream): - """A bidirectional byte stream.""" - - @abstractmethod - async def send_eof(self) -> None: - """ - Send an end-of-file indication to the peer. - - You should not try to send any further data to this stream after calling this - method. This method is idempotent (does nothing on successive calls). - """ - - -#: Type alias for all unreliable bytes-oriented receive streams. -AnyUnreliableByteReceiveStream: TypeAlias = ( - UnreliableObjectReceiveStream[bytes] | ByteReceiveStream -) -#: Type alias for all unreliable bytes-oriented send streams. -AnyUnreliableByteSendStream: TypeAlias = ( - UnreliableObjectSendStream[bytes] | ByteSendStream -) -#: Type alias for all unreliable bytes-oriented streams. -AnyUnreliableByteStream: TypeAlias = UnreliableObjectStream[bytes] | ByteStream -#: Type alias for all bytes-oriented receive streams. -AnyByteReceiveStream: TypeAlias = ObjectReceiveStream[bytes] | ByteReceiveStream -#: Type alias for all bytes-oriented send streams. -AnyByteSendStream: TypeAlias = ObjectSendStream[bytes] | ByteSendStream -#: Type alias for all bytes-oriented streams. -AnyByteStream: TypeAlias = ObjectStream[bytes] | ByteStream - - -class Listener(Generic[T_co], AsyncResource, TypedAttributeProvider): - """An interface for objects that let you accept incoming connections.""" - - @abstractmethod - async def serve( - self, handler: Callable[[T_co], Any], task_group: TaskGroup | None = None - ) -> None: - """ - Accept incoming connections as they come in and start tasks to handle them. - - :param handler: a callable that will be used to handle each accepted connection - :param task_group: the task group that will be used to start tasks for handling - each accepted connection (if omitted, an ad-hoc task group will be created) - """ - - -class ObjectStreamConnectable(Generic[T_co], metaclass=ABCMeta): - @abstractmethod - async def connect(self) -> ObjectStream[T_co]: - """ - Connect to the remote endpoint. - - :return: an object stream connected to the remote end - :raises ConnectionFailed: if the connection fails - """ - - -class ByteStreamConnectable(metaclass=ABCMeta): - @abstractmethod - async def connect(self) -> ByteStream: - """ - Connect to the remote endpoint. - - :return: a bytestream connected to the remote end - :raises ConnectionFailed: if the connection fails - """ - - -#: Type alias for all connectables returning bytestreams or bytes-oriented object streams -AnyByteStreamConnectable: TypeAlias = ( - ObjectStreamConnectable[bytes] | ByteStreamConnectable -) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/abc/_subprocesses.py b/bundle/python-cpu/Lib/site-packages/anyio/abc/_subprocesses.py deleted file mode 100644 index ce0564ceac8aac425675b5c8f7f7205d08061fd3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/abc/_subprocesses.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -from abc import abstractmethod -from signal import Signals - -from ._resources import AsyncResource -from ._streams import ByteReceiveStream, ByteSendStream - - -class Process(AsyncResource): - """An asynchronous version of :class:`subprocess.Popen`.""" - - @abstractmethod - async def wait(self) -> int: - """ - Wait until the process exits. - - :return: the exit code of the process - """ - - @abstractmethod - def terminate(self) -> None: - """ - Terminates the process, gracefully if possible. - - On Windows, this calls ``TerminateProcess()``. - On POSIX systems, this sends ``SIGTERM`` to the process. - - .. seealso:: :meth:`subprocess.Popen.terminate` - """ - - @abstractmethod - def kill(self) -> None: - """ - Kills the process. - - On Windows, this calls ``TerminateProcess()``. - On POSIX systems, this sends ``SIGKILL`` to the process. - - .. seealso:: :meth:`subprocess.Popen.kill` - """ - - @abstractmethod - def send_signal(self, signal: Signals) -> None: - """ - Send a signal to the subprocess. - - .. seealso:: :meth:`subprocess.Popen.send_signal` - - :param signal: the signal number (e.g. :data:`signal.SIGHUP`) - """ - - @property - @abstractmethod - def pid(self) -> int: - """The process ID of the process.""" - - @property - @abstractmethod - def returncode(self) -> int | None: - """ - The return code of the process. If the process has not yet terminated, this will - be ``None``. - """ - - @property - @abstractmethod - def stdin(self) -> ByteSendStream | None: - """The stream for the standard input of the process.""" - - @property - @abstractmethod - def stdout(self) -> ByteReceiveStream | None: - """The stream for the standard output of the process.""" - - @property - @abstractmethod - def stderr(self) -> ByteReceiveStream | None: - """The stream for the standard error output of the process.""" diff --git a/bundle/python-cpu/Lib/site-packages/anyio/abc/_tasks.py b/bundle/python-cpu/Lib/site-packages/anyio/abc/_tasks.py deleted file mode 100644 index 44ee3a70028b609e0c264b10ef5cee2127437fee..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/abc/_tasks.py +++ /dev/null @@ -1,209 +0,0 @@ -from __future__ import annotations - -import sys -from abc import ABCMeta, abstractmethod -from collections.abc import Callable, Coroutine -from contextvars import Context -from types import TracebackType -from typing import TYPE_CHECKING, Any, Literal, Protocol, final, overload - -if sys.version_info >= (3, 13): - from typing import TypeVar -else: - from typing_extensions import TypeVar - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -if TYPE_CHECKING: - from .._core._tasks import CancelScope, TaskHandle - -T_co = TypeVar("T_co", covariant=True) -T_contra = TypeVar("T_contra", contravariant=True, default=None) -PosArgsT = TypeVarTuple("PosArgsT") - - -def get_callable_name(func: Callable, override: object = None) -> str: - if override is not None: - return str(override) - - module = getattr(func, "__module__", None) - qualname = getattr(func, "__qualname__", None) - return ".".join([x for x in (module, qualname) if x]) - - -def call_for_coroutine( - func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], - args: tuple[Unpack[PosArgsT]], - **kwargs: Any, -) -> Coroutine[Any, Any, T_co]: - """ - Call the given function with the given positional and keyword arguments. - - :return: the resulting coroutine - :raises TypeError: if the return value was not a coroutine object - - """ - coro = func(*args, **kwargs) - if not isinstance(coro, Coroutine): - prefix = f"{func.__module__}." if hasattr(func, "__module__") else "" - raise TypeError( - f"Expected {prefix}{func.__qualname__}() to return a coroutine, but " - f"the return value ({coro!r}) is not a coroutine object" - ) - - return coro - - -class TaskStatus(Protocol[T_contra]): - @overload - def started(self: TaskStatus[None]) -> None: ... - - @overload - def started(self, value: T_contra) -> None: ... - - def started(self, value: T_contra | None = None) -> None: - """ - Signal that the task has started. - - :param value: object passed back to the starter of the task - """ - - -class TaskGroup(metaclass=ABCMeta): - """ - Groups several asynchronous tasks together. - - :ivar cancel_scope: the cancel scope inherited by all child tasks - :vartype cancel_scope: CancelScope - - .. note:: On asyncio, support for eager task factories is considered to be - **experimental**. In particular, they don't follow the usual semantics of new - tasks being scheduled on the next iteration of the event loop, and may thus - cause unexpected behavior in code that wasn't written with such semantics in - mind. - """ - - cancel_scope: CancelScope - - def cancel(self, reason: str | None = None) -> None: - """ - Cancel this task group's cancel scope immediately. - - This is a shortcut for calling ``.cancel_scope.cancel()`` on the task group. - - :param reason: a message describing the reason for the cancellation - - .. versionadded:: 4.14.0 - - """ - self.cancel_scope.cancel(reason) - - @abstractmethod - def create_task( - self, - coro: Coroutine[Any, Any, T_co], - *, - name: object = None, - context: Context | None = None, - ) -> TaskHandle[T_co]: - """ - Create a new task from a coroutine object and schedule it to run. - - :param coro: a coroutine object - :param name: optional name to give the task - :param context: optional context to run the task in - :return: a task handle - - .. versionadded:: 4.14.0 - """ - - @final - def start_soon( - self, - func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], - *args: Unpack[PosArgsT], - name: object = None, - ) -> TaskHandle[T_co]: - """ - Start a new task in this task group. - - :param func: a coroutine function - :param args: positional arguments to call the function with - :param name: name of the task, for the purposes of introspection and debugging - :return: a task handle - - .. versionadded:: 3.0 - .. versionchanged:: 4.14.0 - This method now returns a task handle. - - """ - final_name = get_callable_name(func, name) - return self.create_task(call_for_coroutine(func, args), name=final_name) - - @overload - async def start( - self, - func: Callable[..., Coroutine[Any, Any, T_co]], - *args: object, - name: object = None, - return_handle: Literal[False] = ..., - ) -> Any: ... - - @overload - async def start( - self, - func: Callable[..., Coroutine[Any, Any, T_co]], - *args: object, - name: object = None, - return_handle: Literal[True], - ) -> TaskHandle[T_co, Any]: ... - - @abstractmethod - async def start( - self, - func: Callable[..., Coroutine[Any, Any, T_co]], - *args: object, - name: object = None, - return_handle: Literal[False] | Literal[True] = False, - ) -> Any: - """ - Start a new task and wait until it signals for readiness. - - The target callable must accept a keyword argument ``task_status`` (of type - :class:`TaskStatus`). Awaiting on this method will return whatever was passed to - ``task_status.started()`` (``None`` by default). - - .. note:: The :class:`TaskStatus` class is generic, and the type argument should - indicate the type of the value that will be passed to - ``task_status.started()``. - - :param func: a coroutine function that accepts the ``task_status`` keyword - argument - :param args: positional arguments to call the function with - :param name: an optional name for the task, for introspection and debugging - :param return_handle: if ``True``, return a :class:`TaskHandle` which also - contains the start value in ``start_value`` - :return: the value passed to ``task_status.started()`` - :raises RuntimeError: if the task finishes without calling - ``task_status.started()`` - - .. seealso:: :ref:`start_initialize` - - .. versionadded:: 3.0 - """ - - @abstractmethod - async def __aenter__(self) -> TaskGroup: - """Enter the task group context and allow starting new tasks.""" - - @abstractmethod - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - """Exit the task group context waiting for all tasks to finish.""" diff --git a/bundle/python-cpu/Lib/site-packages/anyio/abc/_testing.py b/bundle/python-cpu/Lib/site-packages/anyio/abc/_testing.py deleted file mode 100644 index 2a93fb7cc31533f08f5be52c0528e10147aaac57..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/abc/_testing.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -import types -from abc import ABCMeta, abstractmethod -from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable -from typing import Any, TypeVar - -_T = TypeVar("_T") - - -class TestRunner(metaclass=ABCMeta): - """ - Encapsulates a running event loop. Every call made through this object will use the - same event loop. - """ - - def __enter__(self) -> TestRunner: - return self - - @abstractmethod - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: types.TracebackType | None, - ) -> bool | None: ... - - @abstractmethod - def run_asyncgen_fixture( - self, - fixture_func: Callable[..., AsyncGenerator[_T, Any]], - kwargs: dict[str, Any], - ) -> Iterable[_T]: - """ - Run an async generator fixture. - - :param fixture_func: the fixture function - :param kwargs: keyword arguments to call the fixture function with - :return: an iterator yielding the value yielded from the async generator - """ - - @abstractmethod - def run_fixture( - self, - fixture_func: Callable[..., Coroutine[Any, Any, _T]], - kwargs: dict[str, Any], - ) -> _T: - """ - Run an async fixture. - - :param fixture_func: the fixture function - :param kwargs: keyword arguments to call the fixture function with - :return: the return value of the fixture function - """ - - @abstractmethod - def run_test( - self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] - ) -> None: - """ - Run an async test function. - - :param test_func: the test function - :param kwargs: keyword arguments to call the test function with - """ - - @abstractmethod - def is_running(self) -> bool: - """ - Check if the test runner is running. - - :return: ``True`` if the coroutine is currently being run, ``False`` otherwise. - """ diff --git a/bundle/python-cpu/Lib/site-packages/anyio/from_thread.py b/bundle/python-cpu/Lib/site-packages/anyio/from_thread.py deleted file mode 100644 index 8c7914c2ffff281fd5a0f0273e7d7f5d8a35e459..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/from_thread.py +++ /dev/null @@ -1,582 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "BlockingPortal", - "BlockingPortalProvider", - "check_cancelled", - "run", - "run_sync", - "start_blocking_portal", -) - -import sys -from collections.abc import Awaitable, Callable, Coroutine, Generator -from concurrent.futures import Future -from contextlib import ( - AbstractAsyncContextManager, - AbstractContextManager, - contextmanager, -) -from dataclasses import dataclass, field -from functools import partial -from inspect import isawaitable -from threading import Lock, Thread, current_thread, get_ident -from types import TracebackType -from typing import ( - Any, - Generic, - TypeVar, - cast, - overload, -) - -from ._core._eventloop import ( - get_cancelled_exc_class, - threadlocals, -) -from ._core._eventloop import run as run_eventloop -from ._core._exceptions import NoEventLoopError -from ._core._synchronization import Event -from ._core._tasks import CancelScope, create_task_group -from .abc._tasks import TaskStatus -from .lowlevel import EventLoopToken, current_token - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -T_Retval = TypeVar("T_Retval") -T_co = TypeVar("T_co", covariant=True) -PosArgsT = TypeVarTuple("PosArgsT") - - -def _token_or_error(token: EventLoopToken | None) -> EventLoopToken: - if token is not None: - return token - - try: - return threadlocals.current_token - except AttributeError: - raise NoEventLoopError( - "Not running inside an AnyIO worker thread, and no event loop token was " - "provided" - ) from None - - -def run( - func: Callable[[Unpack[PosArgsT]], Coroutine[Any, Any, T_co]], - *args: Unpack[PosArgsT], - token: EventLoopToken | None = None, -) -> T_co: - """ - Call a coroutine function from a worker thread. - - :param func: a coroutine function - :param args: positional arguments for the callable - :param token: an event loop token to use to get back to the event loop thread - (required if calling this function from outside an AnyIO worker thread) - :return: the return value of the coroutine function - :raises MissingTokenError: if no token was provided and called from outside an - AnyIO worker thread - :raises RunFinishedError: if the event loop tied to ``token`` is no longer running - - .. versionchanged:: 4.11.0 - Added the ``token`` parameter. - - """ - explicit_token = token is not None - token = _token_or_error(token) - return token.backend_class.run_async_from_thread( - func, args, token=token.native_token if explicit_token else None - ) - - -def run_sync( - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - token: EventLoopToken | None = None, -) -> T_Retval: - """ - Call a function in the event loop thread from a worker thread. - - :param func: a callable - :param args: positional arguments for the callable - :param token: an event loop token to use to get back to the event loop thread - (required if calling this function from outside an AnyIO worker thread) - :return: the return value of the callable - :raises MissingTokenError: if no token was provided and called from outside an - AnyIO worker thread - :raises RunFinishedError: if the event loop tied to ``token`` is no longer running - - .. versionchanged:: 4.11.0 - Added the ``token`` parameter. - - """ - explicit_token = token is not None - token = _token_or_error(token) - return token.backend_class.run_sync_from_thread( - func, args, token=token.native_token if explicit_token else None - ) - - -class _BlockingAsyncContextManager(Generic[T_co], AbstractContextManager): - _enter_future: Future[T_co] - _exit_future: Future[bool | None] - _exit_event: Event - _exit_exc_info: tuple[ - type[BaseException] | None, BaseException | None, TracebackType | None - ] = (None, None, None) - - def __init__( - self, async_cm: AbstractAsyncContextManager[T_co], portal: BlockingPortal - ): - self._async_cm = async_cm - self._portal = portal - - async def run_async_cm(self) -> bool | None: - try: - self._exit_event = Event() - value = await self._async_cm.__aenter__() - except BaseException as exc: - self._enter_future.set_exception(exc) - raise - else: - self._enter_future.set_result(value) - - try: - # Wait for the sync context manager to exit. - # This next statement can raise `get_cancelled_exc_class()` if - # something went wrong in a task group in this async context - # manager. - await self._exit_event.wait() - finally: - # In case of cancellation, it could be that we end up here before - # `_BlockingAsyncContextManager.__exit__` is called, and an - # `_exit_exc_info` has been set. - result = await self._async_cm.__aexit__(*self._exit_exc_info) - - return result - - def __enter__(self) -> T_co: - self._enter_future = Future() - self._exit_future = self._portal.start_task_soon(self.run_async_cm) - return self._enter_future.result() - - def __exit__( - self, - __exc_type: type[BaseException] | None, - __exc_value: BaseException | None, - __traceback: TracebackType | None, - ) -> bool | None: - self._exit_exc_info = __exc_type, __exc_value, __traceback - self._portal.call(self._exit_event.set) - return self._exit_future.result() - - -class _BlockingPortalTaskStatus(TaskStatus): - def __init__(self, future: Future): - self._future = future - - def started(self, value: object = None) -> None: - self._future.set_result(value) - - -class BlockingPortal: - """ - An object that lets external threads run code in an asynchronous event loop. - - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - """ - - def __init__(self) -> None: - self._token = current_token() - self._event_loop_thread_id: int | None = get_ident() - self._stop_event = Event() - self._task_group = create_task_group() - - async def __aenter__(self) -> BlockingPortal: - await self._task_group.__aenter__() - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - await self.stop() - return await self._task_group.__aexit__(exc_type, exc_val, exc_tb) - - def _check_running(self) -> None: - if self._event_loop_thread_id is None: - raise RuntimeError("This portal is not running") - if self._event_loop_thread_id == get_ident(): - raise RuntimeError( - "This method cannot be called from the event loop thread" - ) - - async def sleep_until_stopped(self) -> None: - """Sleep until :meth:`stop` is called.""" - await self._stop_event.wait() - - async def stop(self, cancel_remaining: bool = False) -> None: - """ - Signal the portal to shut down. - - This marks the portal as no longer accepting new calls and exits from - :meth:`sleep_until_stopped`. - - :param cancel_remaining: ``True`` to cancel all the remaining tasks, ``False`` - to let them finish before returning - - """ - self._event_loop_thread_id = None - self._stop_event.set() - if cancel_remaining: - self._task_group.cancel_scope.cancel("the blocking portal is shutting down") - - async def _call_func( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - future: Future[T_Retval], - ) -> None: - event_loop_thread_id = self._event_loop_thread_id - - def callback(f: Future[T_Retval]) -> None: - if f.cancelled(): - if event_loop_thread_id == get_ident(): - scope.cancel("the future was cancelled") - elif event_loop_thread_id is not None: - run_sync( - scope.cancel, "the future was cancelled", token=self._token - ) - - try: - retval_or_awaitable = func(*args, **kwargs) - if isawaitable(retval_or_awaitable): - with CancelScope() as scope: - future.add_done_callback(callback) - retval = await retval_or_awaitable - else: - retval = retval_or_awaitable - except get_cancelled_exc_class(): - future.cancel() - future.set_running_or_notify_cancel() - except BaseException as exc: - if not future.cancelled(): - future.set_exception(exc) - - # Let base exceptions fall through - if not isinstance(exc, Exception): - raise - else: - if not future.cancelled(): - future.set_result(retval) - finally: - scope = None # type: ignore[assignment] - - def _spawn_task_from_thread( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - name: object, - future: Future[T_Retval], - ) -> None: - """ - Spawn a new task using the given callable. - - :param func: a callable - :param args: positional arguments to be passed to the callable - :param kwargs: keyword arguments to be passed to the callable - :param name: name of the task (will be coerced to a string if not ``None``) - :param future: a future that will resolve to the return value of the callable, - or the exception raised during its execution - - """ - run_sync( - partial(self._task_group.start_soon, name=name), - self._call_func, - func, - args, - kwargs, - future, - token=self._token, - ) - - @overload - def call( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - *args: Unpack[PosArgsT], - ) -> T_Retval: ... - - @overload - def call( - self, func: Callable[[Unpack[PosArgsT]], T_Retval], *args: Unpack[PosArgsT] - ) -> T_Retval: ... - - def call( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - *args: Unpack[PosArgsT], - ) -> T_Retval: - """ - Call the given function in the event loop thread. - - If the callable returns a coroutine object, it is awaited on. - - :param func: any callable - :raises RuntimeError: if the portal is not running or if this method is called - from within the event loop thread - - """ - return cast(T_Retval, self.start_task_soon(func, *args).result()) - - @overload - def start_task_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - *args: Unpack[PosArgsT], - name: object = None, - ) -> Future[T_Retval]: ... - - @overload - def start_task_soon( - self, - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - name: object = None, - ) -> Future[T_Retval]: ... - - def start_task_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - *args: Unpack[PosArgsT], - name: object = None, - ) -> Future[T_Retval]: - """ - Start a task in the portal's task group. - - The task will be run inside a cancel scope which can be cancelled by cancelling - the returned future. - - :param func: the target function - :param args: positional arguments passed to ``func`` - :param name: name of the task (will be coerced to a string if not ``None``) - :return: a future that resolves with the return value of the callable if the - task completes successfully, or with the exception raised in the task - :raises RuntimeError: if the portal is not running or if this method is called - from within the event loop thread - :rtype: concurrent.futures.Future[T_Retval] - - .. versionadded:: 3.0 - - """ - self._check_running() - f: Future[T_Retval] = Future() - self._spawn_task_from_thread(func, args, {}, name, f) - return f - - def start_task( - self, - func: Callable[..., Awaitable[T_Retval]], - *args: object, - name: object = None, - ) -> tuple[Future[T_Retval], Any]: - """ - Start a task in the portal's task group and wait until it signals for readiness. - - This method works the same way as :meth:`.abc.TaskGroup.start`. - - :param func: the target function - :param args: positional arguments passed to ``func`` - :param name: name of the task (will be coerced to a string if not ``None``) - :return: a tuple of (future, task_status_value) where the ``task_status_value`` - is the value passed to ``task_status.started()`` from within the target - function - :rtype: tuple[concurrent.futures.Future[T_Retval], Any] - - .. versionadded:: 3.0 - - """ - - def task_done(future: Future[T_Retval]) -> None: - if not task_status_future.done(): - if future.cancelled(): - task_status_future.cancel() - elif future.exception(): - task_status_future.set_exception(future.exception()) - else: - exc = RuntimeError( - "Task exited without calling task_status.started()" - ) - task_status_future.set_exception(exc) - - self._check_running() - task_status_future: Future = Future() - task_status = _BlockingPortalTaskStatus(task_status_future) - f: Future = Future() - f.add_done_callback(task_done) - self._spawn_task_from_thread(func, args, {"task_status": task_status}, name, f) - return f, task_status_future.result() - - def wrap_async_context_manager( - self, cm: AbstractAsyncContextManager[T_co] - ) -> AbstractContextManager[T_co]: - """ - Wrap an async context manager as a synchronous context manager via this portal. - - Spawns a task that will call both ``__aenter__()`` and ``__aexit__()``, stopping - in the middle until the synchronous context manager exits. - - :param cm: an asynchronous context manager - :return: a synchronous context manager - - .. versionadded:: 2.1 - - """ - return _BlockingAsyncContextManager(cm, self) - - -@dataclass -class BlockingPortalProvider: - """ - A manager for a blocking portal. Used as a context manager. The first thread to - enter this context manager causes a blocking portal to be started with the specific - parameters, and the last thread to exit causes the portal to be shut down. Thus, - there will be exactly one blocking portal running in this context as long as at - least one thread has entered this context manager. - - The parameters are the same as for :func:`~anyio.run`. - - :param backend: name of the backend - :param backend_options: backend options - - .. versionadded:: 4.4 - """ - - backend: str = "asyncio" - backend_options: dict[str, Any] | None = None - _lock: Lock = field(init=False, default_factory=Lock) - _leases: int = field(init=False, default=0) - _portal: BlockingPortal = field(init=False) - _portal_cm: AbstractContextManager[BlockingPortal] | None = field( - init=False, default=None - ) - - def __enter__(self) -> BlockingPortal: - with self._lock: - if self._portal_cm is None: - self._portal_cm = start_blocking_portal( - self.backend, self.backend_options - ) - self._portal = self._portal_cm.__enter__() - - self._leases += 1 - return self._portal - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - portal_cm: AbstractContextManager[BlockingPortal] | None = None - with self._lock: - assert self._portal_cm - assert self._leases > 0 - self._leases -= 1 - if not self._leases: - portal_cm = self._portal_cm - self._portal_cm = None - del self._portal - - if portal_cm: - portal_cm.__exit__(None, None, None) - - -@contextmanager -def start_blocking_portal( - backend: str = "asyncio", - backend_options: dict[str, Any] | None = None, - *, - name: str | None = None, -) -> Generator[BlockingPortal, Any, None]: - """ - Start a new event loop in a new thread and run a blocking portal in its main task. - - The parameters are the same as for :func:`~anyio.run`. - - :param backend: name of the backend - :param backend_options: backend options - :param name: name of the thread - :return: a context manager that yields a blocking portal - - .. versionchanged:: 3.0 - Usage as a context manager is now required. - - """ - - async def run_portal() -> None: - async with BlockingPortal() as portal_: - if name is None: - current_thread().name = f"{backend}-portal-{id(portal_):x}" - - future.set_result(portal_) - await portal_.sleep_until_stopped() - - def run_blocking_portal() -> None: - if future.set_running_or_notify_cancel(): - try: - run_eventloop( - run_portal, backend=backend, backend_options=backend_options - ) - except BaseException as exc: - if not future.done(): - future.set_exception(exc) - - future: Future[BlockingPortal] = Future() - thread = Thread(target=run_blocking_portal, daemon=True, name=name) - thread.start() - try: - cancel_remaining_tasks = False - portal = future.result() - try: - yield portal - except BaseException: - cancel_remaining_tasks = True - raise - finally: - try: - portal.call(portal.stop, cancel_remaining_tasks) - except RuntimeError: - pass - finally: - thread.join() - - -def check_cancelled() -> None: - """ - Check if the cancel scope of the host task's running the current worker thread has - been cancelled. - - If the host task's current cancel scope has indeed been cancelled, the - backend-specific cancellation exception will be raised. - - :raises RuntimeError: if the current thread was not spawned by - :func:`.to_thread.run_sync` - - """ - try: - token: EventLoopToken = threadlocals.current_token - except AttributeError: - raise NoEventLoopError( - "This function can only be called inside an AnyIO worker thread" - ) from None - - token.backend_class.check_cancelled() diff --git a/bundle/python-cpu/Lib/site-packages/anyio/functools.py b/bundle/python-cpu/Lib/site-packages/anyio/functools.py deleted file mode 100644 index b0bdfb4585efc4e4799388547668fba86fb5c687..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/functools.py +++ /dev/null @@ -1,400 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "AsyncCacheInfo", - "AsyncCacheParameters", - "AsyncLRUCacheWrapper", - "cache", - "lru_cache", - "reduce", -) - -import functools -from collections import OrderedDict -from collections.abc import ( - AsyncIterable, - Awaitable, - Callable, - Coroutine, - Hashable, - Iterable, -) -from functools import update_wrapper -from inspect import iscoroutinefunction -from typing import ( - Any, - Generic, - NamedTuple, - ParamSpec, - TypedDict, - TypeVar, - cast, - final, - overload, -) -from weakref import WeakKeyDictionary - -from ._core._eventloop import current_time -from ._core._synchronization import Lock -from .lowlevel import RunVar, checkpoint - -T = TypeVar("T") -S = TypeVar("S") -P = ParamSpec("P") -lru_cache_items: RunVar[ - WeakKeyDictionary[ - AsyncLRUCacheWrapper[Any, Any], - OrderedDict[ - Hashable, - tuple[_InitialMissingType, Lock, float | None] - | tuple[Any, None, float | None], - ], - ] -] = RunVar("lru_cache_items") - - -class _InitialMissingType: - pass - - -initial_missing: _InitialMissingType = _InitialMissingType() - - -class AsyncCacheInfo(NamedTuple): - hits: int - misses: int - maxsize: int | None - currsize: int - ttl: int | None - - -class AsyncCacheParameters(TypedDict): - maxsize: int | None - typed: bool - always_checkpoint: bool - ttl: int | None - - -class _LRUMethodWrapper(Generic[T]): - def __init__(self, wrapper: AsyncLRUCacheWrapper[..., T], instance: object): - self.__wrapper = wrapper - self.__instance = instance - - def cache_info(self) -> AsyncCacheInfo: - return self.__wrapper.cache_info() - - def cache_parameters(self) -> AsyncCacheParameters: - return self.__wrapper.cache_parameters() - - def cache_clear(self) -> None: - self.__wrapper.cache_clear() - - async def __call__(self, *args: Any, **kwargs: Any) -> T: - if self.__instance is None: - return await self.__wrapper(*args, **kwargs) - - return await self.__wrapper(self.__instance, *args, **kwargs) - - -@final -class AsyncLRUCacheWrapper(Generic[P, T]): - def __init__( - self, - func: Callable[P, Awaitable[T]], - maxsize: int | None, - typed: bool, - always_checkpoint: bool, - ttl: int | None, - ): - self.__wrapped__ = func - self._hits: int = 0 - self._misses: int = 0 - self._maxsize = max(maxsize, 0) if maxsize is not None else None - self._currsize: int = 0 - self._typed = typed - self._always_checkpoint = always_checkpoint - self._ttl = ttl - update_wrapper(self, func) - - def cache_info(self) -> AsyncCacheInfo: - return AsyncCacheInfo( - self._hits, self._misses, self._maxsize, self._currsize, self._ttl - ) - - def cache_parameters(self) -> AsyncCacheParameters: - return { - "maxsize": self._maxsize, - "typed": self._typed, - "always_checkpoint": self._always_checkpoint, - "ttl": self._ttl, - } - - def cache_clear(self) -> None: - if cache := lru_cache_items.get(None): - cache.pop(self, None) - self._hits = self._misses = self._currsize = 0 - - async def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: - # Easy case first: if maxsize == 0, no caching is done - if self._maxsize == 0: - value = await self.__wrapped__(*args, **kwargs) - self._misses += 1 - return value - - # The key is constructed as a flat tuple to avoid memory overhead - key: tuple[Any, ...] = args - if kwargs: - # initial_missing is used as a separator - key += (initial_missing,) + sum(kwargs.items(), ()) - - if self._typed: - key += tuple(type(arg) for arg in args) - if kwargs: - key += (initial_missing,) + tuple(type(val) for val in kwargs.values()) - - try: - cache = lru_cache_items.get() - except LookupError: - cache = WeakKeyDictionary() - lru_cache_items.set(cache) - - try: - cache_entry = cache[self] - except KeyError: - cache_entry = cache[self] = OrderedDict() - - cached_value: T | _InitialMissingType - try: - cached_value, lock, expires_at = cache_entry[key] - except KeyError: - # We're the first task to call this function - cached_value, lock, expires_at = ( - initial_missing, - Lock(fast_acquire=not self._always_checkpoint), - None, - ) - cache_entry[key] = cached_value, lock, expires_at - - if lock is None: - if expires_at is not None and current_time() >= expires_at: - self._currsize -= 1 - cached_value, lock, expires_at = ( - initial_missing, - Lock(fast_acquire=not self._always_checkpoint), - None, - ) - cache_entry[key] = cached_value, lock, expires_at - else: - # The value was already cached - self._hits += 1 - cache_entry.move_to_end(key) - if self._always_checkpoint: - await checkpoint() - - return cast(T, cached_value) - - async with lock: - # Check if another task filled the cache while we acquired the lock - if (cached_value := cache_entry[key][0]) is initial_missing: - self._misses += 1 - if self._maxsize is not None and self._currsize >= self._maxsize: - cache_entry.popitem(last=False) - else: - self._currsize += 1 - - value = await self.__wrapped__(*args, **kwargs) - expires_at = ( - current_time() + self._ttl if self._ttl is not None else None - ) - cache_entry[key] = value, None, expires_at - else: - # Another task filled the cache while we were waiting for the lock - self._hits += 1 - cache_entry.move_to_end(key) - value = cast(T, cached_value) - - return value - - def __get__( - self, instance: object, owner: type | None = None - ) -> _LRUMethodWrapper[T]: - wrapper = _LRUMethodWrapper(self, instance) - update_wrapper(wrapper, self.__wrapped__) - return wrapper - - -class _LRUCacheWrapper: - def __init__( - self, maxsize: int | None, typed: bool, always_checkpoint: bool, ttl: int | None - ): - self._maxsize = maxsize - self._typed = typed - self._always_checkpoint = always_checkpoint - self._ttl = ttl - - @overload - def __call__( # type: ignore[overload-overlap] - self, func: Callable[P, Coroutine[Any, Any, T]], / - ) -> AsyncLRUCacheWrapper[P, T]: ... - - @overload - def __call__( - self, func: Callable[..., T], / - ) -> functools._lru_cache_wrapper[T]: ... - - def __call__( - self, f: Callable[P, Coroutine[Any, Any, T]] | Callable[..., T], / - ) -> AsyncLRUCacheWrapper[P, T] | functools._lru_cache_wrapper[T]: - if iscoroutinefunction(f): - return AsyncLRUCacheWrapper( - f, self._maxsize, self._typed, self._always_checkpoint, self._ttl - ) - - return functools.lru_cache(maxsize=self._maxsize, typed=self._typed)(f) # type: ignore[arg-type] - - -@overload -def cache( # type: ignore[overload-overlap] - func: Callable[P, Coroutine[Any, Any, T]], / -) -> AsyncLRUCacheWrapper[P, T]: ... - - -@overload -def cache(func: Callable[..., T], /) -> functools._lru_cache_wrapper[T]: ... - - -def cache(func: Callable[..., Any] | Callable[P, Coroutine[Any, Any, Any]], /) -> Any: - """ - A convenient shortcut for :func:`lru_cache` with ``maxsize=None``. - - This is the asynchronous equivalent to :func:`functools.cache`. - - """ - return lru_cache(maxsize=None)(func) - - -@overload -def lru_cache( - *, - maxsize: int | None = ..., - typed: bool = ..., - always_checkpoint: bool = ..., - ttl: int | None = ..., -) -> _LRUCacheWrapper: ... - - -@overload -def lru_cache( # type: ignore[overload-overlap] - func: Callable[P, Coroutine[Any, Any, T]], / -) -> AsyncLRUCacheWrapper[P, T]: ... - - -@overload -def lru_cache(func: Callable[..., T], /) -> functools._lru_cache_wrapper[T]: ... - - -def lru_cache( - func: Callable[..., Coroutine[Any, Any, Any]] | Callable[..., Any] | None = None, - /, - *, - maxsize: int | None = 128, - typed: bool = False, - always_checkpoint: bool = False, - ttl: int | None = None, -) -> Any: - """ - An asynchronous version of :func:`functools.lru_cache`. - - If a synchronous function is passed, the standard library - :func:`functools.lru_cache` is applied instead. - - :param always_checkpoint: if ``True``, every call to the cached function will be - guaranteed to yield control to the event loop at least once - :param ttl: time in seconds after which to invalidate cache entries - - .. note:: Caches and locks are managed on a per-event loop basis. - - """ - if func is None: - return _LRUCacheWrapper(maxsize, typed, always_checkpoint, ttl) - - if not callable(func): - raise TypeError("the first argument must be callable") - - return _LRUCacheWrapper(maxsize, typed, always_checkpoint, ttl)(func) - - -@overload -async def reduce( - function: Callable[[T, S], Awaitable[T]], - iterable: Iterable[S] | AsyncIterable[S], - /, - initial: T, -) -> T: ... - - -@overload -async def reduce( - function: Callable[[T, T], Awaitable[T]], - iterable: Iterable[T] | AsyncIterable[T], - /, -) -> T: ... - - -async def reduce( # type: ignore[misc] - function: Callable[[T, T], Awaitable[T]] | Callable[[T, S], Awaitable[T]], - iterable: Iterable[T] | Iterable[S] | AsyncIterable[T] | AsyncIterable[S], - /, - initial: T | _InitialMissingType = initial_missing, -) -> T: - """ - Asynchronous version of :func:`functools.reduce`. - - :param function: a coroutine function that takes two arguments: the accumulated - value and the next element from the iterable - :param iterable: an iterable or async iterable - :param initial: the initial value (if missing, the first element of the iterable is - used as the initial value) - - """ - element: Any - function_called = False - if isinstance(iterable, AsyncIterable): - async_it = iterable.__aiter__() - if initial is initial_missing: - try: - value = cast(T, await async_it.__anext__()) - except StopAsyncIteration: - raise TypeError( - "reduce() of empty sequence with no initial value" - ) from None - else: - value = cast(T, initial) - - async for element in async_it: - value = await function(value, element) - function_called = True - elif isinstance(iterable, Iterable): - it = iter(iterable) - if initial is initial_missing: - try: - value = cast(T, next(it)) - except StopIteration: - raise TypeError( - "reduce() of empty sequence with no initial value" - ) from None - else: - value = cast(T, initial) - - for element in it: - value = await function(value, element) - function_called = True - else: - raise TypeError("reduce() argument 2 must be an iterable or async iterable") - - # Make sure there is at least one checkpoint, even if an empty iterable and an - # initial value were given - if not function_called: - await checkpoint() - - return value diff --git a/bundle/python-cpu/Lib/site-packages/anyio/itertools.py b/bundle/python-cpu/Lib/site-packages/anyio/itertools.py deleted file mode 100644 index 7e5248e4b8f99556cdbb98b024a188d65cdfce83..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/itertools.py +++ /dev/null @@ -1,626 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "accumulate", - "batched", - "Chain", - "combinations", - "combinations_with_replacement", - "compress", - "count", - "cycle", - "dropwhile", - "filterfalse", - "groupby", - "islice", - "pairwise", - "permutations", - "product", - "repeat", - "starmap", - "tee", - "takewhile", - "zip_longest", -) - -import itertools -import operator -import sys -from collections.abc import ( - AsyncGenerator, - AsyncIterable, - AsyncIterator, - Awaitable, - Callable, - Iterable, - Iterator, -) -from dataclasses import dataclass, field -from typing import Any, Generic, TypeVar, cast, overload - -from ._core._synchronization import Lock -from ._core._tasks import CancelScope -from .lowlevel import cancel_shielded_checkpoint, checkpoint, checkpoint_if_cancelled - -T = TypeVar("T") -R = TypeVar("R") -_tee_end = object() - - -@dataclass(eq=False) -class _IterableAsyncIterator(AsyncIterator[T]): - iterator: Iterator[T] - - async def __anext__(self) -> T: - await checkpoint_if_cancelled() - try: - result = next(self.iterator) - except StopIteration: - await cancel_shielded_checkpoint() - raise StopAsyncIteration from None - - await cancel_shielded_checkpoint() - return result - - -def _iterate(iterable: Iterable[T] | AsyncIterable[T]) -> AsyncIterator[T]: - if isinstance(iterable, AsyncIterator): - return iterable - - if isinstance(iterable, AsyncIterable): - return iterable.__aiter__() - - return _IterableAsyncIterator(iter(iterable)) - - -@dataclass(eq=False) -class _TeeLink(Generic[T]): - value: object | None = None - next: _TeeLink[T] | None = None - filled: bool = False - - -@dataclass(eq=False) -class _TeeState(Generic[T]): - iterator: AsyncIterator[T] - lock: Lock = field(default_factory=Lock) - - async def fill(self, link: _TeeLink[T]) -> bool: - if link.filled: - return False - - async with self.lock: - if link.filled: - return True - - link.value = await anext(self.iterator, _tee_end) - if link.value is not _tee_end: - link.next = _TeeLink() - - link.filled = True - return True - - -class _TeeAsyncIterator(AsyncIterator[T]): - _state: _TeeState[T] - _link: _TeeLink[T] - _element_yielded: bool - - def __init__( - self, iterable: Iterable[T] | AsyncIterable[T] | _TeeAsyncIterator[T] - ) -> None: - if isinstance(iterable, _TeeAsyncIterator): - self._state = iterable._state - self._link = iterable._link - else: - self._state = _TeeState(_iterate(iterable)) - self._link = _TeeLink() - - self._element_yielded = False - - async def __anext__(self) -> T: - had_yieldpoint = await self._state.fill(self._link) - if self._link.value is _tee_end: - if not self._element_yielded: - await checkpoint() - - raise StopAsyncIteration - - if not had_yieldpoint: - await checkpoint_if_cancelled() - - self._element_yielded = True - value = cast(T, self._link.value) - next_link = self._link.next - assert next_link is not None - self._link = next_link - if not had_yieldpoint: - await cancel_shielded_checkpoint() - - return value - - -async def _operator_add(x: T, y: T) -> T: - return operator.add(x, y) - - -async def accumulate( - iterable: Iterable[T] | AsyncIterable[T], - function: Callable[[T, T], Awaitable[T]] = _operator_add, - *, - initial: T | None = None, -) -> AsyncGenerator[T, None]: - iterator = _iterate(iterable) - if initial is None: - try: - total = await anext(iterator) - except StopAsyncIteration: - await checkpoint() - return - else: - await checkpoint_if_cancelled() - total = initial - await cancel_shielded_checkpoint() - - yield total - - async for element in iterator: - total = await function(total, element) - yield total - - -async def batched( - iterable: Iterable[T] | AsyncIterable[T], n: int, *, strict: bool = False -) -> AsyncGenerator[tuple[T, ...], None]: - if n < 1: - raise ValueError("n must be at least one") - - iterator = _iterate(iterable) - - while True: - batch: list[T] = [] - for _ in range(n): - try: - batch.append(await anext(iterator)) - except StopAsyncIteration: - if not batch: - await checkpoint() - return - if strict: - raise ValueError("batched(): incomplete batch") from None - - yield tuple(batch) - return - - yield tuple(batch) - - -class Chain: - def __call__( - self, *iterables: Iterable[T] | AsyncIterable[T] - ) -> AsyncGenerator[T, None]: - return self.from_iterable(iterables) - - async def from_iterable( - self, - iterables: ( - Iterable[Iterable[T] | AsyncIterable[T]] - | AsyncIterable[Iterable[T] | AsyncIterable[T]] - ), - ) -> AsyncGenerator[T, None]: - element_yielded = False - outer_iter = _iterate(iterables) - - try: - async for iterable in outer_iter: - async for element in _iterate(iterable): - element_yielded = True - yield element - finally: - aclose = getattr(outer_iter, "aclose", None) - if aclose is not None: - with CancelScope(shield=True): - await aclose() - - if not element_yielded: - await checkpoint() - - -chain: Chain = Chain() - - -async def combinations( - iterable: Iterable[T] | AsyncIterable[T], r: int -) -> AsyncGenerator[tuple[T, ...], None]: - pool: list[T] = [element async for element in _iterate(iterable)] - async for combination in _iterate(itertools.combinations(pool, r)): - yield combination - - -async def combinations_with_replacement( - iterable: Iterable[T] | AsyncIterable[T], r: int -) -> AsyncGenerator[tuple[T, ...], None]: - pool: list[T] = [element async for element in _iterate(iterable)] - async for combination in _iterate(itertools.combinations_with_replacement(pool, r)): - yield combination - - -async def compress( - data: Iterable[T] | AsyncIterable[T], - selectors: Iterable[object] | AsyncIterable[object], -) -> AsyncGenerator[T, None]: - data_iterator = _iterate(data) - selector_iterator = _iterate(selectors) - element_yielded = False - - while True: - try: - datum = await anext(data_iterator) - selector = await anext(selector_iterator) - except StopAsyncIteration: - if not element_yielded: - await checkpoint() - - return - - if selector: - element_yielded = True - yield datum - - -async def count(start: int = 0, step: int = 1) -> AsyncGenerator[int, None]: - n = start - while True: - await checkpoint_if_cancelled() - value = n - n += step - await cancel_shielded_checkpoint() - yield value - - -async def cycle( - iterable: Iterable[T] | AsyncIterable[T], -) -> AsyncGenerator[T, None]: - saved: list[T] = [] - async for element in _iterate(iterable): - saved.append(element) - yield element - - if not saved: - await checkpoint() - return - - while True: - for element in saved: - await checkpoint() - yield element - - -async def dropwhile( - predicate: Callable[[T], Awaitable[object]], - iterable: Iterable[T] | AsyncIterable[T], -) -> AsyncGenerator[T, None]: - element_yielded = False - dropping = True - - async for element in _iterate(iterable): - if dropping and await predicate(element): - continue - - dropping = False - element_yielded = True - yield element - - if not element_yielded: - await checkpoint() - - -async def filterfalse( - predicate: Callable[[T], Awaitable[object]], - iterable: Iterable[T] | AsyncIterable[T], -) -> AsyncGenerator[T, None]: - element_yielded = False - - async for element in _iterate(iterable): - if not await predicate(element): - element_yielded = True - yield element - - if not element_yielded: - await checkpoint() - - -@overload -def groupby( - iterable: Iterable[T] | AsyncIterable[T], -) -> AsyncGenerator[tuple[T, list[T]], None]: ... - - -@overload -def groupby( - iterable: Iterable[T] | AsyncIterable[T], - key: Callable[[T], Awaitable[R]], -) -> AsyncGenerator[tuple[R, list[T]], None]: ... - - -async def groupby( - iterable: Iterable[T] | AsyncIterable[T], - key: Callable[[T], Awaitable[object]] | None = None, -) -> AsyncGenerator[tuple[object, list[T]], None]: - iterator = _iterate(iterable) - try: - element = await anext(iterator) - except StopAsyncIteration: - await checkpoint() - return - - group_key = element if key is None else await key(element) - values = [element] - - async for element in iterator: - next_key = element if key is None else await key(element) - if next_key != group_key: - completed_group = group_key, values - group_key = next_key - values = [element] - yield completed_group - else: - values.append(element) - - yield group_key, values - - -@overload -def islice( - iterable: Iterable[T] | AsyncIterable[T], - stop: int | None, - /, -) -> AsyncGenerator[T, None]: ... - - -@overload -def islice( - iterable: Iterable[T] | AsyncIterable[T], - start: int | None, - stop: int | None, - step: int | None = 1, - /, -) -> AsyncGenerator[T, None]: ... - - -async def islice( - iterable: Iterable[T] | AsyncIterable[T], - *args: int | None, -) -> AsyncGenerator[T, None]: - if not args: - raise TypeError("islice expected at least 2 arguments, got 1") - if len(args) > 3: - raise TypeError(f"islice expected at most 4 arguments, got {len(args) + 1}") - - slice_args = slice(*args) - - start_message = ( - "Indices for islice() must be None or an integer: 0 <= x <= sys.maxsize." - ) - stop_message = ( - "Stop argument for islice() must be None or an integer: 0 <= x <= sys.maxsize." - ) - step_message = "Step for islice() must be a positive integer or None." - - def normalize_index(value: object, message: str) -> int: - try: - index = operator.index(cast(Any, value)) - except TypeError: - raise ValueError(message) from None - - if index < 0 or index > sys.maxsize: - raise ValueError(message) - - return index - - start = ( - 0 - if slice_args.start is None - else normalize_index(slice_args.start, start_message) - ) - stop = ( - None - if slice_args.stop is None - else normalize_index(slice_args.stop, stop_message) - ) - step = ( - 1 if slice_args.step is None else normalize_index(slice_args.step, step_message) - ) - - if step <= 0: - raise ValueError(step_message) - - if stop == 0 or start == stop: - await checkpoint() - return - - iterator = _iterate(iterable) - index = 0 - element_yielded = False - - while stop is None or index < stop: - try: - element = await anext(iterator) - except StopAsyncIteration: - if not element_yielded: - await checkpoint() - - return - - if index >= start and (index - start) % step == 0: - index += 1 - element_yielded = True - yield element - else: - index += 1 - - if not element_yielded: - await checkpoint() - - -async def pairwise( - iterable: Iterable[T] | AsyncIterable[T], -) -> AsyncGenerator[tuple[T, T], None]: - iterator = _iterate(iterable) - try: - previous = await anext(iterator) - except StopAsyncIteration: - await checkpoint() - return - - element_yielded = False - async for element in iterator: - element_yielded = True - pair = (previous, element) - previous = element - yield pair - - if not element_yielded: - await checkpoint() - - -async def permutations( - iterable: Iterable[T] | AsyncIterable[T], r: int | None = None -) -> AsyncGenerator[tuple[T, ...], None]: - pool: list[T] = [element async for element in _iterate(iterable)] - n = len(pool) - if r is None: - r = n - elif not isinstance(r, int): - raise TypeError("Expected int as r") - elif r < 0: - raise ValueError("r must be non-negative") - - async for permutation in _iterate(itertools.permutations(pool, r)): - yield permutation - - -async def product( - *iterables: Iterable[T] | AsyncIterable[T], repeat: int = 1 -) -> AsyncGenerator[tuple[T, ...], None]: - repeat = operator.index(repeat) - if repeat < 0: - raise ValueError("repeat argument cannot be negative") - - pools: list[tuple[T, ...]] = [] - for iterable in iterables: - pool: list[T] = [element async for element in _iterate(iterable)] - pools.append(tuple(pool)) - - async for value in _iterate(itertools.product(*pools, repeat=repeat)): - yield value - - -async def repeat(element: T, times: int | None = None) -> AsyncGenerator[T, None]: - if times is None: - while True: - await checkpoint() - yield element - - remaining = operator.index(cast(Any, times)) - if remaining <= 0: - await checkpoint() - return - - while remaining > 0: - await checkpoint_if_cancelled() - remaining -= 1 - await cancel_shielded_checkpoint() - yield element - - -async def starmap( - function: Callable[..., Awaitable[R]], - iterable: ( - Iterable[Iterable[object] | AsyncIterable[object]] - | AsyncIterable[Iterable[object] | AsyncIterable[object]] - ), -) -> AsyncGenerator[R, None]: - result_yielded = False - - async for args_iterable in _iterate(iterable): - args = [element async for element in _iterate(args_iterable)] - result_yielded = True - yield await function(*args) - - if not result_yielded: - await checkpoint() - - -def tee( - iterable: Iterable[T] | AsyncIterable[T], n: int = 2 -) -> tuple[AsyncIterator[T], ...]: - n = operator.index(cast(Any, n)) - if n < 0: - raise ValueError("n must be >= 0") - if n == 0: - return () - - iterator = _TeeAsyncIterator(iterable) - iterators: list[AsyncIterator[T]] = [iterator] - iterators.extend(_TeeAsyncIterator(iterator) for _ in range(n - 1)) - return tuple(iterators) - - -async def takewhile( - predicate: Callable[[T], Awaitable[object]], - iterable: Iterable[T] | AsyncIterable[T], -) -> AsyncGenerator[T, None]: - element_yielded = False - - async for element in _iterate(iterable): - if not await predicate(element): - if not element_yielded: - await checkpoint() - - return - - element_yielded = True - yield element - - if not element_yielded: - await checkpoint() - - -async def zip_longest( - *iterables: Iterable[object] | AsyncIterable[object], - fillvalue: object = None, -) -> AsyncGenerator[tuple[object, ...], None]: - iterators = [_iterate(iterable) for iterable in iterables] - num_active = len(iterators) - if not num_active: - await checkpoint() - return - - active = [True] * num_active - tuple_yielded = False - - while True: - values: list[object] = [] - for index, iterator in enumerate(iterators): - if not active[index]: - values.append(fillvalue) - continue - - try: - value = await anext(iterator) - except StopAsyncIteration: - active[index] = False - num_active -= 1 - if not num_active: - if not tuple_yielded: - await checkpoint() - - return - - value = fillvalue - - values.append(value) - - tuple_yielded = True - yield tuple(values) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/lowlevel.py b/bundle/python-cpu/Lib/site-packages/anyio/lowlevel.py deleted file mode 100644 index ee111ecc8f22cf54e95bc7df18844e131730e572..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/lowlevel.py +++ /dev/null @@ -1,228 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "EventLoopToken", - "RunvarToken", - "RunVar", - "checkpoint", - "checkpoint_if_cancelled", - "cancel_shielded_checkpoint", - "current_token", -) - -import enum -from dataclasses import dataclass -from types import TracebackType -from typing import TYPE_CHECKING, Any, Generic, Literal, TypeVar, final, overload -from weakref import WeakKeyDictionary - -from ._core._eventloop import get_async_backend - -if TYPE_CHECKING: - from .abc import AsyncBackend - -T = TypeVar("T") -D = TypeVar("D") - - -async def checkpoint() -> None: - """ - Check for cancellation and allow the scheduler to switch to another task. - - Equivalent to (but more efficient than):: - - await checkpoint_if_cancelled() - await cancel_shielded_checkpoint() - - .. versionadded:: 3.0 - - """ - await get_async_backend().checkpoint() - - -async def checkpoint_if_cancelled() -> None: - """ - Enter a checkpoint if the enclosing cancel scope has been cancelled. - - This does not allow the scheduler to switch to a different task. - - .. versionadded:: 3.0 - - """ - await get_async_backend().checkpoint_if_cancelled() - - -async def cancel_shielded_checkpoint() -> None: - """ - Allow the scheduler to switch to another task but without checking for cancellation. - - Equivalent to (but potentially more efficient than):: - - with CancelScope(shield=True): - await checkpoint() - - .. versionadded:: 3.0 - - """ - await get_async_backend().cancel_shielded_checkpoint() - - -@final -@dataclass(frozen=True, repr=False) -class EventLoopToken: - """ - An opaque object that holds a reference to an event loop. - - .. versionadded:: 4.11.0 - """ - - backend_class: type[AsyncBackend] - native_token: object - - -def current_token() -> EventLoopToken: - """ - Return a token object that can be used to call code in the current event loop from - another thread. - - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - .. versionadded:: 4.11.0 - - """ - backend_class = get_async_backend() - raw_token = backend_class.current_token() - return EventLoopToken(backend_class, raw_token) - - -_run_vars: WeakKeyDictionary[object, dict[RunVar[Any], Any]] = WeakKeyDictionary() - - -class _NoValueSet(enum.Enum): - NO_VALUE_SET = enum.auto() - - -class RunvarToken(Generic[T]): - """ - A token that can be used to restore a :class:`RunVar` to its previous value. - - Returned by :meth:`RunVar.set`. Can be used as a context manager to automatically - reset the variable on exit, or passed directly to :meth:`RunVar.reset`. - """ - - __slots__ = "_var", "_value", "_redeemed" - - def __init__(self, var: RunVar[T], value: T | Literal[_NoValueSet.NO_VALUE_SET]): - self._var = var - self._value: T | Literal[_NoValueSet.NO_VALUE_SET] = value - self._redeemed = False - - def __enter__(self) -> RunvarToken[T]: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self._var.reset(self) - - -class RunVar(Generic[T]): - """ - Like a :class:`~contextvars.ContextVar`, except scoped to the running event loop. - - Can be used as a context manager, Just like :class:`~contextvars.ContextVar`, that - will reset the variable to its previous value when the context block is exited. - """ - - __slots__ = "_name", "_default" - - NO_VALUE_SET: Literal[_NoValueSet.NO_VALUE_SET] = _NoValueSet.NO_VALUE_SET - - def __init__( - self, name: str, default: T | Literal[_NoValueSet.NO_VALUE_SET] = NO_VALUE_SET - ): - self._name = name - self._default = default - - @property - def _current_vars(self) -> dict[RunVar[T], T]: - native_token = current_token().native_token - try: - return _run_vars[native_token] - except KeyError: - run_vars = _run_vars[native_token] = {} - return run_vars - - @overload - def get(self, default: D) -> T | D: ... - - @overload - def get(self) -> T: ... - - def get( - self, default: D | Literal[_NoValueSet.NO_VALUE_SET] = NO_VALUE_SET - ) -> T | D: - """ - Return the current value of this run variable. - - :param default: a fallback value to return if no value has been set - :return: the current value, the provided default, or the variable's own default - :raises LookupError: if no value is set and no default is available - - """ - try: - return self._current_vars[self] - except KeyError: - if default is not RunVar.NO_VALUE_SET: - return default - elif self._default is not RunVar.NO_VALUE_SET: - return self._default - - raise LookupError( - f'Run variable "{self._name}" has no value and no default set' - ) - - def set(self, value: T) -> RunvarToken[T]: - """ - Set the value of this run variable for the current event loop. - - :param value: the new value - :return: a token that can be used to restore the previous value - - """ - current_vars = self._current_vars - token = RunvarToken(self, current_vars.get(self, RunVar.NO_VALUE_SET)) - current_vars[self] = value - return token - - def reset(self, token: RunvarToken[T]) -> None: - """ - Restore this run variable to the value it held before the matching :meth:`set`. - - :param token: the token returned by :meth:`set` - :raises ValueError: if the token belongs to a different :class:`RunVar` or the token - has already been used - - """ - if token._var is not self: - raise ValueError("This token does not belong to this RunVar") - - if token._redeemed: - raise ValueError("This token has already been used") - - if token._value is _NoValueSet.NO_VALUE_SET: - try: - del self._current_vars[self] - except KeyError: - pass - else: - self._current_vars[self] = token._value - - token._redeemed = True - - def __repr__(self) -> str: - return f"" diff --git a/bundle/python-cpu/Lib/site-packages/anyio/py.typed b/bundle/python-cpu/Lib/site-packages/anyio/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/anyio/pytest_plugin.py b/bundle/python-cpu/Lib/site-packages/anyio/pytest_plugin.py deleted file mode 100644 index 5c667597d02f268b8f5427d8f80957116992c6ee..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/pytest_plugin.py +++ /dev/null @@ -1,375 +0,0 @@ -from __future__ import annotations - -import dataclasses -import socket -import sys -from collections.abc import Callable, Generator, Iterator -from contextlib import ExitStack, contextmanager -from inspect import isasyncgenfunction, iscoroutinefunction, ismethod -from typing import Any, cast - -import pytest -from _pytest.fixtures import FuncFixtureInfo, SubRequest -from _pytest.outcomes import Exit -from _pytest.python import CallSpec2 -from _pytest.scope import Scope - -from . import get_available_backends -from ._core._eventloop import ( - current_async_library, - get_async_backend, - reset_current_async_library, - set_current_async_library, -) -from ._core._exceptions import iterate_exceptions -from .abc import TestRunner - -if sys.version_info < (3, 11): - from exceptiongroup import ExceptionGroup - -_current_runner: TestRunner | None = None -_runner_stack: ExitStack | None = None -_runner_leases = 0 - - -def extract_backend_and_options(backend: object) -> tuple[str, dict[str, Any]]: - if isinstance(backend, str): - return backend, {} - elif isinstance(backend, tuple) and len(backend) == 2: - if isinstance(backend[0], str) and isinstance(backend[1], dict): - return cast(tuple[str, dict[str, Any]], backend) - - raise TypeError("anyio_backend must be either a string or tuple of (string, dict)") - - -@contextmanager -def get_runner( - backend_name: str, backend_options: dict[str, Any] -) -> Iterator[TestRunner]: - global _current_runner, _runner_leases, _runner_stack - if _current_runner is None: - asynclib = get_async_backend(backend_name) - _runner_stack = ExitStack() - if current_async_library() is None: - # Since we're in control of the event loop, we can cache the name of the - # async library - token = set_current_async_library(backend_name) - _runner_stack.callback(reset_current_async_library, token) - - backend_options = backend_options or {} - _current_runner = _runner_stack.enter_context( - asynclib.create_test_runner(backend_options) - ) - - _runner_leases += 1 - try: - yield _current_runner - finally: - _runner_leases -= 1 - if not _runner_leases: - assert _runner_stack is not None - _runner_stack.close() - _runner_stack = _current_runner = None - - -def pytest_addoption(parser: pytest.Parser) -> None: - parser.addini( - "anyio_mode", - default="strict", - help='AnyIO plugin mode (either "strict" or "auto")', - ) - - -def pytest_configure(config: pytest.Config) -> None: - config.addinivalue_line( - "markers", - "anyio: mark the (coroutine function) test to be run asynchronously via anyio.", - ) - if ( - config.getini("anyio_mode") == "auto" - and config.pluginmanager.has_plugin("asyncio") - and config.getini("asyncio_mode") == "auto" - ): - config.issue_config_time_warning( - pytest.PytestConfigWarning( - "AnyIO auto mode has been enabled together with pytest-asyncio auto " - "mode. This may cause unexpected behavior." - ), - 1, - ) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_fixture_setup(fixturedef: Any, request: Any) -> Generator[Any]: - def wrapper(anyio_backend: Any, request: SubRequest, **kwargs: Any) -> Any: - # Rebind any fixture methods to the request instance - if ( - request.instance - and ismethod(func) - and type(func.__self__) is type(request.instance) - ): - local_func = func.__func__.__get__(request.instance) - else: - local_func = func - - backend_name, backend_options = extract_backend_and_options(anyio_backend) - if has_backend_arg: - kwargs["anyio_backend"] = anyio_backend - - if has_request_arg: - kwargs["request"] = request - - with get_runner(backend_name, backend_options) as runner: - # re-entrant call into the test runner detected. this happens when an async fixture - # is dynamically requested via request.getfixturevalue() from inside a running async - # test or fixture. on asyncio this raises RuntimeError: This event loop is already - # running, on trio the runner deadlocks - the host loop blocks waiting for the - # coroutine to return, but the coroutine is waiting for the host loop. raising here - # prevents the hang and gives a consistent error across backends. - if runner.is_running(): - raise RuntimeError( - "Cannot schedule a coroutine in the test runner while another is already running; " - "likely caused by request.getfixturevalue() on an async fixture." - ) - - if isasyncgenfunction(local_func): - yield from runner.run_asyncgen_fixture(local_func, kwargs) - else: - yield runner.run_fixture(local_func, kwargs) - - # Only apply this to coroutine functions and async generator functions in requests - # that involve the anyio_backend fixture - func = fixturedef.func - if isasyncgenfunction(func) or iscoroutinefunction(func): - if "anyio_backend" in request.fixturenames: - fixturedef.func = wrapper - original_argname = fixturedef.argnames - - if not (has_backend_arg := "anyio_backend" in fixturedef.argnames): - fixturedef.argnames += ("anyio_backend",) - - if not (has_request_arg := "request" in fixturedef.argnames): - fixturedef.argnames += ("request",) - - try: - return (yield) - finally: - fixturedef.func = func - fixturedef.argnames = original_argname - - return (yield) - - -@pytest.hookimpl(tryfirst=True) -def pytest_pycollect_makeitem( - collector: pytest.Module | pytest.Class, name: str, obj: object -) -> None: - if collector.istestfunction(obj, name): - inner_func = obj.hypothesis.inner_test if hasattr(obj, "hypothesis") else obj - if iscoroutinefunction(inner_func): - anyio_auto_mode = collector.config.getini("anyio_mode") == "auto" - marker = collector.get_closest_marker("anyio") - own_markers = getattr(obj, "pytestmark", ()) - if ( - anyio_auto_mode - or marker - or any(marker.name == "anyio" for marker in own_markers) - ): - pytest.mark.usefixtures("anyio_backend")(obj) - - -def pytest_collection_finish(session: pytest.Session) -> None: - for i, item in reversed(list(enumerate(session.items))): - if ( - isinstance(item, pytest.Function) - and iscoroutinefunction(item.function) - and item.get_closest_marker("anyio") is not None - and "anyio_backend" not in item.fixturenames - ): - new_items = [] - try: - cs_fields = {f.name for f in dataclasses.fields(CallSpec2)} - except TypeError: - cs_fields = set() - - for param_index, backend in enumerate(get_available_backends()): - if "_arg2scope" in cs_fields: # pytest >= 8 - callspec = CallSpec2( - params={"anyio_backend": backend}, - indices={"anyio_backend": param_index}, - _arg2scope={"anyio_backend": Scope.Module}, - _idlist=[backend], - marks=[], - ) - else: # pytest 7.x - callspec = CallSpec2( # type: ignore[call-arg] - funcargs={}, - params={"anyio_backend": backend}, - indices={"anyio_backend": param_index}, - arg2scope={"anyio_backend": Scope.Module}, - idlist=[backend], - marks=[], - ) - - fi = item._fixtureinfo - new_names_closure = list(fi.names_closure) - if "anyio_backend" not in new_names_closure: - new_names_closure.append("anyio_backend") - - new_fixtureinfo = FuncFixtureInfo( - argnames=fi.argnames, - initialnames=fi.initialnames, - names_closure=new_names_closure, - name2fixturedefs=fi.name2fixturedefs, - ) - new_item = pytest.Function.from_parent( - item.parent, - name=f"{item.originalname}[{backend}]", - callspec=callspec, - callobj=item.obj, - fixtureinfo=new_fixtureinfo, - keywords=item.keywords, - originalname=item.originalname, - ) - new_items.append(new_item) - - session.items[i : i + 1] = new_items - - -@pytest.hookimpl(tryfirst=True) -def pytest_pyfunc_call(pyfuncitem: Any) -> bool | None: - def run_with_hypothesis(**kwargs: Any) -> None: - with get_runner(backend_name, backend_options) as runner: - runner.run_test(original_func, kwargs) - - backend = pyfuncitem.funcargs.get("anyio_backend") - if backend: - backend_name, backend_options = extract_backend_and_options(backend) - - if hasattr(pyfuncitem.obj, "hypothesis"): - # Wrap the inner test function unless it's already wrapped - original_func = pyfuncitem.obj.hypothesis.inner_test - if original_func.__qualname__ != run_with_hypothesis.__qualname__: - if iscoroutinefunction(original_func): - pyfuncitem.obj.hypothesis.inner_test = run_with_hypothesis - - return None - - if iscoroutinefunction(pyfuncitem.obj): - funcargs = pyfuncitem.funcargs - testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} - with get_runner(backend_name, backend_options) as runner: - try: - runner.run_test(pyfuncitem.obj, testargs) - except ExceptionGroup as excgrp: - for exc in iterate_exceptions(excgrp): - if isinstance(exc, (Exit, KeyboardInterrupt, SystemExit)): - raise exc from excgrp - - raise - - return True - - return None - - -@pytest.fixture(scope="module", params=get_available_backends()) -def anyio_backend(request: Any) -> Any: - return request.param - - -@pytest.fixture -def anyio_backend_name(anyio_backend: Any) -> str: - if isinstance(anyio_backend, str): - return anyio_backend - else: - return anyio_backend[0] - - -@pytest.fixture -def anyio_backend_options(anyio_backend: Any) -> dict[str, Any]: - if isinstance(anyio_backend, str): - return {} - else: - return anyio_backend[1] - - -class FreePortFactory: - """ - Manages port generation based on specified socket kind, ensuring no duplicate - ports are generated. - - This class provides functionality for generating available free ports on the - system. It is initialized with a specific socket kind and can generate ports - for given address families while avoiding reuse of previously generated ports. - - Users should not instantiate this class directly, but use the - ``free_tcp_port_factory`` and ``free_udp_port_factory`` fixtures instead. For simple - uses cases, ``free_tcp_port`` and ``free_udp_port`` can be used instead. - """ - - def __init__(self, kind: socket.SocketKind) -> None: - self._kind = kind - self._generated = set[int]() - - @property - def kind(self) -> socket.SocketKind: - """ - The type of socket connection (e.g., :data:`~socket.SOCK_STREAM` or - :data:`~socket.SOCK_DGRAM`) used to bind for checking port availability - - """ - return self._kind - - def __call__(self, family: socket.AddressFamily | None = None) -> int: - """ - Return an unbound port for the given address family. - - :param family: if omitted, both IPv4 and IPv6 addresses will be tried - :return: a port number - - """ - if family is not None: - families = [family] - else: - families = [socket.AF_INET] - if socket.has_ipv6: - families.append(socket.AF_INET6) - - while True: - port = 0 - with ExitStack() as stack: - for family in families: - sock = stack.enter_context(socket.socket(family, self._kind)) - addr = "::1" if family == socket.AF_INET6 else "127.0.0.1" - try: - sock.bind((addr, port)) - except OSError: - break - - if not port: - port = sock.getsockname()[1] - else: - if port not in self._generated: - self._generated.add(port) - return port - - -@pytest.fixture(scope="session") -def free_tcp_port_factory() -> FreePortFactory: - return FreePortFactory(socket.SOCK_STREAM) - - -@pytest.fixture(scope="session") -def free_udp_port_factory() -> FreePortFactory: - return FreePortFactory(socket.SOCK_DGRAM) - - -@pytest.fixture -def free_tcp_port(free_tcp_port_factory: Callable[[], int]) -> int: - return free_tcp_port_factory() - - -@pytest.fixture -def free_udp_port(free_udp_port_factory: Callable[[], int]) -> int: - return free_udp_port_factory() diff --git a/bundle/python-cpu/Lib/site-packages/anyio/streams/__init__.py b/bundle/python-cpu/Lib/site-packages/anyio/streams/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/anyio/streams/buffered.py b/bundle/python-cpu/Lib/site-packages/anyio/streams/buffered.py deleted file mode 100644 index a3b07f73ac44d5876c239fd0317846564ef06f91..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/streams/buffered.py +++ /dev/null @@ -1,201 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "BufferedByteReceiveStream", - "BufferedByteStream", - "BufferedConnectable", -) - -import sys -from collections.abc import Callable, Iterable, Mapping -from dataclasses import dataclass, field -from typing import Any, SupportsIndex - -from .. import ClosedResourceError, DelimiterNotFound, EndOfStream, IncompleteRead -from ..abc import ( - AnyByteReceiveStream, - AnyByteStream, - AnyByteStreamConnectable, - ByteReceiveStream, - ByteStream, - ByteStreamConnectable, -) - -if sys.version_info >= (3, 12): - from typing import override -else: - from typing_extensions import override - - -@dataclass(eq=False) -class BufferedByteReceiveStream(ByteReceiveStream): - """ - Wraps any bytes-based receive stream and uses a buffer to provide sophisticated - receiving capabilities in the form of a byte stream. - """ - - receive_stream: AnyByteReceiveStream - _buffer: bytearray = field(init=False, default_factory=bytearray) - _closed: bool = field(init=False, default=False) - - async def aclose(self) -> None: - await self.receive_stream.aclose() - self._closed = True - - @property - def buffer(self) -> bytes: - """The bytes currently in the buffer.""" - return bytes(self._buffer) - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return self.receive_stream.extra_attributes - - def feed_data(self, data: Iterable[SupportsIndex], /) -> None: - """ - Append data directly into the buffer. - - Any data in the buffer will be consumed by receive operations before receiving - anything from the wrapped stream. - - :param data: the data to append to the buffer (can be bytes or anything else - that supports ``__index__()``) - - """ - self._buffer.extend(data) - - async def receive(self, max_bytes: int = 65536) -> bytes: - if max_bytes < 1: - raise ValueError("max_bytes must be a positive integer") - - if self._closed: - raise ClosedResourceError - - if self._buffer: - chunk = bytes(self._buffer[:max_bytes]) - del self._buffer[:max_bytes] - return chunk - elif isinstance(self.receive_stream, ByteReceiveStream): - return await self.receive_stream.receive(max_bytes) - else: - # With a bytes-oriented object stream, we need to handle any surplus bytes - # we get from the receive() call - chunk = await self.receive_stream.receive() - if len(chunk) > max_bytes: - # Save the surplus bytes in the buffer - self._buffer.extend(chunk[max_bytes:]) - return chunk[:max_bytes] - else: - return chunk - - async def receive_exactly(self, nbytes: int) -> bytes: - """ - Read exactly the given amount of bytes from the stream. - - :param nbytes: the number of bytes to read - :return: the bytes read - :raises ~anyio.IncompleteRead: if the stream was closed before the requested - amount of bytes could be read from the stream - - """ - while True: - remaining = nbytes - len(self._buffer) - if remaining <= 0: - retval = self._buffer[:nbytes] - del self._buffer[:nbytes] - return bytes(retval) - - try: - if isinstance(self.receive_stream, ByteReceiveStream): - chunk = await self.receive_stream.receive(remaining) - else: - chunk = await self.receive_stream.receive() - except EndOfStream as exc: - raise IncompleteRead from exc - - self._buffer.extend(chunk) - - async def receive_until(self, delimiter: bytes, max_bytes: int) -> bytes: - """ - Read from the stream until the delimiter is found or max_bytes have been read. - - :param delimiter: the marker to look for in the stream - :param max_bytes: maximum number of bytes that will be read before raising - :exc:`~anyio.DelimiterNotFound` - :return: the bytes read (not including the delimiter) - :raises ~anyio.IncompleteRead: if the stream was closed before the delimiter - was found - :raises ~anyio.DelimiterNotFound: if the delimiter is not found within the - bytes read up to the maximum allowed - - """ - delimiter_size = len(delimiter) - offset = 0 - while True: - # Check if the delimiter can be found in the current buffer - index = self._buffer.find(delimiter, offset) - if index >= 0: - found = self._buffer[:index] - del self._buffer[: index + len(delimiter) :] - return bytes(found) - - # Check if the buffer is already at or over the limit - if len(self._buffer) >= max_bytes: - raise DelimiterNotFound(max_bytes) - - # Read more data into the buffer from the socket - try: - data = await self.receive_stream.receive() - except EndOfStream as exc: - raise IncompleteRead from exc - - # Move the offset forward and add the new data to the buffer - offset = max(len(self._buffer) - delimiter_size + 1, 0) - self._buffer.extend(data) - - -class BufferedByteStream(BufferedByteReceiveStream, ByteStream): - """ - A full-duplex variant of :class:`BufferedByteReceiveStream`. All writes are passed - through to the wrapped stream as-is. - """ - - def __init__(self, stream: AnyByteStream): - """ - :param stream: the stream to be wrapped - - """ - super().__init__(stream) - self._stream = stream - - @override - async def send_eof(self) -> None: - await self._stream.send_eof() - - @override - async def send(self, item: bytes) -> None: - await self._stream.send(item) - - -class BufferedConnectable(ByteStreamConnectable): - """ - Wraps a byte stream connectable to produce :class:`BufferedByteStream` connections. - - Use this when you want the streams returned by :meth:`connect` to have the buffered - receive API (e.g. :meth:`~BufferedByteReceiveStream.receive_exactly` and - :meth:`~BufferedByteReceiveStream.receive_until`). - - :param connectable: the byte stream connectable to wrap - """ - - def __init__(self, connectable: AnyByteStreamConnectable): - """ - :param connectable: the connectable to wrap - - """ - self.connectable = connectable - - @override - async def connect(self) -> BufferedByteStream: - stream = await self.connectable.connect() - return BufferedByteStream(stream) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/streams/file.py b/bundle/python-cpu/Lib/site-packages/anyio/streams/file.py deleted file mode 100644 index b9d7ca4f3341a7ad7e44553e4e15134f223cb750..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/streams/file.py +++ /dev/null @@ -1,157 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "FileReadStream", - "FileStreamAttribute", - "FileWriteStream", -) - -from collections.abc import Callable, Mapping -from io import SEEK_SET, UnsupportedOperation -from os import PathLike -from pathlib import Path -from typing import IO, Any - -from .. import ( - BrokenResourceError, - ClosedResourceError, - EndOfStream, - TypedAttributeSet, - to_thread, - typed_attribute, -) -from ..abc import ByteReceiveStream, ByteSendStream - - -class FileStreamAttribute(TypedAttributeSet): - #: the open file descriptor - file: IO[bytes] = typed_attribute() - #: the path of the file on the file system, if available (file must be a real file) - path: Path = typed_attribute() - #: the file number, if available (file must be a real file or a TTY) - fileno: int = typed_attribute() - - -class _BaseFileStream: - def __init__(self, file: IO[bytes]): - self._file = file - - async def aclose(self) -> None: - await to_thread.run_sync(self._file.close) - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - attributes: dict[Any, Callable[[], Any]] = { - FileStreamAttribute.file: lambda: self._file, - } - - if hasattr(self._file, "name"): - attributes[FileStreamAttribute.path] = lambda: Path(self._file.name) - - try: - self._file.fileno() - except UnsupportedOperation: - pass - else: - attributes[FileStreamAttribute.fileno] = lambda: self._file.fileno() - - return attributes - - -class FileReadStream(_BaseFileStream, ByteReceiveStream): - """ - A byte stream that reads from a file in the file system. - - :param file: a file that has been opened for reading in binary mode - - .. versionadded:: 3.0 - """ - - @classmethod - async def from_path(cls, path: str | PathLike[str]) -> FileReadStream: - """ - Create a file read stream by opening the given file. - - :param path: path of the file to read from - - """ - file = await to_thread.run_sync(Path(path).open, "rb") - return cls(file) - - async def receive(self, max_bytes: int = 65536) -> bytes: - if max_bytes < 1: - raise ValueError("max_bytes must be a positive integer") - - try: - data = await to_thread.run_sync(self._file.read, max_bytes) - except ValueError: - raise ClosedResourceError from None - except OSError as exc: - raise BrokenResourceError from exc - - if data: - return data - else: - raise EndOfStream - - async def seek(self, position: int, whence: int = SEEK_SET) -> int: - """ - Seek the file to the given position. - - .. seealso:: :meth:`io.IOBase.seek` - - .. note:: Not all file descriptors are seekable. - - :param position: position to seek the file to - :param whence: controls how ``position`` is interpreted - :return: the new absolute position - :raises OSError: if the file is not seekable - - """ - return await to_thread.run_sync(self._file.seek, position, whence) - - async def tell(self) -> int: - """ - Return the current stream position. - - .. note:: Not all file descriptors are seekable. - - :return: the current absolute position - :raises OSError: if the file is not seekable - - """ - return await to_thread.run_sync(self._file.tell) - - -class FileWriteStream(_BaseFileStream, ByteSendStream): - """ - A byte stream that writes to a file in the file system. - - :param file: a file that has been opened for writing in binary mode - - .. versionadded:: 3.0 - """ - - @classmethod - async def from_path( - cls, path: str | PathLike[str], append: bool = False - ) -> FileWriteStream: - """ - Create a file write stream by opening the given file for writing. - - :param path: path of the file to write to - :param append: if ``True``, open the file for appending; if ``False``, any - existing file at the given path will be truncated - - """ - mode = "ab" if append else "wb" - file = await to_thread.run_sync(Path(path).open, mode) - return cls(file) - - async def send(self, item: bytes) -> None: - try: - await to_thread.run_sync(self._file.write, item) - except ValueError: - raise ClosedResourceError from None - except OSError as exc: - raise BrokenResourceError from exc diff --git a/bundle/python-cpu/Lib/site-packages/anyio/streams/memory.py b/bundle/python-cpu/Lib/site-packages/anyio/streams/memory.py deleted file mode 100644 index d8c205b82636d40064445e68748dc54682ff3e97..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/streams/memory.py +++ /dev/null @@ -1,326 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "MemoryObjectReceiveStream", - "MemoryObjectSendStream", - "MemoryObjectStreamStatistics", -) - -import warnings -from collections import OrderedDict, deque -from dataclasses import dataclass, field -from types import TracebackType -from typing import Generic, NamedTuple, TypeVar - -from .. import ( - BrokenResourceError, - ClosedResourceError, - EndOfStream, - WouldBlock, -) -from .._core._synchronization import Event -from .._core._testing import TaskInfo, get_current_task -from ..abc import ObjectReceiveStream, ObjectSendStream -from ..lowlevel import checkpoint - -T_Item = TypeVar("T_Item") -T_co = TypeVar("T_co", covariant=True) -T_contra = TypeVar("T_contra", contravariant=True) - - -class MemoryObjectStreamStatistics(NamedTuple): - current_buffer_used: int #: number of items stored in the buffer - #: maximum number of items that can be stored on this stream (or :data:`math.inf`) - max_buffer_size: float - open_send_streams: int #: number of unclosed clones of the send stream - open_receive_streams: int #: number of unclosed clones of the receive stream - #: number of tasks blocked on :meth:`MemoryObjectSendStream.send` - tasks_waiting_send: int - #: number of tasks blocked on :meth:`MemoryObjectReceiveStream.receive` - tasks_waiting_receive: int - - -@dataclass(eq=False) -class _MemoryObjectItemReceiver(Generic[T_Item]): - task_info: TaskInfo = field(init=False, default_factory=get_current_task) - item: T_Item = field(init=False) - - def __repr__(self) -> str: - # When item is not defined, we get following error with default __repr__: - # AttributeError: 'MemoryObjectItemReceiver' object has no attribute 'item' - item = getattr(self, "item", None) - return f"{self.__class__.__name__}(task_info={self.task_info}, item={item!r})" - - -@dataclass(eq=False) -class _MemoryObjectStreamState(Generic[T_Item]): - max_buffer_size: float = field() - buffer: deque[T_Item] = field(init=False, default_factory=deque) - open_send_channels: int = field(init=False, default=0) - open_receive_channels: int = field(init=False, default=0) - waiting_receivers: OrderedDict[Event, _MemoryObjectItemReceiver[T_Item]] = field( - init=False, default_factory=OrderedDict - ) - waiting_senders: OrderedDict[Event, T_Item] = field( - init=False, default_factory=OrderedDict - ) - - def statistics(self) -> MemoryObjectStreamStatistics: - return MemoryObjectStreamStatistics( - len(self.buffer), - self.max_buffer_size, - self.open_send_channels, - self.open_receive_channels, - len(self.waiting_senders), - len(self.waiting_receivers), - ) - - -@dataclass(eq=False) -class MemoryObjectReceiveStream(Generic[T_co], ObjectReceiveStream[T_co]): - _state: _MemoryObjectStreamState[T_co] - _closed: bool = field(init=False, default=False) - - def __post_init__(self) -> None: - self._state.open_receive_channels += 1 - - def receive_nowait(self) -> T_co: - """ - Receive the next item if it can be done without waiting. - - :return: the received item - :raises ~anyio.ClosedResourceError: if this send stream has been closed - :raises ~anyio.EndOfStream: if the buffer is empty and this stream has been - closed from the sending end - :raises ~anyio.WouldBlock: if there are no items in the buffer and no tasks - waiting to send - - """ - if self._closed: - raise ClosedResourceError - - if self._state.waiting_senders: - # Get the item from the next sender - send_event, item = self._state.waiting_senders.popitem(last=False) - self._state.buffer.append(item) - send_event.set() - - if self._state.buffer: - return self._state.buffer.popleft() - elif not self._state.open_send_channels: - raise EndOfStream - - raise WouldBlock - - async def receive(self) -> T_co: - await checkpoint() - try: - return self.receive_nowait() - except WouldBlock: - # Add ourselves in the queue - receive_event = Event() - receiver = _MemoryObjectItemReceiver[T_co]() - self._state.waiting_receivers[receive_event] = receiver - - try: - await receive_event.wait() - finally: - self._state.waiting_receivers.pop(receive_event, None) - - try: - return receiver.item - except AttributeError: - raise EndOfStream from None - - def clone(self) -> MemoryObjectReceiveStream[T_co]: - """ - Create a clone of this receive stream. - - Each clone can be closed separately. Only when all clones have been closed will - the receiving end of the memory stream be considered closed by the sending ends. - - :return: the cloned stream - - """ - if self._closed: - raise ClosedResourceError - - return MemoryObjectReceiveStream(_state=self._state) - - def close(self) -> None: - """ - Close the stream. - - This works the exact same way as :meth:`aclose`, but is provided as a special - case for the benefit of synchronous callbacks. - - """ - if not self._closed: - self._closed = True - self._state.open_receive_channels -= 1 - if self._state.open_receive_channels == 0: - send_events = list(self._state.waiting_senders.keys()) - for event in send_events: - event.set() - - async def aclose(self) -> None: - self.close() - - def statistics(self) -> MemoryObjectStreamStatistics: - """ - Return statistics about the current state of this stream. - - .. versionadded:: 3.0 - """ - return self._state.statistics() - - def __enter__(self) -> MemoryObjectReceiveStream[T_co]: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.close() - - def __del__(self) -> None: - if not self._closed: - warnings.warn( - f"Unclosed <{self.__class__.__name__} at {id(self):x}>", - ResourceWarning, - stacklevel=1, - source=self, - ) - - -@dataclass(eq=False) -class MemoryObjectSendStream(Generic[T_contra], ObjectSendStream[T_contra]): - _state: _MemoryObjectStreamState[T_contra] - _closed: bool = field(init=False, default=False) - - def __post_init__(self) -> None: - self._state.open_send_channels += 1 - - def send_nowait(self, item: T_contra) -> None: - """ - Send an item immediately if it can be done without waiting. - - :param item: the item to send - :raises ~anyio.ClosedResourceError: if this send stream has been closed - :raises ~anyio.BrokenResourceError: if the stream has been closed from the - receiving end - :raises ~anyio.WouldBlock: if the buffer is full and there are no tasks waiting - to receive - - """ - if self._closed: - raise ClosedResourceError - if not self._state.open_receive_channels: - raise BrokenResourceError - - while self._state.waiting_receivers: - receive_event, receiver = self._state.waiting_receivers.popitem(last=False) - if not receiver.task_info.has_pending_cancellation(): - receiver.item = item - receive_event.set() - return - - if len(self._state.buffer) < self._state.max_buffer_size: - self._state.buffer.append(item) - else: - raise WouldBlock - - async def send(self, item: T_contra) -> None: - """ - Send an item to the stream. - - If the buffer is full, this method blocks until there is again room in the - buffer or the item can be sent directly to a receiver. - - :param item: the item to send - :raises ~anyio.ClosedResourceError: if this send stream has been closed - :raises ~anyio.BrokenResourceError: if the stream has been closed from the - receiving end - - """ - await checkpoint() - try: - self.send_nowait(item) - except WouldBlock: - # Wait until there's someone on the receiving end - send_event = Event() - self._state.waiting_senders[send_event] = item - try: - await send_event.wait() - except BaseException: - self._state.waiting_senders.pop(send_event, None) - raise - - if send_event in self._state.waiting_senders: - del self._state.waiting_senders[send_event] - raise BrokenResourceError from None - - def clone(self) -> MemoryObjectSendStream[T_contra]: - """ - Create a clone of this send stream. - - Each clone can be closed separately. Only when all clones have been closed will - the sending end of the memory stream be considered closed by the receiving ends. - - :return: the cloned stream - - """ - if self._closed: - raise ClosedResourceError - - return MemoryObjectSendStream(_state=self._state) - - def close(self) -> None: - """ - Close the stream. - - This works the exact same way as :meth:`aclose`, but is provided as a special - case for the benefit of synchronous callbacks. - - """ - if not self._closed: - self._closed = True - self._state.open_send_channels -= 1 - if self._state.open_send_channels == 0: - receive_events = list(self._state.waiting_receivers.keys()) - self._state.waiting_receivers.clear() - for event in receive_events: - event.set() - - async def aclose(self) -> None: - self.close() - - def statistics(self) -> MemoryObjectStreamStatistics: - """ - Return statistics about the current state of this stream. - - .. versionadded:: 3.0 - """ - return self._state.statistics() - - def __enter__(self) -> MemoryObjectSendStream[T_contra]: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.close() - - def __del__(self) -> None: - if not self._closed: - warnings.warn( - f"Unclosed <{self.__class__.__name__} at {id(self):x}>", - ResourceWarning, - stacklevel=1, - source=self, - ) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/streams/stapled.py b/bundle/python-cpu/Lib/site-packages/anyio/streams/stapled.py deleted file mode 100644 index 0a3c53da25ab33e502ac047b619b3f5ebb3c93d3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/streams/stapled.py +++ /dev/null @@ -1,150 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "MultiListener", - "StapledByteStream", - "StapledObjectStream", -) - -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass -from typing import Any, Generic, TypeVar - -from ..abc import ( - ByteReceiveStream, - ByteSendStream, - ByteStream, - Listener, - ObjectReceiveStream, - ObjectSendStream, - ObjectStream, - TaskGroup, -) - -T_Item = TypeVar("T_Item") -T_Stream = TypeVar("T_Stream") - - -@dataclass(eq=False) -class StapledByteStream(ByteStream): - """ - Combines two byte streams into a single, bidirectional byte stream. - - Extra attributes will be provided from both streams, with the receive stream - providing the values in case of a conflict. - - :param ByteSendStream send_stream: the sending byte stream - :param ByteReceiveStream receive_stream: the receiving byte stream - """ - - send_stream: ByteSendStream - receive_stream: ByteReceiveStream - - async def receive(self, max_bytes: int = 65536) -> bytes: - if max_bytes < 1: - raise ValueError("max_bytes must be a positive integer") - - return await self.receive_stream.receive(max_bytes) - - async def send(self, item: bytes) -> None: - await self.send_stream.send(item) - - async def send_eof(self) -> None: - await self.send_stream.aclose() - - async def aclose(self) -> None: - await self.send_stream.aclose() - await self.receive_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self.send_stream.extra_attributes, - **self.receive_stream.extra_attributes, - } - - -@dataclass(eq=False) -class StapledObjectStream(Generic[T_Item], ObjectStream[T_Item]): - """ - Combines two object streams into a single, bidirectional object stream. - - Extra attributes will be provided from both streams, with the receive stream - providing the values in case of a conflict. - - :param ObjectSendStream send_stream: the sending object stream - :param ObjectReceiveStream receive_stream: the receiving object stream - """ - - send_stream: ObjectSendStream[T_Item] - receive_stream: ObjectReceiveStream[T_Item] - - async def receive(self) -> T_Item: - return await self.receive_stream.receive() - - async def send(self, item: T_Item) -> None: - await self.send_stream.send(item) - - async def send_eof(self) -> None: - await self.send_stream.aclose() - - async def aclose(self) -> None: - await self.send_stream.aclose() - await self.receive_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self.send_stream.extra_attributes, - **self.receive_stream.extra_attributes, - } - - -@dataclass(eq=False) -class MultiListener(Generic[T_Stream], Listener[T_Stream]): - """ - Combines multiple listeners into one, serving connections from all of them at once. - - Any MultiListeners in the given collection of listeners will have their listeners - moved into this one. - - Extra attributes are provided from each listener, with each successive listener - overriding any conflicting attributes from the previous one. - - :param listeners: listeners to serve - :type listeners: Sequence[Listener[T_Stream]] - """ - - listeners: Sequence[Listener[T_Stream]] - - def __post_init__(self) -> None: - listeners: list[Listener[T_Stream]] = [] - for listener in self.listeners: - if isinstance(listener, MultiListener): - listeners.extend(listener.listeners) - del listener.listeners[:] # type: ignore[attr-defined] - else: - listeners.append(listener) - - self.listeners = listeners - - async def serve( - self, handler: Callable[[T_Stream], Any], task_group: TaskGroup | None = None - ) -> None: - from .. import create_task_group - - async with create_task_group() as tg: - for listener in self.listeners: - tg.start_soon(listener.serve, handler, task_group) - - async def aclose(self) -> None: - for listener in self.listeners: - await listener.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - attributes: dict = {} - for listener in self.listeners: - attributes.update(listener.extra_attributes) - - return attributes diff --git a/bundle/python-cpu/Lib/site-packages/anyio/streams/text.py b/bundle/python-cpu/Lib/site-packages/anyio/streams/text.py deleted file mode 100644 index 296cd250459f3848bb333301fff1ac32973f219a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/streams/text.py +++ /dev/null @@ -1,176 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "TextConnectable", - "TextReceiveStream", - "TextSendStream", - "TextStream", -) - -import codecs -import sys -from collections.abc import Callable, Mapping -from dataclasses import InitVar, dataclass, field -from typing import Any - -from ..abc import ( - AnyByteReceiveStream, - AnyByteSendStream, - AnyByteStream, - AnyByteStreamConnectable, - ObjectReceiveStream, - ObjectSendStream, - ObjectStream, - ObjectStreamConnectable, -) - -if sys.version_info >= (3, 12): - from typing import override -else: - from typing_extensions import override - - -@dataclass(eq=False) -class TextReceiveStream(ObjectReceiveStream[str]): - """ - Stream wrapper that decodes bytes to strings using the given encoding. - - Decoding is done using :class:`~codecs.IncrementalDecoder` which returns any - completely received unicode characters as soon as they come in. - - :param transport_stream: any bytes-based receive stream - :param encoding: character encoding to use for decoding bytes to strings (defaults - to ``utf-8``) - :param errors: handling scheme for decoding errors (defaults to ``strict``; see the - `codecs module documentation`_ for a comprehensive list of options) - - .. _codecs module documentation: - https://docs.python.org/3/library/codecs.html#codec-objects - """ - - transport_stream: AnyByteReceiveStream - encoding: InitVar[str] = "utf-8" - errors: InitVar[str] = "strict" - _decoder: codecs.IncrementalDecoder = field(init=False) - - def __post_init__(self, encoding: str, errors: str) -> None: - decoder_class = codecs.getincrementaldecoder(encoding) - self._decoder = decoder_class(errors=errors) - - async def receive(self) -> str: - while True: - chunk = await self.transport_stream.receive() - decoded = self._decoder.decode(chunk) - if decoded: - return decoded - - async def aclose(self) -> None: - await self.transport_stream.aclose() - self._decoder.reset() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return self.transport_stream.extra_attributes - - -@dataclass(eq=False) -class TextSendStream(ObjectSendStream[str]): - """ - Sends strings to the wrapped stream as bytes using the given encoding. - - :param AnyByteSendStream transport_stream: any bytes-based send stream - :param str encoding: character encoding to use for encoding strings to bytes - (defaults to ``utf-8``) - :param str errors: handling scheme for encoding errors (defaults to ``strict``; see - the `codecs module documentation`_ for a comprehensive list of options) - - .. _codecs module documentation: - https://docs.python.org/3/library/codecs.html#codec-objects - """ - - transport_stream: AnyByteSendStream - encoding: InitVar[str] = "utf-8" - errors: str = "strict" - _encoder: Callable[..., tuple[bytes, int]] = field(init=False) - - def __post_init__(self, encoding: str) -> None: - self._encoder = codecs.getencoder(encoding) - - async def send(self, item: str) -> None: - encoded = self._encoder(item, self.errors)[0] - await self.transport_stream.send(encoded) - - async def aclose(self) -> None: - await self.transport_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return self.transport_stream.extra_attributes - - -@dataclass(eq=False) -class TextStream(ObjectStream[str]): - """ - A bidirectional stream that decodes bytes to strings on receive and encodes strings - to bytes on send. - - Extra attributes will be provided from both streams, with the receive stream - providing the values in case of a conflict. - - :param AnyByteStream transport_stream: any bytes-based stream - :param str encoding: character encoding to use for encoding/decoding strings to/from - bytes (defaults to ``utf-8``) - :param str errors: handling scheme for encoding errors (defaults to ``strict``; see - the `codecs module documentation`_ for a comprehensive list of options) - - .. _codecs module documentation: - https://docs.python.org/3/library/codecs.html#codec-objects - """ - - transport_stream: AnyByteStream - encoding: InitVar[str] = "utf-8" - errors: InitVar[str] = "strict" - _receive_stream: TextReceiveStream = field(init=False) - _send_stream: TextSendStream = field(init=False) - - def __post_init__(self, encoding: str, errors: str) -> None: - self._receive_stream = TextReceiveStream( - self.transport_stream, encoding=encoding, errors=errors - ) - self._send_stream = TextSendStream( - self.transport_stream, encoding=encoding, errors=errors - ) - - async def receive(self) -> str: - return await self._receive_stream.receive() - - async def send(self, item: str) -> None: - await self._send_stream.send(item) - - async def send_eof(self) -> None: - await self.transport_stream.send_eof() - - async def aclose(self) -> None: - await self._send_stream.aclose() - await self._receive_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self._send_stream.extra_attributes, - **self._receive_stream.extra_attributes, - } - - -class TextConnectable(ObjectStreamConnectable[str]): - def __init__(self, connectable: AnyByteStreamConnectable): - """ - :param connectable: the bytestream endpoint to wrap - - """ - self.connectable = connectable - - @override - async def connect(self) -> TextStream: - stream = await self.connectable.connect() - return TextStream(stream) diff --git a/bundle/python-cpu/Lib/site-packages/anyio/streams/tls.py b/bundle/python-cpu/Lib/site-packages/anyio/streams/tls.py deleted file mode 100644 index 282174c71d6d6672eaff99b919a0bb5cf7d418b2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/streams/tls.py +++ /dev/null @@ -1,436 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "TLSAttribute", - "TLSConnectable", - "TLSListener", - "TLSStream", -) - -import logging -import re -import ssl -import sys -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from functools import wraps -from ssl import SSLContext -from typing import Any, TypeAlias, TypeVar - -from .. import ( - BrokenResourceError, - EndOfStream, - aclose_forcefully, - get_cancelled_exc_class, - to_thread, -) -from .._core._typedattr import TypedAttributeSet, typed_attribute -from ..abc import ( - AnyByteStream, - AnyByteStreamConnectable, - ByteStream, - ByteStreamConnectable, - Listener, - TaskGroup, -) - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -if sys.version_info >= (3, 12): - from typing import override -else: - from typing_extensions import override - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") -_PCTRTT: TypeAlias = tuple[tuple[str, str], ...] -_PCTRTTT: TypeAlias = tuple[_PCTRTT, ...] - - -class TLSAttribute(TypedAttributeSet): - """Contains Transport Layer Security related attributes.""" - - #: the selected ALPN protocol - alpn_protocol: str | None = typed_attribute() - #: the channel binding for type ``tls-unique`` - channel_binding_tls_unique: bytes = typed_attribute() - #: the selected cipher - cipher: tuple[str, str, int] = typed_attribute() - #: the peer certificate in dictionary form (see :meth:`ssl.SSLSocket.getpeercert` - # for more information) - peer_certificate: None | (dict[str, str | _PCTRTTT | _PCTRTT]) = typed_attribute() - #: the peer certificate in binary form - peer_certificate_binary: bytes | None = typed_attribute() - #: ``True`` if this is the server side of the connection - server_side: bool = typed_attribute() - #: ciphers shared by the client during the TLS handshake (``None`` if this is the - #: client side) - shared_ciphers: list[tuple[str, str, int]] | None = typed_attribute() - #: the :class:`~ssl.SSLObject` used for encryption - ssl_object: ssl.SSLObject = typed_attribute() - #: ``True`` if this stream does (and expects) a closing TLS handshake when the - #: stream is being closed - standard_compatible: bool = typed_attribute() - #: the TLS protocol version (e.g. ``TLSv1.2``) - tls_version: str = typed_attribute() - - -@dataclass(eq=False) -class TLSStream(ByteStream): - """ - A stream wrapper that encrypts all sent data and decrypts received data. - - This class has no public initializer; use :meth:`wrap` instead. - All extra attributes from :class:`~TLSAttribute` are supported. - - :var AnyByteStream transport_stream: the wrapped stream - - """ - - transport_stream: AnyByteStream - standard_compatible: bool - _ssl_object: ssl.SSLObject - _read_bio: ssl.MemoryBIO - _write_bio: ssl.MemoryBIO - - @classmethod - async def wrap( - cls, - transport_stream: AnyByteStream, - *, - server_side: bool | None = None, - hostname: str | None = None, - ssl_context: ssl.SSLContext | None = None, - standard_compatible: bool = True, - ) -> TLSStream: - """ - Wrap an existing stream with Transport Layer Security. - - This performs a TLS handshake with the peer. - - :param transport_stream: a bytes-transporting stream to wrap - :param server_side: ``True`` if this is the server side of the connection, - ``False`` if this is the client side (if omitted, will be set to ``False`` - if ``hostname`` has been provided, ``False`` otherwise). Used only to create - a default context when an explicit context has not been provided. - :param hostname: host name of the peer (if host name checking is desired) - :param ssl_context: the SSLContext object to use (if not provided, a secure - default will be created) - :param standard_compatible: if ``False``, skip the closing handshake when - closing the connection, and don't raise an exception if the peer does the - same - :raises ~ssl.SSLError: if the TLS handshake fails - - """ - if server_side is None: - server_side = not hostname - - if not ssl_context: - purpose = ( - ssl.Purpose.CLIENT_AUTH if server_side else ssl.Purpose.SERVER_AUTH - ) - ssl_context = ssl.create_default_context(purpose) - - # Re-enable detection of unexpected EOFs if it was disabled by Python - if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): - ssl_context.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF - - bio_in = ssl.MemoryBIO() - bio_out = ssl.MemoryBIO() - - # Resolve international host names using IDNA 2008. - # Otherwise wrap_bio() would resolve them with IDNA 2003. - if hostname is not None: - from .._core._sockets import idna2008_resolve - - server_hostname: bytes | None = idna2008_resolve(hostname) - else: - server_hostname = None - - # External SSLContext implementations may do blocking I/O in wrap_bio(), - # but the standard library implementation won't - if type(ssl_context) is ssl.SSLContext: - ssl_object = ssl_context.wrap_bio( - bio_in, - bio_out, - server_side=server_side, - server_hostname=server_hostname, - ) - else: - ssl_object = await to_thread.run_sync( - ssl_context.wrap_bio, - bio_in, - bio_out, - server_side, - server_hostname, - None, - ) - - wrapper = cls( - transport_stream=transport_stream, - standard_compatible=standard_compatible, - _ssl_object=ssl_object, - _read_bio=bio_in, - _write_bio=bio_out, - ) - await wrapper._call_sslobject_method(ssl_object.do_handshake) - return wrapper - - async def _call_sslobject_method( - self, func: Callable[[Unpack[PosArgsT]], T_Retval], *args: Unpack[PosArgsT] - ) -> T_Retval: - while True: - try: - result = func(*args) - except ssl.SSLWantReadError: - try: - # Flush any pending writes first - if self._write_bio.pending: - await self.transport_stream.send(self._write_bio.read()) - - data = await self.transport_stream.receive() - except EndOfStream: - self._read_bio.write_eof() - except OSError as exc: - self._read_bio.write_eof() - self._write_bio.write_eof() - raise BrokenResourceError from exc - else: - self._read_bio.write(data) - except ssl.SSLWantWriteError: - await self.transport_stream.send(self._write_bio.read()) - except ssl.SSLSyscallError as exc: - self._read_bio.write_eof() - self._write_bio.write_eof() - raise BrokenResourceError from exc - except ssl.SSLError as exc: - self._read_bio.write_eof() - self._write_bio.write_eof() - if isinstance(exc, ssl.SSLEOFError) or ( - exc.strerror and "UNEXPECTED_EOF_WHILE_READING" in exc.strerror - ): - if self.standard_compatible: - raise BrokenResourceError from exc - else: - raise EndOfStream from None - - raise - else: - # Flush any pending writes first - if self._write_bio.pending: - await self.transport_stream.send(self._write_bio.read()) - - return result - - async def unwrap(self) -> tuple[AnyByteStream, bytes]: - """ - Does the TLS closing handshake. - - :return: a tuple of (wrapped byte stream, bytes left in the read buffer) - - """ - await self._call_sslobject_method(self._ssl_object.unwrap) - self._read_bio.write_eof() - self._write_bio.write_eof() - return self.transport_stream, self._read_bio.read() - - async def aclose(self) -> None: - if self.standard_compatible: - try: - await self.unwrap() - except BaseException: - await aclose_forcefully(self.transport_stream) - raise - - await self.transport_stream.aclose() - - async def receive(self, max_bytes: int = 65536) -> bytes: - if max_bytes < 1: - raise ValueError("max_bytes must be a positive integer") - - data = await self._call_sslobject_method(self._ssl_object.read, max_bytes) - if not data: - raise EndOfStream - - return data - - async def send(self, item: bytes) -> None: - await self._call_sslobject_method(self._ssl_object.write, item) - - async def send_eof(self) -> None: - tls_version = self.extra(TLSAttribute.tls_version) - match = re.match(r"TLSv(\d+)(?:\.(\d+))?", tls_version) - if match: - major, minor = int(match.group(1)), int(match.group(2) or 0) - if (major, minor) < (1, 3): - raise NotImplementedError( - f"send_eof() requires at least TLSv1.3; current " - f"session uses {tls_version}" - ) - - raise NotImplementedError( - "send_eof() has not yet been implemented for TLS streams" - ) - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self.transport_stream.extra_attributes, - TLSAttribute.alpn_protocol: self._ssl_object.selected_alpn_protocol, - TLSAttribute.channel_binding_tls_unique: ( - self._ssl_object.get_channel_binding - ), - TLSAttribute.cipher: self._ssl_object.cipher, - TLSAttribute.peer_certificate: lambda: self._ssl_object.getpeercert(False), - TLSAttribute.peer_certificate_binary: lambda: self._ssl_object.getpeercert( - True - ), - TLSAttribute.server_side: lambda: self._ssl_object.server_side, - TLSAttribute.shared_ciphers: lambda: ( - self._ssl_object.shared_ciphers() - if self._ssl_object.server_side - else None - ), - TLSAttribute.standard_compatible: lambda: self.standard_compatible, - TLSAttribute.ssl_object: lambda: self._ssl_object, - TLSAttribute.tls_version: self._ssl_object.version, - } - - -@dataclass(eq=False) -class TLSListener(Listener[TLSStream]): - """ - A convenience listener that wraps another listener and auto-negotiates a TLS session - on every accepted connection. - - If the TLS handshake times out or raises an exception, - :meth:`handle_handshake_error` is called to do whatever post-mortem processing is - deemed necessary. - - Supports only the :attr:`~TLSAttribute.standard_compatible` extra attribute. - - :param Listener listener: the listener to wrap - :param ssl_context: the SSL context object - :param standard_compatible: a flag passed through to :meth:`TLSStream.wrap` - :param handshake_timeout: time limit for the TLS handshake - (passed to :func:`~anyio.fail_after`) - """ - - listener: Listener[Any] - ssl_context: ssl.SSLContext - standard_compatible: bool = True - handshake_timeout: float = 30 - - @staticmethod - async def handle_handshake_error(exc: BaseException, stream: AnyByteStream) -> None: - """ - Handle an exception raised during the TLS handshake. - - This method does 3 things: - - #. Forcefully closes the original stream - #. Logs the exception (unless it was a cancellation exception) using the - ``anyio.streams.tls`` logger - #. Reraises the exception if it was a base exception or a cancellation exception - - :param exc: the exception - :param stream: the original stream - - """ - await aclose_forcefully(stream) - - # Log all except cancellation exceptions - if not isinstance(exc, get_cancelled_exc_class()): - # CPython (as of 3.11.5) returns incorrect `sys.exc_info()` here when using - # any asyncio implementation, so we explicitly pass the exception to log - # (https://github.com/python/cpython/issues/108668). Trio does not have this - # issue because it works around the CPython bug. - logging.getLogger(__name__).exception( - "Error during TLS handshake", exc_info=exc - ) - - # Only reraise base exceptions and cancellation exceptions - if not isinstance(exc, Exception) or isinstance(exc, get_cancelled_exc_class()): - raise - - async def serve( - self, - handler: Callable[[TLSStream], Any], - task_group: TaskGroup | None = None, - ) -> None: - @wraps(handler) - async def handler_wrapper(stream: AnyByteStream) -> None: - from .. import fail_after - - try: - with fail_after(self.handshake_timeout): - wrapped_stream = await TLSStream.wrap( - stream, - ssl_context=self.ssl_context, - standard_compatible=self.standard_compatible, - ) - except BaseException as exc: - await self.handle_handshake_error(exc, stream) - else: - await handler(wrapped_stream) - - await self.listener.serve(handler_wrapper, task_group) - - async def aclose(self) -> None: - await self.listener.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - TLSAttribute.standard_compatible: lambda: self.standard_compatible, - } - - -class TLSConnectable(ByteStreamConnectable): - """ - Wraps another connectable and does TLS negotiation after a successful connection. - - :param connectable: the connectable to wrap - :param hostname: host name of the server (if host name checking is desired) - :param ssl_context: the SSLContext object to use (if not provided, a secure default - will be created) - :param standard_compatible: if ``False``, skip the closing handshake when closing - the connection, and don't raise an exception if the server does the same - """ - - def __init__( - self, - connectable: AnyByteStreamConnectable, - *, - hostname: str | None = None, - ssl_context: ssl.SSLContext | None = None, - standard_compatible: bool = True, - ) -> None: - self.connectable = connectable - self.ssl_context: SSLContext = ssl_context or ssl.create_default_context( - ssl.Purpose.SERVER_AUTH - ) - if not isinstance(self.ssl_context, ssl.SSLContext): - raise TypeError( - "ssl_context must be an instance of ssl.SSLContext, not " - f"{type(self.ssl_context).__name__}" - ) - self.hostname = hostname - self.standard_compatible = standard_compatible - - @override - async def connect(self) -> TLSStream: - stream = await self.connectable.connect() - try: - return await TLSStream.wrap( - stream, - hostname=self.hostname, - ssl_context=self.ssl_context, - standard_compatible=self.standard_compatible, - ) - except BaseException: - await aclose_forcefully(stream) - raise diff --git a/bundle/python-cpu/Lib/site-packages/anyio/to_interpreter.py b/bundle/python-cpu/Lib/site-packages/anyio/to_interpreter.py deleted file mode 100644 index 694dbe77bc8581032ee72316afe4e0590311ba00..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/to_interpreter.py +++ /dev/null @@ -1,246 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "run_sync", - "current_default_interpreter_limiter", -) - -import atexit -import os -import sys -from collections import deque -from collections.abc import Callable -from typing import Any, Final, TypeVar - -from . import current_time, to_thread -from ._core._exceptions import BrokenWorkerInterpreter -from ._core._synchronization import CapacityLimiter -from .lowlevel import RunVar - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -if sys.version_info >= (3, 14): - from concurrent.interpreters import ExecutionFailed, create - - def _interp_call( - func: Callable[..., Any], args: tuple[Any, ...] - ) -> tuple[Any, bool]: - try: - retval = func(*args) - except BaseException as exc: - return exc, True - else: - return retval, False - - class _Worker: - last_used: float = 0 - - def __init__(self) -> None: - self._interpreter = create() - - def destroy(self) -> None: - self._interpreter.close() - - def call( - self, - func: Callable[..., T_Retval], - args: tuple[Any, ...], - ) -> T_Retval: - try: - res, is_exception = self._interpreter.call(_interp_call, func, args) - except ExecutionFailed as exc: - raise BrokenWorkerInterpreter(exc.excinfo) from exc - - if is_exception: - raise res - - return res -elif sys.version_info >= (3, 13): - import _interpqueues - import _interpreters - - UNBOUND: Final = 2 # I have no clue how this works, but it was used in the stdlib - FMT_UNPICKLED: Final = 0 - FMT_PICKLED: Final = 1 - QUEUE_PICKLE_ARGS: Final = (FMT_PICKLED, UNBOUND) - QUEUE_UNPICKLE_ARGS: Final = (FMT_UNPICKLED, UNBOUND) - - _run_func = compile( - """ -import _interpqueues -from _interpreters import NotShareableError -from pickle import loads, dumps, HIGHEST_PROTOCOL - -QUEUE_PICKLE_ARGS = (1, 2) -QUEUE_UNPICKLE_ARGS = (0, 2) - -item = _interpqueues.get(queue_id)[0] -try: - func, args = loads(item) - retval = func(*args) -except BaseException as exc: - is_exception = True - retval = exc -else: - is_exception = False - -try: - _interpqueues.put(queue_id, (retval, is_exception), *QUEUE_UNPICKLE_ARGS) -except NotShareableError: - retval = dumps(retval, HIGHEST_PROTOCOL) - _interpqueues.put(queue_id, (retval, is_exception), *QUEUE_PICKLE_ARGS) - """, - "", - "exec", - ) - - class _Worker: - last_used: float = 0 - - def __init__(self) -> None: - self._interpreter_id = _interpreters.create() - self._queue_id = _interpqueues.create(1, *QUEUE_UNPICKLE_ARGS) - _interpreters.set___main___attrs( - self._interpreter_id, {"queue_id": self._queue_id} - ) - - def destroy(self) -> None: - _interpqueues.destroy(self._queue_id) - _interpreters.destroy(self._interpreter_id) - - def call( - self, - func: Callable[..., T_Retval], - args: tuple[Any, ...], - ) -> T_Retval: - import pickle - - item = pickle.dumps((func, args), pickle.HIGHEST_PROTOCOL) - _interpqueues.put(self._queue_id, item, *QUEUE_PICKLE_ARGS) - exc_info = _interpreters.exec(self._interpreter_id, _run_func) - if exc_info: - raise BrokenWorkerInterpreter(exc_info) - - res = _interpqueues.get(self._queue_id) - (res, is_exception), fmt = res[:2] - if fmt == FMT_PICKLED: - res = pickle.loads(res) - - if is_exception: - raise res - - return res -else: - - class _Worker: - last_used: float = 0 - - def __init__(self) -> None: - raise RuntimeError("subinterpreters require at least Python 3.13") - - def call( - self, - func: Callable[..., T_Retval], - args: tuple[Any, ...], - ) -> T_Retval: - raise NotImplementedError - - def destroy(self) -> None: - pass - - -DEFAULT_CPU_COUNT: Final = 8 # this is just an arbitrarily selected value -MAX_WORKER_IDLE_TIME = ( - 30 # seconds a subinterpreter can be idle before becoming eligible for pruning -) - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - -_idle_workers = RunVar[deque[_Worker]]("_available_workers") -_default_interpreter_limiter = RunVar[CapacityLimiter]("_default_interpreter_limiter") - - -def _stop_workers(workers: deque[_Worker]) -> None: - for worker in workers: - worker.destroy() - - workers.clear() - - -async def run_sync( - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - limiter: CapacityLimiter | None = None, -) -> T_Retval: - """ - Call the given function with the given arguments in a subinterpreter. - - .. warning:: On Python 3.13, the :mod:`concurrent.interpreters` module was not yet - available, so the code path for that Python version relies on an undocumented, - private API. As such, it is recommended to not rely on this function for anything - mission-critical on Python 3.13. - - :param func: a callable - :param args: the positional arguments for the callable - :param limiter: capacity limiter to use to limit the total number of subinterpreters - running (if omitted, the default limiter is used) - :return: the result of the call - :raises BrokenWorkerInterpreter: if there's an internal error in a subinterpreter - - """ - if limiter is None: - limiter = current_default_interpreter_limiter() - - try: - idle_workers = _idle_workers.get() - except LookupError: - idle_workers = deque() - _idle_workers.set(idle_workers) - atexit.register(_stop_workers, idle_workers) - - async with limiter: - try: - worker = idle_workers.pop() - except IndexError: - worker = _Worker() - - try: - return await to_thread.run_sync( - worker.call, - func, - args, - limiter=limiter, - ) - finally: - # Prune workers that have been idle for too long - now = current_time() - while idle_workers: - if now - idle_workers[0].last_used <= MAX_WORKER_IDLE_TIME: - break - - await to_thread.run_sync(idle_workers.popleft().destroy, limiter=limiter) - - worker.last_used = current_time() - idle_workers.append(worker) - - -def current_default_interpreter_limiter() -> CapacityLimiter: - """ - Return the capacity limiter used by default to limit the number of concurrently - running subinterpreters. - - Defaults to the number of CPU cores. - - :return: a capacity limiter object - - """ - try: - return _default_interpreter_limiter.get() - except LookupError: - limiter = CapacityLimiter(os.cpu_count() or DEFAULT_CPU_COUNT) - _default_interpreter_limiter.set(limiter) - return limiter diff --git a/bundle/python-cpu/Lib/site-packages/anyio/to_process.py b/bundle/python-cpu/Lib/site-packages/anyio/to_process.py deleted file mode 100644 index 8d356fbd2a3df39a9b44eacc3c6c924420c1a7b8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/to_process.py +++ /dev/null @@ -1,269 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "current_default_process_limiter", - "process_worker", - "run_sync", -) - -import os -import pickle -import runpy -import subprocess -import sys -from collections import deque -from collections.abc import Callable -from types import ModuleType -from typing import TypeVar, cast - -from ._core._eventloop import current_time, get_async_backend, get_cancelled_exc_class -from ._core._exceptions import BrokenWorkerProcess -from ._core._subprocesses import open_process -from ._core._synchronization import CapacityLimiter -from ._core._tasks import CancelScope, fail_after -from .abc import ByteReceiveStream, ByteSendStream, Process -from .lowlevel import RunVar, checkpoint_if_cancelled -from .streams.buffered import BufferedByteReceiveStream - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -WORKER_MAX_IDLE_TIME = 300 # 5 minutes - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - -_process_pool_workers: RunVar[set[Process]] = RunVar("_process_pool_workers") -_process_pool_idle_workers: RunVar[deque[tuple[Process, float]]] = RunVar( - "_process_pool_idle_workers" -) -_default_process_limiter: RunVar[CapacityLimiter] = RunVar("_default_process_limiter") - - -async def run_sync( # type: ignore[return] - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - cancellable: bool = False, - limiter: CapacityLimiter | None = None, -) -> T_Retval: - """ - Call the given function with the given arguments in a worker process. - - If the ``cancellable`` option is enabled and the task waiting for its completion is - cancelled, the worker process running it will be abruptly terminated using SIGKILL - (or ``terminateProcess()`` on Windows). - - :param func: a callable - :param args: positional arguments for the callable - :param cancellable: ``True`` to allow cancellation of the operation while it's - running - :param limiter: capacity limiter to use to limit the total amount of processes - running (if omitted, the default limiter is used) - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - :return: an awaitable that yields the return value of the function. - - """ - - async def send_raw_command(pickled_cmd: bytes) -> object: - try: - await stdin.send(pickled_cmd) - response = await buffered.receive_until(b"\n", 50) - status, length = response.split(b" ") - if status not in (b"RETURN", b"EXCEPTION"): - raise RuntimeError( - f"Worker process returned unexpected response: {response!r}" - ) - - pickled_response = await buffered.receive_exactly(int(length)) - except BaseException as exc: - workers.discard(process) - try: - process.kill() - with CancelScope(shield=True): - await process.aclose() - except ProcessLookupError: - pass - - if isinstance(exc, get_cancelled_exc_class()): - raise - else: - raise BrokenWorkerProcess from exc - - retval = pickle.loads(pickled_response) - if status == b"EXCEPTION": - assert isinstance(retval, BaseException) - raise retval - else: - return retval - - # First pickle the request before trying to reserve a worker process - await checkpoint_if_cancelled() - request = pickle.dumps(("run", func, args), protocol=pickle.HIGHEST_PROTOCOL) - - # If this is the first run in this event loop thread, set up the necessary variables - try: - workers = _process_pool_workers.get() - idle_workers = _process_pool_idle_workers.get() - except LookupError: - workers = set() - idle_workers = deque() - _process_pool_workers.set(workers) - _process_pool_idle_workers.set(idle_workers) - get_async_backend().setup_process_pool_exit_at_shutdown(workers) - - async with limiter or current_default_process_limiter(): - # Pop processes from the pool (starting from the most recently used) until we - # find one that hasn't exited yet - process: Process - while idle_workers: - process, idle_since = idle_workers.pop() - if process.returncode is None: - stdin = cast(ByteSendStream, process.stdin) - buffered = BufferedByteReceiveStream( - cast(ByteReceiveStream, process.stdout) - ) - - # Prune any other workers that have been idle for WORKER_MAX_IDLE_TIME - # seconds or longer - now = current_time() - killed_processes: list[Process] = [] - while idle_workers: - if now - idle_workers[0][1] < WORKER_MAX_IDLE_TIME: - break - - process_to_kill, idle_since = idle_workers.popleft() - process_to_kill.kill() - workers.remove(process_to_kill) - killed_processes.append(process_to_kill) - - with CancelScope(shield=True): - for killed_process in killed_processes: - await killed_process.aclose() - - break - - workers.remove(process) - else: - command = [sys.executable, "-u", "-m", __name__] - process = await open_process( - command, stdin=subprocess.PIPE, stdout=subprocess.PIPE - ) - try: - stdin = cast(ByteSendStream, process.stdin) - buffered = BufferedByteReceiveStream( - cast(ByteReceiveStream, process.stdout) - ) - with fail_after(20): - message = await buffered.receive(6) - - if message != b"READY\n": - raise BrokenWorkerProcess( - f"Worker process returned unexpected response: {message!r}" - ) - - main_module_path = getattr(sys.modules["__main__"], "__file__", None) - pickled = pickle.dumps( - ("init", sys.path, main_module_path), - protocol=pickle.HIGHEST_PROTOCOL, - ) - await send_raw_command(pickled) - except (BrokenWorkerProcess, get_cancelled_exc_class()): - raise - except BaseException as exc: - process.kill() - raise BrokenWorkerProcess( - "Error during worker process initialization" - ) from exc - - workers.add(process) - - with CancelScope(shield=not cancellable): - try: - return cast(T_Retval, await send_raw_command(request)) - finally: - if process in workers: - idle_workers.append((process, current_time())) - - -def current_default_process_limiter() -> CapacityLimiter: - """ - Return the capacity limiter that is used by default to limit the number of worker - processes. - - :return: a capacity limiter object - - """ - try: - return _default_process_limiter.get() - except LookupError: - limiter = CapacityLimiter(os.cpu_count() or 2) - _default_process_limiter.set(limiter) - return limiter - - -def process_worker() -> None: - # Redirect standard streams to os.devnull so that user code won't interfere with the - # parent-worker communication - stdin = sys.stdin - stdout = sys.stdout - sys.stdin = open(os.devnull) - sys.stdout = open(os.devnull, "w") - sys.stderr = open(os.devnull, "w") - - stdout.buffer.write(b"READY\n") - while True: - retval = exception = None - try: - command, *args = pickle.load(stdin.buffer) - except EOFError: - return - except BaseException as exc: - exception = exc - else: - if command == "run": - func, args = args - try: - retval = func(*args) - except BaseException as exc: - exception = exc - elif command == "init": - main_module_path: str | None - sys.path, main_module_path = args - del sys.modules["__main__"] - if main_module_path and os.path.isfile(main_module_path): - # Load the parent's main module but as __mp_main__ instead of - # __main__ (like multiprocessing does) to avoid infinite recursion - try: - main = ModuleType("__mp_main__") - main_content = runpy.run_path( - main_module_path, run_name="__mp_main__" - ) - main.__dict__.update(main_content) - sys.modules["__main__"] = sys.modules["__mp_main__"] = main - except BaseException as exc: - exception = exc - try: - if exception is not None: - status = b"EXCEPTION" - pickled = pickle.dumps(exception, pickle.HIGHEST_PROTOCOL) - else: - status = b"RETURN" - pickled = pickle.dumps(retval, pickle.HIGHEST_PROTOCOL) - except BaseException as exc: - exception = exc - status = b"EXCEPTION" - pickled = pickle.dumps(exc, pickle.HIGHEST_PROTOCOL) - - stdout.buffer.write(b"%s %d\n" % (status, len(pickled))) - stdout.buffer.write(pickled) - - # Respect SIGTERM - if isinstance(exception, SystemExit): - raise exception - - -if __name__ == "__main__": - process_worker() diff --git a/bundle/python-cpu/Lib/site-packages/anyio/to_thread.py b/bundle/python-cpu/Lib/site-packages/anyio/to_thread.py deleted file mode 100644 index a01f24f6e776ef30836fb5d82639776820bf194f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/anyio/to_thread.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "run_sync", - "current_default_thread_limiter", -) - -import sys -from collections.abc import Callable -from typing import TYPE_CHECKING, TypeVar -from warnings import warn - -from ._core._eventloop import get_async_backend - -if TYPE_CHECKING: - from ._core._synchronization import CapacityLimiter - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - - -async def run_sync( - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - abandon_on_cancel: bool = False, - cancellable: bool | None = None, - limiter: CapacityLimiter | None = None, -) -> T_Retval: - """ - Call the given function with the given arguments in a worker thread. - - If the ``abandon_on_cancel`` option is enabled and the task waiting for its - completion is cancelled, the thread will still run its course but its - return value (or any raised exception) will be ignored. - - :param func: a callable - :param args: positional arguments for the callable - :param abandon_on_cancel: ``True`` to abandon the thread (leaving it to run - unchecked on own) if the host task is cancelled, ``False`` to ignore - cancellations in the host task until the operation has completed in the worker - thread - :param cancellable: deprecated alias of ``abandon_on_cancel``; will override - ``abandon_on_cancel`` if both parameters are passed - :param limiter: capacity limiter to use to limit the total amount of threads running - (if omitted, the default limiter is used) - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - :return: an awaitable that yields the return value of the function. - - """ - if cancellable is not None: - abandon_on_cancel = cancellable - warn( - "The `cancellable=` keyword argument to `anyio.to_thread.run_sync` is " - "deprecated since AnyIO 4.1.0; use `abandon_on_cancel=` instead", - DeprecationWarning, - stacklevel=2, - ) - - return await get_async_backend().run_sync_in_worker_thread( - func, args, abandon_on_cancel=abandon_on_cancel, limiter=limiter - ) - - -def current_default_thread_limiter() -> CapacityLimiter: - """ - Return the capacity limiter that is used by default to limit the number of - concurrent threads. - - :return: a capacity limiter object - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().current_default_thread_limiter() diff --git a/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/LICENSE.txt b/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/LICENSE.txt deleted file mode 100644 index 79c9825adbacb5d8c6eaee51863b8a40051d97c8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/LICENSE.txt +++ /dev/null @@ -1,20 +0,0 @@ -Copyright 2010 Jason Kirtland - -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/METADATA deleted file mode 100644 index 6d343f57186f8f33f9fd6db264448a753de6e980..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/METADATA +++ /dev/null @@ -1,60 +0,0 @@ -Metadata-Version: 2.3 -Name: blinker -Version: 1.9.0 -Summary: Fast, simple object-to-object and broadcast signaling -Author: Jason Kirtland -Maintainer-email: Pallets Ecosystem -Requires-Python: >=3.9 -Description-Content-Type: text/markdown -Classifier: Development Status :: 5 - Production/Stable -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python -Classifier: Typing :: Typed -Project-URL: Chat, https://discord.gg/pallets -Project-URL: Documentation, https://blinker.readthedocs.io -Project-URL: Source, https://github.com/pallets-eco/blinker/ - -# Blinker - -Blinker provides a fast dispatching system that allows any number of -interested parties to subscribe to events, or "signals". - - -## Pallets Community Ecosystem - -> [!IMPORTANT]\ -> This project is part of the Pallets Community Ecosystem. Pallets is the open -> source organization that maintains Flask; Pallets-Eco enables community -> maintenance of related projects. If you are interested in helping maintain -> this project, please reach out on [the Pallets Discord server][discord]. -> -> [discord]: https://discord.gg/pallets - - -## Example - -Signal receivers can subscribe to specific senders or receive signals -sent by any sender. - -```pycon ->>> from blinker import signal ->>> started = signal('round-started') ->>> def each(round): -... print(f"Round {round}") -... ->>> started.connect(each) - ->>> def round_two(round): -... print("This is round two.") -... ->>> started.connect(round_two, sender=2) - ->>> for round in range(1, 4): -... started.send(round) -... -Round 1! -Round 2! -This is round two. -Round 3! -``` - diff --git a/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/RECORD deleted file mode 100644 index 394215ffc9076c6634d2a763342ecf730369d5f9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/RECORD +++ /dev/null @@ -1,10 +0,0 @@ -blinker-1.9.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -blinker-1.9.0.dist-info/LICENSE.txt,sha256=nrc6HzhZekqhcCXSrhvjg5Ykx5XphdTw6Xac4p-spGc,1054 -blinker-1.9.0.dist-info/METADATA,sha256=uIRiM8wjjbHkCtbCyTvctU37IAZk0kEe5kxAld1dvzA,1633 -blinker-1.9.0.dist-info/RECORD,, -blinker-1.9.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -blinker-1.9.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82 -blinker/__init__.py,sha256=I2EdZqpy4LyjX17Hn1yzJGWCjeLaVaPzsMgHkLfj_cQ,317 -blinker/_utilities.py,sha256=0J7eeXXTUx0Ivf8asfpx0ycVkp0Eqfqnj117x2mYX9E,1675 -blinker/base.py,sha256=QpDuvXXcwJF49lUBcH5BiST46Rz9wSG7VW_p7N_027M,19132 -blinker/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/WHEEL deleted file mode 100644 index e3c6feefa22927866e3fd5575379ea972b432aaf..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/blinker-1.9.0.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: flit 3.10.1 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/blinker/__init__.py b/bundle/python-cpu/Lib/site-packages/blinker/__init__.py deleted file mode 100644 index 1772fa4a543b0288f03d60050f222ed64a83ece0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/blinker/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - -from .base import ANY -from .base import default_namespace -from .base import NamedSignal -from .base import Namespace -from .base import Signal -from .base import signal - -__all__ = [ - "ANY", - "default_namespace", - "NamedSignal", - "Namespace", - "Signal", - "signal", -] diff --git a/bundle/python-cpu/Lib/site-packages/blinker/_utilities.py b/bundle/python-cpu/Lib/site-packages/blinker/_utilities.py deleted file mode 100644 index 000c902a2564d2d4a551edb646a9d654d9e7041b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/blinker/_utilities.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -import collections.abc as c -import inspect -import typing as t -from weakref import ref -from weakref import WeakMethod - -T = t.TypeVar("T") - - -class Symbol: - """A constant symbol, nicer than ``object()``. Repeated calls return the - same instance. - - >>> Symbol('foo') is Symbol('foo') - True - >>> Symbol('foo') - foo - """ - - symbols: t.ClassVar[dict[str, Symbol]] = {} - - def __new__(cls, name: str) -> Symbol: - if name in cls.symbols: - return cls.symbols[name] - - obj = super().__new__(cls) - cls.symbols[name] = obj - return obj - - def __init__(self, name: str) -> None: - self.name = name - - def __repr__(self) -> str: - return self.name - - def __getnewargs__(self) -> tuple[t.Any, ...]: - return (self.name,) - - -def make_id(obj: object) -> c.Hashable: - """Get a stable identifier for a receiver or sender, to be used as a dict - key or in a set. - """ - if inspect.ismethod(obj): - # The id of a bound method is not stable, but the id of the unbound - # function and instance are. - return id(obj.__func__), id(obj.__self__) - - if isinstance(obj, (str, int)): - # Instances with the same value always compare equal and have the same - # hash, even if the id may change. - return obj - - # Assume other types are not hashable but will always be the same instance. - return id(obj) - - -def make_ref(obj: T, callback: c.Callable[[ref[T]], None] | None = None) -> ref[T]: - if inspect.ismethod(obj): - return WeakMethod(obj, callback) # type: ignore[arg-type, return-value] - - return ref(obj, callback) diff --git a/bundle/python-cpu/Lib/site-packages/blinker/base.py b/bundle/python-cpu/Lib/site-packages/blinker/base.py deleted file mode 100644 index d051b94a32f5eecd9f32853d6eaebcca6c7133f6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/blinker/base.py +++ /dev/null @@ -1,512 +0,0 @@ -from __future__ import annotations - -import collections.abc as c -import sys -import typing as t -import weakref -from collections import defaultdict -from contextlib import contextmanager -from functools import cached_property -from inspect import iscoroutinefunction - -from ._utilities import make_id -from ._utilities import make_ref -from ._utilities import Symbol - -F = t.TypeVar("F", bound=c.Callable[..., t.Any]) - -ANY = Symbol("ANY") -"""Symbol for "any sender".""" - -ANY_ID = 0 - - -class Signal: - """A notification emitter. - - :param doc: The docstring for the signal. - """ - - ANY = ANY - """An alias for the :data:`~blinker.ANY` sender symbol.""" - - set_class: type[set[t.Any]] = set - """The set class to use for tracking connected receivers and senders. - Python's ``set`` is unordered. If receivers must be dispatched in the order - they were connected, an ordered set implementation can be used. - - .. versionadded:: 1.7 - """ - - @cached_property - def receiver_connected(self) -> Signal: - """Emitted at the end of each :meth:`connect` call. - - The signal sender is the signal instance, and the :meth:`connect` - arguments are passed through: ``receiver``, ``sender``, and ``weak``. - - .. versionadded:: 1.2 - """ - return Signal(doc="Emitted after a receiver connects.") - - @cached_property - def receiver_disconnected(self) -> Signal: - """Emitted at the end of each :meth:`disconnect` call. - - The sender is the signal instance, and the :meth:`disconnect` arguments - are passed through: ``receiver`` and ``sender``. - - This signal is emitted **only** when :meth:`disconnect` is called - explicitly. This signal cannot be emitted by an automatic disconnect - when a weakly referenced receiver or sender goes out of scope, as the - instance is no longer be available to be used as the sender for this - signal. - - An alternative approach is available by subscribing to - :attr:`receiver_connected` and setting up a custom weakref cleanup - callback on weak receivers and senders. - - .. versionadded:: 1.2 - """ - return Signal(doc="Emitted after a receiver disconnects.") - - def __init__(self, doc: str | None = None) -> None: - if doc: - self.__doc__ = doc - - self.receivers: dict[ - t.Any, weakref.ref[c.Callable[..., t.Any]] | c.Callable[..., t.Any] - ] = {} - """The map of connected receivers. Useful to quickly check if any - receivers are connected to the signal: ``if s.receivers:``. The - structure and data is not part of the public API, but checking its - boolean value is. - """ - - self.is_muted: bool = False - self._by_receiver: dict[t.Any, set[t.Any]] = defaultdict(self.set_class) - self._by_sender: dict[t.Any, set[t.Any]] = defaultdict(self.set_class) - self._weak_senders: dict[t.Any, weakref.ref[t.Any]] = {} - - def connect(self, receiver: F, sender: t.Any = ANY, weak: bool = True) -> F: - """Connect ``receiver`` to be called when the signal is sent by - ``sender``. - - :param receiver: The callable to call when :meth:`send` is called with - the given ``sender``, passing ``sender`` as a positional argument - along with any extra keyword arguments. - :param sender: Any object or :data:`ANY`. ``receiver`` will only be - called when :meth:`send` is called with this sender. If ``ANY``, the - receiver will be called for any sender. A receiver may be connected - to multiple senders by calling :meth:`connect` multiple times. - :param weak: Track the receiver with a :mod:`weakref`. The receiver will - be automatically disconnected when it is garbage collected. When - connecting a receiver defined within a function, set to ``False``, - otherwise it will be disconnected when the function scope ends. - """ - receiver_id = make_id(receiver) - sender_id = ANY_ID if sender is ANY else make_id(sender) - - if weak: - self.receivers[receiver_id] = make_ref( - receiver, self._make_cleanup_receiver(receiver_id) - ) - else: - self.receivers[receiver_id] = receiver - - self._by_sender[sender_id].add(receiver_id) - self._by_receiver[receiver_id].add(sender_id) - - if sender is not ANY and sender_id not in self._weak_senders: - # store a cleanup for weakref-able senders - try: - self._weak_senders[sender_id] = make_ref( - sender, self._make_cleanup_sender(sender_id) - ) - except TypeError: - pass - - if "receiver_connected" in self.__dict__ and self.receiver_connected.receivers: - try: - self.receiver_connected.send( - self, receiver=receiver, sender=sender, weak=weak - ) - except TypeError: - # TODO no explanation or test for this - self.disconnect(receiver, sender) - raise - - return receiver - - def connect_via(self, sender: t.Any, weak: bool = False) -> c.Callable[[F], F]: - """Connect the decorated function to be called when the signal is sent - by ``sender``. - - The decorated function will be called when :meth:`send` is called with - the given ``sender``, passing ``sender`` as a positional argument along - with any extra keyword arguments. - - :param sender: Any object or :data:`ANY`. ``receiver`` will only be - called when :meth:`send` is called with this sender. If ``ANY``, the - receiver will be called for any sender. A receiver may be connected - to multiple senders by calling :meth:`connect` multiple times. - :param weak: Track the receiver with a :mod:`weakref`. The receiver will - be automatically disconnected when it is garbage collected. When - connecting a receiver defined within a function, set to ``False``, - otherwise it will be disconnected when the function scope ends.= - - .. versionadded:: 1.1 - """ - - def decorator(fn: F) -> F: - self.connect(fn, sender, weak) - return fn - - return decorator - - @contextmanager - def connected_to( - self, receiver: c.Callable[..., t.Any], sender: t.Any = ANY - ) -> c.Generator[None, None, None]: - """A context manager that temporarily connects ``receiver`` to the - signal while a ``with`` block executes. When the block exits, the - receiver is disconnected. Useful for tests. - - :param receiver: The callable to call when :meth:`send` is called with - the given ``sender``, passing ``sender`` as a positional argument - along with any extra keyword arguments. - :param sender: Any object or :data:`ANY`. ``receiver`` will only be - called when :meth:`send` is called with this sender. If ``ANY``, the - receiver will be called for any sender. - - .. versionadded:: 1.1 - """ - self.connect(receiver, sender=sender, weak=False) - - try: - yield None - finally: - self.disconnect(receiver) - - @contextmanager - def muted(self) -> c.Generator[None, None, None]: - """A context manager that temporarily disables the signal. No receivers - will be called if the signal is sent, until the ``with`` block exits. - Useful for tests. - """ - self.is_muted = True - - try: - yield None - finally: - self.is_muted = False - - def send( - self, - sender: t.Any | None = None, - /, - *, - _async_wrapper: c.Callable[ - [c.Callable[..., c.Coroutine[t.Any, t.Any, t.Any]]], c.Callable[..., t.Any] - ] - | None = None, - **kwargs: t.Any, - ) -> list[tuple[c.Callable[..., t.Any], t.Any]]: - """Call all receivers that are connected to the given ``sender`` - or :data:`ANY`. Each receiver is called with ``sender`` as a positional - argument along with any extra keyword arguments. Return a list of - ``(receiver, return value)`` tuples. - - The order receivers are called is undefined, but can be influenced by - setting :attr:`set_class`. - - If a receiver raises an exception, that exception will propagate up. - This makes debugging straightforward, with an assumption that correctly - implemented receivers will not raise. - - :param sender: Call receivers connected to this sender, in addition to - those connected to :data:`ANY`. - :param _async_wrapper: Will be called on any receivers that are async - coroutines to turn them into sync callables. For example, could run - the receiver with an event loop. - :param kwargs: Extra keyword arguments to pass to each receiver. - - .. versionchanged:: 1.7 - Added the ``_async_wrapper`` argument. - """ - if self.is_muted: - return [] - - results = [] - - for receiver in self.receivers_for(sender): - if iscoroutinefunction(receiver): - if _async_wrapper is None: - raise RuntimeError("Cannot send to a coroutine function.") - - result = _async_wrapper(receiver)(sender, **kwargs) - else: - result = receiver(sender, **kwargs) - - results.append((receiver, result)) - - return results - - async def send_async( - self, - sender: t.Any | None = None, - /, - *, - _sync_wrapper: c.Callable[ - [c.Callable[..., t.Any]], c.Callable[..., c.Coroutine[t.Any, t.Any, t.Any]] - ] - | None = None, - **kwargs: t.Any, - ) -> list[tuple[c.Callable[..., t.Any], t.Any]]: - """Await all receivers that are connected to the given ``sender`` - or :data:`ANY`. Each receiver is called with ``sender`` as a positional - argument along with any extra keyword arguments. Return a list of - ``(receiver, return value)`` tuples. - - The order receivers are called is undefined, but can be influenced by - setting :attr:`set_class`. - - If a receiver raises an exception, that exception will propagate up. - This makes debugging straightforward, with an assumption that correctly - implemented receivers will not raise. - - :param sender: Call receivers connected to this sender, in addition to - those connected to :data:`ANY`. - :param _sync_wrapper: Will be called on any receivers that are sync - callables to turn them into async coroutines. For example, - could call the receiver in a thread. - :param kwargs: Extra keyword arguments to pass to each receiver. - - .. versionadded:: 1.7 - """ - if self.is_muted: - return [] - - results = [] - - for receiver in self.receivers_for(sender): - if not iscoroutinefunction(receiver): - if _sync_wrapper is None: - raise RuntimeError("Cannot send to a non-coroutine function.") - - result = await _sync_wrapper(receiver)(sender, **kwargs) - else: - result = await receiver(sender, **kwargs) - - results.append((receiver, result)) - - return results - - def has_receivers_for(self, sender: t.Any) -> bool: - """Check if there is at least one receiver that will be called with the - given ``sender``. A receiver connected to :data:`ANY` will always be - called, regardless of sender. Does not check if weakly referenced - receivers are still live. See :meth:`receivers_for` for a stronger - search. - - :param sender: Check for receivers connected to this sender, in addition - to those connected to :data:`ANY`. - """ - if not self.receivers: - return False - - if self._by_sender[ANY_ID]: - return True - - if sender is ANY: - return False - - return make_id(sender) in self._by_sender - - def receivers_for( - self, sender: t.Any - ) -> c.Generator[c.Callable[..., t.Any], None, None]: - """Yield each receiver to be called for ``sender``, in addition to those - to be called for :data:`ANY`. Weakly referenced receivers that are not - live will be disconnected and skipped. - - :param sender: Yield receivers connected to this sender, in addition - to those connected to :data:`ANY`. - """ - # TODO: test receivers_for(ANY) - if not self.receivers: - return - - sender_id = make_id(sender) - - if sender_id in self._by_sender: - ids = self._by_sender[ANY_ID] | self._by_sender[sender_id] - else: - ids = self._by_sender[ANY_ID].copy() - - for receiver_id in ids: - receiver = self.receivers.get(receiver_id) - - if receiver is None: - continue - - if isinstance(receiver, weakref.ref): - strong = receiver() - - if strong is None: - self._disconnect(receiver_id, ANY_ID) - continue - - yield strong - else: - yield receiver - - def disconnect(self, receiver: c.Callable[..., t.Any], sender: t.Any = ANY) -> None: - """Disconnect ``receiver`` from being called when the signal is sent by - ``sender``. - - :param receiver: A connected receiver callable. - :param sender: Disconnect from only this sender. By default, disconnect - from all senders. - """ - sender_id: c.Hashable - - if sender is ANY: - sender_id = ANY_ID - else: - sender_id = make_id(sender) - - receiver_id = make_id(receiver) - self._disconnect(receiver_id, sender_id) - - if ( - "receiver_disconnected" in self.__dict__ - and self.receiver_disconnected.receivers - ): - self.receiver_disconnected.send(self, receiver=receiver, sender=sender) - - def _disconnect(self, receiver_id: c.Hashable, sender_id: c.Hashable) -> None: - if sender_id == ANY_ID: - if self._by_receiver.pop(receiver_id, None) is not None: - for bucket in self._by_sender.values(): - bucket.discard(receiver_id) - - self.receivers.pop(receiver_id, None) - else: - self._by_sender[sender_id].discard(receiver_id) - self._by_receiver[receiver_id].discard(sender_id) - - def _make_cleanup_receiver( - self, receiver_id: c.Hashable - ) -> c.Callable[[weakref.ref[c.Callable[..., t.Any]]], None]: - """Create a callback function to disconnect a weakly referenced - receiver when it is garbage collected. - """ - - def cleanup(ref: weakref.ref[c.Callable[..., t.Any]]) -> None: - # If the interpreter is shutting down, disconnecting can result in a - # weird ignored exception. Don't call it in that case. - if not sys.is_finalizing(): - self._disconnect(receiver_id, ANY_ID) - - return cleanup - - def _make_cleanup_sender( - self, sender_id: c.Hashable - ) -> c.Callable[[weakref.ref[t.Any]], None]: - """Create a callback function to disconnect all receivers for a weakly - referenced sender when it is garbage collected. - """ - assert sender_id != ANY_ID - - def cleanup(ref: weakref.ref[t.Any]) -> None: - self._weak_senders.pop(sender_id, None) - - for receiver_id in self._by_sender.pop(sender_id, ()): - self._by_receiver[receiver_id].discard(sender_id) - - return cleanup - - def _cleanup_bookkeeping(self) -> None: - """Prune unused sender/receiver bookkeeping. Not threadsafe. - - Connecting & disconnecting leaves behind a small amount of bookkeeping - data. Typical workloads using Blinker, for example in most web apps, - Flask, CLI scripts, etc., are not adversely affected by this - bookkeeping. - - With a long-running process performing dynamic signal routing with high - volume, e.g. connecting to function closures, senders are all unique - object instances. Doing all of this over and over may cause memory usage - to grow due to extraneous bookkeeping. (An empty ``set`` for each stale - sender/receiver pair.) - - This method will prune that bookkeeping away, with the caveat that such - pruning is not threadsafe. The risk is that cleanup of a fully - disconnected receiver/sender pair occurs while another thread is - connecting that same pair. If you are in the highly dynamic, unique - receiver/sender situation that has lead you to this method, that failure - mode is perhaps not a big deal for you. - """ - for mapping in (self._by_sender, self._by_receiver): - for ident, bucket in list(mapping.items()): - if not bucket: - mapping.pop(ident, None) - - def _clear_state(self) -> None: - """Disconnect all receivers and senders. Useful for tests.""" - self._weak_senders.clear() - self.receivers.clear() - self._by_sender.clear() - self._by_receiver.clear() - - -class NamedSignal(Signal): - """A named generic notification emitter. The name is not used by the signal - itself, but matches the key in the :class:`Namespace` that it belongs to. - - :param name: The name of the signal within the namespace. - :param doc: The docstring for the signal. - """ - - def __init__(self, name: str, doc: str | None = None) -> None: - super().__init__(doc) - - #: The name of this signal. - self.name: str = name - - def __repr__(self) -> str: - base = super().__repr__() - return f"{base[:-1]}; {self.name!r}>" # noqa: E702 - - -class Namespace(dict[str, NamedSignal]): - """A dict mapping names to signals.""" - - def signal(self, name: str, doc: str | None = None) -> NamedSignal: - """Return the :class:`NamedSignal` for the given ``name``, creating it - if required. Repeated calls with the same name return the same signal. - - :param name: The name of the signal. - :param doc: The docstring of the signal. - """ - if name not in self: - self[name] = NamedSignal(name, doc) - - return self[name] - - -class _PNamespaceSignal(t.Protocol): - def __call__(self, name: str, doc: str | None = None) -> NamedSignal: ... - - -default_namespace: Namespace = Namespace() -"""A default :class:`Namespace` for creating named signals. :func:`signal` -creates a :class:`NamedSignal` in this namespace. -""" - -signal: _PNamespaceSignal = default_namespace.signal -"""Return a :class:`NamedSignal` in :data:`default_namespace` with the given -``name``, creating it if required. Repeated calls with the same name return the -same signal. -""" diff --git a/bundle/python-cpu/Lib/site-packages/blinker/py.typed b/bundle/python-cpu/Lib/site-packages/blinker/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/METADATA deleted file mode 100644 index eb460c2ca89630f6285f4690476e4e04daca9b0e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/METADATA +++ /dev/null @@ -1,78 +0,0 @@ -Metadata-Version: 2.4 -Name: certifi -Version: 2026.7.22 -Summary: Python package for providing Mozilla's CA Bundle. -Home-page: https://github.com/certifi/python-certifi -Author: Kenneth Reitz -Author-email: me@kennethreitz.com -License: MPL-2.0 -Project-URL: Source, https://github.com/certifi/python-certifi -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) -Classifier: Natural Language :: English -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Requires-Python: >=3.7 -License-File: LICENSE -Dynamic: author -Dynamic: author-email -Dynamic: classifier -Dynamic: description -Dynamic: home-page -Dynamic: license -Dynamic: license-file -Dynamic: project-url -Dynamic: requires-python -Dynamic: summary - -Certifi: Python SSL Certificates -================================ - -Certifi provides Mozilla's carefully curated collection of Root Certificates for -validating the trustworthiness of SSL certificates while verifying the identity -of TLS hosts. It has been extracted from the `Requests`_ project. - -Installation ------------- - -``certifi`` is available on PyPI. Simply install it with ``pip``:: - - $ pip install certifi - -Usage ------ - -To reference the installed certificate authority (CA) bundle, you can use the -built-in function:: - - >>> import certifi - - >>> certifi.where() - '/usr/local/lib/python3.7/site-packages/certifi/cacert.pem' - -Or from the command line:: - - $ python -m certifi - /usr/local/lib/python3.7/site-packages/certifi/cacert.pem - -Enjoy! - -.. _`Requests`: https://requests.readthedocs.io/en/latest/ - -Addition/Removal of Certificates --------------------------------- - -Certifi does not support any addition/removal or other modification of the -CA trust store content. This project is intended to provide a reliable and -highly portable root of trust to python deployments. Look to upstream projects -for methods to use alternate trust. diff --git a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/RECORD deleted file mode 100644 index ef8a3fb043614b7b60fd0fc01d8acc70bea20be5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/RECORD +++ /dev/null @@ -1,14 +0,0 @@ -certifi-2026.7.22.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -certifi-2026.7.22.dist-info/METADATA,sha256=71rxY4-7I2dqw8V3ffz8LNnDSP5Bcu1bo9J3ZVskgJA,2474 -certifi-2026.7.22.dist-info/RECORD,, -certifi-2026.7.22.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -certifi-2026.7.22.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91 -certifi-2026.7.22.dist-info/licenses/LICENSE,sha256=6TcW2mucDVpKHfYP5pWzcPBpVgPSH2-D8FPkLPwQyvc,989 -certifi-2026.7.22.dist-info/top_level.txt,sha256=KMu4vUCfsjLrkPbSNdgdekS-pVJzBAJFO__nI8NF6-U,8 -certifi/__init__.py,sha256=elLl9CBfmz0SoxiYzqp-P2g30ZzCMHALyuGp2R0UhsE,94 -certifi/__main__.py,sha256=xBBoj905TUWBLRGANOcf7oi6e-3dMP4cEoG9OyMs11g,243 -certifi/cacert.pem,sha256=nMKndLUZjc_xTZvh5mCR9TiXXYZ84Cmpa84VpV39cw8,240216 -certifi/core.py,sha256=XFXycndG5pf37ayeF8N32HUuDafsyhkVMbO4BAPWHa0,3394 -certifi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -certifi/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -certifi/tests/test_certify.py,sha256=fBedHt839-kSsTNmNzX7RsYzjV0OX4kV0Rw7wofSJ7g,467 diff --git a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/WHEEL deleted file mode 100644 index 1d472b6c22838de58d1c3c0dd2c795a5cde9e415..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/WHEEL +++ /dev/null @@ -1,5 +0,0 @@ -Wheel-Version: 1.0 -Generator: setuptools (83.0.0) -Root-Is-Purelib: true -Tag: py3-none-any - diff --git a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/licenses/LICENSE b/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/licenses/LICENSE deleted file mode 100644 index 62b076cdee58ec8f34034141ba0befd9015b0c7e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/licenses/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -This package contains a modified version of ca-bundle.crt: - -ca-bundle.crt -- Bundle of CA Root Certificates - -This is a bundle of X.509 certificates of public Certificate Authorities -(CA). These were automatically extracted from Mozilla's root certificates -file (certdata.txt). This file can be found in the mozilla source tree: -https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/ckfw/builtins/certdata.txt -It contains the certificates in PEM format and therefore -can be directly used with curl / libcurl / php_curl, or with -an Apache+mod_ssl webserver for SSL client authentication. -Just configure this file as the SSLCACertificateFile.# - -***** BEGIN LICENSE BLOCK ***** -This Source Code Form is subject to the terms of the Mozilla Public License, -v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain -one at http://mozilla.org/MPL/2.0/. - -***** END LICENSE BLOCK ***** -@(#) $RCSfile: certdata.txt,v $ $Revision: 1.80 $ $Date: 2011/11/03 15:11:58 $ diff --git a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/top_level.txt b/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/top_level.txt deleted file mode 100644 index 963eac530b9bc28d704d1bc410299c68e3216d4d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/certifi-2026.7.22.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -certifi diff --git a/bundle/python-cpu/Lib/site-packages/certifi/__init__.py b/bundle/python-cpu/Lib/site-packages/certifi/__init__.py deleted file mode 100644 index 66dae9e5fc75868817e667eadeca1166c6ff5eed..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/certifi/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .core import contents, where - -__all__ = ["contents", "where"] -__version__ = "2026.07.22" diff --git a/bundle/python-cpu/Lib/site-packages/certifi/__main__.py b/bundle/python-cpu/Lib/site-packages/certifi/__main__.py deleted file mode 100644 index 8945b5da857f4a7dec2b84f1225f012f6098418c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/certifi/__main__.py +++ /dev/null @@ -1,12 +0,0 @@ -import argparse - -from certifi import contents, where - -parser = argparse.ArgumentParser() -parser.add_argument("-c", "--contents", action="store_true") -args = parser.parse_args() - -if args.contents: - print(contents()) -else: - print(where()) diff --git a/bundle/python-cpu/Lib/site-packages/certifi/cacert.pem b/bundle/python-cpu/Lib/site-packages/certifi/cacert.pem deleted file mode 100644 index 9a40f439e0d77f3733b76489ae3bb42eaaf021d6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/certifi/cacert.pem +++ /dev/null @@ -1,3959 +0,0 @@ - -# Issuer: CN=COMODO ECC Certification Authority O=COMODO CA Limited -# Subject: CN=COMODO ECC Certification Authority O=COMODO CA Limited -# Label: "COMODO ECC Certification Authority" -# Serial: 41578283867086692638256921589707938090 -# MD5 Fingerprint: 7c:62:ff:74:9d:31:53:5e:68:4a:d5:78:aa:1e:bf:23 -# SHA1 Fingerprint: 9f:74:4e:9f:2b:4d:ba:ec:0f:31:2c:50:b6:56:3b:8e:2d:93:c3:11 -# SHA256 Fingerprint: 17:93:92:7a:06:14:54:97:89:ad:ce:2f:8f:34:f7:f0:b6:6d:0f:3a:e3:a3:b8:4d:21:ec:15:db:ba:4f:ad:c7 ------BEGIN CERTIFICATE----- -MIICiTCCAg+gAwIBAgIQH0evqmIAcFBUTAGem2OZKjAKBggqhkjOPQQDAzCBhTEL -MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE -BxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMT -IkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwMzA2MDAw -MDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdy -ZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09N -T0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlv -biBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQDR3svdcmCFYX7deSR -FtSrYpn1PlILBs5BAH+X4QokPB0BBO490o0JlwzgdeT6+3eKKvUDYEs2ixYjFq0J -cfRK9ChQtP6IHG4/bC8vCVlbpVsLM5niwz2J+Wos77LTBumjQjBAMB0GA1UdDgQW -BBR1cacZSBm8nZ3qQUfflMRId5nTeTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/ -BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjEA7wNbeqy3eApyt4jf/7VGFAkK+qDm -fQjGGoe9GKhzvSbKYAydzpmfz1wPMOG+FDHqAjAU9JM8SaczepBGR7NjfRObTrdv -GDeAU/7dIOA1mjbRxwG55tzd8/8dLDoWV9mSOdY= ------END CERTIFICATE----- - -# Issuer: CN=NetLock Arany (Class Gold) F\u0151tan\xfas\xedtv\xe1ny O=NetLock Kft. OU=Tan\xfas\xedtv\xe1nykiad\xf3k (Certification Services) -# Subject: CN=NetLock Arany (Class Gold) F\u0151tan\xfas\xedtv\xe1ny O=NetLock Kft. OU=Tan\xfas\xedtv\xe1nykiad\xf3k (Certification Services) -# Label: "NetLock Arany (Class Gold) F\u0151tan\xfas\xedtv\xe1ny" -# Serial: 80544274841616 -# MD5 Fingerprint: c5:a1:b7:ff:73:dd:d6:d7:34:32:18:df:fc:3c:ad:88 -# SHA1 Fingerprint: 06:08:3f:59:3f:15:a1:04:a0:69:a4:6b:a9:03:d0:06:b7:97:09:91 -# SHA256 Fingerprint: 6c:61:da:c3:a2:de:f0:31:50:6b:e0:36:d2:a6:fe:40:19:94:fb:d1:3d:f9:c8:d4:66:59:92:74:c4:46:ec:98 ------BEGIN CERTIFICATE----- -MIIEFTCCAv2gAwIBAgIGSUEs5AAQMA0GCSqGSIb3DQEBCwUAMIGnMQswCQYDVQQG -EwJIVTERMA8GA1UEBwwIQnVkYXBlc3QxFTATBgNVBAoMDE5ldExvY2sgS2Z0LjE3 -MDUGA1UECwwuVGFuw7pzw610dsOhbnlraWFkw7NrIChDZXJ0aWZpY2F0aW9uIFNl -cnZpY2VzKTE1MDMGA1UEAwwsTmV0TG9jayBBcmFueSAoQ2xhc3MgR29sZCkgRsWR -dGFuw7pzw610dsOhbnkwHhcNMDgxMjExMTUwODIxWhcNMjgxMjA2MTUwODIxWjCB -pzELMAkGA1UEBhMCSFUxETAPBgNVBAcMCEJ1ZGFwZXN0MRUwEwYDVQQKDAxOZXRM -b2NrIEtmdC4xNzA1BgNVBAsMLlRhbsO6c8OtdHbDoW55a2lhZMOzayAoQ2VydGlm -aWNhdGlvbiBTZXJ2aWNlcykxNTAzBgNVBAMMLE5ldExvY2sgQXJhbnkgKENsYXNz -IEdvbGQpIEbFkXRhbsO6c8OtdHbDoW55MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAxCRec75LbRTDofTjl5Bu0jBFHjzuZ9lk4BqKf8owyoPjIMHj9DrT -lF8afFttvzBPhCf2nx9JvMaZCpDyD/V/Q4Q3Y1GLeqVw/HpYzY6b7cNGbIRwXdrz -AZAj/E4wqX7hJ2Pn7WQ8oLjJM2P+FpD/sLj916jAwJRDC7bVWaaeVtAkH3B5r9s5 -VA1lddkVQZQBr17s9o3x/61k/iCa11zr/qYfCGSji3ZVrR47KGAuhyXoqq8fxmRG -ILdwfzzeSNuWU7c5d+Qa4scWhHaXWy+7GRWF+GmF9ZmnqfI0p6m2pgP8b4Y9VHx2 -BJtr+UBdADTHLpl1neWIA6pN+APSQnbAGwIDAKiLo0UwQzASBgNVHRMBAf8ECDAG -AQH/AgEEMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUzPpnk/C2uNClwB7zU/2M -U9+D15YwDQYJKoZIhvcNAQELBQADggEBAKt/7hwWqZw8UQCgwBEIBaeZ5m8BiFRh -bvG5GK1Krf6BQCOUL/t1fC8oS2IkgYIL9WHxHG64YTjrgfpioTtaYtOUZcTh5m2C -+C8lcLIhJsFyUR+MLMOEkMNaj7rP9KdlpeuY0fsFskZ1FSNqb4VjMIDw1Z4fKRzC -bLBQWV2QWzuoDTDPv31/zvGdg73JRm4gpvlhUbohL3u+pRVjodSVh/GeufOJ8z2F -uLjbvrW5KfnaNwUASZQDhETnv0Mxz3WLJdH0pmT1kvarBes96aULNmLazAZfNou2 -XjG4Kvte9nHfRCaexOYNkbQudZWAUWpLMKawYqGT8ZvYzsRjdT9ZR7E= ------END CERTIFICATE----- - -# Issuer: CN=Microsec e-Szigno Root CA 2009 O=Microsec Ltd. -# Subject: CN=Microsec e-Szigno Root CA 2009 O=Microsec Ltd. -# Label: "Microsec e-Szigno Root CA 2009" -# Serial: 14014712776195784473 -# MD5 Fingerprint: f8:49:f4:03:bc:44:2d:83:be:48:69:7d:29:64:fc:b1 -# SHA1 Fingerprint: 89:df:74:fe:5c:f4:0f:4a:80:f9:e3:37:7d:54:da:91:e1:01:31:8e -# SHA256 Fingerprint: 3c:5f:81:fe:a5:fa:b8:2c:64:bf:a2:ea:ec:af:cd:e8:e0:77:fc:86:20:a7:ca:e5:37:16:3d:f3:6e:db:f3:78 ------BEGIN CERTIFICATE----- -MIIECjCCAvKgAwIBAgIJAMJ+QwRORz8ZMA0GCSqGSIb3DQEBCwUAMIGCMQswCQYD -VQQGEwJIVTERMA8GA1UEBwwIQnVkYXBlc3QxFjAUBgNVBAoMDU1pY3Jvc2VjIEx0 -ZC4xJzAlBgNVBAMMHk1pY3Jvc2VjIGUtU3ppZ25vIFJvb3QgQ0EgMjAwOTEfMB0G -CSqGSIb3DQEJARYQaW5mb0BlLXN6aWduby5odTAeFw0wOTA2MTYxMTMwMThaFw0y -OTEyMzAxMTMwMThaMIGCMQswCQYDVQQGEwJIVTERMA8GA1UEBwwIQnVkYXBlc3Qx -FjAUBgNVBAoMDU1pY3Jvc2VjIEx0ZC4xJzAlBgNVBAMMHk1pY3Jvc2VjIGUtU3pp -Z25vIFJvb3QgQ0EgMjAwOTEfMB0GCSqGSIb3DQEJARYQaW5mb0BlLXN6aWduby5o -dTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOn4j/NjrdqG2KfgQvvP -kd6mJviZpWNwrZuuyjNAfW2WbqEORO7hE52UQlKavXWFdCyoDh2Tthi3jCyoz/tc -cbna7P7ofo/kLx2yqHWH2Leh5TvPmUpG0IMZfcChEhyVbUr02MelTTMuhTlAdX4U -fIASmFDHQWe4oIBhVKZsTh/gnQ4H6cm6M+f+wFUoLAKApxn1ntxVUwOXewdI/5n7 -N4okxFnMUBBjjqqpGrCEGob5X7uxUG6k0QrM1XF+H6cbfPVTbiJfyyvm1HxdrtbC -xkzlBQHZ7Vf8wSN5/PrIJIOV87VqUQHQd9bpEqH5GoP7ghu5sJf0dgYzQ0mg/wu1 -+rUCAwEAAaOBgDB+MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0G -A1UdDgQWBBTLD8bfQkPMPcu1SCOhGnqmKrs0aDAfBgNVHSMEGDAWgBTLD8bfQkPM -Pcu1SCOhGnqmKrs0aDAbBgNVHREEFDASgRBpbmZvQGUtc3ppZ25vLmh1MA0GCSqG -SIb3DQEBCwUAA4IBAQDJ0Q5eLtXMs3w+y/w9/w0olZMEyL/azXm4Q5DwpL7v8u8h -mLzU1F0G9u5C7DBsoKqpyvGvivo/C3NqPuouQH4frlRheesuCDfXI/OMn74dseGk -ddug4lQUsbocKaQY9hK6ohQU4zE1yED/t+AFdlfBHFny+L/k7SViXITwfn4fs775 -tyERzAMBVnCnEJIeGzSBHq2cGsMEPO0CYdYeBvNfOofyK/FFh+U9rNHHV4S9a67c -2Pm2G2JwCz02yULyMtd6YebS2z3PyKnJm9zbWETXbzivf3jTo60adbocwTZ8jx5t -HMN1Rq41Bab2XD0h7lbwyYIiLXpUq3DDfSJlgnCW ------END CERTIFICATE----- - -# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R3 -# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R3 -# Label: "GlobalSign Root CA - R3" -# Serial: 4835703278459759426209954 -# MD5 Fingerprint: c5:df:b8:49:ca:05:13:55:ee:2d:ba:1a:c3:3e:b0:28 -# SHA1 Fingerprint: d6:9b:56:11:48:f0:1c:77:c5:45:78:c1:09:26:df:5b:85:69:76:ad -# SHA256 Fingerprint: cb:b5:22:d7:b7:f1:27:ad:6a:01:13:86:5b:df:1c:d4:10:2e:7d:07:59:af:63:5a:7c:f4:72:0d:c9:63:c5:3b ------BEGIN CERTIFICATE----- -MIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G -A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp -Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4 -MTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG -A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI -hvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8 -RgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT -gHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm -KPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd -QQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ -XriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw -DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o -LkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU -RUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp -jjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK -6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX -mcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs -Mx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH -WD9f ------END CERTIFICATE----- - -# Issuer: CN=Izenpe.com O=IZENPE S.A. -# Subject: CN=Izenpe.com O=IZENPE S.A. -# Label: "Izenpe.com" -# Serial: 917563065490389241595536686991402621 -# MD5 Fingerprint: a6:b0:cd:85:80:da:5c:50:34:a3:39:90:2f:55:67:73 -# SHA1 Fingerprint: 2f:78:3d:25:52:18:a7:4a:65:39:71:b5:2c:a2:9c:45:15:6f:e9:19 -# SHA256 Fingerprint: 25:30:cc:8e:98:32:15:02:ba:d9:6f:9b:1f:ba:1b:09:9e:2d:29:9e:0f:45:48:bb:91:4f:36:3b:c0:d4:53:1f ------BEGIN CERTIFICATE----- -MIIF8TCCA9mgAwIBAgIQALC3WhZIX7/hy/WL1xnmfTANBgkqhkiG9w0BAQsFADA4 -MQswCQYDVQQGEwJFUzEUMBIGA1UECgwLSVpFTlBFIFMuQS4xEzARBgNVBAMMCkl6 -ZW5wZS5jb20wHhcNMDcxMjEzMTMwODI4WhcNMzcxMjEzMDgyNzI1WjA4MQswCQYD -VQQGEwJFUzEUMBIGA1UECgwLSVpFTlBFIFMuQS4xEzARBgNVBAMMCkl6ZW5wZS5j -b20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ03rKDx6sp4boFmVq -scIbRTJxldn+EFvMr+eleQGPicPK8lVx93e+d5TzcqQsRNiekpsUOqHnJJAKClaO -xdgmlOHZSOEtPtoKct2jmRXagaKH9HtuJneJWK3W6wyyQXpzbm3benhB6QiIEn6H -LmYRY2xU+zydcsC8Lv/Ct90NduM61/e0aL6i9eOBbsFGb12N4E3GVFWJGjMxCrFX -uaOKmMPsOzTFlUFpfnXCPCDFYbpRR6AgkJOhkEvzTnyFRVSa0QUmQbC1TR0zvsQD -yCV8wXDbO/QJLVQnSKwv4cSsPsjLkkxTOTcj7NMB+eAJRE1NZMDhDVqHIrytG6P+ -JrUV86f8hBnp7KGItERphIPzidF0BqnMC9bC3ieFUCbKF7jJeodWLBoBHmy+E60Q -rLUk9TiRodZL2vG70t5HtfG8gfZZa88ZU+mNFctKy6lvROUbQc/hhqfK0GqfvEyN -BjNaooXlkDWgYlwWTvDjovoDGrQscbNYLN57C9saD+veIR8GdwYDsMnvmfzAuU8L -hij+0rnq49qlw0dpEuDb8PYZi+17cNcC1u2HGCgsBCRMd+RIihrGO5rUD8r6ddIB -QFqNeb+Lz0vPqhbBleStTIo+F5HUsWLlguWABKQDfo2/2n+iD5dPDNMN+9fR5XJ+ -HMh3/1uaD7euBUbl8agW7EekFwIDAQABo4H2MIHzMIGwBgNVHREEgagwgaWBD2lu -Zm9AaXplbnBlLmNvbaSBkTCBjjFHMEUGA1UECgw+SVpFTlBFIFMuQS4gLSBDSUYg -QTAxMzM3MjYwLVJNZXJjLlZpdG9yaWEtR2FzdGVpeiBUMTA1NSBGNjIgUzgxQzBB -BgNVBAkMOkF2ZGEgZGVsIE1lZGl0ZXJyYW5lbyBFdG9yYmlkZWEgMTQgLSAwMTAx -MCBWaXRvcmlhLUdhc3RlaXowDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC -AQYwHQYDVR0OBBYEFB0cZQ6o8iV7tJHP5LGx5r1VdGwFMA0GCSqGSIb3DQEBCwUA -A4ICAQB4pgwWSp9MiDrAyw6lFn2fuUhfGI8NYjb2zRlrrKvV9pF9rnHzP7MOeIWb -laQnIUdCSnxIOvVFfLMMjlF4rJUT3sb9fbgakEyrkgPH7UIBzg/YsfqikuFgba56 -awmqxinuaElnMIAkejEWOVt+8Rwu3WwJrfIxwYJOubv5vr8qhT/AQKM6WfxZSzwo -JNu0FXWuDYi6LnPAvViH5ULy617uHjAimcs30cQhbIHsvm0m5hzkQiCeR7Csg1lw -LDXWrzY0tM07+DKo7+N4ifuNRSzanLh+QBxh5z6ikixL8s36mLYp//Pye6kfLqCT -VyvehQP5aTfLnnhqBbTFMXiJ7HqnheG5ezzevh55hM6fcA5ZwjUukCox2eRFekGk -LhObNA5me0mrZJfQRsN5nXJQY6aYWwa9SG3YOYNw6DXwBdGqvOPbyALqfP2C2sJb -UjWumDqtujWTI6cfSN01RpiyEGjkpTHCClguGYEQyVB1/OpaFs4R1+7vUIgtYf8/ -QnMFlEPVjjxOAToZpR9GTnfQXeWBIiGH/pR9hNiTrdZoQ0iy2+tzJOeRf1SktoA+ -naM8THLCV8Sg1Mw4J87VBp6iSNnpn86CcDaTmjvfliHjWbcM2pE38P1ZWrOZyGls -QyYBNWNgVYkDOnXYukrZVP/u3oDYLdE41V4tC5h9Pmzb/CaIxw== ------END CERTIFICATE----- - -# Issuer: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc. -# Subject: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc. -# Label: "Go Daddy Root Certificate Authority - G2" -# Serial: 0 -# MD5 Fingerprint: 80:3a:bc:22:c1:e6:fb:8d:9b:3b:27:4a:32:1b:9a:01 -# SHA1 Fingerprint: 47:be:ab:c9:22:ea:e8:0e:78:78:34:62:a7:9f:45:c2:54:fd:e6:8b -# SHA256 Fingerprint: 45:14:0b:32:47:eb:9c:c8:c5:b4:f0:d7:b5:30:91:f7:32:92:08:9e:6e:5a:63:e2:74:9d:d3:ac:a9:19:8e:da ------BEGIN CERTIFICATE----- -MIIDxTCCAq2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx -EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT -EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp -ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAwMFoXDTM3MTIzMTIz -NTk1OVowgYMxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH -EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjExMC8GA1UE -AxMoR28gRGFkZHkgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIw -DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9xYgjx+lk09xvJGKP3gElY6SKD -E6bFIEMBO4Tx5oVJnyfq9oQbTqC023CYxzIBsQU+B07u9PpPL1kwIuerGVZr4oAH -/PMWdYA5UXvl+TW2dE6pjYIT5LY/qQOD+qK+ihVqf94Lw7YZFAXK6sOoBJQ7Rnwy -DfMAZiLIjWltNowRGLfTshxgtDj6AozO091GB94KPutdfMh8+7ArU6SSYmlRJQVh -GkSBjCypQ5Yj36w6gZoOKcUcqeldHraenjAKOc7xiID7S13MMuyFYkMlNAJWJwGR -tDtwKj9useiciAF9n9T521NtYJ2/LOdYq7hfRvzOxBsDPAnrSTFcaUaz4EcCAwEA -AaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE -FDqahQcQZyi27/a9BUFuIMGU2g/eMA0GCSqGSIb3DQEBCwUAA4IBAQCZ21151fmX -WWcDYfF+OwYxdS2hII5PZYe096acvNjpL9DbWu7PdIxztDhC2gV7+AJ1uP2lsdeu -9tfeE8tTEH6KRtGX+rcuKxGrkLAngPnon1rpN5+r5N9ss4UXnT3ZJE95kTXWXwTr -gIOrmgIttRD02JDHBHNA7XIloKmf7J6raBKZV8aPEjoJpL1E/QYVN8Gb5DKj7Tjo -2GTzLH4U/ALqn83/B2gX2yKQOC16jdFU8WnjXzPKej17CuPKf1855eJ1usV2GDPO -LPAvTK33sefOT6jEm0pUBsV/fdUID+Ic/n4XuKxe9tQWskMJDE32p2u0mYRlynqI -4uJEvlz36hz1 ------END CERTIFICATE----- - -# Issuer: CN=Starfield Root Certificate Authority - G2 O=Starfield Technologies, Inc. -# Subject: CN=Starfield Root Certificate Authority - G2 O=Starfield Technologies, Inc. -# Label: "Starfield Root Certificate Authority - G2" -# Serial: 0 -# MD5 Fingerprint: d6:39:81:c6:52:7e:96:69:fc:fc:ca:66:ed:05:f2:96 -# SHA1 Fingerprint: b5:1c:06:7c:ee:2b:0c:3d:f8:55:ab:2d:92:f4:fe:39:d4:e7:0f:0e -# SHA256 Fingerprint: 2c:e1:cb:0b:f9:d2:f9:e1:02:99:3f:be:21:51:52:c3:b2:dd:0c:ab:de:1c:68:e5:31:9b:83:91:54:db:b7:f5 ------BEGIN CERTIFICATE----- -MIID3TCCAsWgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBjzELMAkGA1UEBhMCVVMx -EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT -HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAMTKVN0YXJmaWVs -ZCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAw -MFoXDTM3MTIzMTIzNTk1OVowgY8xCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6 -b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFyZmllbGQgVGVj -aG5vbG9naWVzLCBJbmMuMTIwMAYDVQQDEylTdGFyZmllbGQgUm9vdCBDZXJ0aWZp -Y2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC -ggEBAL3twQP89o/8ArFvW59I2Z154qK3A2FWGMNHttfKPTUuiUP3oWmb3ooa/RMg -nLRJdzIpVv257IzdIvpy3Cdhl+72WoTsbhm5iSzchFvVdPtrX8WJpRBSiUZV9Lh1 -HOZ/5FSuS/hVclcCGfgXcVnrHigHdMWdSL5stPSksPNkN3mSwOxGXn/hbVNMYq/N -Hwtjuzqd+/x5AJhhdM8mgkBj87JyahkNmcrUDnXMN/uLicFZ8WJ/X7NfZTD4p7dN -dloedl40wOiWVpmKs/B/pM293DIxfJHP4F8R+GuqSVzRmZTRouNjWwl2tVZi4Ut0 -HZbUJtQIBFnQmA4O5t78w+wfkPECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO -BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFHwMMh+n2TB/xH1oo2Kooc6rB1snMA0G -CSqGSIb3DQEBCwUAA4IBAQARWfolTwNvlJk7mh+ChTnUdgWUXuEok21iXQnCoKjU -sHU48TRqneSfioYmUeYs0cYtbpUgSpIB7LiKZ3sx4mcujJUDJi5DnUox9g61DLu3 -4jd/IroAow57UvtruzvE03lRTs2Q9GcHGcg8RnoNAX3FWOdt5oUwF5okxBDgBPfg -8n/Uqgr/Qh037ZTlZFkSIHc40zI+OIF1lnP6aI+xy84fxez6nH7PfrHxBy22/L/K -pL/QlwVKvOoYKAKQvVR4CSFx09F9HdkWsKlhPdAKACL8x3vLCWRFCztAgfd9fDL1 -mMpYjn0q7pBZc2T5NnReJaH1ZgUufzkVqSr7UIuOhWn0 ------END CERTIFICATE----- - -# Issuer: CN=Starfield Services Root Certificate Authority - G2 O=Starfield Technologies, Inc. -# Subject: CN=Starfield Services Root Certificate Authority - G2 O=Starfield Technologies, Inc. -# Label: "Starfield Services Root Certificate Authority - G2" -# Serial: 0 -# MD5 Fingerprint: 17:35:74:af:7b:61:1c:eb:f4:f9:3c:e2:ee:40:f9:a2 -# SHA1 Fingerprint: 92:5a:8f:8d:2c:6d:04:e0:66:5f:59:6a:ff:22:d8:63:e8:25:6f:3f -# SHA256 Fingerprint: 56:8d:69:05:a2:c8:87:08:a4:b3:02:51:90:ed:cf:ed:b1:97:4a:60:6a:13:c6:e5:29:0f:cb:2a:e6:3e:da:b5 ------BEGIN CERTIFICATE----- -MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx -EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT -HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs -ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5 -MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD -VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy -ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy -dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI -hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p -OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2 -8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K -Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe -hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk -6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw -DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q -AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI -bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB -ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z -qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd -iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn -0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN -sSi6 ------END CERTIFICATE----- - -# Issuer: CN=Certum Trusted Network CA O=Unizeto Technologies S.A. OU=Certum Certification Authority -# Subject: CN=Certum Trusted Network CA O=Unizeto Technologies S.A. OU=Certum Certification Authority -# Label: "Certum Trusted Network CA" -# Serial: 279744 -# MD5 Fingerprint: d5:e9:81:40:c5:18:69:fc:46:2c:89:75:62:0f:aa:78 -# SHA1 Fingerprint: 07:e0:32:e0:20:b7:2c:3f:19:2f:06:28:a2:59:3a:19:a7:0f:06:9e -# SHA256 Fingerprint: 5c:58:46:8d:55:f5:8e:49:7e:74:39:82:d2:b5:00:10:b6:d1:65:37:4a:cf:83:a7:d4:a3:2d:b7:68:c4:40:8e ------BEGIN CERTIFICATE----- -MIIDuzCCAqOgAwIBAgIDBETAMA0GCSqGSIb3DQEBBQUAMH4xCzAJBgNVBAYTAlBM -MSIwIAYDVQQKExlVbml6ZXRvIFRlY2hub2xvZ2llcyBTLkEuMScwJQYDVQQLEx5D -ZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxIjAgBgNVBAMTGUNlcnR1bSBU -cnVzdGVkIE5ldHdvcmsgQ0EwHhcNMDgxMDIyMTIwNzM3WhcNMjkxMjMxMTIwNzM3 -WjB+MQswCQYDVQQGEwJQTDEiMCAGA1UEChMZVW5pemV0byBUZWNobm9sb2dpZXMg -Uy5BLjEnMCUGA1UECxMeQ2VydHVtIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MSIw -IAYDVQQDExlDZXJ0dW0gVHJ1c3RlZCBOZXR3b3JrIENBMIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEA4/t9o3K6wvDJFIf1awFO4W5AB7ptJ11/91sts1rH -UV+rpDKmYYe2bg+G0jACl/jXaVehGDldamR5xgFZrDwxSjh80gTSSyjoIF87B6LM -TXPb865Px1bVWqeWifrzq2jUI4ZZJ88JJ7ysbnKDHDBy3+Ci6dLhdHUZvSqeexVU -BBvXQzmtVSjF4hq79MDkrjhJM8x2hZ85RdKknvISjFH4fOQtf/WsX+sWn7Et0brM -kUJ3TCXJkDhv2/DM+44el1k+1WBO5gUo7Ul5E0u6SNsv+XLTOcr+H9g0cvW0QM8x -AcPs3hEtF10fuFDRXhmnad4HMyjKUJX5p1TLVIZQRan5SQIDAQABo0IwQDAPBgNV -HRMBAf8EBTADAQH/MB0GA1UdDgQWBBQIds3LB/8k9sXN7buQvOKEN0Z19zAOBgNV -HQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQEFBQADggEBAKaorSLOAT2mo/9i0Eidi15y -sHhE49wcrwn9I0j6vSrEuVUEtRCjjSfeC4Jj0O7eDDd5QVsisrCaQVymcODU0HfL -I9MA4GxWL+FpDQ3Zqr8hgVDZBqWo/5U30Kr+4rP1mS1FhIrlQgnXdAIv94nYmem8 -J9RHjboNRhx3zxSkHLmkMcScKHQDNP8zGSal6Q10tz6XxnboJ5ajZt3hrvJBW8qY -VoNzcOSGGtIxQbovvi0TWnZvTuhOgQ4/WwMioBK+ZlgRSssDxLQqKi2WF+A5VLxI -03YnnZotBqbJ7DnSq9ufmgsnAjUpsUCV5/nonFWIGUbWtzT1fs45mtk48VH3Tyw= ------END CERTIFICATE----- - -# Issuer: CN=TWCA Root Certification Authority O=TAIWAN-CA OU=Root CA -# Subject: CN=TWCA Root Certification Authority O=TAIWAN-CA OU=Root CA -# Label: "TWCA Root Certification Authority" -# Serial: 1 -# MD5 Fingerprint: aa:08:8f:f6:f9:7b:b7:f2:b1:a7:1e:9b:ea:ea:bd:79 -# SHA1 Fingerprint: cf:9e:87:6d:d3:eb:fc:42:26:97:a3:b5:a3:7a:a0:76:a9:06:23:48 -# SHA256 Fingerprint: bf:d8:8f:e1:10:1c:41:ae:3e:80:1b:f8:be:56:35:0e:e9:ba:d1:a6:b9:bd:51:5e:dc:5c:6d:5b:87:11:ac:44 ------BEGIN CERTIFICATE----- -MIIDezCCAmOgAwIBAgIBATANBgkqhkiG9w0BAQUFADBfMQswCQYDVQQGEwJUVzES -MBAGA1UECgwJVEFJV0FOLUNBMRAwDgYDVQQLDAdSb290IENBMSowKAYDVQQDDCFU -V0NBIFJvb3QgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwODI4MDcyNDMz -WhcNMzAxMjMxMTU1OTU5WjBfMQswCQYDVQQGEwJUVzESMBAGA1UECgwJVEFJV0FO -LUNBMRAwDgYDVQQLDAdSb290IENBMSowKAYDVQQDDCFUV0NBIFJvb3QgQ2VydGlm -aWNhdGlvbiBBdXRob3JpdHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB -AQCwfnK4pAOU5qfeCTiRShFAh6d8WWQUe7UREN3+v9XAu1bihSX0NXIP+FPQQeFE -AcK0HMMxQhZHhTMidrIKbw/lJVBPhYa+v5guEGcevhEFhgWQxFnQfHgQsIBct+HH -K3XLfJ+utdGdIzdjp9xCoi2SBBtQwXu4PhvJVgSLL1KbralW6cH/ralYhzC2gfeX -RfwZVzsrb+RH9JlF/h3x+JejiB03HFyP4HYlmlD4oFT/RJB2I9IyxsOrBr/8+7/z -rX2SYgJbKdM1o5OaQ2RgXbL6Mv87BK9NQGr5x+PvI/1ry+UPizgN7gr8/g+YnzAx -3WxSZfmLgb4i4RxYA7qRG4kHAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV -HRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqOFsmjd6LWvJPelSDGRjjCDWmujANBgkq -hkiG9w0BAQUFAAOCAQEAPNV3PdrfibqHDAhUaiBQkr6wQT25JmSDCi/oQMCXKCeC -MErJk/9q56YAf4lCmtYR5VPOL8zy2gXE/uJQxDqGfczafhAJO5I1KlOy/usrBdls -XebQ79NqZp4VKIV66IIArB6nCWlWQtNoURi+VJq/REG6Sb4gumlc7rh3zc5sH62D -lhh9DrUUOYTxKOkto557HnpyWoOzeW/vtPzQCqVYT0bf+215WfKEIlKuD8z7fDvn -aspHYcN6+NOSBB+4IIThNlQWx0DeO4pz3N/GCUzf7Nr/1FNCocnyYh0igzyXxfkZ -YiesZSLX0zzG5Y6yU8xJzrww/nsOM5D77dIUkR8Hrw== ------END CERTIFICATE----- - -# Issuer: O=SECOM Trust Systems CO.,LTD. OU=Security Communication RootCA2 -# Subject: O=SECOM Trust Systems CO.,LTD. OU=Security Communication RootCA2 -# Label: "Security Communication RootCA2" -# Serial: 0 -# MD5 Fingerprint: 6c:39:7d:a4:0e:55:59:b2:3f:d6:41:b1:12:50:de:43 -# SHA1 Fingerprint: 5f:3b:8c:f2:f8:10:b3:7d:78:b4:ce:ec:19:19:c3:73:34:b9:c7:74 -# SHA256 Fingerprint: 51:3b:2c:ec:b8:10:d4:cd:e5:dd:85:39:1a:df:c6:c2:dd:60:d8:7b:b7:36:d2:b5:21:48:4a:a4:7a:0e:be:f6 ------BEGIN CERTIFICATE----- -MIIDdzCCAl+gAwIBAgIBADANBgkqhkiG9w0BAQsFADBdMQswCQYDVQQGEwJKUDEl -MCMGA1UEChMcU0VDT00gVHJ1c3QgU3lzdGVtcyBDTy4sTFRELjEnMCUGA1UECxMe -U2VjdXJpdHkgQ29tbXVuaWNhdGlvbiBSb290Q0EyMB4XDTA5MDUyOTA1MDAzOVoX -DTI5MDUyOTA1MDAzOVowXTELMAkGA1UEBhMCSlAxJTAjBgNVBAoTHFNFQ09NIFRy -dXN0IFN5c3RlbXMgQ08uLExURC4xJzAlBgNVBAsTHlNlY3VyaXR5IENvbW11bmlj -YXRpb24gUm9vdENBMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANAV -OVKxUrO6xVmCxF1SrjpDZYBLx/KWvNs2l9amZIyoXvDjChz335c9S672XewhtUGr -zbl+dp+++T42NKA7wfYxEUV0kz1XgMX5iZnK5atq1LXaQZAQwdbWQonCv/Q4EpVM -VAX3NuRFg3sUZdbcDE3R3n4MqzvEFb46VqZab3ZpUql6ucjrappdUtAtCms1FgkQ -hNBqyjoGADdH5H5XTz+L62e4iKrFvlNVspHEfbmwhRkGeC7bYRr6hfVKkaHnFtWO -ojnflLhwHyg/i/xAXmODPIMqGplrz95Zajv8bxbXH/1KEOtOghY6rCcMU/Gt1SSw -awNQwS08Ft1ENCcadfsCAwEAAaNCMEAwHQYDVR0OBBYEFAqFqXdlBZh8QIH4D5cs -OPEK7DzPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 -DQEBCwUAA4IBAQBMOqNErLlFsceTfsgLCkLfZOoc7llsCLqJX2rKSpWeeo8HxdpF -coJxDjrSzG+ntKEju/Ykn8sX/oymzsLS28yN/HH8AynBbF0zX2S2ZTuJbxh2ePXc -okgfGT+Ok+vx+hfuzU7jBBJV1uXk3fs+BXziHV7Gp7yXT2g69ekuCkO2r1dcYmh8 -t/2jioSgrGK+KwmHNPBqAbubKVY8/gA3zyNs8U6qtnRGEmyR7jTV7JqR50S+kDFy -1UkC9gLl9B/rfNmWVan/7Ir5mUf/NVoCqgTLiluHcSmRvaS0eg29mvVXIwAHIRc/ -SjnRBUkLp7Y3gaVdjKozXoEofKd9J+sAro03 ------END CERTIFICATE----- - -# Issuer: CN=Actalis Authentication Root CA O=Actalis S.p.A./03358520967 -# Subject: CN=Actalis Authentication Root CA O=Actalis S.p.A./03358520967 -# Label: "Actalis Authentication Root CA" -# Serial: 6271844772424770508 -# MD5 Fingerprint: 69:c1:0d:4f:07:a3:1b:c3:fe:56:3d:04:bc:11:f6:a6 -# SHA1 Fingerprint: f3:73:b3:87:06:5a:28:84:8a:f2:f3:4a:ce:19:2b:dd:c7:8e:9c:ac -# SHA256 Fingerprint: 55:92:60:84:ec:96:3a:64:b9:6e:2a:be:01:ce:0b:a8:6a:64:fb:fe:bc:c7:aa:b5:af:c1:55:b3:7f:d7:60:66 ------BEGIN CERTIFICATE----- -MIIFuzCCA6OgAwIBAgIIVwoRl0LE48wwDQYJKoZIhvcNAQELBQAwazELMAkGA1UE -BhMCSVQxDjAMBgNVBAcMBU1pbGFuMSMwIQYDVQQKDBpBY3RhbGlzIFMucC5BLi8w -MzM1ODUyMDk2NzEnMCUGA1UEAwweQWN0YWxpcyBBdXRoZW50aWNhdGlvbiBSb290 -IENBMB4XDTExMDkyMjExMjIwMloXDTMwMDkyMjExMjIwMlowazELMAkGA1UEBhMC -SVQxDjAMBgNVBAcMBU1pbGFuMSMwIQYDVQQKDBpBY3RhbGlzIFMucC5BLi8wMzM1 -ODUyMDk2NzEnMCUGA1UEAwweQWN0YWxpcyBBdXRoZW50aWNhdGlvbiBSb290IENB -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAp8bEpSmkLO/lGMWwUKNv -UTufClrJwkg4CsIcoBh/kbWHuUA/3R1oHwiD1S0eiKD4j1aPbZkCkpAW1V8IbInX -4ay8IMKx4INRimlNAJZaby/ARH6jDuSRzVju3PvHHkVH3Se5CAGfpiEd9UEtL0z9 -KK3giq0itFZljoZUj5NDKd45RnijMCO6zfB9E1fAXdKDa0hMxKufgFpbOr3JpyI/ -gCczWw63igxdBzcIy2zSekciRDXFzMwujt0q7bd9Zg1fYVEiVRvjRuPjPdA1Yprb -rxTIW6HMiRvhMCb8oJsfgadHHwTrozmSBp+Z07/T6k9QnBn+locePGX2oxgkg4YQ -51Q+qDp2JE+BIcXjDwL4k5RHILv+1A7TaLndxHqEguNTVHnd25zS8gebLra8Pu2F -be8lEfKXGkJh90qX6IuxEAf6ZYGyojnP9zz/GPvG8VqLWeICrHuS0E4UT1lF9gxe -KF+w6D9Fz8+vm2/7hNN3WpVvrJSEnu68wEqPSpP4RCHiMUVhUE4Q2OM1fEwZtN4F -v6MGn8i1zeQf1xcGDXqVdFUNaBr8EBtiZJ1t4JWgw5QHVw0U5r0F+7if5t+L4sbn -fpb2U8WANFAoWPASUHEXMLrmeGO89LKtmyuy/uE5jF66CyCU3nuDuP/jVo23Eek7 -jPKxwV2dpAtMK9myGPW1n0sCAwEAAaNjMGEwHQYDVR0OBBYEFFLYiDrIn3hm7Ynz -ezhwlMkCAjbQMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUUtiIOsifeGbt -ifN7OHCUyQICNtAwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3DQEBCwUAA4ICAQAL -e3KHwGCmSUyIWOYdiPcUZEim2FgKDk8TNd81HdTtBjHIgT5q1d07GjLukD0R0i70 -jsNjLiNmsGe+b7bAEzlgqqI0JZN1Ut6nna0Oh4lScWoWPBkdg/iaKWW+9D+a2fDz -WochcYBNy+A4mz+7+uAwTc+G02UQGRjRlwKxK3JCaKygvU5a2hi/a5iB0P2avl4V -SM0RFbnAKVy06Ij3Pjaut2L9HmLecHgQHEhb2rykOLpn7VU+Xlff1ANATIGk0k9j -pwlCCRT8AKnCgHNPLsBA2RF7SOp6AsDT6ygBJlh0wcBzIm2Tlf05fbsq4/aC4yyX -X04fkZT6/iyj2HYauE2yOE+b+h1IYHkm4vP9qdCa6HCPSXrW5b0KDtst842/6+Ok -fcvHlXHo2qN8xcL4dJIEG4aspCJTQLas/kx2z/uUMsA1n3Y/buWQbqCmJqK4LL7R -K4X9p2jIugErsWx0Hbhzlefut8cl8ABMALJ+tguLHPPAUJ4lueAI3jZm/zel0btU -ZCzJJ7VLkn5l/9Mt4blOvH+kQSGQQXemOR/qnuOf0GZvBeyqdn6/axag67XH/JJU -LysRJyU3eExRarDzzFhdFPFqSBX/wge2sY0PjlxQRrM9vwGYT7JZVEc+NHt4bVaT -LnPqZih4zR0Uv6CPLy64Lo7yFIrM6bV8+2ydDKXhlg== ------END CERTIFICATE----- - -# Issuer: CN=Buypass Class 2 Root CA O=Buypass AS-983163327 -# Subject: CN=Buypass Class 2 Root CA O=Buypass AS-983163327 -# Label: "Buypass Class 2 Root CA" -# Serial: 2 -# MD5 Fingerprint: 46:a7:d2:fe:45:fb:64:5a:a8:59:90:9b:78:44:9b:29 -# SHA1 Fingerprint: 49:0a:75:74:de:87:0a:47:fe:58:ee:f6:c7:6b:eb:c6:0b:12:40:99 -# SHA256 Fingerprint: 9a:11:40:25:19:7c:5b:b9:5d:94:e6:3d:55:cd:43:79:08:47:b6:46:b2:3c:df:11:ad:a4:a0:0e:ff:15:fb:48 ------BEGIN CERTIFICATE----- -MIIFWTCCA0GgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBOMQswCQYDVQQGEwJOTzEd -MBsGA1UECgwUQnV5cGFzcyBBUy05ODMxNjMzMjcxIDAeBgNVBAMMF0J1eXBhc3Mg -Q2xhc3MgMiBSb290IENBMB4XDTEwMTAyNjA4MzgwM1oXDTQwMTAyNjA4MzgwM1ow -TjELMAkGA1UEBhMCTk8xHTAbBgNVBAoMFEJ1eXBhc3MgQVMtOTgzMTYzMzI3MSAw -HgYDVQQDDBdCdXlwYXNzIENsYXNzIDIgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEB -BQADggIPADCCAgoCggIBANfHXvfBB9R3+0Mh9PT1aeTuMgHbo4Yf5FkNuud1g1Lr -6hxhFUi7HQfKjK6w3Jad6sNgkoaCKHOcVgb/S2TwDCo3SbXlzwx87vFKu3MwZfPV -L4O2fuPn9Z6rYPnT8Z2SdIrkHJasW4DptfQxh6NR/Md+oW+OU3fUl8FVM5I+GC91 -1K2GScuVr1QGbNgGE41b/+EmGVnAJLqBcXmQRFBoJJRfuLMR8SlBYaNByyM21cHx -MlAQTn/0hpPshNOOvEu/XAFOBz3cFIqUCqTqc/sLUegTBxj6DvEr0VQVfTzh97QZ -QmdiXnfgolXsttlpF9U6r0TtSsWe5HonfOV116rLJeffawrbD02TTqigzXsu8lkB -arcNuAeBfos4GzjmCleZPe4h6KP1DBbdi+w0jpwqHAAVF41og9JwnxgIzRFo1clr -Us3ERo/ctfPYV3Me6ZQ5BL/T3jjetFPsaRyifsSP5BtwrfKi+fv3FmRmaZ9JUaLi -FRhnBkp/1Wy1TbMz4GHrXb7pmA8y1x1LPC5aAVKRCfLf6o3YBkBjqhHk/sM3nhRS -P/TizPJhk9H9Z2vXUq6/aKtAQ6BXNVN48FP4YUIHZMbXb5tMOA1jrGKvNouicwoN -9SG9dKpN6nIDSdvHXx1iY8f93ZHsM+71bbRuMGjeyNYmsHVee7QHIJihdjK4TWxP -AgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMmAd+BikoL1Rpzz -uvdMw964o605MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAU18h -9bqwOlI5LJKwbADJ784g7wbylp7ppHR/ehb8t/W2+xUbP6umwHJdELFx7rxP462s -A20ucS6vxOOto70MEae0/0qyexAQH6dXQbLArvQsWdZHEIjzIVEpMMpghq9Gqx3t -OluwlN5E40EIosHsHdb9T7bWR9AUC8rmyrV7d35BH16Dx7aMOZawP5aBQW9gkOLo -+fsicdl9sz1Gv7SEr5AcD48Saq/v7h56rgJKihcrdv6sVIkkLE8/trKnToyokZf7 -KcZ7XC25y2a2t6hbElGFtQl+Ynhw/qlqYLYdDnkM/crqJIByw5c/8nerQyIKx+u2 -DISCLIBrQYoIwOula9+ZEsuK1V6ADJHgJgg2SMX6OBE1/yWDLfJ6v9r9jv6ly0Us -H8SIU653DtmadsWOLB2jutXsMq7Aqqz30XpN69QH4kj3Io6wpJ9qzo6ysmD0oyLQ -I+uUWnpp3Q+/QFesa1lQ2aOZ4W7+jQF5JyMV3pKdewlNWudLSDBaGOYKbeaP4NK7 -5t98biGCwWg5TbSYWGZizEqQXsP6JwSxeRV0mcy+rSDeJmAc61ZRpqPq5KM/p/9h -3PFaTWwyI0PurKju7koSCTxdccK+efrCh2gdC/1cacwG0Jp9VJkqyTkaGa9LKkPz -Y11aWOIv4x3kqdbQCtCev9eBCfHJxyYNrJgWVqA= ------END CERTIFICATE----- - -# Issuer: CN=Buypass Class 3 Root CA O=Buypass AS-983163327 -# Subject: CN=Buypass Class 3 Root CA O=Buypass AS-983163327 -# Label: "Buypass Class 3 Root CA" -# Serial: 2 -# MD5 Fingerprint: 3d:3b:18:9e:2c:64:5a:e8:d5:88:ce:0e:f9:37:c2:ec -# SHA1 Fingerprint: da:fa:f7:fa:66:84:ec:06:8f:14:50:bd:c7:c2:81:a5:bc:a9:64:57 -# SHA256 Fingerprint: ed:f7:eb:bc:a2:7a:2a:38:4d:38:7b:7d:40:10:c6:66:e2:ed:b4:84:3e:4c:29:b4:ae:1d:5b:93:32:e6:b2:4d ------BEGIN CERTIFICATE----- -MIIFWTCCA0GgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBOMQswCQYDVQQGEwJOTzEd -MBsGA1UECgwUQnV5cGFzcyBBUy05ODMxNjMzMjcxIDAeBgNVBAMMF0J1eXBhc3Mg -Q2xhc3MgMyBSb290IENBMB4XDTEwMTAyNjA4Mjg1OFoXDTQwMTAyNjA4Mjg1OFow -TjELMAkGA1UEBhMCTk8xHTAbBgNVBAoMFEJ1eXBhc3MgQVMtOTgzMTYzMzI3MSAw -HgYDVQQDDBdCdXlwYXNzIENsYXNzIDMgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEB -BQADggIPADCCAgoCggIBAKXaCpUWUOOV8l6ddjEGMnqb8RB2uACatVI2zSRHsJ8Y -ZLya9vrVediQYkwiL944PdbgqOkcLNt4EemOaFEVcsfzM4fkoF0LXOBXByow9c3E -N3coTRiR5r/VUv1xLXA+58bEiuPwKAv0dpihi4dVsjoT/Lc+JzeOIuOoTyrvYLs9 -tznDDgFHmV0ST9tD+leh7fmdvhFHJlsTmKtdFoqwNxxXnUX/iJY2v7vKB3tvh2PX -0DJq1l1sDPGzbjniazEuOQAnFN44wOwZZoYS6J1yFhNkUsepNxz9gjDthBgd9K5c -/3ATAOux9TN6S9ZV+AWNS2mw9bMoNlwUxFFzTWsL8TQH2xc519woe2v1n/MuwU8X -KhDzzMro6/1rqy6any2CbgTUUgGTLT2G/H783+9CHaZr77kgxve9oKeV/afmiSTY -zIw0bOIjL9kSGiG5VZFvC5F5GQytQIgLcOJ60g7YaEi7ghM5EFjp2CoHxhLbWNvS -O1UQRwUVZ2J+GGOmRj8JDlQyXr8NYnon74Do29lLBlo3WiXQCBJ31G8JUJc9yB3D -34xFMFbG02SrZvPAXpacw8Tvw3xrizp5f7NJzz3iiZ+gMEuFuZyUJHmPfWupRWgP -K9Dx2hzLabjKSWJtyNBjYt1gD1iqj6G8BaVmos8bdrKEZLFMOVLAMLrwjEsCsLa3 -AgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFEe4zf/lb+74suwv -Tg75JbCOPGvDMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAACAj -QTUEkMJAYmDv4jVM1z+s4jSQuKFvdvoWFqRINyzpkMLyPPgKn9iB5btb2iUspKdV -cSQy9sgL8rxq+JOssgfCX5/bzMiKqr5qb+FJEMwx14C7u8jYog5kV+qi9cKpMRXS -IGrs/CIBKM+GuIAeqcwRpTzyFrNHnfzSgCHEy9BHcEGhyoMZCCxt8l13nIoUE9Q2 -HJLw5QY33KbmkJs4j1xrG0aGQ0JfPgEHU1RdZX33inOhmlRaHylDFCfChQ+1iHsa -O5S3HWCntZznKWlXWpuTekMwGwPXYshApqr8ZORK15FTAaggiG6cX0S5y2CBNOxv -033aSF/rtJC8LakcC6wc1aJoIIAE1vyxjy+7SjENSoYc6+I2KSb12tjE8nVhz36u -dmNKekBlk4f4HoCMhuWG1o8O/FMsYOgWYRqiPkN7zTlgVGr18okmAWiDSKIz6MkE -kbIRNBE+6tBDGR8Dk5AM/1E9V/RBbuHLoL7ryWPNbczk+DaqaJ3tvV2XcEQNtg41 -3OEMXbugUZTLfhbrES+jkkXITHHZvMmZUldGL1DPvTVp9D0VzgalLA8+9oG6lLvD -u79leNKGef9JOxqDDPDeeOzI8k1MGt6CKfjBWtrt7uYnXuhF0J0cUahoq0Tj0Itq -4/g7u9xN12TyUb7mqqta6THuBrxzvxNiCp/HuZc= ------END CERTIFICATE----- - -# Issuer: CN=T-TeleSec GlobalRoot Class 3 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center -# Subject: CN=T-TeleSec GlobalRoot Class 3 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center -# Label: "T-TeleSec GlobalRoot Class 3" -# Serial: 1 -# MD5 Fingerprint: ca:fb:40:a8:4e:39:92:8a:1d:fe:8e:2f:c4:27:ea:ef -# SHA1 Fingerprint: 55:a6:72:3e:cb:f2:ec:cd:c3:23:74:70:19:9d:2a:be:11:e3:81:d1 -# SHA256 Fingerprint: fd:73:da:d3:1c:64:4f:f1:b4:3b:ef:0c:cd:da:96:71:0b:9c:d9:87:5e:ca:7e:31:70:7a:f3:e9:6d:52:2b:bd ------BEGIN CERTIFICATE----- -MIIDwzCCAqugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBgjELMAkGA1UEBhMCREUx -KzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnByaXNlIFNlcnZpY2VzIEdtYkgxHzAd -BgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50ZXIxJTAjBgNVBAMMHFQtVGVsZVNl -YyBHbG9iYWxSb290IENsYXNzIDMwHhcNMDgxMDAxMTAyOTU2WhcNMzMxMDAxMjM1 -OTU5WjCBgjELMAkGA1UEBhMCREUxKzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnBy -aXNlIFNlcnZpY2VzIEdtYkgxHzAdBgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50 -ZXIxJTAjBgNVBAMMHFQtVGVsZVNlYyBHbG9iYWxSb290IENsYXNzIDMwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC9dZPwYiJvJK7genasfb3ZJNW4t/zN -8ELg63iIVl6bmlQdTQyK9tPPcPRStdiTBONGhnFBSivwKixVA9ZIw+A5OO3yXDw/ -RLyTPWGrTs0NvvAgJ1gORH8EGoel15YUNpDQSXuhdfsaa3Ox+M6pCSzyU9XDFES4 -hqX2iys52qMzVNn6chr3IhUciJFrf2blw2qAsCTz34ZFiP0Zf3WHHx+xGwpzJFu5 -ZeAsVMhg02YXP+HMVDNzkQI6pn97djmiH5a2OK61yJN0HZ65tOVgnS9W0eDrXltM -EnAMbEQgqxHY9Bn20pxSN+f6tsIxO0rUFJmtxxr1XV/6B7h8DR/Wgx6zAgMBAAGj -QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBS1 -A/d2O2GCahKqGFPrAyGUv/7OyjANBgkqhkiG9w0BAQsFAAOCAQEAVj3vlNW92nOy -WL6ukK2YJ5f+AbGwUgC4TeQbIXQbfsDuXmkqJa9c1h3a0nnJ85cp4IaH3gRZD/FZ -1GSFS5mvJQQeyUapl96Cshtwn5z2r3Ex3XsFpSzTucpH9sry9uetuUg/vBa3wW30 -6gmv7PO15wWeph6KU1HWk4HMdJP2udqmJQV0eVp+QD6CSyYRMG7hP0HHRwA11fXT -91Q+gT3aSWqas+8QPebrb9HIIkfLzM8BMZLZGOMivgkeGj5asuRrDFR6fUNOuIml -e9eiPZaGzPImNC1qkp2aGtAw4l1OBLBfiyB+d8E9lYLRRpo7PHi4b6HQDWSieB4p -TpPDpFQUWw== ------END CERTIFICATE----- - -# Issuer: CN=D-TRUST Root Class 3 CA 2 2009 O=D-Trust GmbH -# Subject: CN=D-TRUST Root Class 3 CA 2 2009 O=D-Trust GmbH -# Label: "D-TRUST Root Class 3 CA 2 2009" -# Serial: 623603 -# MD5 Fingerprint: cd:e0:25:69:8d:47:ac:9c:89:35:90:f7:fd:51:3d:2f -# SHA1 Fingerprint: 58:e8:ab:b0:36:15:33:fb:80:f7:9b:1b:6d:29:d3:ff:8d:5f:00:f0 -# SHA256 Fingerprint: 49:e7:a4:42:ac:f0:ea:62:87:05:00:54:b5:25:64:b6:50:e4:f4:9e:42:e3:48:d6:aa:38:e0:39:e9:57:b1:c1 ------BEGIN CERTIFICATE----- -MIIEMzCCAxugAwIBAgIDCYPzMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNVBAYTAkRF -MRUwEwYDVQQKDAxELVRydXN0IEdtYkgxJzAlBgNVBAMMHkQtVFJVU1QgUm9vdCBD -bGFzcyAzIENBIDIgMjAwOTAeFw0wOTExMDUwODM1NThaFw0yOTExMDUwODM1NTha -ME0xCzAJBgNVBAYTAkRFMRUwEwYDVQQKDAxELVRydXN0IEdtYkgxJzAlBgNVBAMM -HkQtVFJVU1QgUm9vdCBDbGFzcyAzIENBIDIgMjAwOTCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBANOySs96R+91myP6Oi/WUEWJNTrGa9v+2wBoqOADER03 -UAifTUpolDWzU9GUY6cgVq/eUXjsKj3zSEhQPgrfRlWLJ23DEE0NkVJD2IfgXU42 -tSHKXzlABF9bfsyjxiupQB7ZNoTWSPOSHjRGICTBpFGOShrvUD9pXRl/RcPHAY9R -ySPocq60vFYJfxLLHLGvKZAKyVXMD9O0Gu1HNVpK7ZxzBCHQqr0ME7UAyiZsxGsM -lFqVlNpQmvH/pStmMaTJOKDfHR+4CS7zp+hnUquVH+BGPtikw8paxTGA6Eian5Rp -/hnd2HN8gcqW3o7tszIFZYQ05ub9VxC1X3a/L7AQDcUCAwEAAaOCARowggEWMA8G -A1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFP3aFMSfMN4hvR5COfyrYyNJ4PGEMA4G -A1UdDwEB/wQEAwIBBjCB0wYDVR0fBIHLMIHIMIGAoH6gfIZ6bGRhcDovL2RpcmVj -dG9yeS5kLXRydXN0Lm5ldC9DTj1ELVRSVVNUJTIwUm9vdCUyMENsYXNzJTIwMyUy -MENBJTIwMiUyMDIwMDksTz1ELVRydXN0JTIwR21iSCxDPURFP2NlcnRpZmljYXRl -cmV2b2NhdGlvbmxpc3QwQ6BBoD+GPWh0dHA6Ly93d3cuZC10cnVzdC5uZXQvY3Js -L2QtdHJ1c3Rfcm9vdF9jbGFzc18zX2NhXzJfMjAwOS5jcmwwDQYJKoZIhvcNAQEL -BQADggEBAH+X2zDI36ScfSF6gHDOFBJpiBSVYEQBrLLpME+bUMJm2H6NMLVwMeni -acfzcNsgFYbQDfC+rAF1hM5+n02/t2A7nPPKHeJeaNijnZflQGDSNiH+0LS4F9p0 -o3/U37CYAqxva2ssJSRyoWXuJVrl5jLn8t+rSfrzkGkj2wTZ51xY/GXUl77M/C4K -zCUqNQT4YJEVdT1B/yMfGchs64JTBKbkTCJNjYy6zltz7GRUUG3RnFX7acM2w4y8 -PIWmawomDeCTmGCufsYkl4phX5GOZpIJhzbNi5stPvZR1FDUWSi9g/LMKHtThm3Y -Johw1+qRzT65ysCQblrGXnRl11z+o+I= ------END CERTIFICATE----- - -# Issuer: CN=D-TRUST Root Class 3 CA 2 EV 2009 O=D-Trust GmbH -# Subject: CN=D-TRUST Root Class 3 CA 2 EV 2009 O=D-Trust GmbH -# Label: "D-TRUST Root Class 3 CA 2 EV 2009" -# Serial: 623604 -# MD5 Fingerprint: aa:c6:43:2c:5e:2d:cd:c4:34:c0:50:4f:11:02:4f:b6 -# SHA1 Fingerprint: 96:c9:1b:0b:95:b4:10:98:42:fa:d0:d8:22:79:fe:60:fa:b9:16:83 -# SHA256 Fingerprint: ee:c5:49:6b:98:8c:e9:86:25:b9:34:09:2e:ec:29:08:be:d0:b0:f3:16:c2:d4:73:0c:84:ea:f1:f3:d3:48:81 ------BEGIN CERTIFICATE----- -MIIEQzCCAyugAwIBAgIDCYP0MA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNVBAYTAkRF -MRUwEwYDVQQKDAxELVRydXN0IEdtYkgxKjAoBgNVBAMMIUQtVFJVU1QgUm9vdCBD -bGFzcyAzIENBIDIgRVYgMjAwOTAeFw0wOTExMDUwODUwNDZaFw0yOTExMDUwODUw -NDZaMFAxCzAJBgNVBAYTAkRFMRUwEwYDVQQKDAxELVRydXN0IEdtYkgxKjAoBgNV -BAMMIUQtVFJVU1QgUm9vdCBDbGFzcyAzIENBIDIgRVYgMjAwOTCCASIwDQYJKoZI -hvcNAQEBBQADggEPADCCAQoCggEBAJnxhDRwui+3MKCOvXwEz75ivJn9gpfSegpn -ljgJ9hBOlSJzmY3aFS3nBfwZcyK3jpgAvDw9rKFs+9Z5JUut8Mxk2og+KbgPCdM0 -3TP1YtHhzRnp7hhPTFiu4h7WDFsVWtg6uMQYZB7jM7K1iXdODL/ZlGsTl28So/6Z -qQTMFexgaDbtCHu39b+T7WYxg4zGcTSHThfqr4uRjRxWQa4iN1438h3Z0S0NL2lR -p75mpoo6Kr3HGrHhFPC+Oh25z1uxav60sUYgovseO3Dvk5h9jHOW8sXvhXCtKSb8 -HgQ+HKDYD8tSg2J87otTlZCpV6LqYQXY+U3EJ/pure3511H3a6UCAwEAAaOCASQw -ggEgMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNOUikxiEyoZLsyvcop9Ntea -HNxnMA4GA1UdDwEB/wQEAwIBBjCB3QYDVR0fBIHVMIHSMIGHoIGEoIGBhn9sZGFw -Oi8vZGlyZWN0b3J5LmQtdHJ1c3QubmV0L0NOPUQtVFJVU1QlMjBSb290JTIwQ2xh -c3MlMjAzJTIwQ0ElMjAyJTIwRVYlMjAyMDA5LE89RC1UcnVzdCUyMEdtYkgsQz1E -RT9jZXJ0aWZpY2F0ZXJldm9jYXRpb25saXN0MEagRKBChkBodHRwOi8vd3d3LmQt -dHJ1c3QubmV0L2NybC9kLXRydXN0X3Jvb3RfY2xhc3NfM19jYV8yX2V2XzIwMDku -Y3JsMA0GCSqGSIb3DQEBCwUAA4IBAQA07XtaPKSUiO8aEXUHL7P+PPoeUSbrh/Yp -3uDx1MYkCenBz1UbtDDZzhr+BlGmFaQt77JLvyAoJUnRpjZ3NOhk31KxEcdzes05 -nsKtjHEh8lprr988TlWvsoRlFIm5d8sqMb7Po23Pb0iUMkZv53GMoKaEGTcH8gNF -CSuGdXzfX2lXANtu2KZyIktQ1HWYVt+3GP9DQ1CuekR78HlR10M9p9OB0/DJT7na -xpeG0ILD5EJt/rDiZE4OJudANCa1CInXCGNjOCd1HjPqbqjdn5lPdE2BiYBL3ZqX -KVwvvoFBuYz/6n1gBp7N1z3TLqMVvKjmJuVvw9y4AyHqnxbxLFS1 ------END CERTIFICATE----- - -# Issuer: CN=CA Disig Root R2 O=Disig a.s. -# Subject: CN=CA Disig Root R2 O=Disig a.s. -# Label: "CA Disig Root R2" -# Serial: 10572350602393338211 -# MD5 Fingerprint: 26:01:fb:d8:27:a7:17:9a:45:54:38:1a:43:01:3b:03 -# SHA1 Fingerprint: b5:61:eb:ea:a4:de:e4:25:4b:69:1a:98:a5:57:47:c2:34:c7:d9:71 -# SHA256 Fingerprint: e2:3d:4a:03:6d:7b:70:e9:f5:95:b1:42:20:79:d2:b9:1e:df:bb:1f:b6:51:a0:63:3e:aa:8a:9d:c5:f8:07:03 ------BEGIN CERTIFICATE----- -MIIFaTCCA1GgAwIBAgIJAJK4iNuwisFjMA0GCSqGSIb3DQEBCwUAMFIxCzAJBgNV -BAYTAlNLMRMwEQYDVQQHEwpCcmF0aXNsYXZhMRMwEQYDVQQKEwpEaXNpZyBhLnMu -MRkwFwYDVQQDExBDQSBEaXNpZyBSb290IFIyMB4XDTEyMDcxOTA5MTUzMFoXDTQy -MDcxOTA5MTUzMFowUjELMAkGA1UEBhMCU0sxEzARBgNVBAcTCkJyYXRpc2xhdmEx -EzARBgNVBAoTCkRpc2lnIGEucy4xGTAXBgNVBAMTEENBIERpc2lnIFJvb3QgUjIw -ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCio8QACdaFXS1tFPbCw3Oe -NcJxVX6B+6tGUODBfEl45qt5WDza/3wcn9iXAng+a0EE6UG9vgMsRfYvZNSrXaNH -PWSb6WiaxswbP7q+sos0Ai6YVRn8jG+qX9pMzk0DIaPY0jSTVpbLTAwAFjxfGs3I -x2ymrdMxp7zo5eFm1tL7A7RBZckQrg4FY8aAamkw/dLukO8NJ9+flXP04SXabBbe -QTg06ov80egEFGEtQX6sx3dOy1FU+16SGBsEWmjGycT6txOgmLcRK7fWV8x8nhfR -yyX+hk4kLlYMeE2eARKmK6cBZW58Yh2EhN/qwGu1pSqVg8NTEQxzHQuyRpDRQjrO -QG6Vrf/GlK1ul4SOfW+eioANSW1z4nuSHsPzwfPrLgVv2RvPN3YEyLRa5Beny912 -H9AZdugsBbPWnDTYltxhh5EF5EQIM8HauQhl1K6yNg3ruji6DOWbnuuNZt2Zz9aJ -QfYEkoopKW1rOhzndX0CcQ7zwOe9yxndnWCywmZgtrEE7snmhrmaZkCo5xHtgUUD -i/ZnWejBBhG93c+AAk9lQHhcR1DIm+YfgXvkRKhbhZri3lrVx/k6RGZL5DJUfORs -nLMOPReisjQS1n6yqEm70XooQL6iFh/f5DcfEXP7kAplQ6INfPgGAVUzfbANuPT1 -rqVCV3w2EYx7XsQDnYx5nQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1Ud -DwEB/wQEAwIBBjAdBgNVHQ4EFgQUtZn4r7CU9eMg1gqtzk5WpC5uQu0wDQYJKoZI -hvcNAQELBQADggIBACYGXnDnZTPIgm7ZnBc6G3pmsgH2eDtpXi/q/075KMOYKmFM -tCQSin1tERT3nLXK5ryeJ45MGcipvXrA1zYObYVybqjGom32+nNjf7xueQgcnYqf -GopTpti72TVVsRHFqQOzVju5hJMiXn7B9hJSi+osZ7z+Nkz1uM/Rs0mSO9MpDpkb -lvdhuDvEK7Z4bLQjb/D907JedR+Zlais9trhxTF7+9FGs9K8Z7RiVLoJ92Owk6Ka -+elSLotgEqv89WBW7xBci8QaQtyDW2QOy7W81k/BfDxujRNt+3vrMNDcTa/F1bal -TFtxyegxvug4BkihGuLq0t4SOVga/4AOgnXmt8kHbA7v/zjxmHHEt38OFdAlab0i -nSvtBfZGR6ztwPDUO+Ls7pZbkBNOHlY667DvlruWIxG68kOGdGSVyCh13x01utI3 -gzhTODY7z2zp+WsO0PsE6E9312UBeIYMej4hYvF/Y3EMyZ9E26gnonW+boE+18Dr -G5gPcFw0sorMwIUY6256s/daoQe/qUKS82Ail+QUoQebTnbAjn39pCXHR+3/H3Os -zMOl6W8KjptlwlCFtaOgUxLMVYdh84GuEEZhvUQhuMI9dM9+JDX6HAcOmz0iyu8x -L4ysEr3vQCj8KWefshNPZiTEUxnpHikV7+ZtsH8tZ/3zbBt1RqPlShfppNcL ------END CERTIFICATE----- - -# Issuer: CN=ACCVRAIZ1 O=ACCV OU=PKIACCV -# Subject: CN=ACCVRAIZ1 O=ACCV OU=PKIACCV -# Label: "ACCVRAIZ1" -# Serial: 6828503384748696800 -# MD5 Fingerprint: d0:a0:5a:ee:05:b6:09:94:21:a1:7d:f1:b2:29:82:02 -# SHA1 Fingerprint: 93:05:7a:88:15:c6:4f:ce:88:2f:fa:91:16:52:28:78:bc:53:64:17 -# SHA256 Fingerprint: 9a:6e:c0:12:e1:a7:da:9d:be:34:19:4d:47:8a:d7:c0:db:18:22:fb:07:1d:f1:29:81:49:6e:d1:04:38:41:13 ------BEGIN CERTIFICATE----- -MIIH0zCCBbugAwIBAgIIXsO3pkN/pOAwDQYJKoZIhvcNAQEFBQAwQjESMBAGA1UE -AwwJQUNDVlJBSVoxMRAwDgYDVQQLDAdQS0lBQ0NWMQ0wCwYDVQQKDARBQ0NWMQsw -CQYDVQQGEwJFUzAeFw0xMTA1MDUwOTM3MzdaFw0zMDEyMzEwOTM3MzdaMEIxEjAQ -BgNVBAMMCUFDQ1ZSQUlaMTEQMA4GA1UECwwHUEtJQUNDVjENMAsGA1UECgwEQUND -VjELMAkGA1UEBhMCRVMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCb -qau/YUqXry+XZpp0X9DZlv3P4uRm7x8fRzPCRKPfmt4ftVTdFXxpNRFvu8gMjmoY -HtiP2Ra8EEg2XPBjs5BaXCQ316PWywlxufEBcoSwfdtNgM3802/J+Nq2DoLSRYWo -G2ioPej0RGy9ocLLA76MPhMAhN9KSMDjIgro6TenGEyxCQ0jVn8ETdkXhBilyNpA -lHPrzg5XPAOBOp0KoVdDaaxXbXmQeOW1tDvYvEyNKKGno6e6Ak4l0Squ7a4DIrhr -IA8wKFSVf+DuzgpmndFALW4ir50awQUZ0m/A8p/4e7MCQvtQqR0tkw8jq8bBD5L/ -0KIV9VMJcRz/RROE5iZe+OCIHAr8Fraocwa48GOEAqDGWuzndN9wrqODJerWx5eH -k6fGioozl2A3ED6XPm4pFdahD9GILBKfb6qkxkLrQaLjlUPTAYVtjrs78yM2x/47 -4KElB0iryYl0/wiPgL/AlmXz7uxLaL2diMMxs0Dx6M/2OLuc5NF/1OVYm3z61PMO -m3WR5LpSLhl+0fXNWhn8ugb2+1KoS5kE3fj5tItQo05iifCHJPqDQsGH+tUtKSpa -cXpkatcnYGMN285J9Y0fkIkyF/hzQ7jSWpOGYdbhdQrqeWZ2iE9x6wQl1gpaepPl -uUsXQA+xtrn13k/c4LOsOxFwYIRKQ26ZIMApcQrAZQIDAQABo4ICyzCCAscwfQYI -KwYBBQUHAQEEcTBvMEwGCCsGAQUFBzAChkBodHRwOi8vd3d3LmFjY3YuZXMvZmls -ZWFkbWluL0FyY2hpdm9zL2NlcnRpZmljYWRvcy9yYWl6YWNjdjEuY3J0MB8GCCsG -AQUFBzABhhNodHRwOi8vb2NzcC5hY2N2LmVzMB0GA1UdDgQWBBTSh7Tj3zcnk1X2 -VuqB5TbMjB4/vTAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNKHtOPfNyeT -VfZW6oHlNsyMHj+9MIIBcwYDVR0gBIIBajCCAWYwggFiBgRVHSAAMIIBWDCCASIG -CCsGAQUFBwICMIIBFB6CARAAQQB1AHQAbwByAGkAZABhAGQAIABkAGUAIABDAGUA -cgB0AGkAZgBpAGMAYQBjAGkA8wBuACAAUgBhAO0AegAgAGQAZQAgAGwAYQAgAEEA -QwBDAFYAIAAoAEEAZwBlAG4AYwBpAGEAIABkAGUAIABUAGUAYwBuAG8AbABvAGcA -7QBhACAAeQAgAEMAZQByAHQAaQBmAGkAYwBhAGMAaQDzAG4AIABFAGwAZQBjAHQA -cgDzAG4AaQBjAGEALAAgAEMASQBGACAAUQA0ADYAMAAxADEANQA2AEUAKQAuACAA -QwBQAFMAIABlAG4AIABoAHQAdABwADoALwAvAHcAdwB3AC4AYQBjAGMAdgAuAGUA -czAwBggrBgEFBQcCARYkaHR0cDovL3d3dy5hY2N2LmVzL2xlZ2lzbGFjaW9uX2Mu -aHRtMFUGA1UdHwROMEwwSqBIoEaGRGh0dHA6Ly93d3cuYWNjdi5lcy9maWxlYWRt -aW4vQXJjaGl2b3MvY2VydGlmaWNhZG9zL3JhaXphY2N2MV9kZXIuY3JsMA4GA1Ud -DwEB/wQEAwIBBjAXBgNVHREEEDAOgQxhY2N2QGFjY3YuZXMwDQYJKoZIhvcNAQEF -BQADggIBAJcxAp/n/UNnSEQU5CmH7UwoZtCPNdpNYbdKl02125DgBS4OxnnQ8pdp -D70ER9m+27Up2pvZrqmZ1dM8MJP1jaGo/AaNRPTKFpV8M9xii6g3+CfYCS0b78gU -JyCpZET/LtZ1qmxNYEAZSUNUY9rizLpm5U9EelvZaoErQNV/+QEnWCzI7UiRfD+m -AM/EKXMRNt6GGT6d7hmKG9Ww7Y49nCrADdg9ZuM8Db3VlFzi4qc1GwQA9j9ajepD -vV+JHanBsMyZ4k0ACtrJJ1vnE5Bc5PUzolVt3OAJTS+xJlsndQAJxGJ3KQhfnlms -tn6tn1QwIgPBHnFk/vk4CpYY3QIUrCPLBhwepH2NDd4nQeit2hW3sCPdK6jT2iWH -7ehVRE2I9DZ+hJp4rPcOVkkO1jMl1oRQQmwgEh0q1b688nCBpHBgvgW1m54ERL5h -I6zppSSMEYCUWqKiuUnSwdzRp+0xESyeGabu4VXhwOrPDYTkF7eifKXeVSUG7szA -h1xA2syVP1XgNce4hL60Xc16gwFy7ofmXx2utYXGJt/mwZrpHgJHnyqobalbz+xF -d3+YJ5oyXSrjhO7FmGYvliAd3djDJ9ew+f7Zfc3Qn48LFFhRny+Lwzgt3uiP1o2H -pPVWQxaZLPSkVrQ0uGE3ycJYgBugl6H8WY3pEfbRD0tVNEYqi4Y7 ------END CERTIFICATE----- - -# Issuer: CN=TWCA Global Root CA O=TAIWAN-CA OU=Root CA -# Subject: CN=TWCA Global Root CA O=TAIWAN-CA OU=Root CA -# Label: "TWCA Global Root CA" -# Serial: 3262 -# MD5 Fingerprint: f9:03:7e:cf:e6:9e:3c:73:7a:2a:90:07:69:ff:2b:96 -# SHA1 Fingerprint: 9c:bb:48:53:f6:a4:f6:d3:52:a4:e8:32:52:55:60:13:f5:ad:af:65 -# SHA256 Fingerprint: 59:76:90:07:f7:68:5d:0f:cd:50:87:2f:9f:95:d5:75:5a:5b:2b:45:7d:81:f3:69:2b:61:0a:98:67:2f:0e:1b ------BEGIN CERTIFICATE----- -MIIFQTCCAymgAwIBAgICDL4wDQYJKoZIhvcNAQELBQAwUTELMAkGA1UEBhMCVFcx -EjAQBgNVBAoTCVRBSVdBTi1DQTEQMA4GA1UECxMHUm9vdCBDQTEcMBoGA1UEAxMT -VFdDQSBHbG9iYWwgUm9vdCBDQTAeFw0xMjA2MjcwNjI4MzNaFw0zMDEyMzExNTU5 -NTlaMFExCzAJBgNVBAYTAlRXMRIwEAYDVQQKEwlUQUlXQU4tQ0ExEDAOBgNVBAsT -B1Jvb3QgQ0ExHDAaBgNVBAMTE1RXQ0EgR2xvYmFsIFJvb3QgQ0EwggIiMA0GCSqG -SIb3DQEBAQUAA4ICDwAwggIKAoICAQCwBdvI64zEbooh745NnHEKH1Jw7W2CnJfF -10xORUnLQEK1EjRsGcJ0pDFfhQKX7EMzClPSnIyOt7h52yvVavKOZsTuKwEHktSz -0ALfUPZVr2YOy+BHYC8rMjk1Ujoog/h7FsYYuGLWRyWRzvAZEk2tY/XTP3VfKfCh -MBwqoJimFb3u/Rk28OKRQ4/6ytYQJ0lM793B8YVwm8rqqFpD/G2Gb3PpN0Wp8DbH -zIh1HrtsBv+baz4X7GGqcXzGHaL3SekVtTzWoWH1EfcFbx39Eb7QMAfCKbAJTibc -46KokWofwpFFiFzlmLhxpRUZyXx1EcxwdE8tmx2RRP1WKKD+u4ZqyPpcC1jcxkt2 -yKsi2XMPpfRaAok/T54igu6idFMqPVMnaR1sjjIsZAAmY2E2TqNGtz99sy2sbZCi -laLOz9qC5wc0GZbpuCGqKX6mOL6OKUohZnkfs8O1CWfe1tQHRvMq2uYiN2DLgbYP -oA/pyJV/v1WRBXrPPRXAb94JlAGD1zQbzECl8LibZ9WYkTunhHiVJqRaCPgrdLQA -BDzfuBSO6N+pjWxnkjMdwLfS7JLIvgm/LCkFbwJrnu+8vyq8W8BQj0FwcYeyTbcE -qYSjMq+u7msXi7Kx/mzhkIyIqJdIzshNy/MGz19qCkKxHh53L46g5pIOBvwFItIm -4TFRfTLcDwIDAQABoyMwITAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB -/zANBgkqhkiG9w0BAQsFAAOCAgEAXzSBdu+WHdXltdkCY4QWwa6gcFGn90xHNcgL -1yg9iXHZqjNB6hQbbCEAwGxCGX6faVsgQt+i0trEfJdLjbDorMjupWkEmQqSpqsn -LhpNgb+E1HAerUf+/UqdM+DyucRFCCEK2mlpc3INvjT+lIutwx4116KD7+U4x6WF -H6vPNOw/KP4M8VeGTslV9xzU2KV9Bnpv1d8Q34FOIWWxtuEXeZVFBs5fzNxGiWNo -RI2T9GRwoD2dKAXDOXC4Ynsg/eTb6QihuJ49CcdP+yz4k3ZB3lLg4VfSnQO8d57+ -nile98FRYB/e2guyLXW3Q0iT5/Z5xoRdgFlglPx4mI88k1HtQJAH32RjJMtOcQWh -15QaiDLxInQirqWm2BJpTGCjAu4r7NRjkgtevi92a6O2JryPA9gK8kxkRr05YuWW -6zRjESjMlfGt7+/cgFhI6Uu46mWs6fyAtbXIRfmswZ/ZuepiiI7E8UuDEq3mi4TW -nsLrgxifarsbJGAzcMzs9zLzXNl5fe+epP7JI8Mk7hWSsT2RTyaGvWZzJBPqpK5j -wa19hAM8EHiGG3njxPPyBJUgriOCxLM6AGK/5jYk4Ve6xx6QddVfP5VhK8E7zeWz -aGHQRiapIVJpLesux+t3zqY6tQMzT3bR51xUAV3LePTJDL/PEo4XLSNolOer/qmy -KwbQBM0= ------END CERTIFICATE----- - -# Issuer: CN=T-TeleSec GlobalRoot Class 2 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center -# Subject: CN=T-TeleSec GlobalRoot Class 2 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center -# Label: "T-TeleSec GlobalRoot Class 2" -# Serial: 1 -# MD5 Fingerprint: 2b:9b:9e:e4:7b:6c:1f:00:72:1a:cc:c1:77:79:df:6a -# SHA1 Fingerprint: 59:0d:2d:7d:88:4f:40:2e:61:7e:a5:62:32:17:65:cf:17:d8:94:e9 -# SHA256 Fingerprint: 91:e2:f5:78:8d:58:10:eb:a7:ba:58:73:7d:e1:54:8a:8e:ca:cd:01:45:98:bc:0b:14:3e:04:1b:17:05:25:52 ------BEGIN CERTIFICATE----- -MIIDwzCCAqugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBgjELMAkGA1UEBhMCREUx -KzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnByaXNlIFNlcnZpY2VzIEdtYkgxHzAd -BgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50ZXIxJTAjBgNVBAMMHFQtVGVsZVNl -YyBHbG9iYWxSb290IENsYXNzIDIwHhcNMDgxMDAxMTA0MDE0WhcNMzMxMDAxMjM1 -OTU5WjCBgjELMAkGA1UEBhMCREUxKzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnBy -aXNlIFNlcnZpY2VzIEdtYkgxHzAdBgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50 -ZXIxJTAjBgNVBAMMHFQtVGVsZVNlYyBHbG9iYWxSb290IENsYXNzIDIwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCqX9obX+hzkeXaXPSi5kfl82hVYAUd -AqSzm1nzHoqvNK38DcLZSBnuaY/JIPwhqgcZ7bBcrGXHX+0CfHt8LRvWurmAwhiC -FoT6ZrAIxlQjgeTNuUk/9k9uN0goOA/FvudocP05l03Sx5iRUKrERLMjfTlH6VJi -1hKTXrcxlkIF+3anHqP1wvzpesVsqXFP6st4vGCvx9702cu+fjOlbpSD8DT6Iavq -jnKgP6TeMFvvhk1qlVtDRKgQFRzlAVfFmPHmBiiRqiDFt1MmUUOyCxGVWOHAD3bZ -wI18gfNycJ5v/hqO2V81xrJvNHy+SE/iWjnX2J14np+GPgNeGYtEotXHAgMBAAGj -QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBS/ -WSA2AHmgoCJrjNXyYdK4LMuCSjANBgkqhkiG9w0BAQsFAAOCAQEAMQOiYQsfdOhy -NsZt+U2e+iKo4YFWz827n+qrkRk4r6p8FU3ztqONpfSO9kSpp+ghla0+AGIWiPAC -uvxhI+YzmzB6azZie60EI4RYZeLbK4rnJVM3YlNfvNoBYimipidx5joifsFvHZVw -IEoHNN/q/xWA5brXethbdXwFeilHfkCoMRN3zUA7tFFHei4R40cR3p1m0IvVVGb6 -g1XqfMIpiRvpb7PO4gWEyS8+eIVibslfwXhjdFjASBgMmTnrpMwatXlajRWc2BQN -9noHV8cigwUtPJslJj0Ys6lDfMjIq2SPDqO/nBudMNva0Bkuqjzx+zOAduTNrRlP -BSeOE6Fuwg== ------END CERTIFICATE----- - -# Issuer: CN=QuoVadis Root CA 1 G3 O=QuoVadis Limited -# Subject: CN=QuoVadis Root CA 1 G3 O=QuoVadis Limited -# Label: "QuoVadis Root CA 1 G3" -# Serial: 687049649626669250736271037606554624078720034195 -# MD5 Fingerprint: a4:bc:5b:3f:fe:37:9a:fa:64:f0:e2:fa:05:3d:0b:ab -# SHA1 Fingerprint: 1b:8e:ea:57:96:29:1a:c9:39:ea:b8:0a:81:1a:73:73:c0:93:79:67 -# SHA256 Fingerprint: 8a:86:6f:d1:b2:76:b5:7e:57:8e:92:1c:65:82:8a:2b:ed:58:e9:f2:f2:88:05:41:34:b7:f1:f4:bf:c9:cc:74 ------BEGIN CERTIFICATE----- -MIIFYDCCA0igAwIBAgIUeFhfLq0sGUvjNwc1NBMotZbUZZMwDQYJKoZIhvcNAQEL -BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc -BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMSBHMzAeFw0xMjAxMTIxNzI3NDRaFw00 -MjAxMTIxNzI3NDRaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM -aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDEgRzMwggIiMA0GCSqG -SIb3DQEBAQUAA4ICDwAwggIKAoICAQCgvlAQjunybEC0BJyFuTHK3C3kEakEPBtV -wedYMB0ktMPvhd6MLOHBPd+C5k+tR4ds7FtJwUrVu4/sh6x/gpqG7D0DmVIB0jWe -rNrwU8lmPNSsAgHaJNM7qAJGr6Qc4/hzWHa39g6QDbXwz8z6+cZM5cOGMAqNF341 -68Xfuw6cwI2H44g4hWf6Pser4BOcBRiYz5P1sZK0/CPTz9XEJ0ngnjybCKOLXSoh -4Pw5qlPafX7PGglTvF0FBM+hSo+LdoINofjSxxR3W5A2B4GbPgb6Ul5jxaYA/qXp -UhtStZI5cgMJYr2wYBZupt0lwgNm3fME0UDiTouG9G/lg6AnhF4EwfWQvTA9xO+o -abw4m6SkltFi2mnAAZauy8RRNOoMqv8hjlmPSlzkYZqn0ukqeI1RPToV7qJZjqlc -3sX5kCLliEVx3ZGZbHqfPT2YfF72vhZooF6uCyP8Wg+qInYtyaEQHeTTRCOQiJ/G -KubX9ZqzWB4vMIkIG1SitZgj7Ah3HJVdYdHLiZxfokqRmu8hqkkWCKi9YSgxyXSt -hfbZxbGL0eUQMk1fiyA6PEkfM4VZDdvLCXVDaXP7a3F98N/ETH3Goy7IlXnLc6KO -Tk0k+17kBL5yG6YnLUlamXrXXAkgt3+UuU/xDRxeiEIbEbfnkduebPRq34wGmAOt -zCjvpUfzUwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB -BjAdBgNVHQ4EFgQUo5fW816iEOGrRZ88F2Q87gFwnMwwDQYJKoZIhvcNAQELBQAD -ggIBABj6W3X8PnrHX3fHyt/PX8MSxEBd1DKquGrX1RUVRpgjpeaQWxiZTOOtQqOC -MTaIzen7xASWSIsBx40Bz1szBpZGZnQdT+3Btrm0DWHMY37XLneMlhwqI2hrhVd2 -cDMT/uFPpiN3GPoajOi9ZcnPP/TJF9zrx7zABC4tRi9pZsMbj/7sPtPKlL92CiUN -qXsCHKnQO18LwIE6PWThv6ctTr1NxNgpxiIY0MWscgKCP6o6ojoilzHdCGPDdRS5 -YCgtW2jgFqlmgiNR9etT2DGbe+m3nUvriBbP+V04ikkwj+3x6xn0dxoxGE1nVGwv -b2X52z3sIexe9PSLymBlVNFxZPT5pqOBMzYzcfCkeF9OrYMh3jRJjehZrJ3ydlo2 -8hP0r+AJx2EqbPfgna67hkooby7utHnNkDPDs3b69fBsnQGQ+p6Q9pxyz0fawx/k -NSBT8lTR32GDpgLiJTjehTItXnOQUl1CxM49S+H5GYQd1aJQzEH7QRTDvdbJWqNj -ZgKAvQU6O0ec7AAmTPWIUb+oI38YB7AL7YsmoWTTYUrrXJ/es69nA7Mf3W1daWhp -q1467HxpvMc7hU6eFbm0FU/DlXpY18ls6Wy58yljXrQs8C097Vpl4KlbQMJImYFt -nh8GKjwStIsPm6Ik8KaN1nrgS7ZklmOVhMJKzRwuJIczYOXD ------END CERTIFICATE----- - -# Issuer: CN=QuoVadis Root CA 2 G3 O=QuoVadis Limited -# Subject: CN=QuoVadis Root CA 2 G3 O=QuoVadis Limited -# Label: "QuoVadis Root CA 2 G3" -# Serial: 390156079458959257446133169266079962026824725800 -# MD5 Fingerprint: af:0c:86:6e:bf:40:2d:7f:0b:3e:12:50:ba:12:3d:06 -# SHA1 Fingerprint: 09:3c:61:f3:8b:8b:dc:7d:55:df:75:38:02:05:00:e1:25:f5:c8:36 -# SHA256 Fingerprint: 8f:e4:fb:0a:f9:3a:4d:0d:67:db:0b:eb:b2:3e:37:c7:1b:f3:25:dc:bc:dd:24:0e:a0:4d:af:58:b4:7e:18:40 ------BEGIN CERTIFICATE----- -MIIFYDCCA0igAwIBAgIURFc0JFuBiZs18s64KztbpybwdSgwDQYJKoZIhvcNAQEL -BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc -BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMiBHMzAeFw0xMjAxMTIxODU5MzJaFw00 -MjAxMTIxODU5MzJaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM -aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDIgRzMwggIiMA0GCSqG -SIb3DQEBAQUAA4ICDwAwggIKAoICAQChriWyARjcV4g/Ruv5r+LrI3HimtFhZiFf -qq8nUeVuGxbULX1QsFN3vXg6YOJkApt8hpvWGo6t/x8Vf9WVHhLL5hSEBMHfNrMW -n4rjyduYNM7YMxcoRvynyfDStNVNCXJJ+fKH46nafaF9a7I6JaltUkSs+L5u+9ym -c5GQYaYDFCDy54ejiK2toIz/pgslUiXnFgHVy7g1gQyjO/Dh4fxaXc6AcW34Sas+ -O7q414AB+6XrW7PFXmAqMaCvN+ggOp+oMiwMzAkd056OXbxMmO7FGmh77FOm6RQ1 -o9/NgJ8MSPsc9PG/Srj61YxxSscfrf5BmrODXfKEVu+lV0POKa2Mq1W/xPtbAd0j -IaFYAI7D0GoT7RPjEiuA3GfmlbLNHiJuKvhB1PLKFAeNilUSxmn1uIZoL1NesNKq -IcGY5jDjZ1XHm26sGahVpkUG0CM62+tlXSoREfA7T8pt9DTEceT/AFr2XK4jYIVz -8eQQsSWu1ZK7E8EM4DnatDlXtas1qnIhO4M15zHfeiFuuDIIfR0ykRVKYnLP43eh -vNURG3YBZwjgQQvD6xVu+KQZ2aKrr+InUlYrAoosFCT5v0ICvybIxo/gbjh9Uy3l -7ZizlWNof/k19N+IxWA1ksB8aRxhlRbQ694Lrz4EEEVlWFA4r0jyWbYW8jwNkALG -cC4BrTwV1wIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB -BjAdBgNVHQ4EFgQU7edvdlq/YOxJW8ald7tyFnGbxD0wDQYJKoZIhvcNAQELBQAD -ggIBAJHfgD9DCX5xwvfrs4iP4VGyvD11+ShdyLyZm3tdquXK4Qr36LLTn91nMX66 -AarHakE7kNQIXLJgapDwyM4DYvmL7ftuKtwGTTwpD4kWilhMSA/ohGHqPHKmd+RC -roijQ1h5fq7KpVMNqT1wvSAZYaRsOPxDMuHBR//47PERIjKWnML2W2mWeyAMQ0Ga -W/ZZGYjeVYg3UQt4XAoeo0L9x52ID8DyeAIkVJOviYeIyUqAHerQbj5hLja7NQ4n -lv1mNDthcnPxFlxHBlRJAHpYErAK74X9sbgzdWqTHBLmYF5vHX/JHyPLhGGfHoJE -+V+tYlUkmlKY7VHnoX6XOuYvHxHaU4AshZ6rNRDbIl9qxV6XU/IyAgkwo1jwDQHV -csaxfGl7w/U2Rcxhbl5MlMVerugOXou/983g7aEOGzPuVBj+D77vfoRrQ+NwmNtd -dbINWQeFFSM51vHfqSYP1kjHs6Yi9TM3WpVHn3u6GBVv/9YUZINJ0gpnIdsPNWNg -KCLjsZWDzYWm3S8P52dSbrsvhXz1SnPnxT7AvSESBT/8twNJAlvIJebiVDj1eYeM -HVOyToV7BjjHLPj4sHKNJeV3UvQDHEimUF+IIDBu8oJDqz2XhOdT+yHBTw8imoa4 -WSr2Rz0ZiC3oheGe7IUIarFsNMkd7EgrO3jtZsSOeWmD3n+M ------END CERTIFICATE----- - -# Issuer: CN=QuoVadis Root CA 3 G3 O=QuoVadis Limited -# Subject: CN=QuoVadis Root CA 3 G3 O=QuoVadis Limited -# Label: "QuoVadis Root CA 3 G3" -# Serial: 268090761170461462463995952157327242137089239581 -# MD5 Fingerprint: df:7d:b9:ad:54:6f:68:a1:df:89:57:03:97:43:b0:d7 -# SHA1 Fingerprint: 48:12:bd:92:3c:a8:c4:39:06:e7:30:6d:27:96:e6:a4:cf:22:2e:7d -# SHA256 Fingerprint: 88:ef:81:de:20:2e:b0:18:45:2e:43:f8:64:72:5c:ea:5f:bd:1f:c2:d9:d2:05:73:07:09:c5:d8:b8:69:0f:46 ------BEGIN CERTIFICATE----- -MIIFYDCCA0igAwIBAgIULvWbAiin23r/1aOp7r0DoM8Sah0wDQYJKoZIhvcNAQEL -BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc -BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMyBHMzAeFw0xMjAxMTIyMDI2MzJaFw00 -MjAxMTIyMDI2MzJaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM -aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDMgRzMwggIiMA0GCSqG -SIb3DQEBAQUAA4ICDwAwggIKAoICAQCzyw4QZ47qFJenMioKVjZ/aEzHs286IxSR -/xl/pcqs7rN2nXrpixurazHb+gtTTK/FpRp5PIpM/6zfJd5O2YIyC0TeytuMrKNu -FoM7pmRLMon7FhY4futD4tN0SsJiCnMK3UmzV9KwCoWdcTzeo8vAMvMBOSBDGzXR -U7Ox7sWTaYI+FrUoRqHe6okJ7UO4BUaKhvVZR74bbwEhELn9qdIoyhA5CcoTNs+c -ra1AdHkrAj80//ogaX3T7mH1urPnMNA3I4ZyYUUpSFlob3emLoG+B01vr87ERROR -FHAGjx+f+IdpsQ7vw4kZ6+ocYfx6bIrc1gMLnia6Et3UVDmrJqMz6nWB2i3ND0/k -A9HvFZcba5DFApCTZgIhsUfei5pKgLlVj7WiL8DWM2fafsSntARE60f75li59wzw -eyuxwHApw0BiLTtIadwjPEjrewl5qW3aqDCYz4ByA4imW0aucnl8CAMhZa634Ryl -sSqiMd5mBPfAdOhx3v89WcyWJhKLhZVXGqtrdQtEPREoPHtht+KPZ0/l7DxMYIBp -VzgeAVuNVejH38DMdyM0SXV89pgR6y3e7UEuFAUCf+D+IOs15xGsIs5XPd7JMG0Q -A4XN8f+MFrXBsj6IbGB/kE+V9/YtrQE5BwT6dYB9v0lQ7e/JxHwc64B+27bQ3RP+ -ydOc17KXqQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB -BjAdBgNVHQ4EFgQUxhfQvKjqAkPyGwaZXSuQILnXnOQwDQYJKoZIhvcNAQELBQAD -ggIBADRh2Va1EodVTd2jNTFGu6QHcrxfYWLopfsLN7E8trP6KZ1/AvWkyaiTt3px -KGmPc+FSkNrVvjrlt3ZqVoAh313m6Tqe5T72omnHKgqwGEfcIHB9UqM+WXzBusnI -FUBhynLWcKzSt/Ac5IYp8M7vaGPQtSCKFWGafoaYtMnCdvvMujAWzKNhxnQT5Wvv -oxXqA/4Ti2Tk08HS6IT7SdEQTXlm66r99I0xHnAUrdzeZxNMgRVhvLfZkXdxGYFg -u/BYpbWcC/ePIlUnwEsBbTuZDdQdm2NnL9DuDcpmvJRPpq3t/O5jrFc/ZSXPsoaP -0Aj/uHYUbt7lJ+yreLVTubY/6CD50qi+YUbKh4yE8/nxoGibIh6BJpsQBJFxwAYf -3KDTuVan45gtf4Od34wrnDKOMpTwATwiKp9Dwi7DmDkHOHv8XgBCH/MyJnmDhPbl -8MFREsALHgQjDFSlTC9JxUrRtm5gDWv8a4uFJGS3iQ6rJUdbPM9+Sb3H6QrG2vd+ -DhcI00iX0HGS8A85PjRqHH3Y8iKuu2n0M7SmSFXRDw4m6Oy2Cy2nhTXN/VnIn9HN -PlopNLk9hM6xZdRZkZFWdSHBd575euFgndOtBBj0fOtek49TSiIp+EgrPk2GrFt/ -ywaZWWDYWGWVjUTR939+J399roD1B0y2PpxxVJkES/1Y+Zj0 ------END CERTIFICATE----- - -# Issuer: CN=DigiCert Assured ID Root G2 O=DigiCert Inc OU=www.digicert.com -# Subject: CN=DigiCert Assured ID Root G2 O=DigiCert Inc OU=www.digicert.com -# Label: "DigiCert Assured ID Root G2" -# Serial: 15385348160840213938643033620894905419 -# MD5 Fingerprint: 92:38:b9:f8:63:24:82:65:2c:57:33:e6:fe:81:8f:9d -# SHA1 Fingerprint: a1:4b:48:d9:43:ee:0a:0e:40:90:4f:3c:e0:a4:c0:91:93:51:5d:3f -# SHA256 Fingerprint: 7d:05:eb:b6:82:33:9f:8c:94:51:ee:09:4e:eb:fe:fa:79:53:a1:14:ed:b2:f4:49:49:45:2f:ab:7d:2f:c1:85 ------BEGIN CERTIFICATE----- -MIIDljCCAn6gAwIBAgIQC5McOtY5Z+pnI7/Dr5r0SzANBgkqhkiG9w0BAQsFADBl -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv -b3QgRzIwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQG -EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl -cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzIwggEi -MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDZ5ygvUj82ckmIkzTz+GoeMVSA -n61UQbVH35ao1K+ALbkKz3X9iaV9JPrjIgwrvJUXCzO/GU1BBpAAvQxNEP4Htecc -biJVMWWXvdMX0h5i89vqbFCMP4QMls+3ywPgym2hFEwbid3tALBSfK+RbLE4E9Hp -EgjAALAcKxHad3A2m67OeYfcgnDmCXRwVWmvo2ifv922ebPynXApVfSr/5Vh88lA -bx3RvpO704gqu52/clpWcTs/1PPRCv4o76Pu2ZmvA9OPYLfykqGxvYmJHzDNw6Yu -YjOuFgJ3RFrngQo8p0Quebg/BLxcoIfhG69Rjs3sLPr4/m3wOnyqi+RnlTGNAgMB -AAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQW -BBTOw0q5mVXyuNtgv6l+vVa1lzan1jANBgkqhkiG9w0BAQsFAAOCAQEAyqVVjOPI -QW5pJ6d1Ee88hjZv0p3GeDgdaZaikmkuOGybfQTUiaWxMTeKySHMq2zNixya1r9I -0jJmwYrA8y8678Dj1JGG0VDjA9tzd29KOVPt3ibHtX2vK0LRdWLjSisCx1BL4Gni -lmwORGYQRI+tBev4eaymG+g3NJ1TyWGqolKvSnAWhsI6yLETcDbYz+70CjTVW0z9 -B5yiutkBclzzTcHdDrEcDcRjvq30FPuJ7KJBDkzMyFdA0G4Dqs0MjomZmWzwPDCv -ON9vvKO+KSAnq3T/EyJ43pdSVR6DtVQgA+6uwE9W3jfMw3+qBCe703e4YtsXfJwo -IhNzbM8m9Yop5w== ------END CERTIFICATE----- - -# Issuer: CN=DigiCert Assured ID Root G3 O=DigiCert Inc OU=www.digicert.com -# Subject: CN=DigiCert Assured ID Root G3 O=DigiCert Inc OU=www.digicert.com -# Label: "DigiCert Assured ID Root G3" -# Serial: 15459312981008553731928384953135426796 -# MD5 Fingerprint: 7c:7f:65:31:0c:81:df:8d:ba:3e:99:e2:5c:ad:6e:fb -# SHA1 Fingerprint: f5:17:a2:4f:9a:48:c6:c9:f8:a2:00:26:9f:dc:0f:48:2c:ab:30:89 -# SHA256 Fingerprint: 7e:37:cb:8b:4c:47:09:0c:ab:36:55:1b:a6:f4:5d:b8:40:68:0f:ba:16:6a:95:2d:b1:00:71:7f:43:05:3f:c2 ------BEGIN CERTIFICATE----- -MIICRjCCAc2gAwIBAgIQC6Fa+h3foLVJRK/NJKBs7DAKBggqhkjOPQQDAzBlMQsw -CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu -ZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3Qg -RzMwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQGEwJV -UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu -Y29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzMwdjAQBgcq -hkjOPQIBBgUrgQQAIgNiAAQZ57ysRGXtzbg/WPuNsVepRC0FFfLvC/8QdJ+1YlJf -Zn4f5dwbRXkLzMZTCp2NXQLZqVneAlr2lSoOjThKiknGvMYDOAdfVdp+CW7if17Q -RSAPWXYQ1qAk8C3eNvJsKTmjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ -BAQDAgGGMB0GA1UdDgQWBBTL0L2p4ZgFUaFNN6KDec6NHSrkhDAKBggqhkjOPQQD -AwNnADBkAjAlpIFFAmsSS3V0T8gj43DydXLefInwz5FyYZ5eEJJZVrmDxxDnOOlY -JjZ91eQ0hjkCMHw2U/Aw5WJjOpnitqM7mzT6HtoQknFekROn3aRukswy1vUhZscv -6pZjamVFkpUBtA== ------END CERTIFICATE----- - -# Issuer: CN=DigiCert Global Root G2 O=DigiCert Inc OU=www.digicert.com -# Subject: CN=DigiCert Global Root G2 O=DigiCert Inc OU=www.digicert.com -# Label: "DigiCert Global Root G2" -# Serial: 4293743540046975378534879503202253541 -# MD5 Fingerprint: e4:a6:8a:c8:54:ac:52:42:46:0a:fd:72:48:1b:2a:44 -# SHA1 Fingerprint: df:3c:24:f9:bf:d6:66:76:1b:26:80:73:fe:06:d1:cc:8d:4f:82:a4 -# SHA256 Fingerprint: cb:3c:cb:b7:60:31:e5:e0:13:8f:8d:d3:9a:23:f9:de:47:ff:c3:5e:43:c1:14:4c:ea:27:d4:6a:5a:b1:cb:5f ------BEGIN CERTIFICATE----- -MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH -MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT -MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j -b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG -9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI -2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx -1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ -q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz -tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ -vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP -BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV -5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY -1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4 -NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG -Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91 -8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe -pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl -MrY= ------END CERTIFICATE----- - -# Issuer: CN=DigiCert Global Root G3 O=DigiCert Inc OU=www.digicert.com -# Subject: CN=DigiCert Global Root G3 O=DigiCert Inc OU=www.digicert.com -# Label: "DigiCert Global Root G3" -# Serial: 7089244469030293291760083333884364146 -# MD5 Fingerprint: f5:5d:a4:50:a5:fb:28:7e:1e:0f:0d:cc:96:57:56:ca -# SHA1 Fingerprint: 7e:04:de:89:6a:3e:66:6d:00:e6:87:d3:3f:fa:d9:3b:e8:3d:34:9e -# SHA256 Fingerprint: 31:ad:66:48:f8:10:41:38:c7:38:f3:9e:a4:32:01:33:39:3e:3a:18:cc:02:29:6e:f9:7c:2a:c9:ef:67:31:d0 ------BEGIN CERTIFICATE----- -MIICPzCCAcWgAwIBAgIQBVVWvPJepDU1w6QP1atFcjAKBggqhkjOPQQDAzBhMQsw -CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu -ZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBHMzAe -Fw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVTMRUw -EwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20x -IDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEczMHYwEAYHKoZIzj0CAQYF -K4EEACIDYgAE3afZu4q4C/sLfyHS8L6+c/MzXRq8NOrexpu80JX28MzQC7phW1FG -fp4tn+6OYwwX7Adw9c+ELkCDnOg/QW07rdOkFFk2eJ0DQ+4QE2xy3q6Ip6FrtUPO -Z9wj/wMco+I+o0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAd -BgNVHQ4EFgQUs9tIpPmhxdiuNkHMEWNpYim8S8YwCgYIKoZIzj0EAwMDaAAwZQIx -AK288mw/EkrRLTnDCgmXc/SINoyIJ7vmiI1Qhadj+Z4y3maTD/HMsQmP3Wyr+mt/ -oAIwOWZbwmSNuJ5Q3KjVSaLtx9zRSX8XAbjIho9OjIgrqJqpisXRAL34VOKa5Vt8 -sycX ------END CERTIFICATE----- - -# Issuer: CN=DigiCert Trusted Root G4 O=DigiCert Inc OU=www.digicert.com -# Subject: CN=DigiCert Trusted Root G4 O=DigiCert Inc OU=www.digicert.com -# Label: "DigiCert Trusted Root G4" -# Serial: 7451500558977370777930084869016614236 -# MD5 Fingerprint: 78:f2:fc:aa:60:1f:2f:b4:eb:c9:37:ba:53:2e:75:49 -# SHA1 Fingerprint: dd:fb:16:cd:49:31:c9:73:a2:03:7d:3f:c8:3a:4d:7d:77:5d:05:e4 -# SHA256 Fingerprint: 55:2f:7b:dc:f1:a7:af:9e:6c:e6:72:01:7f:4f:12:ab:f7:72:40:c7:8e:76:1a:c2:03:d1:d9:d2:0a:c8:99:88 ------BEGIN CERTIFICATE----- -MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBi -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg -RzQwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBiMQswCQYDVQQGEwJV -UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu -Y29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwggIiMA0GCSqG -SIb3DQEBAQUAA4ICDwAwggIKAoICAQC/5pBzaN675F1KPDAiMGkz7MKnJS7JIT3y -ithZwuEppz1Yq3aaza57G4QNxDAf8xukOBbrVsaXbR2rsnnyyhHS5F/WBTxSD1If -xp4VpX6+n6lXFllVcq9ok3DCsrp1mWpzMpTREEQQLt+C8weE5nQ7bXHiLQwb7iDV -ySAdYyktzuxeTsiT+CFhmzTrBcZe7FsavOvJz82sNEBfsXpm7nfISKhmV1efVFiO -DCu3T6cw2Vbuyntd463JT17lNecxy9qTXtyOj4DatpGYQJB5w3jHtrHEtWoYOAMQ -jdjUN6QuBX2I9YI+EJFwq1WCQTLX2wRzKm6RAXwhTNS8rhsDdV14Ztk6MUSaM0C/ -CNdaSaTC5qmgZ92kJ7yhTzm1EVgX9yRcRo9k98FpiHaYdj1ZXUJ2h4mXaXpI8OCi -EhtmmnTK3kse5w5jrubU75KSOp493ADkRSWJtppEGSt+wJS00mFt6zPZxd9LBADM -fRyVw4/3IbKyEbe7f/LVjHAsQWCqsWMYRJUadmJ+9oCw++hkpjPRiQfhvbfmQ6QY -uKZ3AeEPlAwhHbJUKSWJbOUOUlFHdL4mrLZBdd56rF+NP8m800ERElvlEFDrMcXK -chYiCd98THU/Y+whX8QgUWtvsauGi0/C1kVfnSD8oR7FwI+isX4KJpn15GkvmB0t -9dmpsh3lGwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB -hjAdBgNVHQ4EFgQU7NfjgtJxXWRM3y5nP+e6mK4cD08wDQYJKoZIhvcNAQEMBQAD -ggIBALth2X2pbL4XxJEbw6GiAI3jZGgPVs93rnD5/ZpKmbnJeFwMDF/k5hQpVgs2 -SV1EY+CtnJYYZhsjDT156W1r1lT40jzBQ0CuHVD1UvyQO7uYmWlrx8GnqGikJ9yd -+SeuMIW59mdNOj6PWTkiU0TryF0Dyu1Qen1iIQqAyHNm0aAFYF/opbSnr6j3bTWc -fFqK1qI4mfN4i/RN0iAL3gTujJtHgXINwBQy7zBZLq7gcfJW5GqXb5JQbZaNaHqa -sjYUegbyJLkJEVDXCLG4iXqEI2FCKeWjzaIgQdfRnGTZ6iahixTXTBmyUEFxPT9N -cCOGDErcgdLMMpSEDQgJlxxPwO5rIHQw0uA5NBCFIRUBCOhVMt5xSdkoF1BN5r5N -0XWs0Mr7QbhDparTwwVETyw2m+L64kW4I1NsBm9nVX9GtUw/bihaeSbSpKhil9Ie -4u1Ki7wb/UdKDd9nZn6yW0HQO+T0O/QEY+nvwlQAUaCKKsnOeMzV6ocEGLPOr0mI -r/OSmbaz5mEP0oUA51Aa5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1 -/YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tKG48BtieVU+i2iW1bvGjUI+iLUaJW+fCm -gKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP82Z+ ------END CERTIFICATE----- - -# Issuer: CN=COMODO RSA Certification Authority O=COMODO CA Limited -# Subject: CN=COMODO RSA Certification Authority O=COMODO CA Limited -# Label: "COMODO RSA Certification Authority" -# Serial: 101909084537582093308941363524873193117 -# MD5 Fingerprint: 1b:31:b0:71:40:36:cc:14:36:91:ad:c4:3e:fd:ec:18 -# SHA1 Fingerprint: af:e5:d2:44:a8:d1:19:42:30:ff:47:9f:e2:f8:97:bb:cd:7a:8c:b4 -# SHA256 Fingerprint: 52:f0:e1:c4:e5:8e:c6:29:29:1b:60:31:7f:07:46:71:b8:5d:7e:a8:0d:5b:07:27:34:63:53:4b:32:b4:02:34 ------BEGIN CERTIFICATE----- -MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB -hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV -BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5 -MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT -EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR -Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh -dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR -6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X -pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC -9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV -/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf -Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z -+pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w -qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah -SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC -u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf -Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq -crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E -FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB -/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl -wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM -4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV -2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna -FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ -CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK -boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke -jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL -S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb -QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl -0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB -NVOFBkpdn627G190 ------END CERTIFICATE----- - -# Issuer: CN=USERTrust RSA Certification Authority O=The USERTRUST Network -# Subject: CN=USERTrust RSA Certification Authority O=The USERTRUST Network -# Label: "USERTrust RSA Certification Authority" -# Serial: 2645093764781058787591871645665788717 -# MD5 Fingerprint: 1b:fe:69:d1:91:b7:19:33:a3:72:a8:0f:e1:55:e5:b5 -# SHA1 Fingerprint: 2b:8f:1b:57:33:0d:bb:a2:d0:7a:6c:51:f7:0e:e9:0d:da:b9:ad:8e -# SHA256 Fingerprint: e7:93:c9:b0:2f:d8:aa:13:e2:1c:31:22:8a:cc:b0:81:19:64:3b:74:9c:89:89:64:b1:74:6d:46:c3:d4:cb:d2 ------BEGIN CERTIFICATE----- -MIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB -iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl -cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV -BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw -MjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV -BAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU -aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy -dGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B -3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY -tJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/ -Fp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2 -VN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT -79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6 -c0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT -Yo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l -c6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee -UB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE -Hg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd -BgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G -A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF -Up/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO -VWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3 -ATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs -8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR -iQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze -Sf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ -XHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/ -qS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB -VXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB -L6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG -jjxDah2nGN59PRbxYvnKkKj9 ------END CERTIFICATE----- - -# Issuer: CN=USERTrust ECC Certification Authority O=The USERTRUST Network -# Subject: CN=USERTrust ECC Certification Authority O=The USERTRUST Network -# Label: "USERTrust ECC Certification Authority" -# Serial: 123013823720199481456569720443997572134 -# MD5 Fingerprint: fa:68:bc:d9:b5:7f:ad:fd:c9:1d:06:83:28:cc:24:c1 -# SHA1 Fingerprint: d1:cb:ca:5d:b2:d5:2a:7f:69:3b:67:4d:e5:f0:5a:1d:0c:95:7d:f0 -# SHA256 Fingerprint: 4f:f4:60:d5:4b:9c:86:da:bf:bc:fc:57:12:e0:40:0d:2b:ed:3f:bc:4d:4f:bd:aa:86:e0:6a:dc:d2:a9:ad:7a ------BEGIN CERTIFICATE----- -MIICjzCCAhWgAwIBAgIQXIuZxVqUxdJxVt7NiYDMJjAKBggqhkjOPQQDAzCBiDEL -MAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNl -eSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMT -JVVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMjAx -MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT -Ck5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVUaGUg -VVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBFQ0MgQ2VydGlm -aWNhdGlvbiBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQarFRaqflo -I+d61SRvU8Za2EurxtW20eZzca7dnNYMYf3boIkDuAUU7FfO7l0/4iGzzvfUinng -o4N+LZfQYcTxmdwlkWOrfzCjtHDix6EznPO/LlxTsV+zfTJ/ijTjeXmjQjBAMB0G -A1UdDgQWBBQ64QmG1M8ZwpZ2dEl23OA1xmNjmjAOBgNVHQ8BAf8EBAMCAQYwDwYD -VR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjA2Z6EWCNzklwBBHU6+4WMB -zzuqQhFkoJ2UOQIReVx7Hfpkue4WQrO/isIJxOzksU0CMQDpKmFHjFJKS04YcPbW -RNZu9YO6bVi9JNlWSOrvxKJGgYhqOkbRqZtNyWHa0V1Xahg= ------END CERTIFICATE----- - -# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R5 -# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R5 -# Label: "GlobalSign ECC Root CA - R5" -# Serial: 32785792099990507226680698011560947931244 -# MD5 Fingerprint: 9f:ad:3b:1c:02:1e:8a:ba:17:74:38:81:0c:a2:bc:08 -# SHA1 Fingerprint: 1f:24:c6:30:cd:a4:18:ef:20:69:ff:ad:4f:dd:5f:46:3a:1b:69:aa -# SHA256 Fingerprint: 17:9f:bc:14:8a:3d:d0:0f:d2:4e:a1:34:58:cc:43:bf:a7:f5:9c:81:82:d7:83:a5:13:f6:eb:ec:10:0c:89:24 ------BEGIN CERTIFICATE----- -MIICHjCCAaSgAwIBAgIRYFlJ4CYuu1X5CneKcflK2GwwCgYIKoZIzj0EAwMwUDEk -MCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI1MRMwEQYDVQQKEwpH -bG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoX -DTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBD -QSAtIFI1MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu -MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAER0UOlvt9Xb/pOdEh+J8LttV7HpI6SFkc -8GIxLcB6KP4ap1yztsyX50XUWPrRd21DosCHZTQKH3rd6zwzocWdTaRvQZU4f8ke -hOvRnkmSh5SHDDqFSmafnVmTTZdhBoZKo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYD -VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUPeYpSJvqB8ohREom3m7e0oPQn1kwCgYI -KoZIzj0EAwMDaAAwZQIxAOVpEslu28YxuglB4Zf4+/2a4n0Sye18ZNPLBSWLVtmg -515dTguDnFt2KaAJJiFqYgIwcdK1j1zqO+F4CYWodZI7yFz9SO8NdCKoCOJuxUnO -xwy8p2Fp8fc74SrL+SvzZpA3 ------END CERTIFICATE----- - -# Issuer: CN=IdenTrust Commercial Root CA 1 O=IdenTrust -# Subject: CN=IdenTrust Commercial Root CA 1 O=IdenTrust -# Label: "IdenTrust Commercial Root CA 1" -# Serial: 13298821034946342390520003877796839426 -# MD5 Fingerprint: b3:3e:77:73:75:ee:a0:d3:e3:7e:49:63:49:59:bb:c7 -# SHA1 Fingerprint: df:71:7e:aa:4a:d9:4e:c9:55:84:99:60:2d:48:de:5f:bc:f0:3a:25 -# SHA256 Fingerprint: 5d:56:49:9b:e4:d2:e0:8b:cf:ca:d0:8a:3e:38:72:3d:50:50:3b:de:70:69:48:e4:2f:55:60:30:19:e5:28:ae ------BEGIN CERTIFICATE----- -MIIFYDCCA0igAwIBAgIQCgFCgAAAAUUjyES1AAAAAjANBgkqhkiG9w0BAQsFADBK -MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScwJQYDVQQDEx5JZGVu -VHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwHhcNMTQwMTE2MTgxMjIzWhcNMzQw -MTE2MTgxMjIzWjBKMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScw -JQYDVQQDEx5JZGVuVHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwggIiMA0GCSqG -SIb3DQEBAQUAA4ICDwAwggIKAoICAQCnUBneP5k91DNG8W9RYYKyqU+PZ4ldhNlT -3Qwo2dfw/66VQ3KZ+bVdfIrBQuExUHTRgQ18zZshq0PirK1ehm7zCYofWjK9ouuU -+ehcCuz/mNKvcbO0U59Oh++SvL3sTzIwiEsXXlfEU8L2ApeN2WIrvyQfYo3fw7gp -S0l4PJNgiCL8mdo2yMKi1CxUAGc1bnO/AljwpN3lsKImesrgNqUZFvX9t++uP0D1 -bVoE/c40yiTcdCMbXTMTEl3EASX2MN0CXZ/g1Ue9tOsbobtJSdifWwLziuQkkORi -T0/Br4sOdBeo0XKIanoBScy0RnnGF7HamB4HWfp1IYVl3ZBWzvurpWCdxJ35UrCL -vYf5jysjCiN2O/cz4ckA82n5S6LgTrx+kzmEB/dEcH7+B1rlsazRGMzyNeVJSQjK -Vsk9+w8YfYs7wRPCTY/JTw436R+hDmrfYi7LNQZReSzIJTj0+kuniVyc0uMNOYZK -dHzVWYfCP04MXFL0PfdSgvHqo6z9STQaKPNBiDoT7uje/5kdX7rL6B7yuVBgwDHT -c+XvvqDtMwt0viAgxGds8AgDelWAf0ZOlqf0Hj7h9tgJ4TNkK2PXMl6f+cB7D3hv -l7yTmvmcEpB4eoCHFddydJxVdHixuuFucAS6T6C6aMN7/zHwcz09lCqxC0EOoP5N -iGVreTO01wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB -/zAdBgNVHQ4EFgQU7UQZwNPwBovupHu+QucmVMiONnYwDQYJKoZIhvcNAQELBQAD -ggIBAA2ukDL2pkt8RHYZYR4nKM1eVO8lvOMIkPkp165oCOGUAFjvLi5+U1KMtlwH -6oi6mYtQlNeCgN9hCQCTrQ0U5s7B8jeUeLBfnLOic7iPBZM4zY0+sLj7wM+x8uwt -LRvM7Kqas6pgghstO8OEPVeKlh6cdbjTMM1gCIOQ045U8U1mwF10A0Cj7oV+wh93 -nAbowacYXVKV7cndJZ5t+qntozo00Fl72u1Q8zW/7esUTTHHYPTa8Yec4kjixsU3 -+wYQ+nVZZjFHKdp2mhzpgq7vmrlR94gjmmmVYjzlVYA211QC//G5Xc7UI2/YRYRK -W2XviQzdFKcgyxilJbQN+QHwotL0AMh0jqEqSI5l2xPE4iUXfeu+h1sXIFRRk0pT -AwvsXcoz7WL9RccvW9xYoIA55vrX/hMUpu09lEpCdNTDd1lzzY9GvlU47/rokTLq -l1gEIt44w8y8bckzOmoKaT+gyOpyj4xjhiO9bTyWnpXgSUyqorkqG5w2gXjtw+hG -4iZZRHUe2XWJUc0QhJ1hYMtd+ZciTY6Y5uN/9lu7rs3KSoFrXgvzUeF0K+l+J6fZ -mUlO+KWA2yUPHGNiiskzZ2s8EIPGrd6ozRaOjfAHN3Gf8qv8QfXBi+wAN10J5U6A -7/qxXDgGpRtK4dw4LTzcqx+QGtVKnO7RcGzM7vRX+Bi6hG6H ------END CERTIFICATE----- - -# Issuer: CN=IdenTrust Public Sector Root CA 1 O=IdenTrust -# Subject: CN=IdenTrust Public Sector Root CA 1 O=IdenTrust -# Label: "IdenTrust Public Sector Root CA 1" -# Serial: 13298821034946342390521976156843933698 -# MD5 Fingerprint: 37:06:a5:b0:fc:89:9d:ba:f4:6b:8c:1a:64:cd:d5:ba -# SHA1 Fingerprint: ba:29:41:60:77:98:3f:f4:f3:ef:f2:31:05:3b:2e:ea:6d:4d:45:fd -# SHA256 Fingerprint: 30:d0:89:5a:9a:44:8a:26:20:91:63:55:22:d1:f5:20:10:b5:86:7a:ca:e1:2c:78:ef:95:8f:d4:f4:38:9f:2f ------BEGIN CERTIFICATE----- -MIIFZjCCA06gAwIBAgIQCgFCgAAAAUUjz0Z8AAAAAjANBgkqhkiG9w0BAQsFADBN -MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MSowKAYDVQQDEyFJZGVu -VHJ1c3QgUHVibGljIFNlY3RvciBSb290IENBIDEwHhcNMTQwMTE2MTc1MzMyWhcN -MzQwMTE2MTc1MzMyWjBNMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0 -MSowKAYDVQQDEyFJZGVuVHJ1c3QgUHVibGljIFNlY3RvciBSb290IENBIDEwggIi -MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC2IpT8pEiv6EdrCvsnduTyP4o7 -ekosMSqMjbCpwzFrqHd2hCa2rIFCDQjrVVi7evi8ZX3yoG2LqEfpYnYeEe4IFNGy -RBb06tD6Hi9e28tzQa68ALBKK0CyrOE7S8ItneShm+waOh7wCLPQ5CQ1B5+ctMlS -bdsHyo+1W/CD80/HLaXIrcuVIKQxKFdYWuSNG5qrng0M8gozOSI5Cpcu81N3uURF -/YTLNiCBWS2ab21ISGHKTN9T0a9SvESfqy9rg3LvdYDaBjMbXcjaY8ZNzaxmMc3R -3j6HEDbhuaR672BQssvKplbgN6+rNBM5Jeg5ZuSYeqoSmJxZZoY+rfGwyj4GD3vw -EUs3oERte8uojHH01bWRNszwFcYr3lEXsZdMUD2xlVl8BX0tIdUAvwFnol57plzy -9yLxkA2T26pEUWbMfXYD62qoKjgZl3YNa4ph+bz27nb9cCvdKTz4Ch5bQhyLVi9V -GxyhLrXHFub4qjySjmm2AcG1hp2JDws4lFTo6tyePSW8Uybt1as5qsVATFSrsrTZ -2fjXctscvG29ZV/viDUqZi/u9rNl8DONfJhBaUYPQxxp+pu10GFqzcpL2UyQRqsV -WaFHVCkugyhfHMKiq3IXAAaOReyL4jM9f9oZRORicsPfIsbyVtTdX5Vy7W1f90gD -W/3FKqD2cyOEEBsB5wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/ -BAUwAwEB/zAdBgNVHQ4EFgQU43HgntinQtnbcZFrlJPrw6PRFKMwDQYJKoZIhvcN -AQELBQADggIBAEf63QqwEZE4rU1d9+UOl1QZgkiHVIyqZJnYWv6IAcVYpZmxI1Qj -t2odIFflAWJBF9MJ23XLblSQdf4an4EKwt3X9wnQW3IV5B4Jaj0z8yGa5hV+rVHV -DRDtfULAj+7AmgjVQdZcDiFpboBhDhXAuM/FSRJSzL46zNQuOAXeNf0fb7iAaJg9 -TaDKQGXSc3z1i9kKlT/YPyNtGtEqJBnZhbMX73huqVjRI9PHE+1yJX9dsXNw0H8G -lwmEKYBhHfpe/3OsoOOJuBxxFcbeMX8S3OFtm6/n6J91eEyrRjuazr8FGF1NFTwW -mhlQBJqymm9li1JfPFgEKCXAZmExfrngdbkaqIHWchezxQMxNRF4eKLg6TCMf4Df -WN88uieW4oA0beOY02QnrEh+KHdcxiVhJfiFDGX6xDIvpZgF5PgLZxYWxoK4Mhn5 -+bl53B/N66+rDt0b20XkeucC4pVd/GnwU2lhlXV5C15V5jgclKlZM57IcXR5f1GJ -tshquDDIajjDbp7hNxbqBWJMWxJH7ae0s1hWx0nzfxJoCTFx8G34Tkf71oXuxVhA -GaQdp/lLQzfcaFpPz+vCZHTetBXZ9FRUGi8c15dxVJCO2SCdUyt/q4/i6jC8UDfv -8Ue1fXwsBOxonbRJRBD0ckscZOf85muQ3Wl9af0AVqW3rLatt8o+Ae+c ------END CERTIFICATE----- - -# Issuer: CN=CFCA EV ROOT O=China Financial Certification Authority -# Subject: CN=CFCA EV ROOT O=China Financial Certification Authority -# Label: "CFCA EV ROOT" -# Serial: 407555286 -# MD5 Fingerprint: 74:e1:b6:ed:26:7a:7a:44:30:33:94:ab:7b:27:81:30 -# SHA1 Fingerprint: e2:b8:29:4b:55:84:ab:6b:58:c2:90:46:6c:ac:3f:b8:39:8f:84:83 -# SHA256 Fingerprint: 5c:c3:d7:8e:4e:1d:5e:45:54:7a:04:e6:87:3e:64:f9:0c:f9:53:6d:1c:cc:2e:f8:00:f3:55:c4:c5:fd:70:fd ------BEGIN CERTIFICATE----- -MIIFjTCCA3WgAwIBAgIEGErM1jANBgkqhkiG9w0BAQsFADBWMQswCQYDVQQGEwJD -TjEwMC4GA1UECgwnQ2hpbmEgRmluYW5jaWFsIENlcnRpZmljYXRpb24gQXV0aG9y -aXR5MRUwEwYDVQQDDAxDRkNBIEVWIFJPT1QwHhcNMTIwODA4MDMwNzAxWhcNMjkx -MjMxMDMwNzAxWjBWMQswCQYDVQQGEwJDTjEwMC4GA1UECgwnQ2hpbmEgRmluYW5j -aWFsIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MRUwEwYDVQQDDAxDRkNBIEVWIFJP -T1QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDXXWvNED8fBVnVBU03 -sQ7smCuOFR36k0sXgiFxEFLXUWRwFsJVaU2OFW2fvwwbwuCjZ9YMrM8irq93VCpL -TIpTUnrD7i7es3ElweldPe6hL6P3KjzJIx1qqx2hp/Hz7KDVRM8Vz3IvHWOX6Jn5 -/ZOkVIBMUtRSqy5J35DNuF++P96hyk0g1CXohClTt7GIH//62pCfCqktQT+x8Rgp -7hZZLDRJGqgG16iI0gNyejLi6mhNbiyWZXvKWfry4t3uMCz7zEasxGPrb382KzRz -EpR/38wmnvFyXVBlWY9ps4deMm/DGIq1lY+wejfeWkU7xzbh72fROdOXW3NiGUgt -hxwG+3SYIElz8AXSG7Ggo7cbcNOIabla1jj0Ytwli3i/+Oh+uFzJlU9fpy25IGvP -a931DfSCt/SyZi4QKPaXWnuWFo8BGS1sbn85WAZkgwGDg8NNkt0yxoekN+kWzqot -aK8KgWU6cMGbrU1tVMoqLUuFG7OA5nBFDWteNfB/O7ic5ARwiRIlk9oKmSJgamNg -TnYGmE69g60dWIolhdLHZR4tjsbftsbhf4oEIRUpdPA+nJCdDC7xij5aqgwJHsfV -PKPtl8MeNPo4+QgO48BdK4PRVmrJtqhUUy54Mmc9gn900PvhtgVguXDbjgv5E1hv -cWAQUhC5wUEJ73IfZzF4/5YFjQIDAQABo2MwYTAfBgNVHSMEGDAWgBTj/i39KNAL -tbq2osS/BqoFjJP7LzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAd -BgNVHQ4EFgQU4/4t/SjQC7W6tqLEvwaqBYyT+y8wDQYJKoZIhvcNAQELBQADggIB -ACXGumvrh8vegjmWPfBEp2uEcwPenStPuiB/vHiyz5ewG5zz13ku9Ui20vsXiObT -ej/tUxPQ4i9qecsAIyjmHjdXNYmEwnZPNDatZ8POQQaIxffu2Bq41gt/UP+TqhdL -jOztUmCypAbqTuv0axn96/Ua4CUqmtzHQTb3yHQFhDmVOdYLO6Qn+gjYXB74BGBS -ESgoA//vU2YApUo0FmZ8/Qmkrp5nGm9BC2sGE5uPhnEFtC+NiWYzKXZUmhH4J/qy -P5Hgzg0b8zAarb8iXRvTvyUFTeGSGn+ZnzxEk8rUQElsgIfXBDrDMlI1Dlb4pd19 -xIsNER9Tyx6yF7Zod1rg1MvIB671Oi6ON7fQAUtDKXeMOZePglr4UeWJoBjnaH9d -Ci77o0cOPaYjesYBx4/IXr9tgFa+iiS6M+qf4TIRnvHST4D2G0CvOJ4RUHlzEhLN -5mydLIhyPDCBBpEi6lmt2hkuIsKNuYyH4Ga8cyNfIWRjgEj1oDwYPZTISEEdQLpe -/v5WOaHIz16eGWRGENoXkbcFgKyLmZJ956LYBws2J+dIeWCKw9cTXPhyQN9Ky8+Z -AAoACxGV2lZFA4gKn2fQ1XmxqI1AbQ3CekD6819kR5LLU7m7Wc5P/dAVUwHY3+vZ -5nbv0CO7O6l5s9UCKc2Jo5YPSjXnTkLAdc0Hz+Ys63su ------END CERTIFICATE----- - -# Issuer: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed -# Subject: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed -# Label: "OISTE WISeKey Global Root GB CA" -# Serial: 157768595616588414422159278966750757568 -# MD5 Fingerprint: a4:eb:b9:61:28:2e:b7:2f:98:b0:35:26:90:99:51:1d -# SHA1 Fingerprint: 0f:f9:40:76:18:d3:d7:6a:4b:98:f0:a8:35:9e:0c:fd:27:ac:cc:ed -# SHA256 Fingerprint: 6b:9c:08:e8:6e:b0:f7:67:cf:ad:65:cd:98:b6:21:49:e5:49:4a:67:f5:84:5e:7b:d1:ed:01:9f:27:b8:6b:d6 ------BEGIN CERTIFICATE----- -MIIDtTCCAp2gAwIBAgIQdrEgUnTwhYdGs/gjGvbCwDANBgkqhkiG9w0BAQsFADBt -MQswCQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUg -Rm91bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9i -YWwgUm9vdCBHQiBDQTAeFw0xNDEyMDExNTAwMzJaFw0zOTEyMDExNTEwMzFaMG0x -CzAJBgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBG -b3VuZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2Jh -bCBSb290IEdCIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2Be3 -HEokKtaXscriHvt9OO+Y9bI5mE4nuBFde9IllIiCFSZqGzG7qFshISvYD06fWvGx -WuR51jIjK+FTzJlFXHtPrby/h0oLS5daqPZI7H17Dc0hBt+eFf1Biki3IPShehtX -1F1Q/7pn2COZH8g/497/b1t3sWtuuMlk9+HKQUYOKXHQuSP8yYFfTvdv37+ErXNk -u7dCjmn21HYdfp2nuFeKUWdy19SouJVUQHMD9ur06/4oQnc/nSMbsrY9gBQHTC5P -99UKFg29ZkM3fiNDecNAhvVMKdqOmq0NpQSHiB6F4+lT1ZvIiwNjeOvgGUpuuy9r -M2RYk61pv48b74JIxwIDAQABo1EwTzALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUw -AwEB/zAdBgNVHQ4EFgQUNQ/INmNe4qPs+TtmFc5RUuORmj0wEAYJKwYBBAGCNxUB -BAMCAQAwDQYJKoZIhvcNAQELBQADggEBAEBM+4eymYGQfp3FsLAmzYh7KzKNbrgh -cViXfa43FK8+5/ea4n32cZiZBKpDdHij40lhPnOMTZTg+XHEthYOU3gf1qKHLwI5 -gSk8rxWYITD+KJAAjNHhy/peyP34EEY7onhCkRd0VQreUGdNZtGn//3ZwLWoo4rO -ZvUPQ82nK1d7Y0Zqqi5S2PTt4W2tKZB4SLrhI6qjiey1q5bAtEuiHZeeevJuQHHf -aPFlTc58Bd9TZaml8LGXBHAVRgOY1NK/VLSgWH1Sb9pWJmLU2NuJMW8c8CLC02Ic -Nc1MaRVUGpCY3useX8p3x8uOPUNpnJpY0CQ73xtAln41rYHHTnG6iBM= ------END CERTIFICATE----- - -# Issuer: CN=SZAFIR ROOT CA2 O=Krajowa Izba Rozliczeniowa S.A. -# Subject: CN=SZAFIR ROOT CA2 O=Krajowa Izba Rozliczeniowa S.A. -# Label: "SZAFIR ROOT CA2" -# Serial: 357043034767186914217277344587386743377558296292 -# MD5 Fingerprint: 11:64:c1:89:b0:24:b1:8c:b1:07:7e:89:9e:51:9e:99 -# SHA1 Fingerprint: e2:52:fa:95:3f:ed:db:24:60:bd:6e:28:f3:9c:cc:cf:5e:b3:3f:de -# SHA256 Fingerprint: a1:33:9d:33:28:1a:0b:56:e5:57:d3:d3:2b:1c:e7:f9:36:7e:b0:94:bd:5f:a7:2a:7e:50:04:c8:de:d7:ca:fe ------BEGIN CERTIFICATE----- -MIIDcjCCAlqgAwIBAgIUPopdB+xV0jLVt+O2XwHrLdzk1uQwDQYJKoZIhvcNAQEL -BQAwUTELMAkGA1UEBhMCUEwxKDAmBgNVBAoMH0tyYWpvd2EgSXpiYSBSb3psaWN6 -ZW5pb3dhIFMuQS4xGDAWBgNVBAMMD1NaQUZJUiBST09UIENBMjAeFw0xNTEwMTkw -NzQzMzBaFw0zNTEwMTkwNzQzMzBaMFExCzAJBgNVBAYTAlBMMSgwJgYDVQQKDB9L -cmFqb3dhIEl6YmEgUm96bGljemVuaW93YSBTLkEuMRgwFgYDVQQDDA9TWkFGSVIg -Uk9PVCBDQTIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3vD5QqEvN -QLXOYeeWyrSh2gwisPq1e3YAd4wLz32ohswmUeQgPYUM1ljj5/QqGJ3a0a4m7utT -3PSQ1hNKDJA8w/Ta0o4NkjrcsbH/ON7Dui1fgLkCvUqdGw+0w8LBZwPd3BucPbOw -3gAeqDRHu5rr/gsUvTaE2g0gv/pby6kWIK05YO4vdbbnl5z5Pv1+TW9NL++IDWr6 -3fE9biCloBK0TXC5ztdyO4mTp4CEHCdJckm1/zuVnsHMyAHs6A6KCpbns6aH5db5 -BSsNl0BwPLqsdVqc1U2dAgrSS5tmS0YHF2Wtn2yIANwiieDhZNRnvDF5YTy7ykHN -XGoAyDw4jlivAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD -AgEGMB0GA1UdDgQWBBQuFqlKGLXLzPVvUPMjX/hd56zwyDANBgkqhkiG9w0BAQsF -AAOCAQEAtXP4A9xZWx126aMqe5Aosk3AM0+qmrHUuOQn/6mWmc5G4G18TKI4pAZw -8PRBEew/R40/cof5O/2kbytTAOD/OblqBw7rHRz2onKQy4I9EYKL0rufKq8h5mOG -nXkZ7/e7DDWQw4rtTw/1zBLZpD67oPwglV9PJi8RI4NOdQcPv5vRtB3pEAT+ymCP -oky4rc/hkA/NrgrHXXu3UNLUYfrVFdvXn4dRVOul4+vJhaAlIDf7js4MNIThPIGy -d05DpYhfhmehPea0XGG2Ptv+tyjFogeutcrKjSoS75ftwjCkySp6+/NNIxuZMzSg -LvWpCz/UXeHPhJ/iGcJfitYgHuNztw== ------END CERTIFICATE----- - -# Issuer: CN=Certum Trusted Network CA 2 O=Unizeto Technologies S.A. OU=Certum Certification Authority -# Subject: CN=Certum Trusted Network CA 2 O=Unizeto Technologies S.A. OU=Certum Certification Authority -# Label: "Certum Trusted Network CA 2" -# Serial: 44979900017204383099463764357512596969 -# MD5 Fingerprint: 6d:46:9e:d9:25:6d:08:23:5b:5e:74:7d:1e:27:db:f2 -# SHA1 Fingerprint: d3:dd:48:3e:2b:bf:4c:05:e8:af:10:f5:fa:76:26:cf:d3:dc:30:92 -# SHA256 Fingerprint: b6:76:f2:ed:da:e8:77:5c:d3:6c:b0:f6:3c:d1:d4:60:39:61:f4:9e:62:65:ba:01:3a:2f:03:07:b6:d0:b8:04 ------BEGIN CERTIFICATE----- -MIIF0jCCA7qgAwIBAgIQIdbQSk8lD8kyN/yqXhKN6TANBgkqhkiG9w0BAQ0FADCB -gDELMAkGA1UEBhMCUEwxIjAgBgNVBAoTGVVuaXpldG8gVGVjaG5vbG9naWVzIFMu -QS4xJzAlBgNVBAsTHkNlcnR1bSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTEkMCIG -A1UEAxMbQ2VydHVtIFRydXN0ZWQgTmV0d29yayBDQSAyMCIYDzIwMTExMDA2MDgz -OTU2WhgPMjA0NjEwMDYwODM5NTZaMIGAMQswCQYDVQQGEwJQTDEiMCAGA1UEChMZ -VW5pemV0byBUZWNobm9sb2dpZXMgUy5BLjEnMCUGA1UECxMeQ2VydHVtIENlcnRp -ZmljYXRpb24gQXV0aG9yaXR5MSQwIgYDVQQDExtDZXJ0dW0gVHJ1c3RlZCBOZXR3 -b3JrIENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9+Xj45tWA -DGSdhhuWZGc/IjoedQF97/tcZ4zJzFxrqZHmuULlIEub2pt7uZld2ZuAS9eEQCsn -0+i6MLs+CRqnSZXvK0AkwpfHp+6bJe+oCgCXhVqqndwpyeI1B+twTUrWwbNWuKFB -OJvR+zF/j+Bf4bE/D44WSWDXBo0Y+aomEKsq09DRZ40bRr5HMNUuctHFY9rnY3lE -fktjJImGLjQ/KUxSiyqnwOKRKIm5wFv5HdnnJ63/mgKXwcZQkpsCLL2puTRZCr+E -Sv/f/rOf69me4Jgj7KZrdxYq28ytOxykh9xGc14ZYmhFV+SQgkK7QtbwYeDBoz1m -o130GO6IyY0XRSmZMnUCMe4pJshrAua1YkV/NxVaI2iJ1D7eTiew8EAMvE0Xy02i -sx7QBlrd9pPPV3WZ9fqGGmd4s7+W/jTcvedSVuWz5XV710GRBdxdaeOVDUO5/IOW -OZV7bIBaTxNyxtd9KXpEulKkKtVBRgkg/iKgtlswjbyJDNXXcPiHUv3a76xRLgez -Tv7QCdpw75j6VuZt27VXS9zlLCUVyJ4ueE742pyehizKV/Ma5ciSixqClnrDvFAS -adgOWkaLOusm+iPJtrCBvkIApPjW/jAux9JG9uWOdf3yzLnQh1vMBhBgu4M1t15n -3kfsmUjxpKEV/q2MYo45VU85FrmxY53/twIDAQABo0IwQDAPBgNVHRMBAf8EBTAD -AQH/MB0GA1UdDgQWBBS2oVQ5AsOgP46KvPrU+Bym0ToO/TAOBgNVHQ8BAf8EBAMC -AQYwDQYJKoZIhvcNAQENBQADggIBAHGlDs7k6b8/ONWJWsQCYftMxRQXLYtPU2sQ -F/xlhMcQSZDe28cmk4gmb3DWAl45oPePq5a1pRNcgRRtDoGCERuKTsZPpd1iHkTf -CVn0W3cLN+mLIMb4Ck4uWBzrM9DPhmDJ2vuAL55MYIR4PSFk1vtBHxgP58l1cb29 -XN40hz5BsA72udY/CROWFC/emh1auVbONTqwX3BNXuMp8SMoclm2q8KMZiYcdywm -djWLKKdpoPk79SPdhRB0yZADVpHnr7pH1BKXESLjokmUbOe3lEu6LaTaM4tMpkT/ -WjzGHWTYtTHkpjx6qFcL2+1hGsvxznN3Y6SHb0xRONbkX8eftoEq5IVIeVheO/jb -AoJnwTnbw3RLPTYe+SmTiGhbqEQZIfCn6IENLOiTNrQ3ssqwGyZ6miUfmpqAnksq -P/ujmv5zMnHCnsZy4YpoJ/HkD7TETKVhk/iXEAcqMCWpuchxuO9ozC1+9eB+D4Ko -b7a6bINDd82Kkhehnlt4Fj1F4jNy3eFmypnTycUm/Q1oBEauttmbjL4ZvrHG8hnj -XALKLNhvSgfZyTXaQHXyxKcZb55CEJh15pWLYLztxRLXis7VmFxWlgPF7ncGNf/P -5O4/E2Hu29othfDNrp2yGAlFw5Khchf8R7agCyzxxN5DaAhqXzvwdmP7zAYspsbi -DrW5viSP ------END CERTIFICATE----- - -# Issuer: CN=Hellenic Academic and Research Institutions RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority -# Subject: CN=Hellenic Academic and Research Institutions RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority -# Label: "Hellenic Academic and Research Institutions RootCA 2015" -# Serial: 0 -# MD5 Fingerprint: ca:ff:e2:db:03:d9:cb:4b:e9:0f:ad:84:fd:7b:18:ce -# SHA1 Fingerprint: 01:0c:06:95:a6:98:19:14:ff:bf:5f:c6:b0:b6:95:ea:29:e9:12:a6 -# SHA256 Fingerprint: a0:40:92:9a:02:ce:53:b4:ac:f4:f2:ff:c6:98:1c:e4:49:6f:75:5e:6d:45:fe:0b:2a:69:2b:cd:52:52:3f:36 ------BEGIN CERTIFICATE----- -MIIGCzCCA/OgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBpjELMAkGA1UEBhMCR1Ix -DzANBgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5k -IFJlc2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkxQDA+BgNVBAMT -N0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgUm9v -dENBIDIwMTUwHhcNMTUwNzA3MTAxMTIxWhcNNDAwNjMwMTAxMTIxWjCBpjELMAkG -A1UEBhMCR1IxDzANBgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNh -ZGVtaWMgYW5kIFJlc2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkx -QDA+BgNVBAMTN0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1 -dGlvbnMgUm9vdENBIDIwMTUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC -AQDC+Kk/G4n8PDwEXT2QNrCROnk8ZlrvbTkBSRq0t89/TSNTt5AA4xMqKKYx8ZEA -4yjsriFBzh/a/X0SWwGDD7mwX5nh8hKDgE0GPt+sr+ehiGsxr/CL0BgzuNtFajT0 -AoAkKAoCFZVedioNmToUW/bLy1O8E00BiDeUJRtCvCLYjqOWXjrZMts+6PAQZe10 -4S+nfK8nNLspfZu2zwnI5dMK/IhlZXQK3HMcXM1AsRzUtoSMTFDPaI6oWa7CJ06C -ojXdFPQf/7J31Ycvqm59JCfnxssm5uX+Zwdj2EUN3TpZZTlYepKZcj2chF6IIbjV -9Cz82XBST3i4vTwri5WY9bPRaM8gFH5MXF/ni+X1NYEZN9cRCLdmvtNKzoNXADrD -gfgXy5I2XdGj2HUb4Ysn6npIQf1FGQatJ5lOwXBH3bWfgVMS5bGMSF0xQxfjjMZ6 -Y5ZLKTBOhE5iGV48zpeQpX8B653g+IuJ3SWYPZK2fu/Z8VFRfS0myGlZYeCsargq -NhEEelC9MoS+L9xy1dcdFkfkR2YgP/SWxa+OAXqlD3pk9Q0Yh9muiNX6hME6wGko -LfINaFGq46V3xqSQDqE3izEjR8EJCOtu93ib14L8hCCZSRm2Ekax+0VVFqmjZayc -Bw/qa9wfLgZy7IaIEuQt218FL+TwA9MmM+eAws1CoRc0CwIDAQABo0IwQDAPBgNV -HRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUcRVnyMjJvXVd -ctA4GGqd83EkVAswDQYJKoZIhvcNAQELBQADggIBAHW7bVRLqhBYRjTyYtcWNl0I -XtVsyIe9tC5G8jH4fOpCtZMWVdyhDBKg2mF+D1hYc2Ryx+hFjtyp8iY/xnmMsVMI -M4GwVhO+5lFc2JsKT0ucVlMC6U/2DWDqTUJV6HwbISHTGzrMd/K4kPFox/la/vot -9L/J9UUbzjgQKjeKeaO04wlshYaT/4mWJ3iBj2fjRnRUjtkNaeJK9E10A/+yd+2V -Z5fkscWrv2oj6NSU4kQoYsRL4vDY4ilrGnB+JGGTe08DMiUNRSQrlrRGar9KC/ea -j8GsGsVn82800vpzY4zvFrCopEYq+OsS7HK07/grfoxSwIuEVPkvPuNVqNxmsdnh -X9izjFk0WaSrT2y7HxjbdavYy5LNlDhhDgcGH0tGEPEVvo2FXDtKK4F5D7Rpn0lQ -l033DlZdwJVqwjbDG2jJ9SrcR5q+ss7FJej6A7na+RZukYT1HCjI/CbM1xyQVqdf -bzoEvM14iQuODy+jqk+iGxI9FghAD/FGTNeqewjBCvVtJ94Cj8rDtSvK6evIIVM4 -pcw72Hc3MKJP2W/R8kCtQXoXxdZKNYm3QdV8hn9VTYNKpXMgwDqvkPGaJI7ZjnHK -e7iG2rKPmT4dEw0SEe7Uq/DpFXYC5ODfqiAeW2GFZECpkJcNrVPSWh2HagCXZWK0 -vm9qp/UsQu0yrbYhnr68 ------END CERTIFICATE----- - -# Issuer: CN=Hellenic Academic and Research Institutions ECC RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority -# Subject: CN=Hellenic Academic and Research Institutions ECC RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority -# Label: "Hellenic Academic and Research Institutions ECC RootCA 2015" -# Serial: 0 -# MD5 Fingerprint: 81:e5:b4:17:eb:c2:f5:e1:4b:0d:41:7b:49:92:fe:ef -# SHA1 Fingerprint: 9f:f1:71:8d:92:d5:9a:f3:7d:74:97:b4:bc:6f:84:68:0b:ba:b6:66 -# SHA256 Fingerprint: 44:b5:45:aa:8a:25:e6:5a:73:ca:15:dc:27:fc:36:d2:4c:1c:b9:95:3a:06:65:39:b1:15:82:dc:48:7b:48:33 ------BEGIN CERTIFICATE----- -MIICwzCCAkqgAwIBAgIBADAKBggqhkjOPQQDAjCBqjELMAkGA1UEBhMCR1IxDzAN -BgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJl -c2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkxRDBCBgNVBAMTO0hl -bGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgRUNDIFJv -b3RDQSAyMDE1MB4XDTE1MDcwNzEwMzcxMloXDTQwMDYzMDEwMzcxMlowgaoxCzAJ -BgNVBAYTAkdSMQ8wDQYDVQQHEwZBdGhlbnMxRDBCBgNVBAoTO0hlbGxlbmljIEFj -YWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgQ2VydC4gQXV0aG9yaXR5 -MUQwQgYDVQQDEztIZWxsZW5pYyBBY2FkZW1pYyBhbmQgUmVzZWFyY2ggSW5zdGl0 -dXRpb25zIEVDQyBSb290Q0EgMjAxNTB2MBAGByqGSM49AgEGBSuBBAAiA2IABJKg -QehLgoRc4vgxEZmGZE4JJS+dQS8KrjVPdJWyUWRrjWvmP3CV8AVER6ZyOFB2lQJa -jq4onvktTpnvLEhvTCUp6NFxW98dwXU3tNf6e3pCnGoKVlp8aQuqgAkkbH7BRqNC -MEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFLQi -C4KZJAEOnLvkDv2/+5cgk5kqMAoGCCqGSM49BAMCA2cAMGQCMGfOFmI4oqxiRaep -lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof -TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR ------END CERTIFICATE----- - -# Issuer: CN=ISRG Root X1 O=Internet Security Research Group -# Subject: CN=ISRG Root X1 O=Internet Security Research Group -# Label: "ISRG Root X1" -# Serial: 172886928669790476064670243504169061120 -# MD5 Fingerprint: 0c:d2:f9:e0:da:17:73:e9:ed:86:4d:a5:e3:70:e7:4e -# SHA1 Fingerprint: ca:bd:2a:79:a1:07:6a:31:f2:1d:25:36:35:cb:03:9d:43:29:a5:e8 -# SHA256 Fingerprint: 96:bc:ec:06:26:49:76:f3:74:60:77:9a:cf:28:c5:a7:cf:e8:a3:c0:aa:e1:1a:8f:fc:ee:05:c0:bd:df:08:c6 ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw -TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh -cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 -WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu -ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY -MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc -h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ -0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U -A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW -T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH -B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC -B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv -KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn -OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn -jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw -qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI -rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV -HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq -hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL -ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ -3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK -NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 -ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur -TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC -jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc -oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq -4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA -mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d -emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= ------END CERTIFICATE----- - -# Issuer: O=FNMT-RCM OU=AC RAIZ FNMT-RCM -# Subject: O=FNMT-RCM OU=AC RAIZ FNMT-RCM -# Label: "AC RAIZ FNMT-RCM" -# Serial: 485876308206448804701554682760554759 -# MD5 Fingerprint: e2:09:04:b4:d3:bd:d1:a0:14:fd:1a:d2:47:c4:57:1d -# SHA1 Fingerprint: ec:50:35:07:b2:15:c4:95:62:19:e2:a8:9a:5b:42:99:2c:4c:2c:20 -# SHA256 Fingerprint: eb:c5:57:0c:29:01:8c:4d:67:b1:aa:12:7b:af:12:f7:03:b4:61:1e:bc:17:b7:da:b5:57:38:94:17:9b:93:fa ------BEGIN CERTIFICATE----- -MIIFgzCCA2ugAwIBAgIPXZONMGc2yAYdGsdUhGkHMA0GCSqGSIb3DQEBCwUAMDsx -CzAJBgNVBAYTAkVTMREwDwYDVQQKDAhGTk1ULVJDTTEZMBcGA1UECwwQQUMgUkFJ -WiBGTk1ULVJDTTAeFw0wODEwMjkxNTU5NTZaFw0zMDAxMDEwMDAwMDBaMDsxCzAJ -BgNVBAYTAkVTMREwDwYDVQQKDAhGTk1ULVJDTTEZMBcGA1UECwwQQUMgUkFJWiBG -Tk1ULVJDTTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALpxgHpMhm5/ -yBNtwMZ9HACXjywMI7sQmkCpGreHiPibVmr75nuOi5KOpyVdWRHbNi63URcfqQgf -BBckWKo3Shjf5TnUV/3XwSyRAZHiItQDwFj8d0fsjz50Q7qsNI1NOHZnjrDIbzAz -WHFctPVrbtQBULgTfmxKo0nRIBnuvMApGGWn3v7v3QqQIecaZ5JCEJhfTzC8PhxF -tBDXaEAUwED653cXeuYLj2VbPNmaUtu1vZ5Gzz3rkQUCwJaydkxNEJY7kvqcfw+Z -374jNUUeAlz+taibmSXaXvMiwzn15Cou08YfxGyqxRxqAQVKL9LFwag0Jl1mpdIC -IfkYtwb1TplvqKtMUejPUBjFd8g5CSxJkjKZqLsXF3mwWsXmo8RZZUc1g16p6DUL -mbvkzSDGm0oGObVo/CK67lWMK07q87Hj/LaZmtVC+nFNCM+HHmpxffnTtOmlcYF7 -wk5HlqX2doWjKI/pgG6BU6VtX7hI+cL5NqYuSf+4lsKMB7ObiFj86xsc3i1w4peS -MKGJ47xVqCfWS+2QrYv6YyVZLag13cqXM7zlzced0ezvXg5KkAYmY6252TUtB7p2 -ZSysV4999AeU14ECll2jB0nVetBX+RvnU0Z1qrB5QstocQjpYL05ac70r8NWQMet -UqIJ5G+GR4of6ygnXYMgrwTJbFaai0b1AgMBAAGjgYMwgYAwDwYDVR0TAQH/BAUw -AwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFPd9xf3E6Jobd2Sn9R2gzL+H -YJptMD4GA1UdIAQ3MDUwMwYEVR0gADArMCkGCCsGAQUFBwIBFh1odHRwOi8vd3d3 -LmNlcnQuZm5tdC5lcy9kcGNzLzANBgkqhkiG9w0BAQsFAAOCAgEAB5BK3/MjTvDD -nFFlm5wioooMhfNzKWtN/gHiqQxjAb8EZ6WdmF/9ARP67Jpi6Yb+tmLSbkyU+8B1 -RXxlDPiyN8+sD8+Nb/kZ94/sHvJwnvDKuO+3/3Y3dlv2bojzr2IyIpMNOmqOFGYM -LVN0V2Ue1bLdI4E7pWYjJ2cJj+F3qkPNZVEI7VFY/uY5+ctHhKQV8Xa7pO6kO8Rf -77IzlhEYt8llvhjho6Tc+hj507wTmzl6NLrTQfv6MooqtyuGC2mDOL7Nii4LcK2N -JpLuHvUBKwrZ1pebbuCoGRw6IYsMHkCtA+fdZn71uSANA+iW+YJF1DngoABd15jm -fZ5nc8OaKveri6E6FO80vFIOiZiaBECEHX5FaZNXzuvO+FB8TxxuBEOb+dY7Ixjp -6o7RTUaN8Tvkasq6+yO3m/qZASlaWFot4/nUbQ4mrcFuNLwy+AwF+mWj2zs3gyLp -1txyM/1d8iC9djwj2ij3+RvrWWTV3F9yfiD8zYm1kGdNYno/Tq0dwzn+evQoFt9B -9kiABdcPUXmsEKvU7ANm5mqwujGSQkBqvjrTcuFqN1W8rB2Vt2lh8kORdOag0wok -RqEIr9baRRmW1FMdW4R58MD3R++Lj8UGrp1MYp3/RgT408m2ECVAdf4WqslKYIYv -uu8wd+RU4riEmViAqhOLUTpPSPaLtrM= ------END CERTIFICATE----- - -# Issuer: CN=Amazon Root CA 1 O=Amazon -# Subject: CN=Amazon Root CA 1 O=Amazon -# Label: "Amazon Root CA 1" -# Serial: 143266978916655856878034712317230054538369994 -# MD5 Fingerprint: 43:c6:bf:ae:ec:fe:ad:2f:18:c6:88:68:30:fc:c8:e6 -# SHA1 Fingerprint: 8d:a7:f9:65:ec:5e:fc:37:91:0f:1c:6e:59:fd:c1:cc:6a:6e:de:16 -# SHA256 Fingerprint: 8e:cd:e6:88:4f:3d:87:b1:12:5b:a3:1a:c3:fc:b1:3d:70:16:de:7f:57:cc:90:4f:e1:cb:97:c6:ae:98:19:6e ------BEGIN CERTIFICATE----- -MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF -ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 -b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL -MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv -b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj -ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM -9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw -IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 -VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L -93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm -jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC -AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA -A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI -U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs -N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv -o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU -5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy -rqXRfboQnoZsG4q5WTP468SQvvG5 ------END CERTIFICATE----- - -# Issuer: CN=Amazon Root CA 2 O=Amazon -# Subject: CN=Amazon Root CA 2 O=Amazon -# Label: "Amazon Root CA 2" -# Serial: 143266982885963551818349160658925006970653239 -# MD5 Fingerprint: c8:e5:8d:ce:a8:42:e2:7a:c0:2a:5c:7c:9e:26:bf:66 -# SHA1 Fingerprint: 5a:8c:ef:45:d7:a6:98:59:76:7a:8c:8b:44:96:b5:78:cf:47:4b:1a -# SHA256 Fingerprint: 1b:a5:b2:aa:8c:65:40:1a:82:96:01:18:f8:0b:ec:4f:62:30:4d:83:ce:c4:71:3a:19:c3:9c:01:1e:a4:6d:b4 ------BEGIN CERTIFICATE----- -MIIFQTCCAymgAwIBAgITBmyf0pY1hp8KD+WGePhbJruKNzANBgkqhkiG9w0BAQwF -ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 -b24gUm9vdCBDQSAyMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTEL -MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv -b3QgQ0EgMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK2Wny2cSkxK -gXlRmeyKy2tgURO8TW0G/LAIjd0ZEGrHJgw12MBvIITplLGbhQPDW9tK6Mj4kHbZ -W0/jTOgGNk3Mmqw9DJArktQGGWCsN0R5hYGCrVo34A3MnaZMUnbqQ523BNFQ9lXg -1dKmSYXpN+nKfq5clU1Imj+uIFptiJXZNLhSGkOQsL9sBbm2eLfq0OQ6PBJTYv9K -8nu+NQWpEjTj82R0Yiw9AElaKP4yRLuH3WUnAnE72kr3H9rN9yFVkE8P7K6C4Z9r -2UXTu/Bfh+08LDmG2j/e7HJV63mjrdvdfLC6HM783k81ds8P+HgfajZRRidhW+me -z/CiVX18JYpvL7TFz4QuK/0NURBs+18bvBt+xa47mAExkv8LV/SasrlX6avvDXbR -8O70zoan4G7ptGmh32n2M8ZpLpcTnqWHsFcQgTfJU7O7f/aS0ZzQGPSSbtqDT6Zj -mUyl+17vIWR6IF9sZIUVyzfpYgwLKhbcAS4y2j5L9Z469hdAlO+ekQiG+r5jqFoz -7Mt0Q5X5bGlSNscpb/xVA1wf+5+9R+vnSUeVC06JIglJ4PVhHvG/LopyboBZ/1c6 -+XUyo05f7O0oYtlNc/LMgRdg7c3r3NunysV+Ar3yVAhU/bQtCSwXVEqY0VThUWcI -0u1ufm8/0i2BWSlmy5A5lREedCf+3euvAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMB -Af8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSwDPBMMPQFWAJI/TPlUq9LhONm -UjANBgkqhkiG9w0BAQwFAAOCAgEAqqiAjw54o+Ci1M3m9Zh6O+oAA7CXDpO8Wqj2 -LIxyh6mx/H9z/WNxeKWHWc8w4Q0QshNabYL1auaAn6AFC2jkR2vHat+2/XcycuUY -+gn0oJMsXdKMdYV2ZZAMA3m3MSNjrXiDCYZohMr/+c8mmpJ5581LxedhpxfL86kS -k5Nrp+gvU5LEYFiwzAJRGFuFjWJZY7attN6a+yb3ACfAXVU3dJnJUH/jWS5E4ywl -7uxMMne0nxrpS10gxdr9HIcWxkPo1LsmmkVwXqkLN1PiRnsn/eBG8om3zEK2yygm -btmlyTrIQRNg91CMFa6ybRoVGld45pIq2WWQgj9sAq+uEjonljYE1x2igGOpm/Hl -urR8FLBOybEfdF849lHqm/osohHUqS0nGkWxr7JOcQ3AWEbWaQbLU8uz/mtBzUF+ -fUwPfHJ5elnNXkoOrJupmHN5fLT0zLm4BwyydFy4x2+IoZCn9Kr5v2c69BoVYh63 -n749sSmvZ6ES8lgQGVMDMBu4Gon2nL2XA46jCfMdiyHxtN/kHNGfZQIG6lzWE7OE -76KlXIx3KadowGuuQNKotOrN8I1LOJwZmhsoVLiJkO/KdYE+HvJkJMcYr07/R54H -9jVlpNMKVv/1F2Rs76giJUmTtt8AF9pYfl3uxRuw0dFfIRDH+fO6AgonB8Xx1sfT -4PsJYGw= ------END CERTIFICATE----- - -# Issuer: CN=Amazon Root CA 3 O=Amazon -# Subject: CN=Amazon Root CA 3 O=Amazon -# Label: "Amazon Root CA 3" -# Serial: 143266986699090766294700635381230934788665930 -# MD5 Fingerprint: a0:d4:ef:0b:f7:b5:d8:49:95:2a:ec:f5:c4:fc:81:87 -# SHA1 Fingerprint: 0d:44:dd:8c:3c:8c:1a:1a:58:75:64:81:e9:0f:2e:2a:ff:b3:d2:6e -# SHA256 Fingerprint: 18:ce:6c:fe:7b:f1:4e:60:b2:e3:47:b8:df:e8:68:cb:31:d0:2e:bb:3a:da:27:15:69:f5:03:43:b4:6d:b3:a4 ------BEGIN CERTIFICATE----- -MIIBtjCCAVugAwIBAgITBmyf1XSXNmY/Owua2eiedgPySjAKBggqhkjOPQQDAjA5 -MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g -Um9vdCBDQSAzMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG -A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg -Q0EgMzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCmXp8ZBf8ANm+gBG1bG8lKl -ui2yEujSLtf6ycXYqm0fc4E7O5hrOXwzpcVOho6AF2hiRVd9RFgdszflZwjrZt6j -QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSr -ttvXBp43rDCGB5Fwx5zEGbF4wDAKBggqhkjOPQQDAgNJADBGAiEA4IWSoxe3jfkr -BqWTrBqYaGFy+uGh0PsceGCmQ5nFuMQCIQCcAu/xlJyzlvnrxir4tiz+OpAUFteM -YyRIHN8wfdVoOw== ------END CERTIFICATE----- - -# Issuer: CN=Amazon Root CA 4 O=Amazon -# Subject: CN=Amazon Root CA 4 O=Amazon -# Label: "Amazon Root CA 4" -# Serial: 143266989758080763974105200630763877849284878 -# MD5 Fingerprint: 89:bc:27:d5:eb:17:8d:06:6a:69:d5:fd:89:47:b4:cd -# SHA1 Fingerprint: f6:10:84:07:d6:f8:bb:67:98:0c:c2:e2:44:c2:eb:ae:1c:ef:63:be -# SHA256 Fingerprint: e3:5d:28:41:9e:d0:20:25:cf:a6:90:38:cd:62:39:62:45:8d:a5:c6:95:fb:de:a3:c2:2b:0b:fb:25:89:70:92 ------BEGIN CERTIFICATE----- -MIIB8jCCAXigAwIBAgITBmyf18G7EEwpQ+Vxe3ssyBrBDjAKBggqhkjOPQQDAzA5 -MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g -Um9vdCBDQSA0MB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG -A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg -Q0EgNDB2MBAGByqGSM49AgEGBSuBBAAiA2IABNKrijdPo1MN/sGKe0uoe0ZLY7Bi -9i0b2whxIdIA6GO9mif78DluXeo9pcmBqqNbIJhFXRbb/egQbeOc4OO9X4Ri83Bk -M6DLJC9wuoihKqB1+IGuYgbEgds5bimwHvouXKNCMEAwDwYDVR0TAQH/BAUwAwEB -/zAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFNPsxzplbszh2naaVvuc84ZtV+WB -MAoGCCqGSM49BAMDA2gAMGUCMDqLIfG9fhGt0O9Yli/W651+kI0rz2ZVwyzjKKlw -CkcO8DdZEv8tmZQoTipPNU0zWgIxAOp1AE47xDqUEpHJWEadIRNyp4iciuRMStuW -1KyLa2tJElMzrdfkviT8tQp21KW8EA== ------END CERTIFICATE----- - -# Issuer: CN=TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1 O=Turkiye Bilimsel ve Teknolojik Arastirma Kurumu - TUBITAK OU=Kamu Sertifikasyon Merkezi - Kamu SM -# Subject: CN=TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1 O=Turkiye Bilimsel ve Teknolojik Arastirma Kurumu - TUBITAK OU=Kamu Sertifikasyon Merkezi - Kamu SM -# Label: "TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1" -# Serial: 1 -# MD5 Fingerprint: dc:00:81:dc:69:2f:3e:2f:b0:3b:f6:3d:5a:91:8e:49 -# SHA1 Fingerprint: 31:43:64:9b:ec:ce:27:ec:ed:3a:3f:0b:8f:0d:e4:e8:91:dd:ee:ca -# SHA256 Fingerprint: 46:ed:c3:68:90:46:d5:3a:45:3f:b3:10:4a:b8:0d:ca:ec:65:8b:26:60:ea:16:29:dd:7e:86:79:90:64:87:16 ------BEGIN CERTIFICATE----- -MIIEYzCCA0ugAwIBAgIBATANBgkqhkiG9w0BAQsFADCB0jELMAkGA1UEBhMCVFIx -GDAWBgNVBAcTD0dlYnplIC0gS29jYWVsaTFCMEAGA1UEChM5VHVya2l5ZSBCaWxp -bXNlbCB2ZSBUZWtub2xvamlrIEFyYXN0aXJtYSBLdXJ1bXUgLSBUVUJJVEFLMS0w -KwYDVQQLEyRLYW11IFNlcnRpZmlrYXN5b24gTWVya2V6aSAtIEthbXUgU00xNjA0 -BgNVBAMTLVRVQklUQUsgS2FtdSBTTSBTU0wgS29rIFNlcnRpZmlrYXNpIC0gU3Vy -dW0gMTAeFw0xMzExMjUwODI1NTVaFw00MzEwMjUwODI1NTVaMIHSMQswCQYDVQQG -EwJUUjEYMBYGA1UEBxMPR2ViemUgLSBLb2NhZWxpMUIwQAYDVQQKEzlUdXJraXll -IEJpbGltc2VsIHZlIFRla25vbG9qaWsgQXJhc3Rpcm1hIEt1cnVtdSAtIFRVQklU -QUsxLTArBgNVBAsTJEthbXUgU2VydGlmaWthc3lvbiBNZXJrZXppIC0gS2FtdSBT -TTE2MDQGA1UEAxMtVFVCSVRBSyBLYW11IFNNIFNTTCBLb2sgU2VydGlmaWthc2kg -LSBTdXJ1bSAxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr3UwM6q7 -a9OZLBI3hNmNe5eA027n/5tQlT6QlVZC1xl8JoSNkvoBHToP4mQ4t4y86Ij5iySr -LqP1N+RAjhgleYN1Hzv/bKjFxlb4tO2KRKOrbEz8HdDc72i9z+SqzvBV96I01INr -N3wcwv61A+xXzry0tcXtAA9TNypN9E8Mg/uGz8v+jE69h/mniyFXnHrfA2eJLJ2X -YacQuFWQfw4tJzh03+f92k4S400VIgLI4OD8D62K18lUUMw7D8oWgITQUVbDjlZ/ -iSIzL+aFCr2lqBs23tPcLG07xxO9WSMs5uWk99gL7eqQQESolbuT1dCANLZGeA4f -AJNG4e7p+exPFwIDAQABo0IwQDAdBgNVHQ4EFgQUZT/HiobGPN08VFw1+DrtUgxH -V8gwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL -BQADggEBACo/4fEyjq7hmFxLXs9rHmoJ0iKpEsdeV31zVmSAhHqT5Am5EM2fKifh -AHe+SMg1qIGf5LgsyX8OsNJLN13qudULXjS99HMpw+0mFZx+CFOKWI3QSyjfwbPf -IPP54+M638yclNhOT8NrF7f3cuitZjO1JVOr4PhMqZ398g26rrnZqsZr+ZO7rqu4 -lzwDGrpDxpa5RXI4s6ehlj2Re37AIVNMh+3yC1SVUZPVIqUNivGTDj5UDrDYyU7c -8jEyVupk+eq1nRZmQnLzf9OxMUP8pI4X8W0jq5Rm+K37DwhuJi1/FwcJsoz7UMCf -lo3Ptv0AnVoUmr8CRPXBwp8iXqIPoeM= ------END CERTIFICATE----- - -# Issuer: CN=GDCA TrustAUTH R5 ROOT O=GUANG DONG CERTIFICATE AUTHORITY CO.,LTD. -# Subject: CN=GDCA TrustAUTH R5 ROOT O=GUANG DONG CERTIFICATE AUTHORITY CO.,LTD. -# Label: "GDCA TrustAUTH R5 ROOT" -# Serial: 9009899650740120186 -# MD5 Fingerprint: 63:cc:d9:3d:34:35:5c:6f:53:a3:e2:08:70:48:1f:b4 -# SHA1 Fingerprint: 0f:36:38:5b:81:1a:25:c3:9b:31:4e:83:ca:e9:34:66:70:cc:74:b4 -# SHA256 Fingerprint: bf:ff:8f:d0:44:33:48:7d:6a:8a:a6:0c:1a:29:76:7a:9f:c2:bb:b0:5e:42:0f:71:3a:13:b9:92:89:1d:38:93 ------BEGIN CERTIFICATE----- -MIIFiDCCA3CgAwIBAgIIfQmX/vBH6nowDQYJKoZIhvcNAQELBQAwYjELMAkGA1UE -BhMCQ04xMjAwBgNVBAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZ -IENPLixMVEQuMR8wHQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMB4XDTE0 -MTEyNjA1MTMxNVoXDTQwMTIzMTE1NTk1OVowYjELMAkGA1UEBhMCQ04xMjAwBgNV -BAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZIENPLixMVEQuMR8w -HQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMIICIjANBgkqhkiG9w0BAQEF -AAOCAg8AMIICCgKCAgEA2aMW8Mh0dHeb7zMNOwZ+Vfy1YI92hhJCfVZmPoiC7XJj -Dp6L3TQsAlFRwxn9WVSEyfFrs0yw6ehGXTjGoqcuEVe6ghWinI9tsJlKCvLriXBj -TnnEt1u9ol2x8kECK62pOqPseQrsXzrj/e+APK00mxqriCZ7VqKChh/rNYmDf1+u -KU49tm7srsHwJ5uu4/Ts765/94Y9cnrrpftZTqfrlYwiOXnhLQiPzLyRuEH3FMEj -qcOtmkVEs7LXLM3GKeJQEK5cy4KOFxg2fZfmiJqwTTQJ9Cy5WmYqsBebnh52nUpm -MUHfP/vFBu8btn4aRjb3ZGM74zkYI+dndRTVdVeSN72+ahsmUPI2JgaQxXABZG12 -ZuGR224HwGGALrIuL4xwp9E7PLOR5G62xDtw8mySlwnNR30YwPO7ng/Wi64HtloP -zgsMR6flPri9fcebNaBhlzpBdRfMK5Z3KpIhHtmVdiBnaM8Nvd/WHwlqmuLMc3Gk -L30SgLdTMEZeS1SZD2fJpcjyIMGC7J0R38IC+xo70e0gmu9lZJIQDSri3nDxGGeC -jGHeuLzRL5z7D9Ar7Rt2ueQ5Vfj4oR24qoAATILnsn8JuLwwoC8N9VKejveSswoA -HQBUlwbgsQfZxw9cZX08bVlX5O2ljelAU58VS6Bx9hoh49pwBiFYFIeFd3mqgnkC -AwEAAaNCMEAwHQYDVR0OBBYEFOLJQJ9NzuiaoXzPDj9lxSmIahlRMA8GA1UdEwEB -/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQDRSVfg -p8xoWLoBDysZzY2wYUWsEe1jUGn4H3++Fo/9nesLqjJHdtJnJO29fDMylyrHBYZm -DRd9FBUb1Ov9H5r2XpdptxolpAqzkT9fNqyL7FeoPueBihhXOYV0GkLH6VsTX4/5 -COmSdI31R9KrO9b7eGZONn356ZLpBN79SWP8bfsUcZNnL0dKt7n/HipzcEYwv1ry -L3ml4Y0M2fmyYzeMN2WFcGpcWwlyua1jPLHd+PwyvzeG5LuOmCd+uh8W4XAR8gPf -JWIyJyYYMoSf/wA6E7qaTfRPuBRwIrHKK5DOKcFw9C+df/KQHtZa37dG/OaG+svg -IHZ6uqbL9XzeYqWxi+7egmaKTjowHz+Ay60nugxe19CxVsp3cbK1daFQqUBDF8Io -2c9Si1vIY9RCPqAzekYu9wogRlR+ak8x8YF+QnQ4ZXMn7sZ8uI7XpTrXmKGcjBBV -09tL7ECQ8s1uV9JiDnxXk7Gnbc2dg7sq5+W2O3FYrf3RRbxake5TFW/TRQl1brqQ -XR4EzzffHqhmsYzmIGrv/EhOdJhCrylvLmrH+33RZjEizIYAfmaDDEL0vTSSwxrq -T8p+ck0LcIymSLumoRT2+1hEmRSuqguTaaApJUqlyyvdimYHFngVV3Eb7PVHhPOe -MTd61X8kreS8/f3MboPoDKi3QWwH3b08hpcv0g== ------END CERTIFICATE----- - -# Issuer: CN=SSL.com Root Certification Authority RSA O=SSL Corporation -# Subject: CN=SSL.com Root Certification Authority RSA O=SSL Corporation -# Label: "SSL.com Root Certification Authority RSA" -# Serial: 8875640296558310041 -# MD5 Fingerprint: 86:69:12:c0:70:f1:ec:ac:ac:c2:d5:bc:a5:5b:a1:29 -# SHA1 Fingerprint: b7:ab:33:08:d1:ea:44:77:ba:14:80:12:5a:6f:bd:a9:36:49:0c:bb -# SHA256 Fingerprint: 85:66:6a:56:2e:e0:be:5c:e9:25:c1:d8:89:0a:6f:76:a8:7e:c1:6d:4d:7d:5f:29:ea:74:19:cf:20:12:3b:69 ------BEGIN CERTIFICATE----- -MIIF3TCCA8WgAwIBAgIIeyyb0xaAMpkwDQYJKoZIhvcNAQELBQAwfDELMAkGA1UE -BhMCVVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQK -DA9TU0wgQ29ycG9yYXRpb24xMTAvBgNVBAMMKFNTTC5jb20gUm9vdCBDZXJ0aWZp -Y2F0aW9uIEF1dGhvcml0eSBSU0EwHhcNMTYwMjEyMTczOTM5WhcNNDEwMjEyMTcz -OTM5WjB8MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hv -dXN0b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjExMC8GA1UEAwwoU1NMLmNv -bSBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IFJTQTCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBAPkP3aMrfcvQKv7sZ4Wm5y4bunfh4/WvpOz6Sl2R -xFdHaxh3a3by/ZPkPQ/CFp4LZsNWlJ4Xg4XOVu/yFv0AYvUiCVToZRdOQbngT0aX -qhvIuG5iXmmxX9sqAn78bMrzQdjt0Oj8P2FI7bADFB0QDksZ4LtO7IZl/zbzXmcC -C52GVWH9ejjt/uIZALdvoVBidXQ8oPrIJZK0bnoix/geoeOy3ZExqysdBP+lSgQ3 -6YWkMyv94tZVNHwZpEpox7Ko07fKoZOI68GXvIz5HdkihCR0xwQ9aqkpk8zruFvh -/l8lqjRYyMEjVJ0bmBHDOJx+PYZspQ9AhnwC9FwCTyjLrnGfDzrIM/4RJTXq/LrF -YD3ZfBjVsqnTdXgDciLKOsMf7yzlLqn6niy2UUb9rwPW6mBo6oUWNmuF6R7As93E -JNyAKoFBbZQ+yODJgUEAnl6/f8UImKIYLEJAs/lvOCdLToD0PYFH4Ih86hzOtXVc -US4cK38acijnALXRdMbX5J+tB5O2UzU1/Dfkw/ZdFr4hc96SCvigY2q8lpJqPvi8 -ZVWb3vUNiSYE/CUapiVpy8JtynziWV+XrOvvLsi81xtZPCvM8hnIk2snYxnP/Okm -+Mpxm3+T/jRnhE6Z6/yzeAkzcLpmpnbtG3PrGqUNxCITIJRWCk4sbE6x/c+cCbqi -M+2HAgMBAAGjYzBhMB0GA1UdDgQWBBTdBAkHovV6fVJTEpKV7jiAJQ2mWTAPBgNV -HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFN0ECQei9Xp9UlMSkpXuOIAlDaZZMA4G -A1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAIBgRlCn7Jp0cHh5wYfGV -cpNxJK1ok1iOMq8bs3AD/CUrdIWQPXhq9LmLpZc7tRiRux6n+UBbkflVma8eEdBc -Hadm47GUBwwyOabqG7B52B2ccETjit3E+ZUfijhDPwGFpUenPUayvOUiaPd7nNgs -PgohyC0zrL/FgZkxdMF1ccW+sfAjRfSda/wZY52jvATGGAslu1OJD7OAUN5F7kR/ -q5R4ZJjT9ijdh9hwZXT7DrkT66cPYakylszeu+1jTBi7qUD3oFRuIIhxdRjqerQ0 -cuAjJ3dctpDqhiVAq+8zD8ufgr6iIPv2tS0a5sKFsXQP+8hlAqRSAUfdSSLBv9jr -a6x+3uxjMxW3IwiPxg+NQVrdjsW5j+VFP3jbutIbQLH+cU0/4IGiul607BXgk90I -H37hVZkLId6Tngr75qNJvTYw/ud3sqB1l7UtgYgXZSD32pAAn8lSzDLKNXz1PQ/Y -K9f1JmzJBjSWFupwWRoyeXkLtoh/D1JIPb9s2KJELtFOt3JY04kTlf5Eq/jXixtu -nLwsoFvVagCvXzfh1foQC5ichucmj87w7G6KVwuA406ywKBjYZC6VWg3dGq2ktuf -oYYitmUnDuy2n0Jg5GfCtdpBC8TTi2EbvPofkSvXRAdeuims2cXp71NIWuuA8ShY -Ic2wBlX7Jz9TkHCpBB5XJ7k= ------END CERTIFICATE----- - -# Issuer: CN=SSL.com Root Certification Authority ECC O=SSL Corporation -# Subject: CN=SSL.com Root Certification Authority ECC O=SSL Corporation -# Label: "SSL.com Root Certification Authority ECC" -# Serial: 8495723813297216424 -# MD5 Fingerprint: 2e:da:e4:39:7f:9c:8f:37:d1:70:9f:26:17:51:3a:8e -# SHA1 Fingerprint: c3:19:7c:39:24:e6:54:af:1b:c4:ab:20:95:7a:e2:c3:0e:13:02:6a -# SHA256 Fingerprint: 34:17:bb:06:cc:60:07:da:1b:96:1c:92:0b:8a:b4:ce:3f:ad:82:0e:4a:a3:0b:9a:cb:c4:a7:4e:bd:ce:bc:65 ------BEGIN CERTIFICATE----- -MIICjTCCAhSgAwIBAgIIdebfy8FoW6gwCgYIKoZIzj0EAwIwfDELMAkGA1UEBhMC -VVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQKDA9T -U0wgQ29ycG9yYXRpb24xMTAvBgNVBAMMKFNTTC5jb20gUm9vdCBDZXJ0aWZpY2F0 -aW9uIEF1dGhvcml0eSBFQ0MwHhcNMTYwMjEyMTgxNDAzWhcNNDEwMjEyMTgxNDAz -WjB8MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hvdXN0 -b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjExMC8GA1UEAwwoU1NMLmNvbSBS -b290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IEVDQzB2MBAGByqGSM49AgEGBSuB -BAAiA2IABEVuqVDEpiM2nl8ojRfLliJkP9x6jh3MCLOicSS6jkm5BBtHllirLZXI -7Z4INcgn64mMU1jrYor+8FsPazFSY0E7ic3s7LaNGdM0B9y7xgZ/wkWV7Mt/qCPg -CemB+vNH06NjMGEwHQYDVR0OBBYEFILRhXMw5zUE044CkvvlpNHEIejNMA8GA1Ud -EwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUgtGFczDnNQTTjgKS++Wk0cQh6M0wDgYD -VR0PAQH/BAQDAgGGMAoGCCqGSM49BAMCA2cAMGQCMG/n61kRpGDPYbCWe+0F+S8T -kdzt5fxQaxFGRrMcIQBiu77D5+jNB5n5DQtdcj7EqgIwH7y6C+IwJPt8bYBVCpk+ -gA0z5Wajs6O7pdWLjwkspl1+4vAHCGht0nxpbl/f5Wpl ------END CERTIFICATE----- - -# Issuer: CN=SSL.com EV Root Certification Authority RSA R2 O=SSL Corporation -# Subject: CN=SSL.com EV Root Certification Authority RSA R2 O=SSL Corporation -# Label: "SSL.com EV Root Certification Authority RSA R2" -# Serial: 6248227494352943350 -# MD5 Fingerprint: e1:1e:31:58:1a:ae:54:53:02:f6:17:6a:11:7b:4d:95 -# SHA1 Fingerprint: 74:3a:f0:52:9b:d0:32:a0:f4:4a:83:cd:d4:ba:a9:7b:7c:2e:c4:9a -# SHA256 Fingerprint: 2e:7b:f1:6c:c2:24:85:a7:bb:e2:aa:86:96:75:07:61:b0:ae:39:be:3b:2f:e9:d0:cc:6d:4e:f7:34:91:42:5c ------BEGIN CERTIFICATE----- -MIIF6zCCA9OgAwIBAgIIVrYpzTS8ePYwDQYJKoZIhvcNAQELBQAwgYIxCzAJBgNV -BAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEQMA4GA1UEBwwHSG91c3RvbjEYMBYGA1UE -CgwPU1NMIENvcnBvcmF0aW9uMTcwNQYDVQQDDC5TU0wuY29tIEVWIFJvb3QgQ2Vy -dGlmaWNhdGlvbiBBdXRob3JpdHkgUlNBIFIyMB4XDTE3MDUzMTE4MTQzN1oXDTQy -MDUzMDE4MTQzN1owgYIxCzAJBgNVBAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEQMA4G -A1UEBwwHSG91c3RvbjEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMTcwNQYDVQQD -DC5TU0wuY29tIEVWIFJvb3QgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgUlNBIFIy -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAjzZlQOHWTcDXtOlG2mvq -M0fNTPl9fb69LT3w23jhhqXZuglXaO1XPqDQCEGD5yhBJB/jchXQARr7XnAjssuf -OePPxU7Gkm0mxnu7s9onnQqG6YE3Bf7wcXHswxzpY6IXFJ3vG2fThVUCAtZJycxa -4bH3bzKfydQ7iEGonL3Lq9ttewkfokxykNorCPzPPFTOZw+oz12WGQvE43LrrdF9 -HSfvkusQv1vrO6/PgN3B0pYEW3p+pKk8OHakYo6gOV7qd89dAFmPZiw+B6KjBSYR -aZfqhbcPlgtLyEDhULouisv3D5oi53+aNxPN8k0TayHRwMwi8qFG9kRpnMphNQcA -b9ZhCBHqurj26bNg5U257J8UZslXWNvNh2n4ioYSA0e/ZhN2rHd9NCSFg83XqpyQ -Gp8hLH94t2S42Oim9HizVcuE0jLEeK6jj2HdzghTreyI/BXkmg3mnxp3zkyPuBQV -PWKchjgGAGYS5Fl2WlPAApiiECtoRHuOec4zSnaqW4EWG7WK2NAAe15itAnWhmMO -pgWVSbooi4iTsjQc2KRVbrcc0N6ZVTsj9CLg+SlmJuwgUHfbSguPvuUCYHBBXtSu -UDkiFCbLsjtzdFVHB3mBOagwE0TlBIqulhMlQg+5U8Sb/M3kHN48+qvWBkofZ6aY -MBzdLNvcGJVXZsb/XItW9XcCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAfBgNV -HSMEGDAWgBT5YLvU49U09rj1BoAlp3PbRmmonjAdBgNVHQ4EFgQU+WC71OPVNPa4 -9QaAJadz20ZpqJ4wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQBW -s47LCp1Jjr+kxJG7ZhcFUZh1++VQLHqe8RT6q9OKPv+RKY9ji9i0qVQBDb6Thi/5 -Sm3HXvVX+cpVHBK+Rw82xd9qt9t1wkclf7nxY/hoLVUE0fKNsKTPvDxeH3jnpaAg -cLAExbf3cqfeIg29MyVGjGSSJuM+LmOW2puMPfgYCdcDzH2GguDKBAdRUNf/ktUM -79qGn5nX67evaOI5JpS6aLe/g9Pqemc9YmeuJeVy6OLk7K4S9ksrPJ/psEDzOFSz -/bdoyNrGj1E8svuR3Bznm53htw1yj+KkxKl4+esUrMZDBcJlOSgYAsOCsp0FvmXt -ll9ldDz7CTUue5wT/RsPXcdtgTpWD8w74a8CLyKsRspGPKAcTNZEtF4uXBVmCeEm -Kf7GUmG6sXP/wwyc5WxqlD8UykAWlYTzWamsX0xhk23RO8yilQwipmdnRC652dKK -QbNmC1r7fSOl8hqw/96bg5Qu0T/fkreRrwU7ZcegbLHNYhLDkBvjJc40vG93drEQ -w/cFGsDWr3RiSBd3kmmQYRzelYB0VI8YHMPzA9C/pEN1hlMYegouCRw2n5H9gooi -S9EOUCXdywMMF8mDAAhONU2Ki+3wApRmLER/y5UnlhetCTCstnEXbosX9hwJ1C07 -mKVx01QT2WDz9UtmT/rx7iASjbSsV7FFY6GsdqnC+w== ------END CERTIFICATE----- - -# Issuer: CN=SSL.com EV Root Certification Authority ECC O=SSL Corporation -# Subject: CN=SSL.com EV Root Certification Authority ECC O=SSL Corporation -# Label: "SSL.com EV Root Certification Authority ECC" -# Serial: 3182246526754555285 -# MD5 Fingerprint: 59:53:22:65:83:42:01:54:c0:ce:42:b9:5a:7c:f2:90 -# SHA1 Fingerprint: 4c:dd:51:a3:d1:f5:20:32:14:b0:c6:c5:32:23:03:91:c7:46:42:6d -# SHA256 Fingerprint: 22:a2:c1:f7:bd:ed:70:4c:c1:e7:01:b5:f4:08:c3:10:88:0f:e9:56:b5:de:2a:4a:44:f9:9c:87:3a:25:a7:c8 ------BEGIN CERTIFICATE----- -MIIClDCCAhqgAwIBAgIILCmcWxbtBZUwCgYIKoZIzj0EAwIwfzELMAkGA1UEBhMC -VVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQKDA9T -U0wgQ29ycG9yYXRpb24xNDAyBgNVBAMMK1NTTC5jb20gRVYgUm9vdCBDZXJ0aWZp -Y2F0aW9uIEF1dGhvcml0eSBFQ0MwHhcNMTYwMjEyMTgxNTIzWhcNNDEwMjEyMTgx -NTIzWjB/MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hv -dXN0b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjE0MDIGA1UEAwwrU1NMLmNv -bSBFViBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IEVDQzB2MBAGByqGSM49 -AgEGBSuBBAAiA2IABKoSR5CYG/vvw0AHgyBO8TCCogbR8pKGYfL2IWjKAMTH6kMA -VIbc/R/fALhBYlzccBYy3h+Z1MzFB8gIH2EWB1E9fVwHU+M1OIzfzZ/ZLg1Kthku -WnBaBu2+8KGwytAJKaNjMGEwHQYDVR0OBBYEFFvKXuXe0oGqzagtZFG22XKbl+ZP -MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX -5k8wDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMCA2gAMGUCMQCK5kCJN+vp1RPZ -ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg -h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg== ------END CERTIFICATE----- - -# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 -# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 -# Label: "GlobalSign Root CA - R6" -# Serial: 1417766617973444989252670301619537 -# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae -# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1 -# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69 ------BEGIN CERTIFICATE----- -MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg -MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh -bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx -MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET -MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ -KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI -xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k -ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD -aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw -LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw -1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX -k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 -SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h -bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n -WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY -rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce -MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD -AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu -bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN -nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt -Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 -55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj -vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf -cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz -oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp -nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs -pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v -JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R -8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 -5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= ------END CERTIFICATE----- - -# Issuer: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed -# Subject: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed -# Label: "OISTE WISeKey Global Root GC CA" -# Serial: 44084345621038548146064804565436152554 -# MD5 Fingerprint: a9:d6:b9:2d:2f:93:64:f8:a5:69:ca:91:e9:68:07:23 -# SHA1 Fingerprint: e0:11:84:5e:34:de:be:88:81:b9:9c:f6:16:26:d1:96:1f:c3:b9:31 -# SHA256 Fingerprint: 85:60:f9:1c:36:24:da:ba:95:70:b5:fe:a0:db:e3:6f:f1:1a:83:23:be:94:86:85:4f:b3:f3:4a:55:71:19:8d ------BEGIN CERTIFICATE----- -MIICaTCCAe+gAwIBAgIQISpWDK7aDKtARb8roi066jAKBggqhkjOPQQDAzBtMQsw -CQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUgRm91 -bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9iYWwg -Um9vdCBHQyBDQTAeFw0xNzA1MDkwOTQ4MzRaFw00MjA1MDkwOTU4MzNaMG0xCzAJ -BgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBGb3Vu -ZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2JhbCBS -b290IEdDIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAETOlQwMYPchi82PG6s4ni -eUqjFqdrVCTbUf/q9Akkwwsin8tqJ4KBDdLArzHkdIJuyiXZjHWd8dvQmqJLIX4W -p2OQ0jnUsYd4XxiWD1AbNTcPasbc2RNNpI6QN+a9WzGRo1QwUjAOBgNVHQ8BAf8E -BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUSIcUrOPDnpBgOtfKie7T -rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV -57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg -Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 ------END CERTIFICATE----- - -# Issuer: CN=UCA Global G2 Root O=UniTrust -# Subject: CN=UCA Global G2 Root O=UniTrust -# Label: "UCA Global G2 Root" -# Serial: 124779693093741543919145257850076631279 -# MD5 Fingerprint: 80:fe:f0:c4:4a:f0:5c:62:32:9f:1c:ba:78:a9:50:f8 -# SHA1 Fingerprint: 28:f9:78:16:19:7a:ff:18:25:18:aa:44:fe:c1:a0:ce:5c:b6:4c:8a -# SHA256 Fingerprint: 9b:ea:11:c9:76:fe:01:47:64:c1:be:56:a6:f9:14:b5:a5:60:31:7a:bd:99:88:39:33:82:e5:16:1a:a0:49:3c ------BEGIN CERTIFICATE----- -MIIFRjCCAy6gAwIBAgIQXd+x2lqj7V2+WmUgZQOQ7zANBgkqhkiG9w0BAQsFADA9 -MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxGzAZBgNVBAMMElVDQSBH -bG9iYWwgRzIgUm9vdDAeFw0xNjAzMTEwMDAwMDBaFw00MDEyMzEwMDAwMDBaMD0x -CzAJBgNVBAYTAkNOMREwDwYDVQQKDAhVbmlUcnVzdDEbMBkGA1UEAwwSVUNBIEds -b2JhbCBHMiBSb290MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeYr -b3zvJgUno4Ek2m/LAfmZmqkywiKHYUGRO8vDaBsGxUypK8FnFyIdK+35KYmToni9 -kmugow2ifsqTs6bRjDXVdfkX9s9FxeV67HeToI8jrg4aA3++1NDtLnurRiNb/yzm -VHqUwCoV8MmNsHo7JOHXaOIxPAYzRrZUEaalLyJUKlgNAQLx+hVRZ2zA+te2G3/R -VogvGjqNO7uCEeBHANBSh6v7hn4PJGtAnTRnvI3HLYZveT6OqTwXS3+wmeOwcWDc -C/Vkw85DvG1xudLeJ1uK6NjGruFZfc8oLTW4lVYa8bJYS7cSN8h8s+1LgOGN+jIj -tm+3SJUIsUROhYw6AlQgL9+/V087OpAh18EmNVQg7Mc/R+zvWr9LesGtOxdQXGLY -D0tK3Cv6brxzks3sx1DoQZbXqX5t2Okdj4q1uViSukqSKwxW/YDrCPBeKW4bHAyv -j5OJrdu9o54hyokZ7N+1wxrrFv54NkzWbtA+FxyQF2smuvt6L78RHBgOLXMDj6Dl -NaBa4kx1HXHhOThTeEDMg5PXCp6dW4+K5OXgSORIskfNTip1KnvyIvbJvgmRlld6 -iIis7nCs+dwp4wwcOxJORNanTrAmyPPZGpeRaOrvjUYG0lZFWJo8DA+DuAUlwznP -O6Q0ibd5Ei9Hxeepl2n8pndntd978XplFeRhVmUCAwEAAaNCMEAwDgYDVR0PAQH/ -BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIHEjMz15DD/pQwIX4wV -ZyF0Ad/fMA0GCSqGSIb3DQEBCwUAA4ICAQATZSL1jiutROTL/7lo5sOASD0Ee/oj -L3rtNtqyzm325p7lX1iPyzcyochltq44PTUbPrw7tgTQvPlJ9Zv3hcU2tsu8+Mg5 -1eRfB70VVJd0ysrtT7q6ZHafgbiERUlMjW+i67HM0cOU2kTC5uLqGOiiHycFutfl -1qnN3e92mI0ADs0b+gO3joBYDic/UvuUospeZcnWhNq5NXHzJsBPd+aBJ9J3O5oU -b3n09tDh05S60FdRvScFDcH9yBIw7m+NESsIndTUv4BFFJqIRNow6rSn4+7vW4LV -PtateJLbXDzz2K36uGt/xDYotgIVilQsnLAXc47QN6MUPJiVAAwpBVueSUmxX8fj -y88nZY41F7dXyDDZQVu5FLbowg+UMaeUmMxq67XhJ/UQqAHojhJi6IjMtX9Gl8Cb -EGY4GjZGXyJoPd/JxhMnq1MGrKI8hgZlb7F+sSlEmqO6SWkoaY/X5V+tBIZkbxqg -DMUIYs6Ao9Dz7GjevjPHF1t/gMRMTLGmhIrDO7gJzRSBuhjjVFc2/tsvfEehOjPI -+Vg7RE+xygKJBJYoaMVLuCaJu9YzL1DV/pqJuhgyklTGW+Cd+V7lDSKb9triyCGy -YiGqhkCyLmTTX8jjfhFnRR8F/uOi77Oos/N9j/gMHyIfLXC0uAE0djAA5SN4p1bX -UB+K+wb1whnw0A== ------END CERTIFICATE----- - -# Issuer: CN=UCA Extended Validation Root O=UniTrust -# Subject: CN=UCA Extended Validation Root O=UniTrust -# Label: "UCA Extended Validation Root" -# Serial: 106100277556486529736699587978573607008 -# MD5 Fingerprint: a1:f3:5f:43:c6:34:9b:da:bf:8c:7e:05:53:ad:96:e2 -# SHA1 Fingerprint: a3:a1:b0:6f:24:61:23:4a:e3:36:a5:c2:37:fc:a6:ff:dd:f0:d7:3a -# SHA256 Fingerprint: d4:3a:f9:b3:54:73:75:5c:96:84:fc:06:d7:d8:cb:70:ee:5c:28:e7:73:fb:29:4e:b4:1e:e7:17:22:92:4d:24 ------BEGIN CERTIFICATE----- -MIIFWjCCA0KgAwIBAgIQT9Irj/VkyDOeTzRYZiNwYDANBgkqhkiG9w0BAQsFADBH -MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNVBAMMHFVDQSBF -eHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwHhcNMTUwMzEzMDAwMDAwWhcNMzgxMjMx -MDAwMDAwWjBHMQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNV -BAMMHFVDQSBFeHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwggIiMA0GCSqGSIb3DQEB -AQUAA4ICDwAwggIKAoICAQCpCQcoEwKwmeBkqh5DFnpzsZGgdT6o+uM4AHrsiWog -D4vFsJszA1qGxliG1cGFu0/GnEBNyr7uaZa4rYEwmnySBesFK5pI0Lh2PpbIILvS -sPGP2KxFRv+qZ2C0d35qHzwaUnoEPQc8hQ2E0B92CvdqFN9y4zR8V05WAT558aop -O2z6+I9tTcg1367r3CTueUWnhbYFiN6IXSV8l2RnCdm/WhUFhvMJHuxYMjMR83dk -sHYf5BA1FxvyDrFspCqjc/wJHx4yGVMR59mzLC52LqGj3n5qiAno8geK+LLNEOfi -c0CTuwjRP+H8C5SzJe98ptfRr5//lpr1kXuYC3fUfugH0mK1lTnj8/FtDw5lhIpj -VMWAtuCeS31HJqcBCF3RiJ7XwzJE+oJKCmhUfzhTA8ykADNkUVkLo4KRel7sFsLz -KuZi2irbWWIQJUoqgQtHB0MGcIfS+pMRKXpITeuUx3BNr2fVUbGAIAEBtHoIppB/ -TuDvB0GHr2qlXov7z1CymlSvw4m6WC31MJixNnI5fkkE/SmnTHnkBVfblLkWU41G -sx2VYVdWf6/wFlthWG82UBEL2KwrlRYaDh8IzTY0ZRBiZtWAXxQgXy0MoHgKaNYs -1+lvK9JKBZP8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0YIrO7o1dfdRUVjzyAfd5LQD -fwIDAQABo0IwQDAdBgNVHQ4EFgQU2XQ65DA9DfcS3H5aBZ8eNJr34RQwDwYDVR0T -AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBADaN -l8xCFWQpN5smLNb7rhVpLGsaGvdftvkHTFnq88nIua7Mui563MD1sC3AO6+fcAUR -ap8lTwEpcOPlDOHqWnzcSbvBHiqB9RZLcpHIojG5qtr8nR/zXUACE/xOHAbKsxSQ -VBcZEhrxH9cMaVr2cXj0lH2RC47skFSOvG+hTKv8dGT9cZr4QQehzZHkPJrgmzI5 -c6sq1WnIeJEmMX3ixzDx/BR4dxIOE/TdFpS/S2d7cFOFyrC78zhNLJA5wA3CXWvp -4uXViI3WLL+rG761KIcSF3Ru/H38j9CHJrAb+7lsq+KePRXBOy5nAliRn+/4Qh8s -t2j1da3Ptfb/EX3C8CSlrdP6oDyp+l3cpaDvRKS+1ujl5BOWF3sGPjLtx7dCvHaj -2GU4Kzg1USEODm8uNBNA4StnDG1KQTAYI1oyVZnJF+A83vbsea0rWBmirSwiGpWO -vpaQXUJXxPkUAzUrHC1RVwinOt4/5Mi0A3PCwSaAuwtCH60NryZy2sy+s6ODWA2C -xR9GUeOcGMyNm43sSet1UNWMKFnKdDTajAshqx7qG+XH/RU+wBeq+yNuJkbL+vmx -cmtpzyKEC2IPrNkZAJSidjzULZrtBJ4tBmIQN1IchXIbJ+XMxjHsN+xjWZsLHXbM -fjKaiJUINlK73nZfdklJrX+9ZSCyycErdhh2n1ax ------END CERTIFICATE----- - -# Issuer: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 -# Subject: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 -# Label: "Certigna Root CA" -# Serial: 269714418870597844693661054334862075617 -# MD5 Fingerprint: 0e:5c:30:62:27:eb:5b:bc:d7:ae:62:ba:e9:d5:df:77 -# SHA1 Fingerprint: 2d:0d:52:14:ff:9e:ad:99:24:01:74:20:47:6e:6c:85:27:27:f5:43 -# SHA256 Fingerprint: d4:8d:3d:23:ee:db:50:a4:59:e5:51:97:60:1c:27:77:4b:9d:7b:18:c9:4d:5a:05:95:11:a1:02:50:b9:31:68 ------BEGIN CERTIFICATE----- -MIIGWzCCBEOgAwIBAgIRAMrpG4nxVQMNo+ZBbcTjpuEwDQYJKoZIhvcNAQELBQAw -WjELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCURoaW15b3RpczEcMBoGA1UECwwTMDAw -MiA0ODE0NjMwODEwMDAzNjEZMBcGA1UEAwwQQ2VydGlnbmEgUm9vdCBDQTAeFw0x -MzEwMDEwODMyMjdaFw0zMzEwMDEwODMyMjdaMFoxCzAJBgNVBAYTAkZSMRIwEAYD -VQQKDAlEaGlteW90aXMxHDAaBgNVBAsMEzAwMDIgNDgxNDYzMDgxMDAwMzYxGTAX -BgNVBAMMEENlcnRpZ25hIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw -ggIKAoICAQDNGDllGlmx6mQWDoyUJJV8g9PFOSbcDO8WV43X2KyjQn+Cyu3NW9sO -ty3tRQgXstmzy9YXUnIo245Onoq2C/mehJpNdt4iKVzSs9IGPjA5qXSjklYcoW9M -CiBtnyN6tMbaLOQdLNyzKNAT8kxOAkmhVECe5uUFoC2EyP+YbNDrihqECB63aCPu -I9Vwzm1RaRDuoXrC0SIxwoKF0vJVdlB8JXrJhFwLrN1CTivngqIkicuQstDuI7pm -TLtipPlTWmR7fJj6o0ieD5Wupxj0auwuA0Wv8HT4Ks16XdG+RCYyKfHx9WzMfgIh -C59vpD++nVPiz32pLHxYGpfhPTc3GGYo0kDFUYqMwy3OU4gkWGQwFsWq4NYKpkDf -ePb1BHxpE4S80dGnBs8B92jAqFe7OmGtBIyT46388NtEbVncSVmurJqZNjBBe3Yz -IoejwpKGbvlw7q6Hh5UbxHq9MfPU0uWZ/75I7HX1eBYdpnDBfzwboZL7z8g81sWT -Co/1VTp2lc5ZmIoJlXcymoO6LAQ6l73UL77XbJuiyn1tJslV1c/DeVIICZkHJC1k -JWumIWmbat10TWuXekG9qxf5kBdIjzb5LdXF2+6qhUVB+s06RbFo5jZMm5BX7CO5 -hwjCxAnxl4YqKE3idMDaxIzb3+KhF1nOJFl0Mdp//TBt2dzhauH8XwIDAQABo4IB -GjCCARYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE -FBiHVuBud+4kNTxOc5of1uHieX4rMB8GA1UdIwQYMBaAFBiHVuBud+4kNTxOc5of -1uHieX4rMEQGA1UdIAQ9MDswOQYEVR0gADAxMC8GCCsGAQUFBwIBFiNodHRwczov -L3d3d3cuY2VydGlnbmEuZnIvYXV0b3JpdGVzLzBtBgNVHR8EZjBkMC+gLaArhilo -dHRwOi8vY3JsLmNlcnRpZ25hLmZyL2NlcnRpZ25hcm9vdGNhLmNybDAxoC+gLYYr -aHR0cDovL2NybC5kaGlteW90aXMuY29tL2NlcnRpZ25hcm9vdGNhLmNybDANBgkq -hkiG9w0BAQsFAAOCAgEAlLieT/DjlQgi581oQfccVdV8AOItOoldaDgvUSILSo3L -6btdPrtcPbEo/uRTVRPPoZAbAh1fZkYJMyjhDSSXcNMQH+pkV5a7XdrnxIxPTGRG -HVyH41neQtGbqH6mid2PHMkwgu07nM3A6RngatgCdTer9zQoKJHyBApPNeNgJgH6 -0BGM+RFq7q89w1DTj18zeTyGqHNFkIwgtnJzFyO+B2XleJINugHA64wcZr+shncB -lA2c5uk5jR+mUYyZDDl34bSb+hxnV29qao6pK0xXeXpXIs/NX2NGjVxZOob4Mkdi -o2cNGJHc+6Zr9UhhcyNZjgKnvETq9Emd8VRY+WCv2hikLyhF3HqgiIZd8zvn/yk1 -gPxkQ5Tm4xxvvq0OKmOZK8l+hfZx6AYDlf7ej0gcWtSS6Cvu5zHbugRqh5jnxV/v -faci9wHYTfmJ0A6aBVmknpjZbyvKcL5kwlWj9Omvw5Ip3IgWJJk8jSaYtlu3zM63 -Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh -jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw -3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0= ------END CERTIFICATE----- - -# Issuer: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI -# Subject: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI -# Label: "emSign Root CA - G1" -# Serial: 235931866688319308814040 -# MD5 Fingerprint: 9c:42:84:57:dd:cb:0b:a7:2e:95:ad:b6:f3:da:bc:ac -# SHA1 Fingerprint: 8a:c7:ad:8f:73:ac:4e:c1:b5:75:4d:a5:40:f4:fc:cf:7c:b5:8e:8c -# SHA256 Fingerprint: 40:f6:af:03:46:a9:9a:a1:cd:1d:55:5a:4e:9c:ce:62:c7:f9:63:46:03:ee:40:66:15:83:3d:c8:c8:d0:03:67 ------BEGIN CERTIFICATE----- -MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD -VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU -ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH -MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO -MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv -Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN -BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz -f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO -8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq -d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM -tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt -Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB -o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD -AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x -PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM -wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d -GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH -6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby -RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx -iN66zB+Afko= ------END CERTIFICATE----- - -# Issuer: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI -# Subject: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI -# Label: "emSign ECC Root CA - G3" -# Serial: 287880440101571086945156 -# MD5 Fingerprint: ce:0b:72:d1:9f:88:8e:d0:50:03:e8:e3:b8:8b:67:40 -# SHA1 Fingerprint: 30:43:fa:4f:f2:57:dc:a0:c3:80:ee:2e:58:ea:78:b2:3f:e6:bb:c1 -# SHA256 Fingerprint: 86:a1:ec:ba:08:9c:4a:8d:3b:be:27:34:c6:12:ba:34:1d:81:3e:04:3c:f9:e8:a8:62:cd:5c:57:a3:6b:be:6b ------BEGIN CERTIFICATE----- -MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG -EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo -bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g -RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ -TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s -b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw -djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 -WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS -fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB -zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq -hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB -CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD -+JbNR6iC8hZVdyR+EhCVBCyj ------END CERTIFICATE----- - -# Issuer: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI -# Subject: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI -# Label: "emSign Root CA - C1" -# Serial: 825510296613316004955058 -# MD5 Fingerprint: d8:e3:5d:01:21:fa:78:5a:b0:df:ba:d2:ee:2a:5f:68 -# SHA1 Fingerprint: e7:2e:f1:df:fc:b2:09:28:cf:5d:d4:d5:67:37:b1:51:cb:86:4f:01 -# SHA256 Fingerprint: 12:56:09:aa:30:1d:a0:a2:49:b9:7a:82:39:cb:6a:34:21:6f:44:dc:ac:9f:39:54:b1:42:92:f2:e8:c8:60:8f ------BEGIN CERTIFICATE----- -MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG -A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg -SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw -MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln -biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v -dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ -BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ -HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH -3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH -GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c -xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1 -aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq -TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL -BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87 -/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4 -kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG -YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT -+xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo -WXzhriKi4gp6D/piq1JM4fHfyr6DDUI= ------END CERTIFICATE----- - -# Issuer: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI -# Subject: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI -# Label: "emSign ECC Root CA - C3" -# Serial: 582948710642506000014504 -# MD5 Fingerprint: 3e:53:b3:a3:81:ee:d7:10:f8:d3:b0:1d:17:92:f5:d5 -# SHA1 Fingerprint: b6:af:43:c2:9b:81:53:7d:f6:ef:6b:c3:1f:1f:60:15:0c:ee:48:66 -# SHA256 Fingerprint: bc:4d:80:9b:15:18:9d:78:db:3e:1d:8c:f4:f9:72:6a:79:5d:a1:64:3c:a5:f1:35:8e:1d:db:0e:dc:0d:7e:b3 ------BEGIN CERTIFICATE----- -MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG -EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx -IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw -MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln -biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND -IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci -MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti -sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O -BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB -Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c -3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J -0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ== ------END CERTIFICATE----- - -# Issuer: CN=Hongkong Post Root CA 3 O=Hongkong Post -# Subject: CN=Hongkong Post Root CA 3 O=Hongkong Post -# Label: "Hongkong Post Root CA 3" -# Serial: 46170865288971385588281144162979347873371282084 -# MD5 Fingerprint: 11:fc:9f:bd:73:30:02:8a:fd:3f:f3:58:b9:cb:20:f0 -# SHA1 Fingerprint: 58:a2:d0:ec:20:52:81:5b:c1:f3:f8:64:02:24:4e:c2:8e:02:4b:02 -# SHA256 Fingerprint: 5a:2f:c0:3f:0c:83:b0:90:bb:fa:40:60:4b:09:88:44:6c:76:36:18:3d:f9:84:6e:17:10:1a:44:7f:b8:ef:d6 ------BEGIN CERTIFICATE----- -MIIFzzCCA7egAwIBAgIUCBZfikyl7ADJk0DfxMauI7gcWqQwDQYJKoZIhvcNAQEL -BQAwbzELMAkGA1UEBhMCSEsxEjAQBgNVBAgTCUhvbmcgS29uZzESMBAGA1UEBxMJ -SG9uZyBLb25nMRYwFAYDVQQKEw1Ib25na29uZyBQb3N0MSAwHgYDVQQDExdIb25n -a29uZyBQb3N0IFJvb3QgQ0EgMzAeFw0xNzA2MDMwMjI5NDZaFw00MjA2MDMwMjI5 -NDZaMG8xCzAJBgNVBAYTAkhLMRIwEAYDVQQIEwlIb25nIEtvbmcxEjAQBgNVBAcT -CUhvbmcgS29uZzEWMBQGA1UEChMNSG9uZ2tvbmcgUG9zdDEgMB4GA1UEAxMXSG9u -Z2tvbmcgUG9zdCBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQCziNfqzg8gTr7m1gNt7ln8wlffKWihgw4+aMdoWJwcYEuJQwy51BWy7sFO -dem1p+/l6TWZ5Mwc50tfjTMwIDNT2aa71T4Tjukfh0mtUC1Qyhi+AViiE3CWu4mI -VoBc+L0sPOFMV4i707mV78vH9toxdCim5lSJ9UExyuUmGs2C4HDaOym71QP1mbpV -9WTRYA6ziUm4ii8F0oRFKHyPaFASePwLtVPLwpgchKOesL4jpNrcyCse2m5FHomY -2vkALgbpDDtw1VAliJnLzXNg99X/NWfFobxeq81KuEXryGgeDQ0URhLj0mRiikKY -vLTGCAj4/ahMZJx2Ab0vqWwzD9g/KLg8aQFChn5pwckGyuV6RmXpwtZQQS4/t+Tt -bNe/JgERohYpSms0BpDsE9K2+2p20jzt8NYt3eEV7KObLyzJPivkaTv/ciWxNoZb -x39ri1UbSsUgYT2uy1DhCDq+sI9jQVMwCFk8mB13umOResoQUGC/8Ne8lYePl8X+ -l2oBlKN8W4UdKjk60FSh0Tlxnf0h+bV78OLgAo9uliQlLKAeLKjEiafv7ZkGL7YK -TE/bosw3Gq9HhS2KX8Q0NEwA/RiTZxPRN+ZItIsGxVd7GYYKecsAyVKvQv83j+Gj -Hno9UKtjBucVtT+2RTeUN7F+8kjDf8V1/peNRY8apxpyKBpADwIDAQABo2MwYTAP -BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBQXnc0e -i9Y5K3DTXNSguB+wAPzFYTAdBgNVHQ4EFgQUF53NHovWOStw01zUoLgfsAD8xWEw -DQYJKoZIhvcNAQELBQADggIBAFbVe27mIgHSQpsY1Q7XZiNc4/6gx5LS6ZStS6LG -7BJ8dNVI0lkUmcDrudHr9EgwW62nV3OZqdPlt9EuWSRY3GguLmLYauRwCy0gUCCk -MpXRAJi70/33MvJJrsZ64Ee+bs7Lo3I6LWldy8joRTnU+kLBEUx3XZL7av9YROXr -gZ6voJmtvqkBZss4HTzfQx/0TW60uhdG/H39h4F5ag0zD/ov+BS5gLNdTaqX4fnk -GMX41TiMJjz98iji7lpJiCzfeT2OnpA8vUFKOt1b9pq0zj8lMH8yfaIDlNDceqFS -3m6TjRgm/VWsvY+b0s+v54Ysyx8Jb6NvqYTUc79NoXQbTiNg8swOqn+knEwlqLJm -Ozj/2ZQw9nKEvmhVEA/GcywWaZMH/rFF7buiVWqw2rVKAiUnhde3t4ZEFolsgCs+ -l6mc1X5VTMbeRRAc6uk7nwNT7u56AQIWeNTowr5GdogTPyK7SBIdUgC0An4hGh6c -JfTzPV4e0hz5sy229zdcxsshTrD3mUcYhcErulWuBurQB7Lcq9CClnXO0lD+mefP -L5/ndtFhKvshuzHQqp9HpLIiyhY6UFfEW0NnxWViA0kB60PZ2Pierc+xYw5F9KBa -LJstxabArahH9CdMOA0uG0k7UvToiIMrVCjU8jVStDKDYmlkDJGcn5fqdBb9HxEG -mpv0 ------END CERTIFICATE----- - -# Issuer: CN=Microsoft ECC Root Certificate Authority 2017 O=Microsoft Corporation -# Subject: CN=Microsoft ECC Root Certificate Authority 2017 O=Microsoft Corporation -# Label: "Microsoft ECC Root Certificate Authority 2017" -# Serial: 136839042543790627607696632466672567020 -# MD5 Fingerprint: dd:a1:03:e6:4a:93:10:d1:bf:f0:19:42:cb:fe:ed:67 -# SHA1 Fingerprint: 99:9a:64:c3:7f:f4:7d:9f:ab:95:f1:47:69:89:14:60:ee:c4:c3:c5 -# SHA256 Fingerprint: 35:8d:f3:9d:76:4a:f9:e1:b7:66:e9:c9:72:df:35:2e:e1:5c:fa:c2:27:af:6a:d1:d7:0e:8e:4a:6e:dc:ba:02 ------BEGIN CERTIFICATE----- -MIICWTCCAd+gAwIBAgIQZvI9r4fei7FK6gxXMQHC7DAKBggqhkjOPQQDAzBlMQsw -CQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYD -VQQDEy1NaWNyb3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIw -MTcwHhcNMTkxMjE4MjMwNjQ1WhcNNDIwNzE4MjMxNjA0WjBlMQswCQYDVQQGEwJV -UzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1NaWNy -b3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwdjAQBgcq -hkjOPQIBBgUrgQQAIgNiAATUvD0CQnVBEyPNgASGAlEvaqiBYgtlzPbKnR5vSmZR -ogPZnZH6thaxjG7efM3beaYvzrvOcS/lpaso7GMEZpn4+vKTEAXhgShC48Zo9OYb -hGBKia/teQ87zvH2RPUBeMCjVDBSMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8E -BTADAQH/MB0GA1UdDgQWBBTIy5lycFIM+Oa+sgRXKSrPQhDtNTAQBgkrBgEEAYI3 -FQEEAwIBADAKBggqhkjOPQQDAwNoADBlAjBY8k3qDPlfXu5gKcs68tvWMoQZP3zV -L8KxzJOuULsJMsbG7X7JNpQS5GiFBqIb0C8CMQCZ6Ra0DvpWSNSkMBaReNtUjGUB -iudQZsIxtzm6uBoiB078a1QWIP8rtedMDE2mT3M= ------END CERTIFICATE----- - -# Issuer: CN=Microsoft RSA Root Certificate Authority 2017 O=Microsoft Corporation -# Subject: CN=Microsoft RSA Root Certificate Authority 2017 O=Microsoft Corporation -# Label: "Microsoft RSA Root Certificate Authority 2017" -# Serial: 40975477897264996090493496164228220339 -# MD5 Fingerprint: 10:ff:00:ff:cf:c9:f8:c7:7a:c0:ee:35:8e:c9:0f:47 -# SHA1 Fingerprint: 73:a5:e6:4a:3b:ff:83:16:ff:0e:dc:cc:61:8a:90:6e:4e:ae:4d:74 -# SHA256 Fingerprint: c7:41:f7:0f:4b:2a:8d:88:bf:2e:71:c1:41:22:ef:53:ef:10:eb:a0:cf:a5:e6:4c:fa:20:f4:18:85:30:73:e0 ------BEGIN CERTIFICATE----- -MIIFqDCCA5CgAwIBAgIQHtOXCV/YtLNHcB6qvn9FszANBgkqhkiG9w0BAQwFADBl -MQswCQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYw -NAYDVQQDEy1NaWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 -IDIwMTcwHhcNMTkxMjE4MjI1MTIyWhcNNDIwNzE4MjMwMDIzWjBlMQswCQYDVQQG -EwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1N -aWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwggIi -MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDKW76UM4wplZEWCpW9R2LBifOZ -Nt9GkMml7Xhqb0eRaPgnZ1AzHaGm++DlQ6OEAlcBXZxIQIJTELy/xztokLaCLeX0 -ZdDMbRnMlfl7rEqUrQ7eS0MdhweSE5CAg2Q1OQT85elss7YfUJQ4ZVBcF0a5toW1 -HLUX6NZFndiyJrDKxHBKrmCk3bPZ7Pw71VdyvD/IybLeS2v4I2wDwAW9lcfNcztm -gGTjGqwu+UcF8ga2m3P1eDNbx6H7JyqhtJqRjJHTOoI+dkC0zVJhUXAoP8XFWvLJ -jEm7FFtNyP9nTUwSlq31/niol4fX/V4ggNyhSyL71Imtus5Hl0dVe49FyGcohJUc -aDDv70ngNXtk55iwlNpNhTs+VcQor1fznhPbRiefHqJeRIOkpcrVE7NLP8TjwuaG -YaRSMLl6IE9vDzhTyzMMEyuP1pq9KsgtsRx9S1HKR9FIJ3Jdh+vVReZIZZ2vUpC6 -W6IYZVcSn2i51BVrlMRpIpj0M+Dt+VGOQVDJNE92kKz8OMHY4Xu54+OU4UZpyw4K -UGsTuqwPN1q3ErWQgR5WrlcihtnJ0tHXUeOrO8ZV/R4O03QK0dqq6mm4lyiPSMQH -+FJDOvTKVTUssKZqwJz58oHhEmrARdlns87/I6KJClTUFLkqqNfs+avNJVgyeY+Q -W5g5xAgGwax/Dj0ApQIDAQABo1QwUjAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/ -BAUwAwEB/zAdBgNVHQ4EFgQUCctZf4aycI8awznjwNnpv7tNsiMwEAYJKwYBBAGC -NxUBBAMCAQAwDQYJKoZIhvcNAQEMBQADggIBAKyvPl3CEZaJjqPnktaXFbgToqZC -LgLNFgVZJ8og6Lq46BrsTaiXVq5lQ7GPAJtSzVXNUzltYkyLDVt8LkS/gxCP81OC -gMNPOsduET/m4xaRhPtthH80dK2Jp86519efhGSSvpWhrQlTM93uCupKUY5vVau6 -tZRGrox/2KJQJWVggEbbMwSubLWYdFQl3JPk+ONVFT24bcMKpBLBaYVu32TxU5nh -SnUgnZUP5NbcA/FZGOhHibJXWpS2qdgXKxdJ5XbLwVaZOjex/2kskZGT4d9Mozd2 -TaGf+G0eHdP67Pv0RR0Tbc/3WeUiJ3IrhvNXuzDtJE3cfVa7o7P4NHmJweDyAmH3 -pvwPuxwXC65B2Xy9J6P9LjrRk5Sxcx0ki69bIImtt2dmefU6xqaWM/5TkshGsRGR -xpl/j8nWZjEgQRCHLQzWwa80mMpkg/sTV9HB8Dx6jKXB/ZUhoHHBk2dxEuqPiApp -GWSZI1b7rCoucL5mxAyE7+WL85MB+GqQk2dLsmijtWKP6T+MejteD+eMuMZ87zf9 -dOLITzNy4ZQ5bb0Sr74MTnB8G2+NszKTc0QWbej09+CVgI+WXTik9KveCjCHk9hN -AHFiRSdLOkKEW39lt2c0Ui2cFmuqqNh7o0JMcccMyj6D5KbvtwEwXlGjefVwaaZB -RA+GsCyRxj3qrg+E ------END CERTIFICATE----- - -# Issuer: CN=e-Szigno Root CA 2017 O=Microsec Ltd. -# Subject: CN=e-Szigno Root CA 2017 O=Microsec Ltd. -# Label: "e-Szigno Root CA 2017" -# Serial: 411379200276854331539784714 -# MD5 Fingerprint: de:1f:f6:9e:84:ae:a7:b4:21:ce:1e:58:7d:d1:84:98 -# SHA1 Fingerprint: 89:d4:83:03:4f:9e:9a:48:80:5f:72:37:d4:a9:a6:ef:cb:7c:1f:d1 -# SHA256 Fingerprint: be:b0:0b:30:83:9b:9b:c3:2c:32:e4:44:79:05:95:06:41:f2:64:21:b1:5e:d0:89:19:8b:51:8a:e2:ea:1b:99 ------BEGIN CERTIFICATE----- -MIICQDCCAeWgAwIBAgIMAVRI7yH9l1kN9QQKMAoGCCqGSM49BAMCMHExCzAJBgNV -BAYTAkhVMREwDwYDVQQHDAhCdWRhcGVzdDEWMBQGA1UECgwNTWljcm9zZWMgTHRk -LjEXMBUGA1UEYQwOVkFUSFUtMjM1ODQ0OTcxHjAcBgNVBAMMFWUtU3ppZ25vIFJv -b3QgQ0EgMjAxNzAeFw0xNzA4MjIxMjA3MDZaFw00MjA4MjIxMjA3MDZaMHExCzAJ -BgNVBAYTAkhVMREwDwYDVQQHDAhCdWRhcGVzdDEWMBQGA1UECgwNTWljcm9zZWMg -THRkLjEXMBUGA1UEYQwOVkFUSFUtMjM1ODQ0OTcxHjAcBgNVBAMMFWUtU3ppZ25v -IFJvb3QgQ0EgMjAxNzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABJbcPYrYsHtv -xie+RJCxs1YVe45DJH0ahFnuY2iyxl6H0BVIHqiQrb1TotreOpCmYF9oMrWGQd+H -Wyx7xf58etqjYzBhMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0G -A1UdDgQWBBSHERUI0arBeAyxr87GyZDvvzAEwDAfBgNVHSMEGDAWgBSHERUI0arB -eAyxr87GyZDvvzAEwDAKBggqhkjOPQQDAgNJADBGAiEAtVfd14pVCzbhhkT61Nlo -jbjcI4qKDdQvfepz7L9NbKgCIQDLpbQS+ue16M9+k/zzNY9vTlp8tLxOsvxyqltZ -+efcMQ== ------END CERTIFICATE----- - -# Issuer: O=CERTSIGN SA OU=certSIGN ROOT CA G2 -# Subject: O=CERTSIGN SA OU=certSIGN ROOT CA G2 -# Label: "certSIGN Root CA G2" -# Serial: 313609486401300475190 -# MD5 Fingerprint: 8c:f1:75:8a:c6:19:cf:94:b7:f7:65:20:87:c3:97:c7 -# SHA1 Fingerprint: 26:f9:93:b4:ed:3d:28:27:b0:b9:4b:a7:e9:15:1d:a3:8d:92:e5:32 -# SHA256 Fingerprint: 65:7c:fe:2f:a7:3f:aa:38:46:25:71:f3:32:a2:36:3a:46:fc:e7:02:09:51:71:07:02:cd:fb:b6:ee:da:33:05 ------BEGIN CERTIFICATE----- -MIIFRzCCAy+gAwIBAgIJEQA0tk7GNi02MA0GCSqGSIb3DQEBCwUAMEExCzAJBgNV -BAYTAlJPMRQwEgYDVQQKEwtDRVJUU0lHTiBTQTEcMBoGA1UECxMTY2VydFNJR04g -Uk9PVCBDQSBHMjAeFw0xNzAyMDYwOTI3MzVaFw00MjAyMDYwOTI3MzVaMEExCzAJ -BgNVBAYTAlJPMRQwEgYDVQQKEwtDRVJUU0lHTiBTQTEcMBoGA1UECxMTY2VydFNJ -R04gUk9PVCBDQSBHMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMDF -dRmRfUR0dIf+DjuW3NgBFszuY5HnC2/OOwppGnzC46+CjobXXo9X69MhWf05N0Iw -vlDqtg+piNguLWkh59E3GE59kdUWX2tbAMI5Qw02hVK5U2UPHULlj88F0+7cDBrZ -uIt4ImfkabBoxTzkbFpG583H+u/E7Eu9aqSs/cwoUe+StCmrqzWaTOTECMYmzPhp -n+Sc8CnTXPnGFiWeI8MgwT0PPzhAsP6CRDiqWhqKa2NYOLQV07YRaXseVO6MGiKs -cpc/I1mbySKEwQdPzH/iV8oScLumZfNpdWO9lfsbl83kqK/20U6o2YpxJM02PbyW -xPFsqa7lzw1uKA2wDrXKUXt4FMMgL3/7FFXhEZn91QqhngLjYl/rNUssuHLoPj1P -rCy7Lobio3aP5ZMqz6WryFyNSwb/EkaseMsUBzXgqd+L6a8VTxaJW732jcZZroiF -DsGJ6x9nxUWO/203Nit4ZoORUSs9/1F3dmKh7Gc+PoGD4FapUB8fepmrY7+EF3fx -DTvf95xhszWYijqy7DwaNz9+j5LP2RIUZNoQAhVB/0/E6xyjyfqZ90bp4RjZsbgy -LcsUDFDYg2WD7rlcz8sFWkz6GZdr1l0T08JcVLwyc6B49fFtHsufpaafItzRUZ6C -eWRgKRM+o/1Pcmqr4tTluCRVLERLiohEnMqE0yo7AgMBAAGjQjBAMA8GA1UdEwEB -/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSCIS1mxteg4BXrzkwJ -d8RgnlRuAzANBgkqhkiG9w0BAQsFAAOCAgEAYN4auOfyYILVAzOBywaK8SJJ6ejq -kX/GM15oGQOGO0MBzwdw5AgeZYWR5hEit/UCI46uuR59H35s5r0l1ZUa8gWmr4UC -b6741jH/JclKyMeKqdmfS0mbEVeZkkMR3rYzpMzXjWR91M08KCy0mpbqTfXERMQl -qiCA2ClV9+BB/AYm/7k29UMUA2Z44RGx2iBfRgB4ACGlHgAoYXhvqAEBj500mv/0 -OJD7uNGzcgbJceaBxXntC6Z58hMLnPddDnskk7RI24Zf3lCGeOdA5jGokHZwYa+c -NywRtYK3qq4kNFtyDGkNzVmf9nGvnAvRCjj5BiKDUyUM/FHE5r7iOZULJK2v0ZXk -ltd0ZGtxTgI8qoXzIKNDOXZbbFD+mpwUHmUUihW9o4JFWklWatKcsWMy5WHgUyIO -pwpJ6st+H6jiYoD2EEVSmAYY3qXNL3+q1Ok+CHLsIwMCPKaq2LxndD0UF/tUSxfj -03k9bWtJySgOLnRQvwzZRjoQhsmnP+mg7H/rpXdYaXHmgwo38oZJar55CJD2AhZk -PuXaTH4MNMn5X7azKFGnpyuqSfqNZSlO42sTp5SjLVFteAxEy9/eCG/Oo2Sr05WE -1LlSVHJ7liXMvGnjSG4N0MedJ5qq+BOS3R7fY581qRY27Iy4g/Q9iY/NtBde17MX -QRBdJ3NghVdJIgc= ------END CERTIFICATE----- - -# Issuer: CN=NAVER Global Root Certification Authority O=NAVER BUSINESS PLATFORM Corp. -# Subject: CN=NAVER Global Root Certification Authority O=NAVER BUSINESS PLATFORM Corp. -# Label: "NAVER Global Root Certification Authority" -# Serial: 9013692873798656336226253319739695165984492813 -# MD5 Fingerprint: c8:7e:41:f6:25:3b:f5:09:b3:17:e8:46:3d:bf:d0:9b -# SHA1 Fingerprint: 8f:6b:f2:a9:27:4a:da:14:a0:c4:f4:8e:61:27:f9:c0:1e:78:5d:d1 -# SHA256 Fingerprint: 88:f4:38:dc:f8:ff:d1:fa:8f:42:91:15:ff:e5:f8:2a:e1:e0:6e:0c:70:c3:75:fa:ad:71:7b:34:a4:9e:72:65 ------BEGIN CERTIFICATE----- -MIIFojCCA4qgAwIBAgIUAZQwHqIL3fXFMyqxQ0Rx+NZQTQ0wDQYJKoZIhvcNAQEM -BQAwaTELMAkGA1UEBhMCS1IxJjAkBgNVBAoMHU5BVkVSIEJVU0lORVNTIFBMQVRG -T1JNIENvcnAuMTIwMAYDVQQDDClOQVZFUiBHbG9iYWwgUm9vdCBDZXJ0aWZpY2F0 -aW9uIEF1dGhvcml0eTAeFw0xNzA4MTgwODU4NDJaFw0zNzA4MTgyMzU5NTlaMGkx -CzAJBgNVBAYTAktSMSYwJAYDVQQKDB1OQVZFUiBCVVNJTkVTUyBQTEFURk9STSBD -b3JwLjEyMDAGA1UEAwwpTkFWRVIgR2xvYmFsIFJvb3QgQ2VydGlmaWNhdGlvbiBB -dXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC21PGTXLVA -iQqrDZBbUGOukJR0F0Vy1ntlWilLp1agS7gvQnXp2XskWjFlqxcX0TM62RHcQDaH -38dq6SZeWYp34+hInDEW+j6RscrJo+KfziFTowI2MMtSAuXaMl3Dxeb57hHHi8lE -HoSTGEq0n+USZGnQJoViAbbJAh2+g1G7XNr4rRVqmfeSVPc0W+m/6imBEtRTkZaz -kVrd/pBzKPswRrXKCAfHcXLJZtM0l/aM9BhK4dA9WkW2aacp+yPOiNgSnABIqKYP -szuSjXEOdMWLyEz59JuOuDxp7W87UC9Y7cSw0BwbagzivESq2M0UXZR4Yb8Obtoq -vC8MC3GmsxY/nOb5zJ9TNeIDoKAYv7vxvvTWjIcNQvcGufFt7QSUqP620wbGQGHf -nZ3zVHbOUzoBppJB7ASjjw2i1QnK1sua8e9DXcCrpUHPXFNwcMmIpi3Ua2FzUCaG -YQ5fG8Ir4ozVu53BA0K6lNpfqbDKzE0K70dpAy8i+/Eozr9dUGWokG2zdLAIx6yo -0es+nPxdGoMuK8u180SdOqcXYZaicdNwlhVNt0xz7hlcxVs+Qf6sdWA7G2POAN3a -CJBitOUt7kinaxeZVL6HSuOpXgRM6xBtVNbv8ejyYhbLgGvtPe31HzClrkvJE+2K -AQHJuFFYwGY6sWZLxNUxAmLpdIQM201GLQIDAQABo0IwQDAdBgNVHQ4EFgQU0p+I -36HNLL3s9TsBAZMzJ7LrYEswDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB -Af8wDQYJKoZIhvcNAQEMBQADggIBADLKgLOdPVQG3dLSLvCkASELZ0jKbY7gyKoN -qo0hV4/GPnrK21HUUrPUloSlWGB/5QuOH/XcChWB5Tu2tyIvCZwTFrFsDDUIbatj -cu3cvuzHV+YwIHHW1xDBE1UBjCpD5EHxzzp6U5LOogMFDTjfArsQLtk70pt6wKGm -+LUx5vR1yblTmXVHIloUFcd4G7ad6Qz4G3bxhYTeodoS76TiEJd6eN4MUZeoIUCL -hr0N8F5OSza7OyAfikJW4Qsav3vQIkMsRIz75Sq0bBwcupTgE34h5prCy8VCZLQe -lHsIJchxzIdFV4XTnyliIoNRlwAYl3dqmJLJfGBs32x9SuRwTMKeuB330DTHD8z7 -p/8Dvq1wkNoL3chtl1+afwkyQf3NosxabUzyqkn+Zvjp2DXrDige7kgvOtB5CTh8 -piKCk5XQA76+AqAF3SAi428diDRgxuYKuQl1C/AH6GmWNcf7I4GOODm4RStDeKLR -LBT/DShycpWbXgnbiUSYqqFJu3FS8r/2/yehNq+4tneI3TqkbZs0kNwUXTC/t+sX -5Ie3cdCh13cV1ELX8vMxmV2b3RZtP+oGI/hGoiLtk/bdmuYqh7GYVPEi92tF4+KO -dh2ajcQGjTa3FPOdVGm3jjzVpG2Tgbet9r1ke8LJaDmgkpzNNIaRkPpkUZ3+/uul -9XXeifdy ------END CERTIFICATE----- - -# Issuer: CN=AC RAIZ FNMT-RCM SERVIDORES SEGUROS O=FNMT-RCM OU=Ceres -# Subject: CN=AC RAIZ FNMT-RCM SERVIDORES SEGUROS O=FNMT-RCM OU=Ceres -# Label: "AC RAIZ FNMT-RCM SERVIDORES SEGUROS" -# Serial: 131542671362353147877283741781055151509 -# MD5 Fingerprint: 19:36:9c:52:03:2f:d2:d1:bb:23:cc:dd:1e:12:55:bb -# SHA1 Fingerprint: 62:ff:d9:9e:c0:65:0d:03:ce:75:93:d2:ed:3f:2d:32:c9:e3:e5:4a -# SHA256 Fingerprint: 55:41:53:b1:3d:2c:f9:dd:b7:53:bf:be:1a:4e:0a:e0:8d:0a:a4:18:70:58:fe:60:a2:b8:62:b2:e4:b8:7b:cb ------BEGIN CERTIFICATE----- -MIICbjCCAfOgAwIBAgIQYvYybOXE42hcG2LdnC6dlTAKBggqhkjOPQQDAzB4MQsw -CQYDVQQGEwJFUzERMA8GA1UECgwIRk5NVC1SQ00xDjAMBgNVBAsMBUNlcmVzMRgw -FgYDVQRhDA9WQVRFUy1RMjgyNjAwNEoxLDAqBgNVBAMMI0FDIFJBSVogRk5NVC1S -Q00gU0VSVklET1JFUyBTRUdVUk9TMB4XDTE4MTIyMDA5MzczM1oXDTQzMTIyMDA5 -MzczM1oweDELMAkGA1UEBhMCRVMxETAPBgNVBAoMCEZOTVQtUkNNMQ4wDAYDVQQL -DAVDZXJlczEYMBYGA1UEYQwPVkFURVMtUTI4MjYwMDRKMSwwKgYDVQQDDCNBQyBS -QUlaIEZOTVQtUkNNIFNFUlZJRE9SRVMgU0VHVVJPUzB2MBAGByqGSM49AgEGBSuB -BAAiA2IABPa6V1PIyqvfNkpSIeSX0oNnnvBlUdBeh8dHsVnyV0ebAAKTRBdp20LH -sbI6GA60XYyzZl2hNPk2LEnb80b8s0RpRBNm/dfF/a82Tc4DTQdxz69qBdKiQ1oK -Um8BA06Oi6NCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD -VR0OBBYEFAG5L++/EYZg8k/QQW6rcx/n0m5JMAoGCCqGSM49BAMDA2kAMGYCMQCu -SuMrQMN0EfKVrRYj3k4MGuZdpSRea0R7/DjiT8ucRRcRTBQnJlU5dUoDzBOQn5IC -MQD6SmxgiHPz7riYYqnOK8LZiqZwMR2vsJRM60/G49HzYqc8/5MuB1xJAWdpEgJy -v+c= ------END CERTIFICATE----- - -# Issuer: CN=GlobalSign Root R46 O=GlobalSign nv-sa -# Subject: CN=GlobalSign Root R46 O=GlobalSign nv-sa -# Label: "GlobalSign Root R46" -# Serial: 1552617688466950547958867513931858518042577 -# MD5 Fingerprint: c4:14:30:e4:fa:66:43:94:2a:6a:1b:24:5f:19:d0:ef -# SHA1 Fingerprint: 53:a2:b0:4b:ca:6b:d6:45:e6:39:8a:8e:c4:0d:d2:bf:77:c3:a2:90 -# SHA256 Fingerprint: 4f:a3:12:6d:8d:3a:11:d1:c4:85:5a:4f:80:7c:ba:d6:cf:91:9d:3a:5a:88:b0:3b:ea:2c:63:72:d9:3c:40:c9 ------BEGIN CERTIFICATE----- -MIIFWjCCA0KgAwIBAgISEdK7udcjGJ5AXwqdLdDfJWfRMA0GCSqGSIb3DQEBDAUA -MEYxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMRwwGgYD -VQQDExNHbG9iYWxTaWduIFJvb3QgUjQ2MB4XDTE5MDMyMDAwMDAwMFoXDTQ2MDMy -MDAwMDAwMFowRjELMAkGA1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYt -c2ExHDAaBgNVBAMTE0dsb2JhbFNpZ24gUm9vdCBSNDYwggIiMA0GCSqGSIb3DQEB -AQUAA4ICDwAwggIKAoICAQCsrHQy6LNl5brtQyYdpokNRbopiLKkHWPd08EsCVeJ -OaFV6Wc0dwxu5FUdUiXSE2te4R2pt32JMl8Nnp8semNgQB+msLZ4j5lUlghYruQG -vGIFAha/r6gjA7aUD7xubMLL1aa7DOn2wQL7Id5m3RerdELv8HQvJfTqa1VbkNud -316HCkD7rRlr+/fKYIje2sGP1q7Vf9Q8g+7XFkyDRTNrJ9CG0Bwta/OrffGFqfUo -0q3v84RLHIf8E6M6cqJaESvWJ3En7YEtbWaBkoe0G1h6zD8K+kZPTXhc+CtI4wSE -y132tGqzZfxCnlEmIyDLPRT5ge1lFgBPGmSXZgjPjHvjK8Cd+RTyG/FWaha/LIWF -zXg4mutCagI0GIMXTpRW+LaCtfOW3T3zvn8gdz57GSNrLNRyc0NXfeD412lPFzYE -+cCQYDdF3uYM2HSNrpyibXRdQr4G9dlkbgIQrImwTDsHTUB+JMWKmIJ5jqSngiCN -I/onccnfxkF0oE32kRbcRoxfKWMxWXEM2G/CtjJ9++ZdU6Z+Ffy7dXxd7Pj2Fxzs -x2sZy/N78CsHpdlseVR2bJ0cpm4O6XkMqCNqo98bMDGfsVR7/mrLZqrcZdCinkqa -ByFrgY/bxFn63iLABJzjqls2k+g9vXqhnQt2sQvHnf3PmKgGwvgqo6GDoLclcqUC -4wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV -HQ4EFgQUA1yrc4GHqMywptWU4jaWSf8FmSwwDQYJKoZIhvcNAQEMBQADggIBAHx4 -7PYCLLtbfpIrXTncvtgdokIzTfnvpCo7RGkerNlFo048p9gkUbJUHJNOxO97k4Vg -JuoJSOD1u8fpaNK7ajFxzHmuEajwmf3lH7wvqMxX63bEIaZHU1VNaL8FpO7XJqti -2kM3S+LGteWygxk6x9PbTZ4IevPuzz5i+6zoYMzRx6Fcg0XERczzF2sUyQQCPtIk -pnnpHs6i58FZFZ8d4kuaPp92CC1r2LpXFNqD6v6MVenQTqnMdzGxRBF6XLE+0xRF -FRhiJBPSy03OXIPBNvIQtQ6IbbjhVp+J3pZmOUdkLG5NrmJ7v2B0GbhWrJKsFjLt -rWhV/pi60zTe9Mlhww6G9kuEYO4Ne7UyWHmRVSyBQ7N0H3qqJZ4d16GLuc1CLgSk -ZoNNiTW2bKg2SnkheCLQQrzRQDGQob4Ez8pn7fXwgNNgyYMqIgXQBztSvwyeqiv5 -u+YfjyW6hY0XHgL+XVAEV8/+LbzvXMAaq7afJMbfc2hIkCwU9D9SGuTSyxTDYWnP -4vkYxboznxSjBF25cfe1lNj2M8FawTSLfJvdkzrnE6JwYZ+vj+vYxXX4M2bUdGc6 -N3ec592kD3ZDZopD8p/7DEJ4Y9HiD2971KE9dJeFt0g5QdYg/NA6s/rob8SKunE3 -vouXsXgxT7PntgMTzlSdriVZzH81Xwj3QEUxeCp6 ------END CERTIFICATE----- - -# Issuer: CN=GlobalSign Root E46 O=GlobalSign nv-sa -# Subject: CN=GlobalSign Root E46 O=GlobalSign nv-sa -# Label: "GlobalSign Root E46" -# Serial: 1552617690338932563915843282459653771421763 -# MD5 Fingerprint: b5:b8:66:ed:de:08:83:e3:c9:e2:01:34:06:ac:51:6f -# SHA1 Fingerprint: 39:b4:6c:d5:fe:80:06:eb:e2:2f:4a:bb:08:33:a0:af:db:b9:dd:84 -# SHA256 Fingerprint: cb:b9:c4:4d:84:b8:04:3e:10:50:ea:31:a6:9f:51:49:55:d7:bf:d2:e2:c6:b4:93:01:01:9a:d6:1d:9f:50:58 ------BEGIN CERTIFICATE----- -MIICCzCCAZGgAwIBAgISEdK7ujNu1LzmJGjFDYQdmOhDMAoGCCqGSM49BAMDMEYx -CzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMRwwGgYDVQQD -ExNHbG9iYWxTaWduIFJvb3QgRTQ2MB4XDTE5MDMyMDAwMDAwMFoXDTQ2MDMyMDAw -MDAwMFowRjELMAkGA1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2Ex -HDAaBgNVBAMTE0dsb2JhbFNpZ24gUm9vdCBFNDYwdjAQBgcqhkjOPQIBBgUrgQQA -IgNiAAScDrHPt+ieUnd1NPqlRqetMhkytAepJ8qUuwzSChDH2omwlwxwEwkBjtjq -R+q+soArzfwoDdusvKSGN+1wCAB16pMLey5SnCNoIwZD7JIvU4Tb+0cUB+hflGdd -yXqBPCCjQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud -DgQWBBQxCpCPtsad0kRLgLWi5h+xEk8blTAKBggqhkjOPQQDAwNoADBlAjEA31SQ -7Zvvi5QCkxeCmb6zniz2C5GMn0oUsfZkvLtoURMMA/cVi4RguYv/Uo7njLwcAjA8 -+RHUjE7AwWHCFUyqqx0LMV87HOIAl0Qx5v5zli/altP+CAezNIm8BZ/3Hobui3A= ------END CERTIFICATE----- - -# Issuer: CN=ANF Secure Server Root CA O=ANF Autoridad de Certificacion OU=ANF CA Raiz -# Subject: CN=ANF Secure Server Root CA O=ANF Autoridad de Certificacion OU=ANF CA Raiz -# Label: "ANF Secure Server Root CA" -# Serial: 996390341000653745 -# MD5 Fingerprint: 26:a6:44:5a:d9:af:4e:2f:b2:1d:b6:65:b0:4e:e8:96 -# SHA1 Fingerprint: 5b:6e:68:d0:cc:15:b6:a0:5f:1e:c1:5f:ae:02:fc:6b:2f:5d:6f:74 -# SHA256 Fingerprint: fb:8f:ec:75:91:69:b9:10:6b:1e:51:16:44:c6:18:c5:13:04:37:3f:6c:06:43:08:8d:8b:ef:fd:1b:99:75:99 ------BEGIN CERTIFICATE----- -MIIF7zCCA9egAwIBAgIIDdPjvGz5a7EwDQYJKoZIhvcNAQELBQAwgYQxEjAQBgNV -BAUTCUc2MzI4NzUxMDELMAkGA1UEBhMCRVMxJzAlBgNVBAoTHkFORiBBdXRvcmlk -YWQgZGUgQ2VydGlmaWNhY2lvbjEUMBIGA1UECxMLQU5GIENBIFJhaXoxIjAgBgNV -BAMTGUFORiBTZWN1cmUgU2VydmVyIFJvb3QgQ0EwHhcNMTkwOTA0MTAwMDM4WhcN -MzkwODMwMTAwMDM4WjCBhDESMBAGA1UEBRMJRzYzMjg3NTEwMQswCQYDVQQGEwJF -UzEnMCUGA1UEChMeQU5GIEF1dG9yaWRhZCBkZSBDZXJ0aWZpY2FjaW9uMRQwEgYD -VQQLEwtBTkYgQ0EgUmFpejEiMCAGA1UEAxMZQU5GIFNlY3VyZSBTZXJ2ZXIgUm9v -dCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANvrayvmZFSVgpCj -cqQZAZ2cC4Ffc0m6p6zzBE57lgvsEeBbphzOG9INgxwruJ4dfkUyYA8H6XdYfp9q -yGFOtibBTI3/TO80sh9l2Ll49a2pcbnvT1gdpd50IJeh7WhM3pIXS7yr/2WanvtH -2Vdy8wmhrnZEE26cLUQ5vPnHO6RYPUG9tMJJo8gN0pcvB2VSAKduyK9o7PQUlrZX -H1bDOZ8rbeTzPvY1ZNoMHKGESy9LS+IsJJ1tk0DrtSOOMspvRdOoiXsezx76W0OL -zc2oD2rKDF65nkeP8Nm2CgtYZRczuSPkdxl9y0oukntPLxB3sY0vaJxizOBQ+OyR -p1RMVwnVdmPF6GUe7m1qzwmd+nxPrWAI/VaZDxUse6mAq4xhj0oHdkLePfTdsiQz -W7i1o0TJrH93PB0j7IKppuLIBkwC/qxcmZkLLxCKpvR/1Yd0DVlJRfbwcVw5Kda/ -SiOL9V8BY9KHcyi1Swr1+KuCLH5zJTIdC2MKF4EA/7Z2Xue0sUDKIbvVgFHlSFJn -LNJhiQcND85Cd8BEc5xEUKDbEAotlRyBr+Qc5RQe8TZBAQIvfXOn3kLMTOmJDVb3 -n5HUA8ZsyY/b2BzgQJhdZpmYgG4t/wHFzstGH6wCxkPmrqKEPMVOHj1tyRRM4y5B -u8o5vzY8KhmqQYdOpc5LMnndkEl/AgMBAAGjYzBhMB8GA1UdIwQYMBaAFJxf0Gxj -o1+TypOYCK2Mh6UsXME3MB0GA1UdDgQWBBScX9BsY6Nfk8qTmAitjIelLFzBNzAO -BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC -AgEATh65isagmD9uw2nAalxJUqzLK114OMHVVISfk/CHGT0sZonrDUL8zPB1hT+L -9IBdeeUXZ701guLyPI59WzbLWoAAKfLOKyzxj6ptBZNscsdW699QIyjlRRA96Gej -rw5VD5AJYu9LWaL2U/HANeQvwSS9eS9OICI7/RogsKQOLHDtdD+4E5UGUcjohybK -pFtqFiGS3XNgnhAY3jyB6ugYw3yJ8otQPr0R4hUDqDZ9MwFsSBXXiJCZBMXM5gf0 -vPSQ7RPi6ovDj6MzD8EpTBNO2hVWcXNyglD2mjN8orGoGjR0ZVzO0eurU+AagNjq -OknkJjCb5RyKqKkVMoaZkgoQI1YS4PbOTOK7vtuNknMBZi9iPrJyJ0U27U1W45eZ -/zo1PqVUSlJZS2Db7v54EX9K3BR5YLZrZAPbFYPhor72I5dQ8AkzNqdxliXzuUJ9 -2zg/LFis6ELhDtjTO0wugumDLmsx2d1Hhk9tl5EuT+IocTUW0fJz/iUrB0ckYyfI -+PbZa/wSMVYIwFNCr5zQM378BvAxRAMU8Vjq8moNqRGyg77FGr8H6lnco4g175x2 -MjxNBiLOFeXdntiP2t7SxDnlF4HPOEfrf4htWRvfn0IUrn7PqLBmZdo3r5+qPeoo -tt7VMVgWglvquxl1AnMaykgaIZOQCo6ThKd9OyMYkomgjaw= ------END CERTIFICATE----- - -# Issuer: CN=Certum EC-384 CA O=Asseco Data Systems S.A. OU=Certum Certification Authority -# Subject: CN=Certum EC-384 CA O=Asseco Data Systems S.A. OU=Certum Certification Authority -# Label: "Certum EC-384 CA" -# Serial: 160250656287871593594747141429395092468 -# MD5 Fingerprint: b6:65:b3:96:60:97:12:a1:ec:4e:e1:3d:a3:c6:c9:f1 -# SHA1 Fingerprint: f3:3e:78:3c:ac:df:f4:a2:cc:ac:67:55:69:56:d7:e5:16:3c:e1:ed -# SHA256 Fingerprint: 6b:32:80:85:62:53:18:aa:50:d1:73:c9:8d:8b:da:09:d5:7e:27:41:3d:11:4c:f7:87:a0:f5:d0:6c:03:0c:f6 ------BEGIN CERTIFICATE----- -MIICZTCCAeugAwIBAgIQeI8nXIESUiClBNAt3bpz9DAKBggqhkjOPQQDAzB0MQsw -CQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBTLkEuMScw -JQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxGTAXBgNVBAMT -EENlcnR1bSBFQy0zODQgQ0EwHhcNMTgwMzI2MDcyNDU0WhcNNDMwMzI2MDcyNDU0 -WjB0MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBT -LkEuMScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxGTAX -BgNVBAMTEENlcnR1bSBFQy0zODQgQ0EwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAATE -KI6rGFtqvm5kN2PkzeyrOvfMobgOgknXhimfoZTy42B4mIF4Bk3y7JoOV2CDn7Tm -Fy8as10CW4kjPMIRBSqniBMY81CE1700LCeJVf/OTOffph8oxPBUw7l8t1Ot68Kj -QjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI0GZnQkdjrzife81r1HfS+8 -EF9LMA4GA1UdDwEB/wQEAwIBBjAKBggqhkjOPQQDAwNoADBlAjADVS2m5hjEfO/J -UG7BJw+ch69u1RsIGL2SKcHvlJF40jocVYli5RsJHrpka/F2tNQCMQC0QoSZ/6vn -nvuRlydd3LBbMHHOXjgaatkl5+r3YZJW+OraNsKHZZYuciUvf9/DE8k= ------END CERTIFICATE----- - -# Issuer: CN=Certum Trusted Root CA O=Asseco Data Systems S.A. OU=Certum Certification Authority -# Subject: CN=Certum Trusted Root CA O=Asseco Data Systems S.A. OU=Certum Certification Authority -# Label: "Certum Trusted Root CA" -# Serial: 40870380103424195783807378461123655149 -# MD5 Fingerprint: 51:e1:c2:e7:fe:4c:84:af:59:0e:2f:f4:54:6f:ea:29 -# SHA1 Fingerprint: c8:83:44:c0:18:ae:9f:cc:f1:87:b7:8f:22:d1:c5:d7:45:84:ba:e5 -# SHA256 Fingerprint: fe:76:96:57:38:55:77:3e:37:a9:5e:7a:d4:d9:cc:96:c3:01:57:c1:5d:31:76:5b:a9:b1:57:04:e1:ae:78:fd ------BEGIN CERTIFICATE----- -MIIFwDCCA6igAwIBAgIQHr9ZULjJgDdMBvfrVU+17TANBgkqhkiG9w0BAQ0FADB6 -MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBTLkEu -MScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxHzAdBgNV -BAMTFkNlcnR1bSBUcnVzdGVkIFJvb3QgQ0EwHhcNMTgwMzE2MTIxMDEzWhcNNDMw -MzE2MTIxMDEzWjB6MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEg -U3lzdGVtcyBTLkEuMScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRo -b3JpdHkxHzAdBgNVBAMTFkNlcnR1bSBUcnVzdGVkIFJvb3QgQ0EwggIiMA0GCSqG -SIb3DQEBAQUAA4ICDwAwggIKAoICAQDRLY67tzbqbTeRn06TpwXkKQMlzhyC93yZ -n0EGze2jusDbCSzBfN8pfktlL5On1AFrAygYo9idBcEq2EXxkd7fO9CAAozPOA/q -p1x4EaTByIVcJdPTsuclzxFUl6s1wB52HO8AU5853BSlLCIls3Jy/I2z5T4IHhQq -NwuIPMqw9MjCoa68wb4pZ1Xi/K1ZXP69VyywkI3C7Te2fJmItdUDmj0VDT06qKhF -8JVOJVkdzZhpu9PMMsmN74H+rX2Ju7pgE8pllWeg8xn2A1bUatMn4qGtg/BKEiJ3 -HAVz4hlxQsDsdUaakFjgao4rpUYwBI4Zshfjvqm6f1bxJAPXsiEodg42MEx51UGa -mqi4NboMOvJEGyCI98Ul1z3G4z5D3Yf+xOr1Uz5MZf87Sst4WmsXXw3Hw09Omiqi -7VdNIuJGmj8PkTQkfVXjjJU30xrwCSss0smNtA0Aq2cpKNgB9RkEth2+dv5yXMSF -ytKAQd8FqKPVhJBPC/PgP5sZ0jeJP/J7UhyM9uH3PAeXjA6iWYEMspA90+NZRu0P -qafegGtaqge2Gcu8V/OXIXoMsSt0Puvap2ctTMSYnjYJdmZm/Bo/6khUHL4wvYBQ -v3y1zgD2DGHZ5yQD4OMBgQ692IU0iL2yNqh7XAjlRICMb/gv1SHKHRzQ+8S1h9E6 -Tsd2tTVItQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSM+xx1 -vALTn04uSNn5YFSqxLNP+jAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQENBQAD -ggIBAEii1QALLtA/vBzVtVRJHlpr9OTy4EA34MwUe7nJ+jW1dReTagVphZzNTxl4 -WxmB82M+w85bj/UvXgF2Ez8sALnNllI5SW0ETsXpD4YN4fqzX4IS8TrOZgYkNCvo -zMrnadyHncI013nR03e4qllY/p0m+jiGPp2Kh2RX5Rc64vmNueMzeMGQ2Ljdt4NR -5MTMI9UGfOZR0800McD2RrsLrfw9EAUqO0qRJe6M1ISHgCq8CYyqOhNf6DR5UMEQ -GfnTKB7U0VEwKbOukGfWHwpjscWpxkIxYxeU72nLL/qMFH3EQxiJ2fAyQOaA4kZf -5ePBAFmo+eggvIksDkc0C+pXwlM2/KfUrzHN/gLldfq5Jwn58/U7yn2fqSLLiMmq -0Uc9NneoWWRrJ8/vJ8HjJLWG965+Mk2weWjROeiQWMODvA8s1pfrzgzhIMfatz7D -P78v3DSk+yshzWePS/Tj6tQ/50+6uaWTRRxmHyH6ZF5v4HaUMst19W7l9o/HuKTM -qJZ9ZPskWkoDbGs4xugDQ5r3V7mzKWmTOPQD8rv7gmsHINFSH5pkAnuYZttcTVoP -0ISVoDwUQwbKytu4QTbaakRnh6+v40URFWkIsr4WOZckbxJF0WddCajJFdr60qZf -E2Efv4WstK2tBZQIgx51F9NxO5NQI1mg7TyRVJ12AMXDuDjb ------END CERTIFICATE----- - -# Issuer: CN=TunTrust Root CA O=Agence Nationale de Certification Electronique -# Subject: CN=TunTrust Root CA O=Agence Nationale de Certification Electronique -# Label: "TunTrust Root CA" -# Serial: 108534058042236574382096126452369648152337120275 -# MD5 Fingerprint: 85:13:b9:90:5b:36:5c:b6:5e:b8:5a:f8:e0:31:57:b4 -# SHA1 Fingerprint: cf:e9:70:84:0f:e0:73:0f:9d:f6:0c:7f:2c:4b:ee:20:46:34:9c:bb -# SHA256 Fingerprint: 2e:44:10:2a:b5:8c:b8:54:19:45:1c:8e:19:d9:ac:f3:66:2c:af:bc:61:4b:6a:53:96:0a:30:f7:d0:e2:eb:41 ------BEGIN CERTIFICATE----- -MIIFszCCA5ugAwIBAgIUEwLV4kBMkkaGFmddtLu7sms+/BMwDQYJKoZIhvcNAQEL -BQAwYTELMAkGA1UEBhMCVE4xNzA1BgNVBAoMLkFnZW5jZSBOYXRpb25hbGUgZGUg -Q2VydGlmaWNhdGlvbiBFbGVjdHJvbmlxdWUxGTAXBgNVBAMMEFR1blRydXN0IFJv -b3QgQ0EwHhcNMTkwNDI2MDg1NzU2WhcNNDQwNDI2MDg1NzU2WjBhMQswCQYDVQQG -EwJUTjE3MDUGA1UECgwuQWdlbmNlIE5hdGlvbmFsZSBkZSBDZXJ0aWZpY2F0aW9u -IEVsZWN0cm9uaXF1ZTEZMBcGA1UEAwwQVHVuVHJ1c3QgUm9vdCBDQTCCAiIwDQYJ -KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMPN0/y9BFPdDCA61YguBUtB9YOCfvdZ -n56eY+hz2vYGqU8ftPkLHzmMmiDQfgbU7DTZhrx1W4eI8NLZ1KMKsmwb60ksPqxd -2JQDoOw05TDENX37Jk0bbjBU2PWARZw5rZzJJQRNmpA+TkBuimvNKWfGzC3gdOgF -VwpIUPp6Q9p+7FuaDmJ2/uqdHYVy7BG7NegfJ7/Boce7SBbdVtfMTqDhuazb1YMZ -GoXRlJfXyqNlC/M4+QKu3fZnz8k/9YosRxqZbwUN/dAdgjH8KcwAWJeRTIAAHDOF -li/LQcKLEITDCSSJH7UP2dl3RxiSlGBcx5kDPP73lad9UKGAwqmDrViWVSHbhlnU -r8a83YFuB9tgYv7sEG7aaAH0gxupPqJbI9dkxt/con3YS7qC0lH4Zr8GRuR5KiY2 -eY8fTpkdso8MDhz/yV3A/ZAQprE38806JG60hZC/gLkMjNWb1sjxVj8agIl6qeIb -MlEsPvLfe/ZdeikZjuXIvTZxi11Mwh0/rViizz1wTaZQmCXcI/m4WEEIcb9PuISg -jwBUFfyRbVinljvrS5YnzWuioYasDXxU5mZMZl+QviGaAkYt5IPCgLnPSz7ofzwB -7I9ezX/SKEIBlYrilz0QIX32nRzFNKHsLA4KUiwSVXAkPcvCFDVDXSdOvsC9qnyW -5/yeYa1E0wCXAgMBAAGjYzBhMB0GA1UdDgQWBBQGmpsfU33x9aTI04Y+oXNZtPdE -ITAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFAaamx9TffH1pMjThj6hc1m0 -90QhMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAqgVutt0Vyb+z -xiD2BkewhpMl0425yAA/l/VSJ4hxyXT968pk21vvHl26v9Hr7lxpuhbI87mP0zYu -QEkHDVneixCwSQXi/5E/S7fdAo74gShczNxtr18UnH1YeA32gAm56Q6XKRm4t+v4 -FstVEuTGfbvE7Pi1HE4+Z7/FXxttbUcoqgRYYdZ2vyJ/0Adqp2RT8JeNnYA/u8EH -22Wv5psymsNUk8QcCMNE+3tjEUPRahphanltkE8pjkcFwRJpadbGNjHh/PqAulxP -xOu3Mqz4dWEX1xAZufHSCe96Qp1bWgvUxpVOKs7/B9dPfhgGiPEZtdmYu65xxBzn -dFlY7wyJz4sfdZMaBBSSSFCp61cpABbjNhzI+L/wM9VBD8TMPN3pM0MBkRArHtG5 -Xc0yGYuPjCB31yLEQtyEFpslbei0VXF/sHyz03FJuc9SpAQ/3D2gu68zngowYI7b -nV2UqL1g52KAdoGDDIzMMEZJ4gzSqK/rYXHv5yJiqfdcZGyfFoxnNidF9Ql7v/YQ -CvGwjVRDjAS6oz/v4jXH+XTgbzRB0L9zZVcg+ZtnemZoJE6AZb0QmQZZ8mWvuMZH -u/2QeItBcy6vVR/cO5JyboTT0GFMDcx2V+IthSIVNg3rAZ3r2OvEhJn7wAzMMujj -d9qDRIueVSjAi1jTkD5OGwDxFa2DK5o= ------END CERTIFICATE----- - -# Issuer: CN=HARICA TLS RSA Root CA 2021 O=Hellenic Academic and Research Institutions CA -# Subject: CN=HARICA TLS RSA Root CA 2021 O=Hellenic Academic and Research Institutions CA -# Label: "HARICA TLS RSA Root CA 2021" -# Serial: 76817823531813593706434026085292783742 -# MD5 Fingerprint: 65:47:9b:58:86:dd:2c:f0:fc:a2:84:1f:1e:96:c4:91 -# SHA1 Fingerprint: 02:2d:05:82:fa:88:ce:14:0c:06:79:de:7f:14:10:e9:45:d7:a5:6d -# SHA256 Fingerprint: d9:5d:0e:8e:da:79:52:5b:f9:be:b1:1b:14:d2:10:0d:32:94:98:5f:0c:62:d9:fa:bd:9c:d9:99:ec:cb:7b:1d ------BEGIN CERTIFICATE----- -MIIFpDCCA4ygAwIBAgIQOcqTHO9D88aOk8f0ZIk4fjANBgkqhkiG9w0BAQsFADBs -MQswCQYDVQQGEwJHUjE3MDUGA1UECgwuSGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJl -c2VhcmNoIEluc3RpdHV0aW9ucyBDQTEkMCIGA1UEAwwbSEFSSUNBIFRMUyBSU0Eg -Um9vdCBDQSAyMDIxMB4XDTIxMDIxOTEwNTUzOFoXDTQ1MDIxMzEwNTUzN1owbDEL -MAkGA1UEBhMCR1IxNzA1BgNVBAoMLkhlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNl -YXJjaCBJbnN0aXR1dGlvbnMgQ0ExJDAiBgNVBAMMG0hBUklDQSBUTFMgUlNBIFJv -b3QgQ0EgMjAyMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAIvC569l -mwVnlskNJLnQDmT8zuIkGCyEf3dRywQRNrhe7Wlxp57kJQmXZ8FHws+RFjZiPTgE -4VGC/6zStGndLuwRo0Xua2s7TL+MjaQenRG56Tj5eg4MmOIjHdFOY9TnuEFE+2uv -a9of08WRiFukiZLRgeaMOVig1mlDqa2YUlhu2wr7a89o+uOkXjpFc5gH6l8Cct4M -pbOfrqkdtx2z/IpZ525yZa31MJQjB/OCFks1mJxTuy/K5FrZx40d/JiZ+yykgmvw -Kh+OC19xXFyuQnspiYHLA6OZyoieC0AJQTPb5lh6/a6ZcMBaD9YThnEvdmn8kN3b -LW7R8pv1GmuebxWMevBLKKAiOIAkbDakO/IwkfN4E8/BPzWr8R0RI7VDIp4BkrcY -AuUR0YLbFQDMYTfBKnya4dC6s1BG7oKsnTH4+yPiAwBIcKMJJnkVU2DzOFytOOqB -AGMUuTNe3QvboEUHGjMJ+E20pwKmafTCWQWIZYVWrkvL4N48fS0ayOn7H6NhStYq -E613TBoYm5EPWNgGVMWX+Ko/IIqmhaZ39qb8HOLubpQzKoNQhArlT4b4UEV4AIHr -W2jjJo3Me1xR9BQsQL4aYB16cmEdH2MtiKrOokWQCPxrvrNQKlr9qEgYRtaQQJKQ -CoReaDH46+0N0x3GfZkYVVYnZS6NRcUk7M7jAgMBAAGjQjBAMA8GA1UdEwEB/wQF -MAMBAf8wHQYDVR0OBBYEFApII6ZgpJIKM+qTW8VX6iVNvRLuMA4GA1UdDwEB/wQE -AwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAPpBIqm5iFSVmewzVjIuJndftTgfvnNAU -X15QvWiWkKQUEapobQk1OUAJ2vQJLDSle1mESSmXdMgHHkdt8s4cUCbjnj1AUz/3 -f5Z2EMVGpdAgS1D0NTsY9FVqQRtHBmg8uwkIYtlfVUKqrFOFrJVWNlar5AWMxaja -H6NpvVMPxP/cyuN+8kyIhkdGGvMA9YCRotxDQpSbIPDRzbLrLFPCU3hKTwSUQZqP -JzLB5UkZv/HywouoCjkxKLR9YjYsTewfM7Z+d21+UPCfDtcRj88YxeMn/ibvBZ3P -zzfF0HvaO7AWhAw6k9a+F9sPPg4ZeAnHqQJyIkv3N3a6dcSFA1pj1bF1BcK5vZSt -jBWZp5N99sXzqnTPBIWUmAD04vnKJGW/4GKvyMX6ssmeVkjaef2WdhW+o45WxLM0 -/L5H9MG0qPzVMIho7suuyWPEdr6sOBjhXlzPrjoiUevRi7PzKzMHVIf6tLITe7pT -BGIBnfHAT+7hOtSLIBD6Alfm78ELt5BGnBkpjNxvoEppaZS3JGWg/6w/zgH7IS79 -aPib8qXPMThcFarmlwDB31qlpzmq6YR/PFGoOtmUW4y/Twhx5duoXNTSpv4Ao8YW -xw/ogM4cKGR0GQjTQuPOAF1/sdwTsOEFy9EgqoZ0njnnkf3/W9b3raYvAwtt41dU -63ZTGI0RmLo= ------END CERTIFICATE----- - -# Issuer: CN=HARICA TLS ECC Root CA 2021 O=Hellenic Academic and Research Institutions CA -# Subject: CN=HARICA TLS ECC Root CA 2021 O=Hellenic Academic and Research Institutions CA -# Label: "HARICA TLS ECC Root CA 2021" -# Serial: 137515985548005187474074462014555733966 -# MD5 Fingerprint: ae:f7:4c:e5:66:35:d1:b7:9b:8c:22:93:74:d3:4b:b0 -# SHA1 Fingerprint: bc:b0:c1:9d:e9:98:92:70:19:38:57:e9:8d:a7:b4:5d:6e:ee:01:48 -# SHA256 Fingerprint: 3f:99:cc:47:4a:cf:ce:4d:fe:d5:87:94:66:5e:47:8d:15:47:73:9f:2e:78:0f:1b:b4:ca:9b:13:30:97:d4:01 ------BEGIN CERTIFICATE----- -MIICVDCCAdugAwIBAgIQZ3SdjXfYO2rbIvT/WeK/zjAKBggqhkjOPQQDAzBsMQsw -CQYDVQQGEwJHUjE3MDUGA1UECgwuSGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJlc2Vh -cmNoIEluc3RpdHV0aW9ucyBDQTEkMCIGA1UEAwwbSEFSSUNBIFRMUyBFQ0MgUm9v -dCBDQSAyMDIxMB4XDTIxMDIxOTExMDExMFoXDTQ1MDIxMzExMDEwOVowbDELMAkG -A1UEBhMCR1IxNzA1BgNVBAoMLkhlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJj -aCBJbnN0aXR1dGlvbnMgQ0ExJDAiBgNVBAMMG0hBUklDQSBUTFMgRUNDIFJvb3Qg -Q0EgMjAyMTB2MBAGByqGSM49AgEGBSuBBAAiA2IABDgI/rGgltJ6rK9JOtDA4MM7 -KKrxcm1lAEeIhPyaJmuqS7psBAqIXhfyVYf8MLA04jRYVxqEU+kw2anylnTDUR9Y -STHMmE5gEYd103KUkE+bECUqqHgtvpBBWJAVcqeht6NCMEAwDwYDVR0TAQH/BAUw -AwEB/zAdBgNVHQ4EFgQUyRtTgRL+BNUW0aq8mm+3oJUZbsowDgYDVR0PAQH/BAQD -AgGGMAoGCCqGSM49BAMDA2cAMGQCMBHervjcToiwqfAircJRQO9gcS3ujwLEXQNw -SaSS6sUUiHCm0w2wqsosQJz76YJumgIwK0eaB8bRwoF8yguWGEEbo/QwCZ61IygN -nxS2PFOiTAZpffpskcYqSUXm7LcT4Tps ------END CERTIFICATE----- - -# Issuer: CN=Autoridad de Certificacion Firmaprofesional CIF A62634068 -# Subject: CN=Autoridad de Certificacion Firmaprofesional CIF A62634068 -# Label: "Autoridad de Certificacion Firmaprofesional CIF A62634068" -# Serial: 1977337328857672817 -# MD5 Fingerprint: 4e:6e:9b:54:4c:ca:b7:fa:48:e4:90:b1:15:4b:1c:a3 -# SHA1 Fingerprint: 0b:be:c2:27:22:49:cb:39:aa:db:35:5c:53:e3:8c:ae:78:ff:b6:fe -# SHA256 Fingerprint: 57:de:05:83:ef:d2:b2:6e:03:61:da:99:da:9d:f4:64:8d:ef:7e:e8:44:1c:3b:72:8a:fa:9b:cd:e0:f9:b2:6a ------BEGIN CERTIFICATE----- -MIIGFDCCA/ygAwIBAgIIG3Dp0v+ubHEwDQYJKoZIhvcNAQELBQAwUTELMAkGA1UE -BhMCRVMxQjBABgNVBAMMOUF1dG9yaWRhZCBkZSBDZXJ0aWZpY2FjaW9uIEZpcm1h -cHJvZmVzaW9uYWwgQ0lGIEE2MjYzNDA2ODAeFw0xNDA5MjMxNTIyMDdaFw0zNjA1 -MDUxNTIyMDdaMFExCzAJBgNVBAYTAkVTMUIwQAYDVQQDDDlBdXRvcmlkYWQgZGUg -Q2VydGlmaWNhY2lvbiBGaXJtYXByb2Zlc2lvbmFsIENJRiBBNjI2MzQwNjgwggIi -MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDKlmuO6vj78aI14H9M2uDDUtd9 -thDIAl6zQyrET2qyyhxdKJp4ERppWVevtSBC5IsP5t9bpgOSL/UR5GLXMnE42QQM -cas9UX4PB99jBVzpv5RvwSmCwLTaUbDBPLutN0pcyvFLNg4kq7/DhHf9qFD0sefG -L9ItWY16Ck6WaVICqjaY7Pz6FIMMNx/Jkjd/14Et5cS54D40/mf0PmbR0/RAz15i -NA9wBj4gGFrO93IbJWyTdBSTo3OxDqqHECNZXyAFGUftaI6SEspd/NYrspI8IM/h -X68gvqB2f3bl7BqGYTM+53u0P6APjqK5am+5hyZvQWyIplD9amML9ZMWGxmPsu2b -m8mQ9QEM3xk9Dz44I8kvjwzRAv4bVdZO0I08r0+k8/6vKtMFnXkIoctXMbScyJCy -Z/QYFpM6/EfY0XiWMR+6KwxfXZmtY4laJCB22N/9q06mIqqdXuYnin1oKaPnirja -EbsXLZmdEyRG98Xi2J+Of8ePdG1asuhy9azuJBCtLxTa/y2aRnFHvkLfuwHb9H/T -KI8xWVvTyQKmtFLKbpf7Q8UIJm+K9Lv9nyiqDdVF8xM6HdjAeI9BZzwelGSuewvF -6NkBiDkal4ZkQdU7hwxu+g/GvUgUvzlN1J5Bto+WHWOWk9mVBngxaJ43BjuAiUVh -OSPHG0SjFeUc+JIwuwIDAQABo4HvMIHsMB0GA1UdDgQWBBRlzeurNR4APn7VdMAc -tHNHDhpkLzASBgNVHRMBAf8ECDAGAQH/AgEBMIGmBgNVHSAEgZ4wgZswgZgGBFUd -IAAwgY8wLwYIKwYBBQUHAgEWI2h0dHA6Ly93d3cuZmlybWFwcm9mZXNpb25hbC5j -b20vY3BzMFwGCCsGAQUFBwICMFAeTgBQAGEAcwBlAG8AIABkAGUAIABsAGEAIABC -AG8AbgBhAG4AbwB2AGEAIAA0ADcAIABCAGEAcgBjAGUAbABvAG4AYQAgADAAOAAw -ADEANzAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQELBQADggIBAHSHKAIrdx9m -iWTtj3QuRhy7qPj4Cx2Dtjqn6EWKB7fgPiDL4QjbEwj4KKE1soCzC1HA01aajTNF -Sa9J8OA9B3pFE1r/yJfY0xgsfZb43aJlQ3CTkBW6kN/oGbDbLIpgD7dvlAceHabJ -hfa9NPhAeGIQcDq+fUs5gakQ1JZBu/hfHAsdCPKxsIl68veg4MSPi3i1O1ilI45P -Vf42O+AMt8oqMEEgtIDNrvx2ZnOorm7hfNoD6JQg5iKj0B+QXSBTFCZX2lSX3xZE -EAEeiGaPcjiT3SC3NL7X8e5jjkd5KAb881lFJWAiMxujX6i6KtoaPc1A6ozuBRWV -1aUsIC+nmCjuRfzxuIgALI9C2lHVnOUTaHFFQ4ueCyE8S1wF3BqfmI7avSKecs2t -CsvMo2ebKHTEm9caPARYpoKdrcd7b/+Alun4jWq9GJAd/0kakFI3ky88Al2CdgtR -5xbHV/g4+afNmyJU72OwFW1TZQNKXkqgsqeOSQBZONXH9IBk9W6VULgRfhVwOEqw -f9DEMnDAGf/JOC0ULGb0QkTmVXYbgBVX/8Cnp6o5qtjTcNAuuuuUavpfNIbnYrX9 -ivAwhZTJryQCL2/W3Wf+47BVTwSYT6RBVuKT0Gro1vP7ZeDOdcQxWQzugsgMYDNK -GbqEZycPvEJdvSRUDewdcAZfpLz6IHxV ------END CERTIFICATE----- - -# Issuer: CN=vTrus ECC Root CA O=iTrusChina Co.,Ltd. -# Subject: CN=vTrus ECC Root CA O=iTrusChina Co.,Ltd. -# Label: "vTrus ECC Root CA" -# Serial: 630369271402956006249506845124680065938238527194 -# MD5 Fingerprint: de:4b:c1:f5:52:8c:9b:43:e1:3e:8f:55:54:17:8d:85 -# SHA1 Fingerprint: f6:9c:db:b0:fc:f6:02:13:b6:52:32:a6:a3:91:3f:16:70:da:c3:e1 -# SHA256 Fingerprint: 30:fb:ba:2c:32:23:8e:2a:98:54:7a:f9:79:31:e5:50:42:8b:9b:3f:1c:8e:eb:66:33:dc:fa:86:c5:b2:7d:d3 ------BEGIN CERTIFICATE----- -MIICDzCCAZWgAwIBAgIUbmq8WapTvpg5Z6LSa6Q75m0c1towCgYIKoZIzj0EAwMw -RzELMAkGA1UEBhMCQ04xHDAaBgNVBAoTE2lUcnVzQ2hpbmEgQ28uLEx0ZC4xGjAY -BgNVBAMTEXZUcnVzIEVDQyBSb290IENBMB4XDTE4MDczMTA3MjY0NFoXDTQzMDcz -MTA3MjY0NFowRzELMAkGA1UEBhMCQ04xHDAaBgNVBAoTE2lUcnVzQ2hpbmEgQ28u -LEx0ZC4xGjAYBgNVBAMTEXZUcnVzIEVDQyBSb290IENBMHYwEAYHKoZIzj0CAQYF -K4EEACIDYgAEZVBKrox5lkqqHAjDo6LN/llWQXf9JpRCux3NCNtzslt188+cToL0 -v/hhJoVs1oVbcnDS/dtitN9Ti72xRFhiQgnH+n9bEOf+QP3A2MMrMudwpremIFUd -e4BdS49nTPEQo0IwQDAdBgNVHQ4EFgQUmDnNvtiyjPeyq+GtJK97fKHbH88wDwYD -VR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwCgYIKoZIzj0EAwMDaAAwZQIw -V53dVvHH4+m4SVBrm2nDb+zDfSXkV5UTQJtS0zvzQBm8JsctBp61ezaf9SXUY2sA -AjEA6dPGnlaaKsyh2j/IZivTWJwghfqrkYpwcBE4YGQLYgmRWAD5Tfs0aNoJrSEG -GJTO ------END CERTIFICATE----- - -# Issuer: CN=vTrus Root CA O=iTrusChina Co.,Ltd. -# Subject: CN=vTrus Root CA O=iTrusChina Co.,Ltd. -# Label: "vTrus Root CA" -# Serial: 387574501246983434957692974888460947164905180485 -# MD5 Fingerprint: b8:c9:37:df:fa:6b:31:84:64:c5:ea:11:6a:1b:75:fc -# SHA1 Fingerprint: 84:1a:69:fb:f5:cd:1a:25:34:13:3d:e3:f8:fc:b8:99:d0:c9:14:b7 -# SHA256 Fingerprint: 8a:71:de:65:59:33:6f:42:6c:26:e5:38:80:d0:0d:88:a1:8d:a4:c6:a9:1f:0d:cb:61:94:e2:06:c5:c9:63:87 ------BEGIN CERTIFICATE----- -MIIFVjCCAz6gAwIBAgIUQ+NxE9izWRRdt86M/TX9b7wFjUUwDQYJKoZIhvcNAQEL -BQAwQzELMAkGA1UEBhMCQ04xHDAaBgNVBAoTE2lUcnVzQ2hpbmEgQ28uLEx0ZC4x -FjAUBgNVBAMTDXZUcnVzIFJvb3QgQ0EwHhcNMTgwNzMxMDcyNDA1WhcNNDMwNzMx -MDcyNDA1WjBDMQswCQYDVQQGEwJDTjEcMBoGA1UEChMTaVRydXNDaGluYSBDby4s -THRkLjEWMBQGA1UEAxMNdlRydXMgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEBBQAD -ggIPADCCAgoCggIBAL1VfGHTuB0EYgWgrmy3cLRB6ksDXhA/kFocizuwZotsSKYc -IrrVQJLuM7IjWcmOvFjai57QGfIvWcaMY1q6n6MLsLOaXLoRuBLpDLvPbmyAhykU -AyyNJJrIZIO1aqwTLDPxn9wsYTwaP3BVm60AUn/PBLn+NvqcwBauYv6WTEN+VRS+ -GrPSbcKvdmaVayqwlHeFXgQPYh1jdfdr58tbmnDsPmcF8P4HCIDPKNsFxhQnL4Z9 -8Cfe/+Z+M0jnCx5Y0ScrUw5XSmXX+6KAYPxMvDVTAWqXcoKv8R1w6Jz1717CbMdH -flqUhSZNO7rrTOiwCcJlwp2dCZtOtZcFrPUGoPc2BX70kLJrxLT5ZOrpGgrIDajt -J8nU57O5q4IikCc9Kuh8kO+8T/3iCiSn3mUkpF3qwHYw03dQ+A0Em5Q2AXPKBlim -0zvc+gRGE1WKyURHuFE5Gi7oNOJ5y1lKCn+8pu8fA2dqWSslYpPZUxlmPCdiKYZN -pGvu/9ROutW04o5IWgAZCfEF2c6Rsffr6TlP9m8EQ5pV9T4FFL2/s1m02I4zhKOQ -UqqzApVg+QxMaPnu1RcN+HFXtSXkKe5lXa/R7jwXC1pDxaWG6iSe4gUH3DRCEpHW -OXSuTEGC2/KmSNGzm/MzqvOmwMVO9fSddmPmAsYiS8GVP1BkLFTltvA8Kc9XAgMB -AAGjQjBAMB0GA1UdDgQWBBRUYnBj8XWEQ1iO0RYgscasGrz2iTAPBgNVHRMBAf8E -BTADAQH/MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAKbqSSaet -8PFww+SX8J+pJdVrnjT+5hpk9jprUrIQeBqfTNqK2uwcN1LgQkv7bHbKJAs5EhWd -nxEt/Hlk3ODg9d3gV8mlsnZwUKT+twpw1aA08XXXTUm6EdGz2OyC/+sOxL9kLX1j -bhd47F18iMjrjld22VkE+rxSH0Ws8HqA7Oxvdq6R2xCOBNyS36D25q5J08FsEhvM -Kar5CKXiNxTKsbhm7xqC5PD48acWabfbqWE8n/Uxy+QARsIvdLGx14HuqCaVvIiv -TDUHKgLKeBRtRytAVunLKmChZwOgzoy8sHJnxDHO2zTlJQNgJXtxmOTAGytfdELS -S8VZCAeHvsXDf+eW2eHcKJfWjwXj9ZtOyh1QRwVTsMo554WgicEFOwE30z9J4nfr -I8iIZjs9OXYhRvHsXyO466JmdXTBQPfYaJqT4i2pLr0cox7IdMakLXogqzu4sEb9 -b91fUlV1YvCXoHzXOP0l382gmxDPi7g4Xl7FtKYCNqEeXxzP4padKar9mK5S4fNB -UvupLnKWnyfjqnN9+BojZns7q2WwMgFLFT49ok8MKzWixtlnEjUwzXYuFrOZnk1P -Ti07NEPhmg4NpGaXutIcSkwsKouLgU9xGqndXHt7CMUADTdA43x7VF8vhV929ven -sBxXVsFy6K2ir40zSbofitzmdHxghm+Hl3s= ------END CERTIFICATE----- - -# Issuer: CN=ISRG Root X2 O=Internet Security Research Group -# Subject: CN=ISRG Root X2 O=Internet Security Research Group -# Label: "ISRG Root X2" -# Serial: 87493402998870891108772069816698636114 -# MD5 Fingerprint: d3:9e:c4:1e:23:3c:a6:df:cf:a3:7e:6d:e0:14:e6:e5 -# SHA1 Fingerprint: bd:b1:b9:3c:d5:97:8d:45:c6:26:14:55:f8:db:95:c7:5a:d1:53:af -# SHA256 Fingerprint: 69:72:9b:8e:15:a8:6e:fc:17:7a:57:af:b7:17:1d:fc:64:ad:d2:8c:2f:ca:8c:f1:50:7e:34:45:3c:cb:14:70 ------BEGIN CERTIFICATE----- -MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw -CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg -R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00 -MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT -ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw -EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW -+1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9 -ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T -AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI -zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW -tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1 -/q4AaOeMSQ+2b1tbFfLn ------END CERTIFICATE----- - -# Issuer: CN=HiPKI Root CA - G1 O=Chunghwa Telecom Co., Ltd. -# Subject: CN=HiPKI Root CA - G1 O=Chunghwa Telecom Co., Ltd. -# Label: "HiPKI Root CA - G1" -# Serial: 60966262342023497858655262305426234976 -# MD5 Fingerprint: 69:45:df:16:65:4b:e8:68:9a:8f:76:5f:ff:80:9e:d3 -# SHA1 Fingerprint: 6a:92:e4:a8:ee:1b:ec:96:45:37:e3:29:57:49:cd:96:e3:e5:d2:60 -# SHA256 Fingerprint: f0:15:ce:3c:c2:39:bf:ef:06:4b:e9:f1:d2:c4:17:e1:a0:26:4a:0a:94:be:1f:0c:8d:12:18:64:eb:69:49:cc ------BEGIN CERTIFICATE----- -MIIFajCCA1KgAwIBAgIQLd2szmKXlKFD6LDNdmpeYDANBgkqhkiG9w0BAQsFADBP -MQswCQYDVQQGEwJUVzEjMCEGA1UECgwaQ2h1bmdod2EgVGVsZWNvbSBDby4sIEx0 -ZC4xGzAZBgNVBAMMEkhpUEtJIFJvb3QgQ0EgLSBHMTAeFw0xOTAyMjIwOTQ2MDRa -Fw0zNzEyMzExNTU5NTlaME8xCzAJBgNVBAYTAlRXMSMwIQYDVQQKDBpDaHVuZ2h3 -YSBUZWxlY29tIENvLiwgTHRkLjEbMBkGA1UEAwwSSGlQS0kgUm9vdCBDQSAtIEcx -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA9B5/UnMyDHPkvRN0o9Qw -qNCuS9i233VHZvR85zkEHmpwINJaR3JnVfSl6J3VHiGh8Ge6zCFovkRTv4354twv -Vcg3Px+kwJyz5HdcoEb+d/oaoDjq7Zpy3iu9lFc6uux55199QmQ5eiY29yTw1S+6 -lZgRZq2XNdZ1AYDgr/SEYYwNHl98h5ZeQa/rh+r4XfEuiAU+TCK72h8q3VJGZDnz -Qs7ZngyzsHeXZJzA9KMuH5UHsBffMNsAGJZMoYFL3QRtU6M9/Aes1MU3guvklQgZ -KILSQjqj2FPseYlgSGDIcpJQ3AOPgz+yQlda22rpEZfdhSi8MEyr48KxRURHH+CK -FgeW0iEPU8DtqX7UTuybCeyvQqww1r/REEXgphaypcXTT3OUM3ECoWqj1jOXTyFj -HluP2cFeRXF3D4FdXyGarYPM+l7WjSNfGz1BryB1ZlpK9p/7qxj3ccC2HTHsOyDr -y+K49a6SsvfhhEvyovKTmiKe0xRvNlS9H15ZFblzqMF8b3ti6RZsR1pl8w4Rm0bZ -/W3c1pzAtH2lsN0/Vm+h+fbkEkj9Bn8SV7apI09bA8PgcSojt/ewsTu8mL3WmKgM -a/aOEmem8rJY5AIJEzypuxC00jBF8ez3ABHfZfjcK0NVvxaXxA/VLGGEqnKG/uY6 -fsI/fe78LxQ+5oXdUG+3Se0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAdBgNV -HQ4EFgQU8ncX+l6o/vY9cdVouslGDDjYr7AwDgYDVR0PAQH/BAQDAgGGMA0GCSqG -SIb3DQEBCwUAA4ICAQBQUfB13HAE4/+qddRxosuej6ip0691x1TPOhwEmSKsxBHi -7zNKpiMdDg1H2DfHb680f0+BazVP6XKlMeJ45/dOlBhbQH3PayFUhuaVevvGyuqc -SE5XCV0vrPSltJczWNWseanMX/mF+lLFjfiRFOs6DRfQUsJ748JzjkZ4Bjgs6Fza -ZsT0pPBWGTMpWmWSBUdGSquEwx4noR8RkpkndZMPvDY7l1ePJlsMu5wP1G4wB9Tc -XzZoZjmDlicmisjEOf6aIW/Vcobpf2Lll07QJNBAsNB1CI69aO4I1258EHBGG3zg -iLKecoaZAeO/n0kZtCW+VmWuF2PlHt/o/0elv+EmBYTksMCv5wiZqAxeJoBF1Pho -L5aPruJKHJwWDBNvOIf2u8g0X5IDUXlwpt/L9ZlNec1OvFefQ05rLisY+GpzjLrF -Ne85akEez3GoorKGB1s6yeHvP2UEgEcyRHCVTjFnanRbEEV16rCf0OY1/k6fi8wr -kkVbbiVghUbN0aqwdmaTd5a+g744tiROJgvM7XpWGuDpWsZkrUx6AEhEL7lAuxM+ -vhV4nYWBSipX3tUZQ9rbyltHhoMLP7YNdnhzeSJesYAfz77RP1YQmCuVh6EfnWQU -YDksswBVLuT1sw5XxJFBAJw/6KXf6vb/yPCtbVKoF6ubYfwSUTXkJf2vqmqGOQ== ------END CERTIFICATE----- - -# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R4 -# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R4 -# Label: "GlobalSign ECC Root CA - R4" -# Serial: 159662223612894884239637590694 -# MD5 Fingerprint: 26:29:f8:6d:e1:88:bf:a2:65:7f:aa:c4:cd:0f:7f:fc -# SHA1 Fingerprint: 6b:a0:b0:98:e1:71:ef:5a:ad:fe:48:15:80:77:10:f4:bd:6f:0b:28 -# SHA256 Fingerprint: b0:85:d7:0b:96:4f:19:1a:73:e4:af:0d:54:ae:7a:0e:07:aa:fd:af:9b:71:dd:08:62:13:8a:b7:32:5a:24:a2 ------BEGIN CERTIFICATE----- -MIIB3DCCAYOgAwIBAgINAgPlfvU/k/2lCSGypjAKBggqhkjOPQQDAjBQMSQwIgYD -VQQLExtHbG9iYWxTaWduIEVDQyBSb290IENBIC0gUjQxEzARBgNVBAoTCkdsb2Jh -bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTIxMTEzMDAwMDAwWhcNMzgw -MTE5MDMxNDA3WjBQMSQwIgYDVQQLExtHbG9iYWxTaWduIEVDQyBSb290IENBIC0g -UjQxEzARBgNVBAoTCkdsb2JhbFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wWTAT -BgcqhkjOPQIBBggqhkjOPQMBBwNCAAS4xnnTj2wlDp8uORkcA6SumuU5BwkWymOx -uYb4ilfBV85C+nOh92VC/x7BALJucw7/xyHlGKSq2XE/qNS5zowdo0IwQDAOBgNV -HQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUVLB7rUW44kB/ -+wpu+74zyTyjhNUwCgYIKoZIzj0EAwIDRwAwRAIgIk90crlgr/HmnKAWBVBfw147 -bmF0774BxL4YSFlhgjICICadVGNA3jdgUM/I2O2dgq43mLyjj0xMqTQrbO/7lZsm ------END CERTIFICATE----- - -# Issuer: CN=GTS Root R1 O=Google Trust Services LLC -# Subject: CN=GTS Root R1 O=Google Trust Services LLC -# Label: "GTS Root R1" -# Serial: 159662320309726417404178440727 -# MD5 Fingerprint: 05:fe:d0:bf:71:a8:a3:76:63:da:01:e0:d8:52:dc:40 -# SHA1 Fingerprint: e5:8c:1c:c4:91:3b:38:63:4b:e9:10:6e:e3:ad:8e:6b:9d:d9:81:4a -# SHA256 Fingerprint: d9:47:43:2a:bd:e7:b7:fa:90:fc:2e:6b:59:10:1b:12:80:e0:e1:c7:e4:e4:0f:a3:c6:88:7f:ff:57:a7:f4:cf ------BEGIN CERTIFICATE----- -MIIFVzCCAz+gAwIBAgINAgPlk28xsBNJiGuiFzANBgkqhkiG9w0BAQwFADBHMQsw -CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU -MBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw -MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp -Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEBAQUA -A4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaMf/vo -27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vXmX7w -Cl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7zUjw -TcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0Pfybl -qAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtcvfaH -szVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4Zor8 -Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUspzBmk -MiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOORc92 -wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYWk70p -aDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+DVrN -VjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgFlQID -AQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E -FgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBAJ+qQibb -C5u+/x6Wki4+omVKapi6Ist9wTrYggoGxval3sBOh2Z5ofmmWJyq+bXmYOfg6LEe -QkEzCzc9zolwFcq1JKjPa7XSQCGYzyI0zzvFIoTgxQ6KfF2I5DUkzps+GlQebtuy -h6f88/qBVRRiClmpIgUxPoLW7ttXNLwzldMXG+gnoot7TiYaelpkttGsN/H9oPM4 -7HLwEXWdyzRSjeZ2axfG34arJ45JK3VmgRAhpuo+9K4l/3wV3s6MJT/KYnAK9y8J -ZgfIPxz88NtFMN9iiMG1D53Dn0reWVlHxYciNuaCp+0KueIHoI17eko8cdLiA6Ef -MgfdG+RCzgwARWGAtQsgWSl4vflVy2PFPEz0tv/bal8xa5meLMFrUKTX5hgUvYU/ -Z6tGn6D/Qqc6f1zLXbBwHSs09dR2CQzreExZBfMzQsNhFRAbd03OIozUhfJFfbdT -6u9AWpQKXCBfTkBdYiJ23//OYb2MI3jSNwLgjt7RETeJ9r/tSQdirpLsQBqvFAnZ -0E6yove+7u7Y/9waLd64NnHi/Hm3lCXRSHNboTXns5lndcEZOitHTtNCjv0xyBZm -2tIMPNuzjsmhDYAPexZ3FL//2wmUspO8IFgV6dtxQ/PeEMMA3KgqlbbC1j+Qa3bb -bP6MvPJwNQzcmRk13NfIRmPVNnGuV/u3gm3c ------END CERTIFICATE----- - -# Issuer: CN=GTS Root R3 O=Google Trust Services LLC -# Subject: CN=GTS Root R3 O=Google Trust Services LLC -# Label: "GTS Root R3" -# Serial: 159662495401136852707857743206 -# MD5 Fingerprint: 3e:e7:9d:58:02:94:46:51:94:e5:e0:22:4a:8b:e7:73 -# SHA1 Fingerprint: ed:e5:71:80:2b:c8:92:b9:5b:83:3c:d2:32:68:3f:09:cd:a0:1e:46 -# SHA256 Fingerprint: 34:d8:a7:3e:e2:08:d9:bc:db:0d:95:65:20:93:4b:4e:40:e6:94:82:59:6e:8b:6f:73:c8:42:6b:01:0a:6f:48 ------BEGIN CERTIFICATE----- -MIICCTCCAY6gAwIBAgINAgPluILrIPglJ209ZjAKBggqhkjOPQQDAzBHMQswCQYD -VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG -A1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw -WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz -IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQAIgNi -AAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout736G -jOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2ADDL2 -4CejQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW -BBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEA9uEglRR7 -VKOQFhG/hMjqb2sXnh5GmCCbn9MN2azTL818+FsuVbu/3ZL3pAzcMeGiAjEA/Jdm -ZuVDFhOD3cffL74UOO0BzrEXGhF16b0DjyZ+hOXJYKaV11RZt+cRLInUue4X ------END CERTIFICATE----- - -# Issuer: CN=GTS Root R4 O=Google Trust Services LLC -# Subject: CN=GTS Root R4 O=Google Trust Services LLC -# Label: "GTS Root R4" -# Serial: 159662532700760215368942768210 -# MD5 Fingerprint: 43:96:83:77:19:4d:76:b3:9d:65:52:e4:1d:22:a5:e8 -# SHA1 Fingerprint: 77:d3:03:67:b5:e0:0c:15:f6:0c:38:61:df:7c:e1:3b:92:46:4d:47 -# SHA256 Fingerprint: 34:9d:fa:40:58:c5:e2:63:12:3b:39:8a:e7:95:57:3c:4e:13:13:c8:3f:e6:8f:93:55:6c:d5:e8:03:1b:3c:7d ------BEGIN CERTIFICATE----- -MIICCTCCAY6gAwIBAgINAgPlwGjvYxqccpBQUjAKBggqhkjOPQQDAzBHMQswCQYD -VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG -A1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw -WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz -IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQAIgNi -AATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzuhXyi -QHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/lxKvR -HYqjQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW -BBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNpADBmAjEA6ED/g94D -9J+uHXqnLrmvT/aDHQ4thQEd0dlq7A/Cr8deVl5c1RxYIigL9zC2L7F8AjEA8GE8 -p/SgguMh1YQdc4acLa/KNJvxn7kjNuK8YAOdgLOaVsjh4rsUecrNIdSUtUlD ------END CERTIFICATE----- - -# Issuer: CN=Telia Root CA v2 O=Telia Finland Oyj -# Subject: CN=Telia Root CA v2 O=Telia Finland Oyj -# Label: "Telia Root CA v2" -# Serial: 7288924052977061235122729490515358 -# MD5 Fingerprint: 0e:8f:ac:aa:82:df:85:b1:f4:dc:10:1c:fc:99:d9:48 -# SHA1 Fingerprint: b9:99:cd:d1:73:50:8a:c4:47:05:08:9c:8c:88:fb:be:a0:2b:40:cd -# SHA256 Fingerprint: 24:2b:69:74:2f:cb:1e:5b:2a:bf:98:89:8b:94:57:21:87:54:4e:5b:4d:99:11:78:65:73:62:1f:6a:74:b8:2c ------BEGIN CERTIFICATE----- -MIIFdDCCA1ygAwIBAgIPAWdfJ9b+euPkrL4JWwWeMA0GCSqGSIb3DQEBCwUAMEQx -CzAJBgNVBAYTAkZJMRowGAYDVQQKDBFUZWxpYSBGaW5sYW5kIE95ajEZMBcGA1UE -AwwQVGVsaWEgUm9vdCBDQSB2MjAeFw0xODExMjkxMTU1NTRaFw00MzExMjkxMTU1 -NTRaMEQxCzAJBgNVBAYTAkZJMRowGAYDVQQKDBFUZWxpYSBGaW5sYW5kIE95ajEZ -MBcGA1UEAwwQVGVsaWEgUm9vdCBDQSB2MjCCAiIwDQYJKoZIhvcNAQEBBQADggIP -ADCCAgoCggIBALLQPwe84nvQa5n44ndp586dpAO8gm2h/oFlH0wnrI4AuhZ76zBq -AMCzdGh+sq/H1WKzej9Qyow2RCRj0jbpDIX2Q3bVTKFgcmfiKDOlyzG4OiIjNLh9 -vVYiQJ3q9HsDrWj8soFPmNB06o3lfc1jw6P23pLCWBnglrvFxKk9pXSW/q/5iaq9 -lRdU2HhE8Qx3FZLgmEKnpNaqIJLNwaCzlrI6hEKNfdWV5Nbb6WLEWLN5xYzTNTOD -n3WhUidhOPFZPY5Q4L15POdslv5e2QJltI5c0BE0312/UqeBAMN/mUWZFdUXyApT -7GPzmX3MaRKGwhfwAZ6/hLzRUssbkmbOpFPlob/E2wnW5olWK8jjfN7j/4nlNW4o -6GwLI1GpJQXrSPjdscr6bAhR77cYbETKJuFzxokGgeWKrLDiKca5JLNrRBH0pUPC -TEPlcDaMtjNXepUugqD0XBCzYYP2AgWGLnwtbNwDRm41k9V6lS/eINhbfpSQBGq6 -WT0EBXWdN6IOLj3rwaRSg/7Qa9RmjtzG6RJOHSpXqhC8fF6CfaamyfItufUXJ63R -DolUK5X6wK0dmBR4M0KGCqlztft0DbcbMBnEWg4cJ7faGND/isgFuvGqHKI3t+ZI -pEYslOqodmJHixBTB0hXbOKSTbauBcvcwUpej6w9GU7C7WB1K9vBykLVAgMBAAGj -YzBhMB8GA1UdIwQYMBaAFHKs5DN5qkWH9v2sHZ7Wxy+G2CQ5MB0GA1UdDgQWBBRy -rOQzeapFh/b9rB2e1scvhtgkOTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUw -AwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAoDtZpwmUPjaE0n4vOaWWl/oRrfxn83EJ -8rKJhGdEr7nv7ZbsnGTbMjBvZ5qsfl+yqwE2foH65IRe0qw24GtixX1LDoJt0nZi -0f6X+J8wfBj5tFJ3gh1229MdqfDBmgC9bXXYfef6xzijnHDoRnkDry5023X4blMM -A8iZGok1GTzTyVR8qPAs5m4HeW9q4ebqkYJpCh3DflminmtGFZhb069GHWLIzoBS -SRE/yQQSwxN8PzuKlts8oB4KtItUsiRnDe+Cy748fdHif64W1lZYudogsYMVoe+K -TTJvQS8TUoKU1xrBeKJR3Stwbbca+few4GeXVtt8YVMJAygCQMez2P2ccGrGKMOF -6eLtGpOg3kuYooQ+BXcBlj37tCAPnHICehIv1aO6UXivKitEZU61/Qrowc15h2Er -3oBXRb9n8ZuRXqWk7FlIEA04x7D6w0RtBPV4UBySllva9bguulvP5fBqnUsvWHMt -Ty3EHD70sz+rFQ47GUGKpMFXEmZxTPpT41frYpUJnlTd0cI8Vzy9OK2YZLe4A5pT -VmBds9hCG1xLEooc6+t9xnppxyd/pPiL8uSUZodL6ZQHCRJ5irLrdATczvREWeAW -ysUsWNc8e89ihmpQfTU2Zqf7N+cox9jQraVplI/owd8k+BsHMYeB2F326CjYSlKA -rBPuUBQemMc= ------END CERTIFICATE----- - -# Issuer: CN=D-TRUST BR Root CA 1 2020 O=D-Trust GmbH -# Subject: CN=D-TRUST BR Root CA 1 2020 O=D-Trust GmbH -# Label: "D-TRUST BR Root CA 1 2020" -# Serial: 165870826978392376648679885835942448534 -# MD5 Fingerprint: b5:aa:4b:d5:ed:f7:e3:55:2e:8f:72:0a:f3:75:b8:ed -# SHA1 Fingerprint: 1f:5b:98:f0:e3:b5:f7:74:3c:ed:e6:b0:36:7d:32:cd:f4:09:41:67 -# SHA256 Fingerprint: e5:9a:aa:81:60:09:c2:2b:ff:5b:25:ba:d3:7d:f3:06:f0:49:79:7c:1f:81:d8:5a:b0:89:e6:57:bd:8f:00:44 ------BEGIN CERTIFICATE----- -MIIC2zCCAmCgAwIBAgIQfMmPK4TX3+oPyWWa00tNljAKBggqhkjOPQQDAzBIMQsw -CQYDVQQGEwJERTEVMBMGA1UEChMMRC1UcnVzdCBHbWJIMSIwIAYDVQQDExlELVRS -VVNUIEJSIFJvb3QgQ0EgMSAyMDIwMB4XDTIwMDIxMTA5NDUwMFoXDTM1MDIxMTA5 -NDQ1OVowSDELMAkGA1UEBhMCREUxFTATBgNVBAoTDEQtVHJ1c3QgR21iSDEiMCAG -A1UEAxMZRC1UUlVTVCBCUiBSb290IENBIDEgMjAyMDB2MBAGByqGSM49AgEGBSuB -BAAiA2IABMbLxyjR+4T1mu9CFCDhQ2tuda38KwOE1HaTJddZO0Flax7mNCq7dPYS -zuht56vkPE4/RAiLzRZxy7+SmfSk1zxQVFKQhYN4lGdnoxwJGT11NIXe7WB9xwy0 -QVK5buXuQqOCAQ0wggEJMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFHOREKv/ -VbNafAkl1bK6CKBrqx9tMA4GA1UdDwEB/wQEAwIBBjCBxgYDVR0fBIG+MIG7MD6g -PKA6hjhodHRwOi8vY3JsLmQtdHJ1c3QubmV0L2NybC9kLXRydXN0X2JyX3Jvb3Rf -Y2FfMV8yMDIwLmNybDB5oHegdYZzbGRhcDovL2RpcmVjdG9yeS5kLXRydXN0Lm5l -dC9DTj1ELVRSVVNUJTIwQlIlMjBSb290JTIwQ0ElMjAxJTIwMjAyMCxPPUQtVHJ1 -c3QlMjBHbWJILEM9REU/Y2VydGlmaWNhdGVyZXZvY2F0aW9ubGlzdDAKBggqhkjO -PQQDAwNpADBmAjEAlJAtE/rhY/hhY+ithXhUkZy4kzg+GkHaQBZTQgjKL47xPoFW -wKrY7RjEsK70PvomAjEA8yjixtsrmfu3Ubgko6SUeho/5jbiA1czijDLgsfWFBHV -dWNbFJWcHwHP2NVypw87 ------END CERTIFICATE----- - -# Issuer: CN=D-TRUST EV Root CA 1 2020 O=D-Trust GmbH -# Subject: CN=D-TRUST EV Root CA 1 2020 O=D-Trust GmbH -# Label: "D-TRUST EV Root CA 1 2020" -# Serial: 126288379621884218666039612629459926992 -# MD5 Fingerprint: 8c:2d:9d:70:9f:48:99:11:06:11:fb:e9:cb:30:c0:6e -# SHA1 Fingerprint: 61:db:8c:21:59:69:03:90:d8:7c:9c:12:86:54:cf:9d:3d:f4:dd:07 -# SHA256 Fingerprint: 08:17:0d:1a:a3:64:53:90:1a:2f:95:92:45:e3:47:db:0c:8d:37:ab:aa:bc:56:b8:1a:a1:00:dc:95:89:70:db ------BEGIN CERTIFICATE----- -MIIC2zCCAmCgAwIBAgIQXwJB13qHfEwDo6yWjfv/0DAKBggqhkjOPQQDAzBIMQsw -CQYDVQQGEwJERTEVMBMGA1UEChMMRC1UcnVzdCBHbWJIMSIwIAYDVQQDExlELVRS -VVNUIEVWIFJvb3QgQ0EgMSAyMDIwMB4XDTIwMDIxMTEwMDAwMFoXDTM1MDIxMTA5 -NTk1OVowSDELMAkGA1UEBhMCREUxFTATBgNVBAoTDEQtVHJ1c3QgR21iSDEiMCAG -A1UEAxMZRC1UUlVTVCBFViBSb290IENBIDEgMjAyMDB2MBAGByqGSM49AgEGBSuB -BAAiA2IABPEL3YZDIBnfl4XoIkqbz52Yv7QFJsnL46bSj8WeeHsxiamJrSc8ZRCC -/N/DnU7wMyPE0jL1HLDfMxddxfCxivnvubcUyilKwg+pf3VlSSowZ/Rk99Yad9rD -wpdhQntJraOCAQ0wggEJMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFH8QARY3 -OqQo5FD4pPfsazK2/umLMA4GA1UdDwEB/wQEAwIBBjCBxgYDVR0fBIG+MIG7MD6g -PKA6hjhodHRwOi8vY3JsLmQtdHJ1c3QubmV0L2NybC9kLXRydXN0X2V2X3Jvb3Rf -Y2FfMV8yMDIwLmNybDB5oHegdYZzbGRhcDovL2RpcmVjdG9yeS5kLXRydXN0Lm5l -dC9DTj1ELVRSVVNUJTIwRVYlMjBSb290JTIwQ0ElMjAxJTIwMjAyMCxPPUQtVHJ1 -c3QlMjBHbWJILEM9REU/Y2VydGlmaWNhdGVyZXZvY2F0aW9ubGlzdDAKBggqhkjO -PQQDAwNpADBmAjEAyjzGKnXCXnViOTYAYFqLwZOZzNnbQTs7h5kXO9XMT8oi96CA -y/m0sRtW9XLS/BnRAjEAkfcwkz8QRitxpNA7RJvAKQIFskF3UfN5Wp6OFKBOQtJb -gfM0agPnIjhQW+0ZT0MW ------END CERTIFICATE----- - -# Issuer: CN=DigiCert TLS ECC P384 Root G5 O=DigiCert, Inc. -# Subject: CN=DigiCert TLS ECC P384 Root G5 O=DigiCert, Inc. -# Label: "DigiCert TLS ECC P384 Root G5" -# Serial: 13129116028163249804115411775095713523 -# MD5 Fingerprint: d3:71:04:6a:43:1c:db:a6:59:e1:a8:a3:aa:c5:71:ed -# SHA1 Fingerprint: 17:f3:de:5e:9f:0f:19:e9:8e:f6:1f:32:26:6e:20:c4:07:ae:30:ee -# SHA256 Fingerprint: 01:8e:13:f0:77:25:32:cf:80:9b:d1:b1:72:81:86:72:83:fc:48:c6:e1:3b:e9:c6:98:12:85:4a:49:0c:1b:05 ------BEGIN CERTIFICATE----- -MIICGTCCAZ+gAwIBAgIQCeCTZaz32ci5PhwLBCou8zAKBggqhkjOPQQDAzBOMQsw -CQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJjAkBgNVBAMTHURp -Z2lDZXJ0IFRMUyBFQ0MgUDM4NCBSb290IEc1MB4XDTIxMDExNTAwMDAwMFoXDTQ2 -MDExNDIzNTk1OVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkRpZ2lDZXJ0LCBJ -bmMuMSYwJAYDVQQDEx1EaWdpQ2VydCBUTFMgRUNDIFAzODQgUm9vdCBHNTB2MBAG -ByqGSM49AgEGBSuBBAAiA2IABMFEoc8Rl1Ca3iOCNQfN0MsYndLxf3c1TzvdlHJS -7cI7+Oz6e2tYIOyZrsn8aLN1udsJ7MgT9U7GCh1mMEy7H0cKPGEQQil8pQgO4CLp -0zVozptjn4S1mU1YoI71VOeVyaNCMEAwHQYDVR0OBBYEFMFRRVBZqz7nLFr6ICIS -B4CIfBFqMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49 -BAMDA2gAMGUCMQCJao1H5+z8blUD2WdsJk6Dxv3J+ysTvLd6jLRl0mlpYxNjOyZQ -LgGheQaRnUi/wr4CMEfDFXuxoJGZSZOoPHzoRgaLLPIxAJSdYsiJvRmEFOml+wG4 -DXZDjC5Ty3zfDBeWUA== ------END CERTIFICATE----- - -# Issuer: CN=DigiCert TLS RSA4096 Root G5 O=DigiCert, Inc. -# Subject: CN=DigiCert TLS RSA4096 Root G5 O=DigiCert, Inc. -# Label: "DigiCert TLS RSA4096 Root G5" -# Serial: 11930366277458970227240571539258396554 -# MD5 Fingerprint: ac:fe:f7:34:96:a9:f2:b3:b4:12:4b:e4:27:41:6f:e1 -# SHA1 Fingerprint: a7:88:49:dc:5d:7c:75:8c:8c:de:39:98:56:b3:aa:d0:b2:a5:71:35 -# SHA256 Fingerprint: 37:1a:00:dc:05:33:b3:72:1a:7e:eb:40:e8:41:9e:70:79:9d:2b:0a:0f:2c:1d:80:69:31:65:f7:ce:c4:ad:75 ------BEGIN CERTIFICATE----- -MIIFZjCCA06gAwIBAgIQCPm0eKj6ftpqMzeJ3nzPijANBgkqhkiG9w0BAQwFADBN -MQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJTAjBgNVBAMT -HERpZ2lDZXJ0IFRMUyBSU0E0MDk2IFJvb3QgRzUwHhcNMjEwMTE1MDAwMDAwWhcN -NDYwMTE0MjM1OTU5WjBNMQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQs -IEluYy4xJTAjBgNVBAMTHERpZ2lDZXJ0IFRMUyBSU0E0MDk2IFJvb3QgRzUwggIi -MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCz0PTJeRGd/fxmgefM1eS87IE+ -ajWOLrfn3q/5B03PMJ3qCQuZvWxX2hhKuHisOjmopkisLnLlvevxGs3npAOpPxG0 -2C+JFvuUAT27L/gTBaF4HI4o4EXgg/RZG5Wzrn4DReW+wkL+7vI8toUTmDKdFqgp -wgscONyfMXdcvyej/Cestyu9dJsXLfKB2l2w4SMXPohKEiPQ6s+d3gMXsUJKoBZM -pG2T6T867jp8nVid9E6P/DsjyG244gXazOvswzH016cpVIDPRFtMbzCe88zdH5RD -nU1/cHAN1DrRN/BsnZvAFJNY781BOHW8EwOVfH/jXOnVDdXifBBiqmvwPXbzP6Po -sMH976pXTayGpxi0KcEsDr9kvimM2AItzVwv8n/vFfQMFawKsPHTDU9qTXeXAaDx -Zre3zu/O7Oyldcqs4+Fj97ihBMi8ez9dLRYiVu1ISf6nL3kwJZu6ay0/nTvEF+cd -Lvvyz6b84xQslpghjLSR6Rlgg/IwKwZzUNWYOwbpx4oMYIwo+FKbbuH2TbsGJJvX -KyY//SovcfXWJL5/MZ4PbeiPT02jP/816t9JXkGPhvnxd3lLG7SjXi/7RgLQZhNe -XoVPzthwiHvOAbWWl9fNff2C+MIkwcoBOU+NosEUQB+cZtUMCUbW8tDRSHZWOkPL -tgoRObqME2wGtZ7P6wIDAQABo0IwQDAdBgNVHQ4EFgQUUTMc7TZArxfTJc1paPKv -TiM+s0EwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcN -AQEMBQADggIBAGCmr1tfV9qJ20tQqcQjNSH/0GEwhJG3PxDPJY7Jv0Y02cEhJhxw -GXIeo8mH/qlDZJY6yFMECrZBu8RHANmfGBg7sg7zNOok992vIGCukihfNudd5N7H -PNtQOa27PShNlnx2xlv0wdsUpasZYgcYQF+Xkdycx6u1UQ3maVNVzDl92sURVXLF -O4uJ+DQtpBflF+aZfTCIITfNMBc9uPK8qHWgQ9w+iUuQrm0D4ByjoJYJu32jtyoQ -REtGBzRj7TG5BO6jm5qu5jF49OokYTurWGT/u4cnYiWB39yhL/btp/96j1EuMPik -AdKFOV8BmZZvWltwGUb+hmA+rYAQCd05JS9Yf7vSdPD3Rh9GOUrYU9DzLjtxpdRv -/PNn5AeP3SYZ4Y1b+qOTEZvpyDrDVWiakuFSdjjo4bq9+0/V77PnSIMx8IIh47a+ -p6tv75/fTM8BuGJqIz3nCU2AG3swpMPdB380vqQmsvZB6Akd4yCYqjdP//fx4ilw -MUc/dNAUFvohigLVigmUdy7yWSiLfFCSCmZ4OIN1xLVaqBHG5cGdZlXPU8Sv13WF -qUITVuwhd4GTWgzqltlJyqEI8pc7bZsEGCREjnwB8twl2F6GmrE52/WRMmrRpnCK -ovfepEWFJqgejF0pW8hL2JpqA15w8oVPbEtoL8pU9ozaMv7Da4M/OMZ+ ------END CERTIFICATE----- - -# Issuer: CN=Certainly Root R1 O=Certainly -# Subject: CN=Certainly Root R1 O=Certainly -# Label: "Certainly Root R1" -# Serial: 188833316161142517227353805653483829216 -# MD5 Fingerprint: 07:70:d4:3e:82:87:a0:fa:33:36:13:f4:fa:33:e7:12 -# SHA1 Fingerprint: a0:50:ee:0f:28:71:f4:27:b2:12:6d:6f:50:96:25:ba:cc:86:42:af -# SHA256 Fingerprint: 77:b8:2c:d8:64:4c:43:05:f7:ac:c5:cb:15:6b:45:67:50:04:03:3d:51:c6:0c:62:02:a8:e0:c3:34:67:d3:a0 ------BEGIN CERTIFICATE----- -MIIFRzCCAy+gAwIBAgIRAI4P+UuQcWhlM1T01EQ5t+AwDQYJKoZIhvcNAQELBQAw -PTELMAkGA1UEBhMCVVMxEjAQBgNVBAoTCUNlcnRhaW5seTEaMBgGA1UEAxMRQ2Vy -dGFpbmx5IFJvb3QgUjEwHhcNMjEwNDAxMDAwMDAwWhcNNDYwNDAxMDAwMDAwWjA9 -MQswCQYDVQQGEwJVUzESMBAGA1UEChMJQ2VydGFpbmx5MRowGAYDVQQDExFDZXJ0 -YWlubHkgUm9vdCBSMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANA2 -1B/q3avk0bbm+yLA3RMNansiExyXPGhjZjKcA7WNpIGD2ngwEc/csiu+kr+O5MQT -vqRoTNoCaBZ0vrLdBORrKt03H2As2/X3oXyVtwxwhi7xOu9S98zTm/mLvg7fMbed -aFySpvXl8wo0tf97ouSHocavFwDvA5HtqRxOcT3Si2yJ9HiG5mpJoM610rCrm/b0 -1C7jcvk2xusVtyWMOvwlDbMicyF0yEqWYZL1LwsYpfSt4u5BvQF5+paMjRcCMLT5 -r3gajLQ2EBAHBXDQ9DGQilHFhiZ5shGIXsXwClTNSaa/ApzSRKft43jvRl5tcdF5 -cBxGX1HpyTfcX35pe0HfNEXgO4T0oYoKNp43zGJS4YkNKPl6I7ENPT2a/Z2B7yyQ -wHtETrtJ4A5KVpK8y7XdeReJkd5hiXSSqOMyhb5OhaRLWcsrxXiOcVTQAjeZjOVJ -6uBUcqQRBi8LjMFbvrWhsFNunLhgkR9Za/kt9JQKl7XsxXYDVBtlUrpMklZRNaBA -2CnbrlJ2Oy0wQJuK0EJWtLeIAaSHO1OWzaMWj/Nmqhexx2DgwUMFDO6bW2BvBlyH -Wyf5QBGenDPBt+U1VwV/J84XIIwc/PH72jEpSe31C4SnT8H2TsIonPru4K8H+zMR -eiFPCyEQtkA6qyI6BJyLm4SGcprSp6XEtHWRqSsjAgMBAAGjQjBAMA4GA1UdDwEB -/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTgqj8ljZ9EXME66C6u -d0yEPmcM9DANBgkqhkiG9w0BAQsFAAOCAgEAuVevuBLaV4OPaAszHQNTVfSVcOQr -PbA56/qJYv331hgELyE03fFo8NWWWt7CgKPBjcZq91l3rhVkz1t5BXdm6ozTaw3d -8VkswTOlMIAVRQdFGjEitpIAq5lNOo93r6kiyi9jyhXWx8bwPWz8HA2YEGGeEaIi -1wrykXprOQ4vMMM2SZ/g6Q8CRFA3lFV96p/2O7qUpUzpvD5RtOjKkjZUbVwlKNrd -rRT90+7iIgXr0PK3aBLXWopBGsaSpVo7Y0VPv+E6dyIvXL9G+VoDhRNCX8reU9di -taY1BMJH/5n9hN9czulegChB8n3nHpDYT3Y+gjwN/KUD+nsa2UUeYNrEjvn8K8l7 -lcUq/6qJ34IxD3L/DCfXCh5WAFAeDJDBlrXYFIW7pw0WwfgHJBu6haEaBQmAupVj -yTrsJZ9/nbqkRxWbRHDxakvWOF5D8xh+UG7pWijmZeZ3Gzr9Hb4DJqPb1OG7fpYn -Kx3upPvaJVQTA945xsMfTZDsjxtK0hzthZU4UHlG1sGQUDGpXJpuHfUzVounmdLy -yCwzk5Iwx06MZTMQZBf9JBeW0Y3COmor6xOLRPIh80oat3df1+2IpHLlOR+Vnb5n -wXARPbv0+Em34yaXOp/SX3z7wJl8OSngex2/DaeP0ik0biQVy96QXr8axGbqwua6 -OV+KmalBWQewLK8= ------END CERTIFICATE----- - -# Issuer: CN=Certainly Root E1 O=Certainly -# Subject: CN=Certainly Root E1 O=Certainly -# Label: "Certainly Root E1" -# Serial: 8168531406727139161245376702891150584 -# MD5 Fingerprint: 0a:9e:ca:cd:3e:52:50:c6:36:f3:4b:a3:ed:a7:53:e9 -# SHA1 Fingerprint: f9:e1:6d:dc:01:89:cf:d5:82:45:63:3e:c5:37:7d:c2:eb:93:6f:2b -# SHA256 Fingerprint: b4:58:5f:22:e4:ac:75:6a:4e:86:12:a1:36:1c:5d:9d:03:1a:93:fd:84:fe:bb:77:8f:a3:06:8b:0f:c4:2d:c2 ------BEGIN CERTIFICATE----- -MIIB9zCCAX2gAwIBAgIQBiUzsUcDMydc+Y2aub/M+DAKBggqhkjOPQQDAzA9MQsw -CQYDVQQGEwJVUzESMBAGA1UEChMJQ2VydGFpbmx5MRowGAYDVQQDExFDZXJ0YWlu -bHkgUm9vdCBFMTAeFw0yMTA0MDEwMDAwMDBaFw00NjA0MDEwMDAwMDBaMD0xCzAJ -BgNVBAYTAlVTMRIwEAYDVQQKEwlDZXJ0YWlubHkxGjAYBgNVBAMTEUNlcnRhaW5s -eSBSb290IEUxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE3m/4fxzf7flHh4axpMCK -+IKXgOqPyEpeKn2IaKcBYhSRJHpcnqMXfYqGITQYUBsQ3tA3SybHGWCA6TS9YBk2 -QNYphwk8kXr2vBMj3VlOBF7PyAIcGFPBMdjaIOlEjeR2o0IwQDAOBgNVHQ8BAf8E -BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU8ygYy2R17ikq6+2uI1g4 -hevIIgcwCgYIKoZIzj0EAwMDaAAwZQIxALGOWiDDshliTd6wT99u0nCK8Z9+aozm -ut6Dacpps6kFtZaSF4fC0urQe87YQVt8rgIwRt7qy12a7DLCZRawTDBcMPPaTnOG -BtjOiQRINzf43TNRnXCve1XYAS59BWQOhriR ------END CERTIFICATE----- - -# Issuer: CN=Security Communication ECC RootCA1 O=SECOM Trust Systems CO.,LTD. -# Subject: CN=Security Communication ECC RootCA1 O=SECOM Trust Systems CO.,LTD. -# Label: "Security Communication ECC RootCA1" -# Serial: 15446673492073852651 -# MD5 Fingerprint: 7e:43:b0:92:68:ec:05:43:4c:98:ab:5d:35:2e:7e:86 -# SHA1 Fingerprint: b8:0e:26:a9:bf:d2:b2:3b:c0:ef:46:c9:ba:c7:bb:f6:1d:0d:41:41 -# SHA256 Fingerprint: e7:4f:bd:a5:5b:d5:64:c4:73:a3:6b:44:1a:a7:99:c8:a6:8e:07:74:40:e8:28:8b:9f:a1:e5:0e:4b:ba:ca:11 ------BEGIN CERTIFICATE----- -MIICODCCAb6gAwIBAgIJANZdm7N4gS7rMAoGCCqGSM49BAMDMGExCzAJBgNVBAYT -AkpQMSUwIwYDVQQKExxTRUNPTSBUcnVzdCBTeXN0ZW1zIENPLixMVEQuMSswKQYD -VQQDEyJTZWN1cml0eSBDb21tdW5pY2F0aW9uIEVDQyBSb290Q0ExMB4XDTE2MDYx -NjA1MTUyOFoXDTM4MDExODA1MTUyOFowYTELMAkGA1UEBhMCSlAxJTAjBgNVBAoT -HFNFQ09NIFRydXN0IFN5c3RlbXMgQ08uLExURC4xKzApBgNVBAMTIlNlY3VyaXR5 -IENvbW11bmljYXRpb24gRUNDIFJvb3RDQTEwdjAQBgcqhkjOPQIBBgUrgQQAIgNi -AASkpW9gAwPDvTH00xecK4R1rOX9PVdu12O/5gSJko6BnOPpR27KkBLIE+Cnnfdl -dB9sELLo5OnvbYUymUSxXv3MdhDYW72ixvnWQuRXdtyQwjWpS4g8EkdtXP9JTxpK -ULGjQjBAMB0GA1UdDgQWBBSGHOf+LaVKiwj+KBH6vqNm+GBZLzAOBgNVHQ8BAf8E -BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjAVXUI9/Lbu -9zuxNuie9sRGKEkz0FhDKmMpzE2xtHqiuQ04pV1IKv3LsnNdo4gIxwwCMQDAqy0O -be0YottT6SXbVQjgUMzfRGEWgqtJsLKB7HOHeLRMsmIbEvoWTSVLY70eN9k= ------END CERTIFICATE----- - -# Issuer: CN=BJCA Global Root CA1 O=BEIJING CERTIFICATE AUTHORITY -# Subject: CN=BJCA Global Root CA1 O=BEIJING CERTIFICATE AUTHORITY -# Label: "BJCA Global Root CA1" -# Serial: 113562791157148395269083148143378328608 -# MD5 Fingerprint: 42:32:99:76:43:33:36:24:35:07:82:9b:28:f9:d0:90 -# SHA1 Fingerprint: d5:ec:8d:7b:4c:ba:79:f4:e7:e8:cb:9d:6b:ae:77:83:10:03:21:6a -# SHA256 Fingerprint: f3:89:6f:88:fe:7c:0a:88:27:66:a7:fa:6a:d2:74:9f:b5:7a:7f:3e:98:fb:76:9c:1f:a7:b0:9c:2c:44:d5:ae ------BEGIN CERTIFICATE----- -MIIFdDCCA1ygAwIBAgIQVW9l47TZkGobCdFsPsBsIDANBgkqhkiG9w0BAQsFADBU -MQswCQYDVQQGEwJDTjEmMCQGA1UECgwdQkVJSklORyBDRVJUSUZJQ0FURSBBVVRI -T1JJVFkxHTAbBgNVBAMMFEJKQ0EgR2xvYmFsIFJvb3QgQ0ExMB4XDTE5MTIxOTAz -MTYxN1oXDTQ0MTIxMjAzMTYxN1owVDELMAkGA1UEBhMCQ04xJjAkBgNVBAoMHUJF -SUpJTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZMR0wGwYDVQQDDBRCSkNBIEdsb2Jh -bCBSb290IENBMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAPFmCL3Z -xRVhy4QEQaVpN3cdwbB7+sN3SJATcmTRuHyQNZ0YeYjjlwE8R4HyDqKYDZ4/N+AZ -spDyRhySsTphzvq3Rp4Dhtczbu33RYx2N95ulpH3134rhxfVizXuhJFyV9xgw8O5 -58dnJCNPYwpj9mZ9S1WnP3hkSWkSl+BMDdMJoDIwOvqfwPKcxRIqLhy1BDPapDgR -at7GGPZHOiJBhyL8xIkoVNiMpTAK+BcWyqw3/XmnkRd4OJmtWO2y3syJfQOcs4ll -5+M7sSKGjwZteAf9kRJ/sGsciQ35uMt0WwfCyPQ10WRjeulumijWML3mG90Vr4Tq -nMfK9Q7q8l0ph49pczm+LiRvRSGsxdRpJQaDrXpIhRMsDQa4bHlW/KNnMoH1V6XK -V0Jp6VwkYe/iMBhORJhVb3rCk9gZtt58R4oRTklH2yiUAguUSiz5EtBP6DF+bHq/ -pj+bOT0CFqMYs2esWz8sgytnOYFcuX6U1WTdno9uruh8W7TXakdI136z1C2OVnZO -z2nxbkRs1CTqjSShGL+9V/6pmTW12xB3uD1IutbB5/EjPtffhZ0nPNRAvQoMvfXn -jSXWgXSHRtQpdaJCbPdzied9v3pKH9MiyRVVz99vfFXQpIsHETdfg6YmV6YBW37+ -WGgHqel62bno/1Afq8K0wM7o6v0PvY1NuLxxAgMBAAGjQjBAMB0GA1UdDgQWBBTF -7+3M2I0hxkjk49cULqcWk+WYATAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQE -AwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAUoKsITQfI/Ki2Pm4rzc2IInRNwPWaZ+4 -YRC6ojGYWUfo0Q0lHhVBDOAqVdVXUsv45Mdpox1NcQJeXyFFYEhcCY5JEMEE3Kli -awLwQ8hOnThJdMkycFRtwUf8jrQ2ntScvd0g1lPJGKm1Vrl2i5VnZu69mP6u775u -+2D2/VnGKhs/I0qUJDAnyIm860Qkmss9vk/Ves6OF8tiwdneHg56/0OGNFK8YT88 -X7vZdrRTvJez/opMEi4r89fO4aL/3Xtw+zuhTaRjAv04l5U/BXCga99igUOLtFkN -SoxUnMW7gZ/NfaXvCyUeOiDbHPwfmGcCCtRzRBPbUYQaVQNW4AB+dAb/OMRyHdOo -P2gxXdMJxy6MW2Pg6Nwe0uxhHvLe5e/2mXZgLR6UcnHGCyoyx5JO1UbXHfmpGQrI -+pXObSOYqgs4rZpWDW+N8TEAiMEXnM0ZNjX+VVOg4DwzX5Ze4jLp3zO7Bkqp2IRz -znfSxqxx4VyjHQy7Ct9f4qNx2No3WqB4K/TUfet27fJhcKVlmtOJNBir+3I+17Q9 -eVzYH6Eze9mCUAyTF6ps3MKCuwJXNq+YJyo5UOGwifUll35HaBC07HPKs5fRJNz2 -YqAo07WjuGS3iGJCz51TzZm+ZGiPTx4SSPfSKcOYKMryMguTjClPPGAyzQWWYezy -r/6zcCwupvI= ------END CERTIFICATE----- - -# Issuer: CN=BJCA Global Root CA2 O=BEIJING CERTIFICATE AUTHORITY -# Subject: CN=BJCA Global Root CA2 O=BEIJING CERTIFICATE AUTHORITY -# Label: "BJCA Global Root CA2" -# Serial: 58605626836079930195615843123109055211 -# MD5 Fingerprint: 5e:0a:f6:47:5f:a6:14:e8:11:01:95:3f:4d:01:eb:3c -# SHA1 Fingerprint: f4:27:86:eb:6e:b8:6d:88:31:67:02:fb:ba:66:a4:53:00:aa:7a:a6 -# SHA256 Fingerprint: 57:4d:f6:93:1e:27:80:39:66:7b:72:0a:fd:c1:60:0f:c2:7e:b6:6d:d3:09:29:79:fb:73:85:64:87:21:28:82 ------BEGIN CERTIFICATE----- -MIICJTCCAaugAwIBAgIQLBcIfWQqwP6FGFkGz7RK6zAKBggqhkjOPQQDAzBUMQsw -CQYDVQQGEwJDTjEmMCQGA1UECgwdQkVJSklORyBDRVJUSUZJQ0FURSBBVVRIT1JJ -VFkxHTAbBgNVBAMMFEJKQ0EgR2xvYmFsIFJvb3QgQ0EyMB4XDTE5MTIxOTAzMTgy -MVoXDTQ0MTIxMjAzMTgyMVowVDELMAkGA1UEBhMCQ04xJjAkBgNVBAoMHUJFSUpJ -TkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZMR0wGwYDVQQDDBRCSkNBIEdsb2JhbCBS -b290IENBMjB2MBAGByqGSM49AgEGBSuBBAAiA2IABJ3LgJGNU2e1uVCxA/jlSR9B -IgmwUVJY1is0j8USRhTFiy8shP8sbqjV8QnjAyEUxEM9fMEsxEtqSs3ph+B99iK+ -+kpRuDCK/eHeGBIK9ke35xe/J4rUQUyWPGCWwf0VHKNCMEAwHQYDVR0OBBYEFNJK -sVF/BvDRgh9Obl+rg/xI1LCRMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD -AgEGMAoGCCqGSM49BAMDA2gAMGUCMBq8W9f+qdJUDkpd0m2xQNz0Q9XSSpkZElaA -94M04TVOSG0ED1cxMDAtsaqdAzjbBgIxAMvMh1PLet8gUXOQwKhbYdDFUDn9hf7B -43j4ptZLvZuHjw/l1lOWqzzIQNph91Oj9w== ------END CERTIFICATE----- - -# Issuer: CN=Sectigo Public Server Authentication Root E46 O=Sectigo Limited -# Subject: CN=Sectigo Public Server Authentication Root E46 O=Sectigo Limited -# Label: "Sectigo Public Server Authentication Root E46" -# Serial: 88989738453351742415770396670917916916 -# MD5 Fingerprint: 28:23:f8:b2:98:5c:37:16:3b:3e:46:13:4e:b0:b3:01 -# SHA1 Fingerprint: ec:8a:39:6c:40:f0:2e:bc:42:75:d4:9f:ab:1c:1a:5b:67:be:d2:9a -# SHA256 Fingerprint: c9:0f:26:f0:fb:1b:40:18:b2:22:27:51:9b:5c:a2:b5:3e:2c:a5:b3:be:5c:f1:8e:fe:1b:ef:47:38:0c:53:83 ------BEGIN CERTIFICATE----- -MIICOjCCAcGgAwIBAgIQQvLM2htpN0RfFf51KBC49DAKBggqhkjOPQQDAzBfMQsw -CQYDVQQGEwJHQjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1T -ZWN0aWdvIFB1YmxpYyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBFNDYwHhcN -MjEwMzIyMDAwMDAwWhcNNDYwMzIxMjM1OTU5WjBfMQswCQYDVQQGEwJHQjEYMBYG -A1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1TZWN0aWdvIFB1YmxpYyBT -ZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBFNDYwdjAQBgcqhkjOPQIBBgUrgQQA -IgNiAAR2+pmpbiDt+dd34wc7qNs9Xzjoq1WmVk/WSOrsfy2qw7LFeeyZYX8QeccC -WvkEN/U0NSt3zn8gj1KjAIns1aeibVvjS5KToID1AZTc8GgHHs3u/iVStSBDHBv+ -6xnOQ6OjQjBAMB0GA1UdDgQWBBTRItpMWfFLXyY4qp3W7usNw/upYTAOBgNVHQ8B -Af8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNnADBkAjAn7qRa -qCG76UeXlImldCBteU/IvZNeWBj7LRoAasm4PdCkT0RHlAFWovgzJQxC36oCMB3q -4S6ILuH5px0CMk7yn2xVdOOurvulGu7t0vzCAxHrRVxgED1cf5kDW21USAGKcw== ------END CERTIFICATE----- - -# Issuer: CN=Sectigo Public Server Authentication Root R46 O=Sectigo Limited -# Subject: CN=Sectigo Public Server Authentication Root R46 O=Sectigo Limited -# Label: "Sectigo Public Server Authentication Root R46" -# Serial: 156256931880233212765902055439220583700 -# MD5 Fingerprint: 32:10:09:52:00:d5:7e:6c:43:df:15:c0:b1:16:93:e5 -# SHA1 Fingerprint: ad:98:f9:f3:e4:7d:75:3b:65:d4:82:b3:a4:52:17:bb:6e:f5:e4:38 -# SHA256 Fingerprint: 7b:b6:47:a6:2a:ee:ac:88:bf:25:7a:a5:22:d0:1f:fe:a3:95:e0:ab:45:c7:3f:93:f6:56:54:ec:38:f2:5a:06 ------BEGIN CERTIFICATE----- -MIIFijCCA3KgAwIBAgIQdY39i658BwD6qSWn4cetFDANBgkqhkiG9w0BAQwFADBf -MQswCQYDVQQGEwJHQjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQD -Ey1TZWN0aWdvIFB1YmxpYyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBSNDYw -HhcNMjEwMzIyMDAwMDAwWhcNNDYwMzIxMjM1OTU5WjBfMQswCQYDVQQGEwJHQjEY -MBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1TZWN0aWdvIFB1Ymxp -YyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBSNDYwggIiMA0GCSqGSIb3DQEB -AQUAA4ICDwAwggIKAoICAQCTvtU2UnXYASOgHEdCSe5jtrch/cSV1UgrJnwUUxDa -ef0rty2k1Cz66jLdScK5vQ9IPXtamFSvnl0xdE8H/FAh3aTPaE8bEmNtJZlMKpnz -SDBh+oF8HqcIStw+KxwfGExxqjWMrfhu6DtK2eWUAtaJhBOqbchPM8xQljeSM9xf -iOefVNlI8JhD1mb9nxc4Q8UBUQvX4yMPFF1bFOdLvt30yNoDN9HWOaEhUTCDsG3X -ME6WW5HwcCSrv0WBZEMNvSE6Lzzpng3LILVCJ8zab5vuZDCQOc2TZYEhMbUjUDM3 -IuM47fgxMMxF/mL50V0yeUKH32rMVhlATc6qu/m1dkmU8Sf4kaWD5QazYw6A3OAS -VYCmO2a0OYctyPDQ0RTp5A1NDvZdV3LFOxxHVp3i1fuBYYzMTYCQNFu31xR13NgE -SJ/AwSiItOkcyqex8Va3e0lMWeUgFaiEAin6OJRpmkkGj80feRQXEgyDet4fsZfu -+Zd4KKTIRJLpfSYFplhym3kT2BFfrsU4YjRosoYwjviQYZ4ybPUHNs2iTG7sijbt -8uaZFURww3y8nDnAtOFr94MlI1fZEoDlSfB1D++N6xybVCi0ITz8fAr/73trdf+L -HaAZBav6+CuBQug4urv7qv094PPK306Xlynt8xhW6aWWrL3DkJiy4Pmi1KZHQ3xt -zwIDAQABo0IwQDAdBgNVHQ4EFgQUVnNYZJX5khqwEioEYnmhQBWIIUkwDgYDVR0P -AQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAC9c -mTz8Bl6MlC5w6tIyMY208FHVvArzZJ8HXtXBc2hkeqK5Duj5XYUtqDdFqij0lgVQ -YKlJfp/imTYpE0RHap1VIDzYm/EDMrraQKFz6oOht0SmDpkBm+S8f74TlH7Kph52 -gDY9hAaLMyZlbcp+nv4fjFg4exqDsQ+8FxG75gbMY/qB8oFM2gsQa6H61SilzwZA -Fv97fRheORKkU55+MkIQpiGRqRxOF3yEvJ+M0ejf5lG5Nkc/kLnHvALcWxxPDkjB -JYOcCj+esQMzEhonrPcibCTRAUH4WAP+JWgiH5paPHxsnnVI84HxZmduTILA7rpX -DhjvLpr3Etiga+kFpaHpaPi8TD8SHkXoUsCjvxInebnMMTzD9joiFgOgyY9mpFui -TdaBJQbpdqQACj7LzTWb4OE4y2BThihCQRxEV+ioratF4yUQvNs+ZUH7G6aXD+u5 -dHn5HrwdVw1Hr8Mvn4dGp+smWg9WY7ViYG4A++MnESLn/pmPNPW56MORcr3Ywx65 -LvKRRFHQV80MNNVIIb/bE/FmJUNS0nAiNs2fxBx1IK1jcmMGDw4nztJqDby1ORrp -0XZ60Vzk50lJLVU3aPAaOpg+VBeHVOmmJ1CJeyAvP/+/oYtKR5j/K3tJPsMpRmAY -QqszKbrAKbkTidOIijlBO8n9pu0f9GBj39ItVQGL ------END CERTIFICATE----- - -# Issuer: CN=SSL.com TLS RSA Root CA 2022 O=SSL Corporation -# Subject: CN=SSL.com TLS RSA Root CA 2022 O=SSL Corporation -# Label: "SSL.com TLS RSA Root CA 2022" -# Serial: 148535279242832292258835760425842727825 -# MD5 Fingerprint: d8:4e:c6:59:30:d8:fe:a0:d6:7a:5a:2c:2c:69:78:da -# SHA1 Fingerprint: ec:2c:83:40:72:af:26:95:10:ff:0e:f2:03:ee:31:70:f6:78:9d:ca -# SHA256 Fingerprint: 8f:af:7d:2e:2c:b4:70:9b:b8:e0:b3:36:66:bf:75:a5:dd:45:b5:de:48:0f:8e:a8:d4:bf:e6:be:bc:17:f2:ed ------BEGIN CERTIFICATE----- -MIIFiTCCA3GgAwIBAgIQb77arXO9CEDii02+1PdbkTANBgkqhkiG9w0BAQsFADBO -MQswCQYDVQQGEwJVUzEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMSUwIwYDVQQD -DBxTU0wuY29tIFRMUyBSU0EgUm9vdCBDQSAyMDIyMB4XDTIyMDgyNTE2MzQyMloX -DTQ2MDgxOTE2MzQyMVowTjELMAkGA1UEBhMCVVMxGDAWBgNVBAoMD1NTTCBDb3Jw -b3JhdGlvbjElMCMGA1UEAwwcU1NMLmNvbSBUTFMgUlNBIFJvb3QgQ0EgMjAyMjCC -AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANCkCXJPQIgSYT41I57u9nTP -L3tYPc48DRAokC+X94xI2KDYJbFMsBFMF3NQ0CJKY7uB0ylu1bUJPiYYf7ISf5OY -t6/wNr/y7hienDtSxUcZXXTzZGbVXcdotL8bHAajvI9AI7YexoS9UcQbOcGV0ins -S657Lb85/bRi3pZ7QcacoOAGcvvwB5cJOYF0r/c0WRFXCsJbwST0MXMwgsadugL3 -PnxEX4MN8/HdIGkWCVDi1FW24IBydm5MR7d1VVm0U3TZlMZBrViKMWYPHqIbKUBO -L9975hYsLfy/7PO0+r4Y9ptJ1O4Fbtk085zx7AGL0SDGD6C1vBdOSHtRwvzpXGk3 -R2azaPgVKPC506QVzFpPulJwoxJF3ca6TvvC0PeoUidtbnm1jPx7jMEWTO6Af77w -dr5BUxIzrlo4QqvXDz5BjXYHMtWrifZOZ9mxQnUjbvPNQrL8VfVThxc7wDNY8VLS -+YCk8OjwO4s4zKTGkH8PnP2L0aPP2oOnaclQNtVcBdIKQXTbYxE3waWglksejBYS -d66UNHsef8JmAOSqg+qKkK3ONkRN0VHpvB/zagX9wHQfJRlAUW7qglFA35u5CCoG -AtUjHBPW6dvbxrB6y3snm/vg1UYk7RBLY0ulBY+6uB0rpvqR4pJSvezrZ5dtmi2f -gTIFZzL7SAg/2SW4BCUvAgMBAAGjYzBhMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0j -BBgwFoAU+y437uOEeicuzRk1sTN8/9REQrkwHQYDVR0OBBYEFPsuN+7jhHonLs0Z -NbEzfP/UREK5MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAjYlt -hEUY8U+zoO9opMAdrDC8Z2awms22qyIZZtM7QbUQnRC6cm4pJCAcAZli05bg4vsM -QtfhWsSWTVTNj8pDU/0quOr4ZcoBwq1gaAafORpR2eCNJvkLTqVTJXojpBzOCBvf -R4iyrT7gJ4eLSYwfqUdYe5byiB0YrrPRpgqU+tvT5TgKa3kSM/tKWTcWQA673vWJ -DPFs0/dRa1419dvAJuoSc06pkZCmF8NsLzjUo3KUQyxi4U5cMj29TH0ZR6LDSeeW -P4+a0zvkEdiLA9z2tmBVGKaBUfPhqBVq6+AL8BQx1rmMRTqoENjwuSfr98t67wVy -lrXEj5ZzxOhWc5y8aVFjvO9nHEMaX3cZHxj4HCUp+UmZKbaSPaKDN7EgkaibMOlq -bLQjk2UEqxHzDh1TJElTHaE/nUiSEeJ9DU/1172iWD54nR4fK/4huxoTtrEoZP2w -AgDHbICivRZQIA9ygV/MlP+7mea6kMvq+cYMwq7FGc4zoWtcu358NFcXrfA/rs3q -r5nsLFR+jM4uElZI7xc7P0peYNLcdDa8pUNjyw9bowJWCZ4kLOGGgYz+qxcs+sji -Mho6/4UIyYOf8kpIEFR3N+2ivEC+5BB09+Rbu7nzifmPQdjH5FCQNYA+HLhNkNPU -98OwoX6EyneSMSy4kLGCenROmxMmtNVQZlR4rmA= ------END CERTIFICATE----- - -# Issuer: CN=SSL.com TLS ECC Root CA 2022 O=SSL Corporation -# Subject: CN=SSL.com TLS ECC Root CA 2022 O=SSL Corporation -# Label: "SSL.com TLS ECC Root CA 2022" -# Serial: 26605119622390491762507526719404364228 -# MD5 Fingerprint: 99:d7:5c:f1:51:36:cc:e9:ce:d9:19:2e:77:71:56:c5 -# SHA1 Fingerprint: 9f:5f:d9:1a:54:6d:f5:0c:71:f0:ee:7a:bd:17:49:98:84:73:e2:39 -# SHA256 Fingerprint: c3:2f:fd:9f:46:f9:36:d1:6c:36:73:99:09:59:43:4b:9a:d6:0a:af:bb:9e:7c:f3:36:54:f1:44:cc:1b:a1:43 ------BEGIN CERTIFICATE----- -MIICOjCCAcCgAwIBAgIQFAP1q/s3ixdAW+JDsqXRxDAKBggqhkjOPQQDAzBOMQsw -CQYDVQQGEwJVUzEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMSUwIwYDVQQDDBxT -U0wuY29tIFRMUyBFQ0MgUm9vdCBDQSAyMDIyMB4XDTIyMDgyNTE2MzM0OFoXDTQ2 -MDgxOTE2MzM0N1owTjELMAkGA1UEBhMCVVMxGDAWBgNVBAoMD1NTTCBDb3Jwb3Jh -dGlvbjElMCMGA1UEAwwcU1NMLmNvbSBUTFMgRUNDIFJvb3QgQ0EgMjAyMjB2MBAG -ByqGSM49AgEGBSuBBAAiA2IABEUpNXP6wrgjzhR9qLFNoFs27iosU8NgCTWyJGYm -acCzldZdkkAZDsalE3D07xJRKF3nzL35PIXBz5SQySvOkkJYWWf9lCcQZIxPBLFN -SeR7T5v15wj4A4j3p8OSSxlUgaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSME -GDAWgBSJjy+j6CugFFR781a4Jl9nOAuc0DAdBgNVHQ4EFgQUiY8vo+groBRUe/NW -uCZfZzgLnNAwDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMDA2gAMGUCMFXjIlbp -15IkWE8elDIPDAI2wv2sdDJO4fscgIijzPvX6yv/N33w7deedWo1dlJF4AIxAMeN -b0Igj762TVntd00pxCAgRWSGOlDGxK0tk/UYfXLtqc/ErFc2KAhl3zx5Zn6g6g== ------END CERTIFICATE----- - -# Issuer: CN=Atos TrustedRoot Root CA ECC TLS 2021 O=Atos -# Subject: CN=Atos TrustedRoot Root CA ECC TLS 2021 O=Atos -# Label: "Atos TrustedRoot Root CA ECC TLS 2021" -# Serial: 81873346711060652204712539181482831616 -# MD5 Fingerprint: 16:9f:ad:f1:70:ad:79:d6:ed:29:b4:d1:c5:79:70:a8 -# SHA1 Fingerprint: 9e:bc:75:10:42:b3:02:f3:81:f4:f7:30:62:d4:8f:c3:a7:51:b2:dd -# SHA256 Fingerprint: b2:fa:e5:3e:14:cc:d7:ab:92:12:06:47:01:ae:27:9c:1d:89:88:fa:cb:77:5f:a8:a0:08:91:4e:66:39:88:a8 ------BEGIN CERTIFICATE----- -MIICFTCCAZugAwIBAgIQPZg7pmY9kGP3fiZXOATvADAKBggqhkjOPQQDAzBMMS4w -LAYDVQQDDCVBdG9zIFRydXN0ZWRSb290IFJvb3QgQ0EgRUNDIFRMUyAyMDIxMQ0w -CwYDVQQKDARBdG9zMQswCQYDVQQGEwJERTAeFw0yMTA0MjIwOTI2MjNaFw00MTA0 -MTcwOTI2MjJaMEwxLjAsBgNVBAMMJUF0b3MgVHJ1c3RlZFJvb3QgUm9vdCBDQSBF -Q0MgVExTIDIwMjExDTALBgNVBAoMBEF0b3MxCzAJBgNVBAYTAkRFMHYwEAYHKoZI -zj0CAQYFK4EEACIDYgAEloZYKDcKZ9Cg3iQZGeHkBQcfl+3oZIK59sRxUM6KDP/X -tXa7oWyTbIOiaG6l2b4siJVBzV3dscqDY4PMwL502eCdpO5KTlbgmClBk1IQ1SQ4 -AjJn8ZQSb+/Xxd4u/RmAo0IwQDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR2 -KCXWfeBmmnoJsmo7jjPXNtNPojAOBgNVHQ8BAf8EBAMCAYYwCgYIKoZIzj0EAwMD -aAAwZQIwW5kp85wxtolrbNa9d+F851F+uDrNozZffPc8dz7kUK2o59JZDCaOMDtu -CCrCp1rIAjEAmeMM56PDr9NJLkaCI2ZdyQAUEv049OGYa3cpetskz2VAv9LcjBHo -9H1/IISpQuQo ------END CERTIFICATE----- - -# Issuer: CN=Atos TrustedRoot Root CA RSA TLS 2021 O=Atos -# Subject: CN=Atos TrustedRoot Root CA RSA TLS 2021 O=Atos -# Label: "Atos TrustedRoot Root CA RSA TLS 2021" -# Serial: 111436099570196163832749341232207667876 -# MD5 Fingerprint: d4:d3:46:b8:9a:c0:9c:76:5d:9e:3a:c3:b9:99:31:d2 -# SHA1 Fingerprint: 18:52:3b:0d:06:37:e4:d6:3a:df:23:e4:98:fb:5b:16:fb:86:74:48 -# SHA256 Fingerprint: 81:a9:08:8e:a5:9f:b3:64:c5:48:a6:f8:55:59:09:9b:6f:04:05:ef:bf:18:e5:32:4e:c9:f4:57:ba:00:11:2f ------BEGIN CERTIFICATE----- -MIIFZDCCA0ygAwIBAgIQU9XP5hmTC/srBRLYwiqipDANBgkqhkiG9w0BAQwFADBM -MS4wLAYDVQQDDCVBdG9zIFRydXN0ZWRSb290IFJvb3QgQ0EgUlNBIFRMUyAyMDIx -MQ0wCwYDVQQKDARBdG9zMQswCQYDVQQGEwJERTAeFw0yMTA0MjIwOTIxMTBaFw00 -MTA0MTcwOTIxMDlaMEwxLjAsBgNVBAMMJUF0b3MgVHJ1c3RlZFJvb3QgUm9vdCBD -QSBSU0EgVExTIDIwMjExDTALBgNVBAoMBEF0b3MxCzAJBgNVBAYTAkRFMIICIjAN -BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtoAOxHm9BYx9sKOdTSJNy/BBl01Z -4NH+VoyX8te9j2y3I49f1cTYQcvyAh5x5en2XssIKl4w8i1mx4QbZFc4nXUtVsYv -Ye+W/CBGvevUez8/fEc4BKkbqlLfEzfTFRVOvV98r61jx3ncCHvVoOX3W3WsgFWZ -kmGbzSoXfduP9LVq6hdKZChmFSlsAvFr1bqjM9xaZ6cF4r9lthawEO3NUDPJcFDs -GY6wx/J0W2tExn2WuZgIWWbeKQGb9Cpt0xU6kGpn8bRrZtkh68rZYnxGEFzedUln -nkL5/nWpo63/dgpnQOPF943HhZpZnmKaau1Fh5hnstVKPNe0OwANwI8f4UDErmwh -3El+fsqyjW22v5MvoVw+j8rtgI5Y4dtXz4U2OLJxpAmMkokIiEjxQGMYsluMWuPD -0xeqqxmjLBvk1cbiZnrXghmmOxYsL3GHX0WelXOTwkKBIROW1527k2gV+p2kHYzy -geBYBr3JtuP2iV2J+axEoctr+hbxx1A9JNr3w+SH1VbxT5Aw+kUJWdo0zuATHAR8 -ANSbhqRAvNncTFd+rrcztl524WWLZt+NyteYr842mIycg5kDcPOvdO3GDjbnvezB -c6eUWsuSZIKmAMFwoW4sKeFYV+xafJlrJaSQOoD0IJ2azsct+bJLKZWD6TWNp0lI -pw9MGZHQ9b8Q4HECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU -dEmZ0f+0emhFdcN+tNzMzjkz2ggwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB -DAUAA4ICAQAjQ1MkYlxt/T7Cz1UAbMVWiLkO3TriJQ2VSpfKgInuKs1l+NsW4AmS -4BjHeJi78+xCUvuppILXTdiK/ORO/auQxDh1MoSf/7OwKwIzNsAQkG8dnK/haZPs -o0UvFJ/1TCplQ3IM98P4lYsU84UgYt1UU90s3BiVaU+DR3BAM1h3Egyi61IxHkzJ -qM7F78PRreBrAwA0JrRUITWXAdxfG/F851X6LWh3e9NpzNMOa7pNdkTWwhWaJuyw -xfW70Xp0wmzNxbVe9kzmWy2B27O3Opee7c9GslA9hGCZcbUztVdF5kJHdWoOsAgM -rr3e97sPWD2PAzHoPYJQyi9eDF20l74gNAf0xBLh7tew2VktafcxBPTy+av5EzH4 -AXcOPUIjJsyacmdRIXrMPIWo6iFqO9taPKU0nprALN+AnCng33eU0aKAQv9qTFsR -0PXNor6uzFFcw9VUewyu1rkGd4Di7wcaaMxZUa1+XGdrudviB0JbuAEFWDlN5LuY -o7Ey7Nmj1m+UI/87tyll5gfp77YZ6ufCOB0yiJA8EytuzO+rdwY0d4RPcuSBhPm5 -dDTedk+SKlOxJTnbPP/lPqYO5Wue/9vsL3SD3460s6neFE3/MaNFcyT6lSnMEpcE -oji2jbDwN/zIIX8/syQbPYtuzE2wFg2WHYMfRsCbvUOZ58SWLs5fyQ== ------END CERTIFICATE----- - -# Issuer: CN=TrustAsia Global Root CA G3 O=TrustAsia Technologies, Inc. -# Subject: CN=TrustAsia Global Root CA G3 O=TrustAsia Technologies, Inc. -# Label: "TrustAsia Global Root CA G3" -# Serial: 576386314500428537169965010905813481816650257167 -# MD5 Fingerprint: 30:42:1b:b7:bb:81:75:35:e4:16:4f:53:d2:94:de:04 -# SHA1 Fingerprint: 63:cf:b6:c1:27:2b:56:e4:88:8e:1c:23:9a:b6:2e:81:47:24:c3:c7 -# SHA256 Fingerprint: e0:d3:22:6a:eb:11:63:c2:e4:8f:f9:be:3b:50:b4:c6:43:1b:e7:bb:1e:ac:c5:c3:6b:5d:5e:c5:09:03:9a:08 ------BEGIN CERTIFICATE----- -MIIFpTCCA42gAwIBAgIUZPYOZXdhaqs7tOqFhLuxibhxkw8wDQYJKoZIhvcNAQEM -BQAwWjELMAkGA1UEBhMCQ04xJTAjBgNVBAoMHFRydXN0QXNpYSBUZWNobm9sb2dp -ZXMsIEluYy4xJDAiBgNVBAMMG1RydXN0QXNpYSBHbG9iYWwgUm9vdCBDQSBHMzAe -Fw0yMTA1MjAwMjEwMTlaFw00NjA1MTkwMjEwMTlaMFoxCzAJBgNVBAYTAkNOMSUw -IwYDVQQKDBxUcnVzdEFzaWEgVGVjaG5vbG9naWVzLCBJbmMuMSQwIgYDVQQDDBtU -cnVzdEFzaWEgR2xvYmFsIFJvb3QgQ0EgRzMwggIiMA0GCSqGSIb3DQEBAQUAA4IC -DwAwggIKAoICAQDAMYJhkuSUGwoqZdC+BqmHO1ES6nBBruL7dOoKjbmzTNyPtxNS -T1QY4SxzlZHFZjtqz6xjbYdT8PfxObegQ2OwxANdV6nnRM7EoYNl9lA+sX4WuDqK -AtCWHwDNBSHvBm3dIZwZQ0WhxeiAysKtQGIXBsaqvPPW5vxQfmZCHzyLpnl5hkA1 -nyDvP+uLRx+PjsXUjrYsyUQE49RDdT/VP68czH5GX6zfZBCK70bwkPAPLfSIC7Ep -qq+FqklYqL9joDiR5rPmd2jE+SoZhLsO4fWvieylL1AgdB4SQXMeJNnKziyhWTXA -yB1GJ2Faj/lN03J5Zh6fFZAhLf3ti1ZwA0pJPn9pMRJpxx5cynoTi+jm9WAPzJMs -hH/x/Gr8m0ed262IPfN2dTPXS6TIi/n1Q1hPy8gDVI+lhXgEGvNz8teHHUGf59gX -zhqcD0r83ERoVGjiQTz+LISGNzzNPy+i2+f3VANfWdP3kXjHi3dqFuVJhZBFcnAv -kV34PmVACxmZySYgWmjBNb9Pp1Hx2BErW+Canig7CjoKH8GB5S7wprlppYiU5msT -f9FkPz2ccEblooV7WIQn3MSAPmeamseaMQ4w7OYXQJXZRe0Blqq/DPNL0WP3E1jA -uPP6Z92bfW1K/zJMtSU7/xxnD4UiWQWRkUF3gdCFTIcQcf+eQxuulXUtgQIDAQAB -o2MwYTAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFEDk5PIj7zjKsK5Xf/Ih -MBY027ySMB0GA1UdDgQWBBRA5OTyI+84yrCuV3/yITAWNNu8kjAOBgNVHQ8BAf8E -BAMCAQYwDQYJKoZIhvcNAQEMBQADggIBACY7UeFNOPMyGLS0XuFlXsSUT9SnYaP4 -wM8zAQLpw6o1D/GUE3d3NZ4tVlFEbuHGLige/9rsR82XRBf34EzC4Xx8MnpmyFq2 -XFNFV1pF1AWZLy4jVe5jaN/TG3inEpQGAHUNcoTpLrxaatXeL1nHo+zSh2bbt1S1 -JKv0Q3jbSwTEb93mPmY+KfJLaHEih6D4sTNjduMNhXJEIlU/HHzp/LgV6FL6qj6j -ITk1dImmasI5+njPtqzn59ZW/yOSLlALqbUHM/Q4X6RJpstlcHboCoWASzY9M/eV -VHUl2qzEc4Jl6VL1XP04lQJqaTDFHApXB64ipCz5xUG3uOyfT0gA+QEEVcys+TIx -xHWVBqB/0Y0n3bOppHKH/lmLmnp0Ft0WpWIp6zqW3IunaFnT63eROfjXy9mPX1on -AX1daBli2MjN9LdyR75bl87yraKZk62Uy5P2EgmVtqvXO9A/EcswFi55gORngS1d -7XB4tmBZrOFdRWOPyN9yaFvqHbgB8X7754qz41SgOAngPN5C8sLtLpvzHzW2Ntjj -gKGLzZlkD8Kqq7HK9W+eQ42EVJmzbsASZthwEPEGNTNDqJwuuhQxzhB/HIbjj9LV -+Hfsm6vxL2PZQl/gZ4FkkfGXL/xuJvYz+NO1+MRiqzFRJQJ6+N1rZdVtTTDIZbpo -FGWsJwt0ivKH ------END CERTIFICATE----- - -# Issuer: CN=TrustAsia Global Root CA G4 O=TrustAsia Technologies, Inc. -# Subject: CN=TrustAsia Global Root CA G4 O=TrustAsia Technologies, Inc. -# Label: "TrustAsia Global Root CA G4" -# Serial: 451799571007117016466790293371524403291602933463 -# MD5 Fingerprint: 54:dd:b2:d7:5f:d8:3e:ed:7c:e0:0b:2e:cc:ed:eb:eb -# SHA1 Fingerprint: 57:73:a5:61:5d:80:b2:e6:ac:38:82:fc:68:07:31:ac:9f:b5:92:5a -# SHA256 Fingerprint: be:4b:56:cb:50:56:c0:13:6a:52:6d:f4:44:50:8d:aa:36:a0:b5:4f:42:e4:ac:38:f7:2a:f4:70:e4:79:65:4c ------BEGIN CERTIFICATE----- -MIICVTCCAdygAwIBAgIUTyNkuI6XY57GU4HBdk7LKnQV1tcwCgYIKoZIzj0EAwMw -WjELMAkGA1UEBhMCQ04xJTAjBgNVBAoMHFRydXN0QXNpYSBUZWNobm9sb2dpZXMs -IEluYy4xJDAiBgNVBAMMG1RydXN0QXNpYSBHbG9iYWwgUm9vdCBDQSBHNDAeFw0y -MTA1MjAwMjEwMjJaFw00NjA1MTkwMjEwMjJaMFoxCzAJBgNVBAYTAkNOMSUwIwYD -VQQKDBxUcnVzdEFzaWEgVGVjaG5vbG9naWVzLCBJbmMuMSQwIgYDVQQDDBtUcnVz -dEFzaWEgR2xvYmFsIFJvb3QgQ0EgRzQwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAATx -s8045CVD5d4ZCbuBeaIVXxVjAd7Cq92zphtnS4CDr5nLrBfbK5bKfFJV4hrhPVbw -LxYI+hW8m7tH5j/uqOFMjPXTNvk4XatwmkcN4oFBButJ+bAp3TPsUKV/eSm4IJij -YzBhMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUpbtKl86zK3+kMd6Xg1mD -pm9xy94wHQYDVR0OBBYEFKW7SpfOsyt/pDHel4NZg6ZvccveMA4GA1UdDwEB/wQE -AwIBBjAKBggqhkjOPQQDAwNnADBkAjBe8usGzEkxn0AAbbd+NvBNEU/zy4k6LHiR -UKNbwMp1JvK/kF0LgoxgKJ/GcJpo5PECMFxYDlZ2z1jD1xCMuo6u47xkdUfFVZDj -/bpV6wfEU6s3qe4hsiFbYI89MvHVI5TWWA== ------END CERTIFICATE----- - -# Issuer: CN=Telekom Security TLS ECC Root 2020 O=Deutsche Telekom Security GmbH -# Subject: CN=Telekom Security TLS ECC Root 2020 O=Deutsche Telekom Security GmbH -# Label: "Telekom Security TLS ECC Root 2020" -# Serial: 72082518505882327255703894282316633856 -# MD5 Fingerprint: c1:ab:fe:6a:10:2c:03:8d:bc:1c:22:32:c0:85:a7:fd -# SHA1 Fingerprint: c0:f8:96:c5:a9:3b:01:06:21:07:da:18:42:48:bc:e9:9d:88:d5:ec -# SHA256 Fingerprint: 57:8a:f4:de:d0:85:3f:4e:59:98:db:4a:ea:f9:cb:ea:8d:94:5f:60:b6:20:a3:8d:1a:3c:13:b2:bc:7b:a8:e1 ------BEGIN CERTIFICATE----- -MIICQjCCAcmgAwIBAgIQNjqWjMlcsljN0AFdxeVXADAKBggqhkjOPQQDAzBjMQsw -CQYDVQQGEwJERTEnMCUGA1UECgweRGV1dHNjaGUgVGVsZWtvbSBTZWN1cml0eSBH -bWJIMSswKQYDVQQDDCJUZWxla29tIFNlY3VyaXR5IFRMUyBFQ0MgUm9vdCAyMDIw -MB4XDTIwMDgyNTA3NDgyMFoXDTQ1MDgyNTIzNTk1OVowYzELMAkGA1UEBhMCREUx -JzAlBgNVBAoMHkRldXRzY2hlIFRlbGVrb20gU2VjdXJpdHkgR21iSDErMCkGA1UE -AwwiVGVsZWtvbSBTZWN1cml0eSBUTFMgRUNDIFJvb3QgMjAyMDB2MBAGByqGSM49 -AgEGBSuBBAAiA2IABM6//leov9Wq9xCazbzREaK9Z0LMkOsVGJDZos0MKiXrPk/O -tdKPD/M12kOLAoC+b1EkHQ9rK8qfwm9QMuU3ILYg/4gND21Ju9sGpIeQkpT0CdDP -f8iAC8GXs7s1J8nCG6NCMEAwHQYDVR0OBBYEFONyzG6VmUex5rNhTNHLq+O6zd6f -MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cA -MGQCMHVSi7ekEE+uShCLsoRbQuHmKjYC2qBuGT8lv9pZMo7k+5Dck2TOrbRBR2Di -z6fLHgIwN0GMZt9Ba9aDAEH9L1r3ULRn0SyocddDypwnJJGDSA3PzfdUga/sf+Rn -27iQ7t0l ------END CERTIFICATE----- - -# Issuer: CN=Telekom Security TLS RSA Root 2023 O=Deutsche Telekom Security GmbH -# Subject: CN=Telekom Security TLS RSA Root 2023 O=Deutsche Telekom Security GmbH -# Label: "Telekom Security TLS RSA Root 2023" -# Serial: 44676229530606711399881795178081572759 -# MD5 Fingerprint: bf:5b:eb:54:40:cd:48:71:c4:20:8d:7d:de:0a:42:f2 -# SHA1 Fingerprint: 54:d3:ac:b3:bd:57:56:f6:85:9d:ce:e5:c3:21:e2:d4:ad:83:d0:93 -# SHA256 Fingerprint: ef:c6:5c:ad:bb:59:ad:b6:ef:e8:4d:a2:23:11:b3:56:24:b7:1b:3b:1e:a0:da:8b:66:55:17:4e:c8:97:86:46 ------BEGIN CERTIFICATE----- -MIIFszCCA5ugAwIBAgIQIZxULej27HF3+k7ow3BXlzANBgkqhkiG9w0BAQwFADBj -MQswCQYDVQQGEwJERTEnMCUGA1UECgweRGV1dHNjaGUgVGVsZWtvbSBTZWN1cml0 -eSBHbWJIMSswKQYDVQQDDCJUZWxla29tIFNlY3VyaXR5IFRMUyBSU0EgUm9vdCAy -MDIzMB4XDTIzMDMyODEyMTY0NVoXDTQ4MDMyNzIzNTk1OVowYzELMAkGA1UEBhMC -REUxJzAlBgNVBAoMHkRldXRzY2hlIFRlbGVrb20gU2VjdXJpdHkgR21iSDErMCkG -A1UEAwwiVGVsZWtvbSBTZWN1cml0eSBUTFMgUlNBIFJvb3QgMjAyMzCCAiIwDQYJ -KoZIhvcNAQEBBQADggIPADCCAgoCggIBAO01oYGA88tKaVvC+1GDrib94W7zgRJ9 -cUD/h3VCKSHtgVIs3xLBGYSJwb3FKNXVS2xE1kzbB5ZKVXrKNoIENqil/Cf2SfHV -cp6R+SPWcHu79ZvB7JPPGeplfohwoHP89v+1VmLhc2o0mD6CuKyVU/QBoCcHcqMA -U6DksquDOFczJZSfvkgdmOGjup5czQRxUX11eKvzWarE4GC+j4NSuHUaQTXtvPM6 -Y+mpFEXX5lLRbtLevOP1Czvm4MS9Q2QTps70mDdsipWol8hHD/BeEIvnHRz+sTug -BTNoBUGCwQMrAcjnj02r6LX2zWtEtefdi+zqJbQAIldNsLGyMcEWzv/9FIS3R/qy -8XDe24tsNlikfLMR0cN3f1+2JeANxdKz+bi4d9s3cXFH42AYTyS2dTd4uaNir73J -co4vzLuu2+QVUhkHM/tqty1LkCiCc/4YizWN26cEar7qwU02OxY2kTLvtkCJkUPg -8qKrBC7m8kwOFjQgrIfBLX7JZkcXFBGk8/ehJImr2BrIoVyxo/eMbcgByU/J7MT8 -rFEz0ciD0cmfHdRHNCk+y7AO+oMLKFjlKdw/fKifybYKu6boRhYPluV75Gp6SG12 -mAWl3G0eQh5C2hrgUve1g8Aae3g1LDj1H/1Joy7SWWO/gLCMk3PLNaaZlSJhZQNg -+y+TS/qanIA7AgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUtqeX -gj10hZv3PJ+TmpV5dVKMbUcwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBS2 -p5eCPXSFm/c8n5OalXl1UoxtRzANBgkqhkiG9w0BAQwFAAOCAgEAqMxhpr51nhVQ -pGv7qHBFfLp+sVr8WyP6Cnf4mHGCDG3gXkaqk/QeoMPhk9tLrbKmXauw1GLLXrtm -9S3ul0A8Yute1hTWjOKWi0FpkzXmuZlrYrShF2Y0pmtjxrlO8iLpWA1WQdH6DErw -M807u20hOq6OcrXDSvvpfeWxm4bu4uB9tPcy/SKE8YXJN3nptT+/XOR0so8RYgDd -GGah2XsjX/GO1WfoVNpbOms2b/mBsTNHM3dA+VKq3dSDz4V4mZqTuXNnQkYRIer+ -CqkbGmVps4+uFrb2S1ayLfmlyOw7YqPta9BO1UAJpB+Y1zqlklkg5LB9zVtzaL1t -xKITDmcZuI1CfmwMmm6gJC3VRRvcxAIU/oVbZZfKTpBQCHpCNfnqwmbU+AGuHrS+ -w6jv/naaoqYfRvaE7fzbzsQCzndILIyy7MMAo+wsVRjBfhnu4S/yrYObnqsZ38aK -L4x35bcF7DvB7L6Gs4a8wPfc5+pbrrLMtTWGS9DiP7bY+A4A7l3j941Y/8+LN+lj -X273CXE2whJdV/LItM3z7gLfEdxquVeEHVlNjM7IDiPCtyaaEBRx/pOyiriA8A4Q -ntOoUAw3gi/q4Iqd4Sw5/7W0cwDk90imc6y/st53BIe0o82bNSQ3+pCTE4FCxpgm -dTdmQRCsu/WU48IxK63nI1bMNSWSs1A= ------END CERTIFICATE----- - -# Issuer: CN=TWCA CYBER Root CA O=TAIWAN-CA OU=Root CA -# Subject: CN=TWCA CYBER Root CA O=TAIWAN-CA OU=Root CA -# Label: "TWCA CYBER Root CA" -# Serial: 85076849864375384482682434040119489222 -# MD5 Fingerprint: 0b:33:a0:97:52:95:d4:a9:fd:bb:db:6e:a3:55:5b:51 -# SHA1 Fingerprint: f6:b1:1c:1a:83:38:e9:7b:db:b3:a8:c8:33:24:e0:2d:9c:7f:26:66 -# SHA256 Fingerprint: 3f:63:bb:28:14:be:17:4e:c8:b6:43:9c:f0:8d:6d:56:f0:b7:c4:05:88:3a:56:48:a3:34:42:4d:6b:3e:c5:58 ------BEGIN CERTIFICATE----- -MIIFjTCCA3WgAwIBAgIQQAE0jMIAAAAAAAAAATzyxjANBgkqhkiG9w0BAQwFADBQ -MQswCQYDVQQGEwJUVzESMBAGA1UEChMJVEFJV0FOLUNBMRAwDgYDVQQLEwdSb290 -IENBMRswGQYDVQQDExJUV0NBIENZQkVSIFJvb3QgQ0EwHhcNMjIxMTIyMDY1NDI5 -WhcNNDcxMTIyMTU1OTU5WjBQMQswCQYDVQQGEwJUVzESMBAGA1UEChMJVEFJV0FO -LUNBMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJUV0NBIENZQkVSIFJvb3Qg -Q0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDG+Moe2Qkgfh1sTs6P -40czRJzHyWmqOlt47nDSkvgEs1JSHWdyKKHfi12VCv7qze33Kc7wb3+szT3vsxxF -avcokPFhV8UMxKNQXd7UtcsZyoC5dc4pztKFIuwCY8xEMCDa6pFbVuYdHNWdZsc/ -34bKS1PE2Y2yHer43CdTo0fhYcx9tbD47nORxc5zb87uEB8aBs/pJ2DFTxnk684i -JkXXYJndzk834H/nY62wuFm40AZoNWDTNq5xQwTxaWV4fPMf88oon1oglWa0zbfu -j3ikRRjpJi+NmykosaS3Om251Bw4ckVYsV7r8Cibt4LK/c/WMw+f+5eesRycnupf -Xtuq3VTpMCEobY5583WSjCb+3MX2w7DfRFlDo7YDKPYIMKoNM+HvnKkHIuNZW0CP -2oi3aQiotyMuRAlZN1vH4xfyIutuOVLF3lSnmMlLIJXcRolftBL5hSmO68gnFSDA -S9TMfAxsNAwmmyYxpjyn9tnQS6Jk/zuZQXLB4HCX8SS7K8R0IrGsayIyJNN4KsDA -oS/xUgXJP+92ZuJF2A09rZXIx4kmyA+upwMu+8Ff+iDhcK2wZSA3M2Cw1a/XDBzC -kHDXShi8fgGwsOsVHkQGzaRP6AzRwyAQ4VRlnrZR0Bp2a0JaWHY06rc3Ga4udfmW -5cFZ95RXKSWNOkyrTZpB0F8mAwIDAQABo2MwYTAOBgNVHQ8BAf8EBAMCAQYwDwYD -VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBSdhWEUfMFib5do5E83QOGt4A1WNzAd -BgNVHQ4EFgQUnYVhFHzBYm+XaORPN0DhreANVjcwDQYJKoZIhvcNAQEMBQADggIB -AGSPesRiDrWIzLjHhg6hShbNcAu3p4ULs3a2D6f/CIsLJc+o1IN1KriWiLb73y0t -tGlTITVX1olNc79pj3CjYcya2x6a4CD4bLubIp1dhDGaLIrdaqHXKGnK/nZVekZn -68xDiBaiA9a5F/gZbG0jAn/xX9AKKSM70aoK7akXJlQKTcKlTfjF/biBzysseKNn -TKkHmvPfXvt89YnNdJdhEGoHK4Fa0o635yDRIG4kqIQnoVesqlVYL9zZyvpoBJ7t -RCT5dEA7IzOrg1oYJkK2bVS1FmAwbLGg+LhBoF1JSdJlBTrq/p1hvIbZv97Tujqx -f36SNI7JAG7cmL3c7IAFrQI932XtCwP39xaEBDG6k5TY8hL4iuO/Qq+n1M0RFxbI -Qh0UqEL20kCGoE8jypZFVmAGzbdVAaYBlGX+bgUJurSkquLvWL69J1bY73NxW0Qz -8ppy6rBePm6pUlvscG21h483XjyMnM7k8M4MZ0HMzvaAq07MTFb1wWFZk7Q+ptq4 -NxKfKjLji7gh7MMrZQzvIt6IKTtM1/r+t+FHvpw+PoP7UV31aPcuIYXcv/Fa4nzX -xeSDwWrruoBa3lwtcHb4yOWHh8qgnaHlIhInD0Q9HWzq1MKLL295q39QpsQZp6F6 -t5b5wR9iWqJDB0BeJsas7a5wFsWqynKKTbDPAYsDP27X ------END CERTIFICATE----- - -# Issuer: CN=SecureSign Root CA14 O=Cybertrust Japan Co., Ltd. -# Subject: CN=SecureSign Root CA14 O=Cybertrust Japan Co., Ltd. -# Label: "SecureSign Root CA14" -# Serial: 575790784512929437950770173562378038616896959179 -# MD5 Fingerprint: 71:0d:72:fa:92:19:65:5e:89:04:ac:16:33:f0:bc:d5 -# SHA1 Fingerprint: dd:50:c0:f7:79:b3:64:2e:74:a2:b8:9d:9f:d3:40:dd:bb:f0:f2:4f -# SHA256 Fingerprint: 4b:00:9c:10:34:49:4f:9a:b5:6b:ba:3b:a1:d6:27:31:fc:4d:20:d8:95:5a:dc:ec:10:a9:25:60:72:61:e3:38 ------BEGIN CERTIFICATE----- -MIIFcjCCA1qgAwIBAgIUZNtaDCBO6Ncpd8hQJ6JaJ90t8sswDQYJKoZIhvcNAQEM -BQAwUTELMAkGA1UEBhMCSlAxIzAhBgNVBAoTGkN5YmVydHJ1c3QgSmFwYW4gQ28u -LCBMdGQuMR0wGwYDVQQDExRTZWN1cmVTaWduIFJvb3QgQ0ExNDAeFw0yMDA0MDgw -NzA2MTlaFw00NTA0MDgwNzA2MTlaMFExCzAJBgNVBAYTAkpQMSMwIQYDVQQKExpD -eWJlcnRydXN0IEphcGFuIENvLiwgTHRkLjEdMBsGA1UEAxMUU2VjdXJlU2lnbiBS -b290IENBMTQwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDF0nqh1oq/ -FjHQmNE6lPxauG4iwWL3pwon71D2LrGeaBLwbCRjOfHw3xDG3rdSINVSW0KZnvOg -vlIfX8xnbacuUKLBl422+JX1sLrcneC+y9/3OPJH9aaakpUqYllQC6KxNedlsmGy -6pJxaeQp8E+BgQQ8sqVb1MWoWWd7VRxJq3qdwudzTe/NCcLEVxLbAQ4jeQkHO6Lo -/IrPj8BGJJw4J+CDnRugv3gVEOuGTgpa/d/aLIJ+7sr2KeH6caH3iGicnPCNvg9J -kdjqOvn90Ghx2+m1K06Ckm9mH+Dw3EzsytHqunQG+bOEkJTRX45zGRBdAuVwpcAQ -0BB8b8VYSbSwbprafZX1zNoCr7gsfXmPvkPx+SgojQlD+Ajda8iLLCSxjVIHvXib -y8posqTdDEx5YMaZ0ZPxMBoH064iwurO8YQJzOAUbn8/ftKChazcqRZOhaBgy/ac -18izju3Gm5h1DVXoX+WViwKkrkMpKBGk5hIwAUt1ax5mnXkvpXYvHUC0bcl9eQjs -0Wq2XSqypWa9a4X0dFbD9ed1Uigspf9mR6XU/v6eVL9lfgHWMI+lNpyiUBzuOIAB -SMbHdPTGrMNASRZhdCyvjG817XsYAFs2PJxQDcqSMxDxJklt33UkN4Ii1+iW/RVL -ApY+B3KVfqs9TC7XyvDf4Fg/LS8EmjijAQIDAQABo0IwQDAPBgNVHRMBAf8EBTAD -AQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUBpOjCl4oaTeqYR3r6/wtbyPk -86AwDQYJKoZIhvcNAQEMBQADggIBAJaAcgkGfpzMkwQWu6A6jZJOtxEaCnFxEM0E -rX+lRVAQZk5KQaID2RFPeje5S+LGjzJmdSX7684/AykmjbgWHfYfM25I5uj4V7Ib -ed87hwriZLoAymzvftAj63iP/2SbNDefNWWipAA9EiOWWF3KY4fGoweITedpdopT -zfFP7ELyk+OZpDc8h7hi2/DsHzc/N19DzFGdtfCXwreFamgLRB7lUe6TzktuhsHS -DCRZNhqfLJGP4xjblJUK7ZGqDpncllPjYYPGFrojutzdfhrGe0K22VoF3Jpf1d+4 -2kd92jjbrDnVHmtsKheMYc2xbXIBw8MgAGJoFjHVdqqGuw6qnsb58Nn4DSEC5MUo -FlkRudlpcyqSeLiSV5sI8jrlL5WwWLdrIBRtFO8KvH7YVdiI2i/6GaX7i+B/OfVy -K4XELKzvGUWSTLNhB9xNH27SgRNcmvMSZ4PPmz+Ln52kuaiWA3rF7iDeM9ovnhp6 -dB7h7sxaOgTdsxoEqBRjrLdHEoOabPXm6RUVkRqEGQ6UROcSjiVbgGcZ3GOTEAtl -Lor6CZpO2oYofaphNdgOpygau1LgePhsumywbrmHXumZNTfxPWQrqaA0k89jL9WB -365jJ6UeTo3cKXhZ+PmhIIynJkBugnLNeLLIjzwec+fBH7/PzqUqm9tEZDKgu39c -JRNItX+S ------END CERTIFICATE----- - -# Issuer: CN=SecureSign Root CA15 O=Cybertrust Japan Co., Ltd. -# Subject: CN=SecureSign Root CA15 O=Cybertrust Japan Co., Ltd. -# Label: "SecureSign Root CA15" -# Serial: 126083514594751269499665114766174399806381178503 -# MD5 Fingerprint: 13:30:fc:c4:62:a6:a9:de:b5:c1:68:af:b5:d2:31:47 -# SHA1 Fingerprint: cb:ba:83:c8:c1:5a:5d:f1:f9:73:6f:ca:d7:ef:28:13:06:4a:07:7d -# SHA256 Fingerprint: e7:78:f0:f0:95:fe:84:37:29:cd:1a:00:82:17:9e:53:14:a9:c2:91:44:28:05:e1:fb:1d:8f:b6:b8:88:6c:3a ------BEGIN CERTIFICATE----- -MIICIzCCAamgAwIBAgIUFhXHw9hJp75pDIqI7fBw+d23PocwCgYIKoZIzj0EAwMw -UTELMAkGA1UEBhMCSlAxIzAhBgNVBAoTGkN5YmVydHJ1c3QgSmFwYW4gQ28uLCBM -dGQuMR0wGwYDVQQDExRTZWN1cmVTaWduIFJvb3QgQ0ExNTAeFw0yMDA0MDgwODMy -NTZaFw00NTA0MDgwODMyNTZaMFExCzAJBgNVBAYTAkpQMSMwIQYDVQQKExpDeWJl -cnRydXN0IEphcGFuIENvLiwgTHRkLjEdMBsGA1UEAxMUU2VjdXJlU2lnbiBSb290 -IENBMTUwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQLUHSNZDKZmbPSYAi4Io5GdCx4 -wCtELW1fHcmuS1Iggz24FG1Th2CeX2yF2wYUleDHKP+dX+Sq8bOLbe1PL0vJSpSR -ZHX+AezB2Ot6lHhWGENfa4HL9rzatAy2KZMIaY+jQjBAMA8GA1UdEwEB/wQFMAMB -Af8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBTrQciu/NWeUUj1vYv0hyCTQSvT -9DAKBggqhkjOPQQDAwNoADBlAjEA2S6Jfl5OpBEHvVnCB96rMjhTKkZEBhd6zlHp -4P9mLQlO4E/0BdGF9jVg3PVys0Z9AjBEmEYagoUeYWmJSwdLZrWeqrqgHkHZAXQ6 -bkU6iYAZezKYVWOr62Nuk22rGwlgMU4= ------END CERTIFICATE----- - -# Issuer: CN=D-TRUST BR Root CA 2 2023 O=D-Trust GmbH -# Subject: CN=D-TRUST BR Root CA 2 2023 O=D-Trust GmbH -# Label: "D-TRUST BR Root CA 2 2023" -# Serial: 153168538924886464690566649552453098598 -# MD5 Fingerprint: e1:09:ed:d3:60:d4:56:1b:47:1f:b7:0c:5f:1b:5f:85 -# SHA1 Fingerprint: 2d:b0:70:ee:71:94:af:69:68:17:db:79:ce:58:9f:a0:6b:96:f7:87 -# SHA256 Fingerprint: 05:52:e6:f8:3f:df:65:e8:fa:96:70:e6:66:df:28:a4:e2:13:40:b5:10:cb:e5:25:66:f9:7c:4f:b9:4b:2b:d1 ------BEGIN CERTIFICATE----- -MIIFqTCCA5GgAwIBAgIQczswBEhb2U14LnNLyaHcZjANBgkqhkiG9w0BAQ0FADBI -MQswCQYDVQQGEwJERTEVMBMGA1UEChMMRC1UcnVzdCBHbWJIMSIwIAYDVQQDExlE -LVRSVVNUIEJSIFJvb3QgQ0EgMiAyMDIzMB4XDTIzMDUwOTA4NTYzMVoXDTM4MDUw -OTA4NTYzMFowSDELMAkGA1UEBhMCREUxFTATBgNVBAoTDEQtVHJ1c3QgR21iSDEi -MCAGA1UEAxMZRC1UUlVTVCBCUiBSb290IENBIDIgMjAyMzCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBAK7/CVmRgApKaOYkP7in5Mg6CjoWzckjYaCTcfKr -i3OPoGdlYNJUa2NRb0kz4HIHE304zQaSBylSa053bATTlfrdTIzZXcFhfUvnKLNE -gXtRr90zsWh81k5M/itoucpmacTsXld/9w3HnDY25QdgrMBM6ghs7wZ8T1soegj8 -k12b9py0i4a6Ibn08OhZWiihNIQaJZG2tY/vsvmA+vk9PBFy2OMvhnbFeSzBqZCT -Rphny4NqoFAjpzv2gTng7fC5v2Xx2Mt6++9zA84A9H3X4F07ZrjcjrqDy4d2A/wl -2ecjbwb9Z/Pg/4S8R7+1FhhGaRTMBffb00msa8yr5LULQyReS2tNZ9/WtT5PeB+U -cSTq3nD88ZP+npNa5JRal1QMNXtfbO4AHyTsA7oC9Xb0n9Sa7YUsOCIvx9gvdhFP -/Wxc6PWOJ4d/GUohR5AdeY0cW/jPSoXk7bNbjb7EZChdQcRurDhaTyN0dKkSw/bS -uREVMweR2Ds3OmMwBtHFIjYoYiMQ4EbMl6zWK11kJNXuHA7e+whadSr2Y23OC0K+ -0bpwHJwh5Q8xaRfX/Aq03u2AnMuStIv13lmiWAmlY0cL4UEyNEHZmrHZqLAbWt4N -DfTisl01gLmB1IRpkQLLddCNxbU9CZEJjxShFHR5PtbJFR2kWVki3PaKRT08EtY+ -XTIvAgMBAAGjgY4wgYswDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUZ5Dw1t61 -GNVGKX5cq/ieCLxklRAwDgYDVR0PAQH/BAQDAgEGMEkGA1UdHwRCMEAwPqA8oDqG -OGh0dHA6Ly9jcmwuZC10cnVzdC5uZXQvY3JsL2QtdHJ1c3RfYnJfcm9vdF9jYV8y -XzIwMjMuY3JsMA0GCSqGSIb3DQEBDQUAA4ICAQA097N3U9swFrktpSHxQCF16+tI -FoE9c+CeJyrrd6kTpGoKWloUMz1oH4Guaf2Mn2VsNELZLdB/eBaxOqwjMa1ef67n -riv6uvw8l5VAk1/DLQOj7aRvU9f6QA4w9QAgLABMjDu0ox+2v5Eyq6+SmNMW5tTR -VFxDWy6u71cqqLRvpO8NVhTaIasgdp4D/Ca4nj8+AybmTNudX0KEPUUDAxxZiMrc -LmEkWqTqJwtzEr5SswrPMhfiHocaFpVIbVrg0M8JkiZmkdijYQ6qgYF/6FKC0ULn -4B0Y+qSFNueG4A3rvNTJ1jxD8V1Jbn6Bm2m1iWKPiFLY1/4nwSPFyysCu7Ff/vtD -hQNGvl3GyiEm/9cCnnRK3PgTFbGBVzbLZVzRHTF36SXDw7IyN9XxmAnkbWOACKsG -koHU6XCPpz+y7YaMgmo1yEJagtFSGkUPFaUA8JR7ZSdXOUPPfH/mvTWze/EZTN46 -ls/pdu4D58JDUjxqgejBWoC9EV2Ta/vH5mQ/u2kc6d0li690yVRAysuTEwrt+2aS -Ecr1wPrYg1UDfNPFIkZ1cGt5SAYqgpq/5usWDiJFAbzdNpQ0qTUmiteXue4Icr80 -knCDgKs4qllo3UCkGJCy89UDyibK79XH4I9TjvAA46jtn/mtd+ArY0+ew+43u3gJ -hJ65bvspmZDogNOfJA== ------END CERTIFICATE----- - -# Issuer: CN=TrustAsia TLS ECC Root CA O=TrustAsia Technologies, Inc. -# Subject: CN=TrustAsia TLS ECC Root CA O=TrustAsia Technologies, Inc. -# Label: "TrustAsia TLS ECC Root CA" -# Serial: 310892014698942880364840003424242768478804666567 -# MD5 Fingerprint: 09:48:04:77:d2:fc:65:93:71:66:b1:11:95:4f:06:8c -# SHA1 Fingerprint: b5:ec:39:f3:a1:66:37:ae:c3:05:94:57:e2:be:11:be:b7:a1:7f:36 -# SHA256 Fingerprint: c0:07:6b:9e:f0:53:1f:b1:a6:56:d6:7c:4e:be:97:cd:5d:ba:a4:1e:f4:45:98:ac:c2:48:98:78:c9:2d:87:11 ------BEGIN CERTIFICATE----- -MIICMTCCAbegAwIBAgIUNnThTXxlE8msg1UloD5Sfi9QaMcwCgYIKoZIzj0EAwMw -WDELMAkGA1UEBhMCQ04xJTAjBgNVBAoTHFRydXN0QXNpYSBUZWNobm9sb2dpZXMs -IEluYy4xIjAgBgNVBAMTGVRydXN0QXNpYSBUTFMgRUNDIFJvb3QgQ0EwHhcNMjQw -NTE1MDU0MTU2WhcNNDQwNTE1MDU0MTU1WjBYMQswCQYDVQQGEwJDTjElMCMGA1UE -ChMcVHJ1c3RBc2lhIFRlY2hub2xvZ2llcywgSW5jLjEiMCAGA1UEAxMZVHJ1c3RB -c2lhIFRMUyBFQ0MgUm9vdCBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABLh/pVs/ -AT598IhtrimY4ZtcU5nb9wj/1WrgjstEpvDBjL1P1M7UiFPoXlfXTr4sP/MSpwDp -guMqWzJ8S5sUKZ74LYO1644xST0mYekdcouJtgq7nDM1D9rs3qlKH8kzsaNCMEAw -DwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQULIVTu7FDzTLqnqOH/qKYqKaT6RAw -DgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2gAMGUCMFRH18MtYYZI9HlaVQ01 -L18N9mdsd0AaRuf4aFtOJx24mH1/k78ITcTaRTChD15KeAIxAKORh/IRM4PDwYqR -OkwrULG9IpRdNYlzg8WbGf60oenUoWa2AaU2+dhoYSi3dOGiMQ== ------END CERTIFICATE----- - -# Issuer: CN=TrustAsia TLS RSA Root CA O=TrustAsia Technologies, Inc. -# Subject: CN=TrustAsia TLS RSA Root CA O=TrustAsia Technologies, Inc. -# Label: "TrustAsia TLS RSA Root CA" -# Serial: 160405846464868906657516898462547310235378010780 -# MD5 Fingerprint: 3b:9e:c3:86:0f:34:3c:6b:c5:46:c4:8e:1d:e7:19:12 -# SHA1 Fingerprint: a5:46:50:c5:62:ea:95:9a:1a:a7:04:6f:17:58:c7:29:53:3d:03:fa -# SHA256 Fingerprint: 06:c0:8d:7d:af:d8:76:97:1e:b1:12:4f:e6:7f:84:7e:c0:c7:a1:58:d3:ea:53:cb:e9:40:e2:ea:97:91:f4:c3 ------BEGIN CERTIFICATE----- -MIIFgDCCA2igAwIBAgIUHBjYz+VTPyI1RlNUJDxsR9FcSpwwDQYJKoZIhvcNAQEM -BQAwWDELMAkGA1UEBhMCQ04xJTAjBgNVBAoTHFRydXN0QXNpYSBUZWNobm9sb2dp -ZXMsIEluYy4xIjAgBgNVBAMTGVRydXN0QXNpYSBUTFMgUlNBIFJvb3QgQ0EwHhcN -MjQwNTE1MDU0MTU3WhcNNDQwNTE1MDU0MTU2WjBYMQswCQYDVQQGEwJDTjElMCMG -A1UEChMcVHJ1c3RBc2lhIFRlY2hub2xvZ2llcywgSW5jLjEiMCAGA1UEAxMZVHJ1 -c3RBc2lhIFRMUyBSU0EgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC -AgoCggIBAMMWuBtqpERz5dZO9LnPWwvB0ZqB9WOwj0PBuwhaGnrhB3YmH49pVr7+ -NmDQDIPNlOrnxS1cLwUWAp4KqC/lYCZUlviYQB2srp10Zy9U+5RjmOMmSoPGlbYJ -Q1DNDX3eRA5gEk9bNb2/mThtfWza4mhzH/kxpRkQcwUqwzIZheo0qt1CHjCNP561 -HmHVb70AcnKtEj+qpklz8oYVlQwQX1Fkzv93uMltrOXVmPGZLmzjyUT5tUMnCE32 -ft5EebuyjBza00tsLtbDeLdM1aTk2tyKjg7/D8OmYCYozza/+lcK7Fs/6TAWe8Tb -xNRkoDD75f0dcZLdKY9BWN4ArTr9PXwaqLEX8E40eFgl1oUh63kd0Nyrz2I8sMeX -i9bQn9P+PN7F4/w6g3CEIR0JwqH8uyghZVNgepBtljhb//HXeltt08lwSUq6HTrQ -UNoyIBnkiz/r1RYmNzz7dZ6wB3C4FGB33PYPXFIKvF1tjVEK2sUYyJtt3LCDs3+j -TnhMmCWr8n4uIF6CFabW2I+s5c0yhsj55NqJ4js+k8UTav/H9xj8Z7XvGCxUq0DT -bE3txci3OE9kxJRMT6DNrqXGJyV1J23G2pyOsAWZ1SgRxSHUuPzHlqtKZFlhaxP8 -S8ySpg+kUb8OWJDZgoM5pl+z+m6Ss80zDoWo8SnTq1mt1tve1CuBAgMBAAGjQjBA -MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLgHkXlcBvRG/XtZylomkadFK/hT -MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQwFAAOCAgEAIZtqBSBdGBanEqT3 -Rz/NyjuujsCCztxIJXgXbODgcMTWltnZ9r96nBO7U5WS/8+S4PPFJzVXqDuiGev4 -iqME3mmL5Dw8veWv0BIb5Ylrc5tvJQJLkIKvQMKtuppgJFqBTQUYo+IzeXoLH5Pt -7DlK9RME7I10nYEKqG/odv6LTytpEoYKNDbdgptvT+Bz3Ul/KD7JO6NXBNiT2Twp -2xIQaOHEibgGIOcberyxk2GaGUARtWqFVwHxtlotJnMnlvm5P1vQiJ3koP26TpUJ -g3933FEFlJ0gcXax7PqJtZwuhfG5WyRasQmr2soaB82G39tp27RIGAAtvKLEiUUj -pQ7hRGU+isFqMB3iYPg6qocJQrmBktwliJiJ8Xw18WLK7nn4GS/+X/jbh87qqA8M -pugLoDzga5SYnH+tBuYc6kIQX+ImFTw3OffXvO645e8D7r0i+yiGNFjEWn9hongP -XvPKnbwbPKfILfanIhHKA9jnZwqKDss1jjQ52MjqjZ9k4DewbNfFj8GQYSbbJIwe -SsCI3zWQzj8C9GRh3sfIB5XeMhg6j6JCQCTl1jNdfK7vsU1P1FeQNWrcrgSXSYk0 -ly4wBOeY99sLAZDBHwo/+ML+TvrbmnNzFrwFuHnYWa8G5z9nODmxfKuU4CkUpijy -323imttUQ/hHWKNddBWcwauwxzQ= ------END CERTIFICATE----- - -# Issuer: CN=D-TRUST EV Root CA 2 2023 O=D-Trust GmbH -# Subject: CN=D-TRUST EV Root CA 2 2023 O=D-Trust GmbH -# Label: "D-TRUST EV Root CA 2 2023" -# Serial: 139766439402180512324132425437959641711 -# MD5 Fingerprint: 96:b4:78:09:f0:09:cb:77:eb:bb:1b:4d:6f:36:bc:b6 -# SHA1 Fingerprint: a5:5b:d8:47:6c:8f:19:f7:4c:f4:6d:6b:b6:c2:79:82:22:df:54:8b -# SHA256 Fingerprint: 8e:82:21:b2:e7:d4:00:78:36:a1:67:2f:0d:cc:29:9c:33:bc:07:d3:16:f1:32:fa:1a:20:6d:58:71:50:f1:ce ------BEGIN CERTIFICATE----- -MIIFqTCCA5GgAwIBAgIQaSYJfoBLTKCnjHhiU19abzANBgkqhkiG9w0BAQ0FADBI -MQswCQYDVQQGEwJERTEVMBMGA1UEChMMRC1UcnVzdCBHbWJIMSIwIAYDVQQDExlE -LVRSVVNUIEVWIFJvb3QgQ0EgMiAyMDIzMB4XDTIzMDUwOTA5MTAzM1oXDTM4MDUw -OTA5MTAzMlowSDELMAkGA1UEBhMCREUxFTATBgNVBAoTDEQtVHJ1c3QgR21iSDEi -MCAGA1UEAxMZRC1UUlVTVCBFViBSb290IENBIDIgMjAyMzCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBANiOo4mAC7JXUtypU0w3uX9jFxPvp1sjW2l1sJkK -F8GLxNuo4MwxusLyzV3pt/gdr2rElYfXR8mV2IIEUD2BCP/kPbOx1sWy/YgJ25yE -7CUXFId/MHibaljJtnMoPDT3mfd/06b4HEV8rSyMlD/YZxBTfiLNTiVR8CUkNRFe -EMbsh2aJgWi6zCudR3Mfvc2RpHJqnKIbGKBv7FD0fUDCqDDPvXPIEysQEx6Lmqg6 -lHPTGGkKSv/BAQP/eX+1SH977ugpbzZMlWGG2Pmic4ruri+W7mjNPU0oQvlFKzIb -RlUWaqZLKfm7lVa/Rh3sHZMdwGWyH6FDrlaeoLGPaxK3YG14C8qKXO0elg6DpkiV -jTujIcSuWMYAsoS0I6SWhjW42J7YrDRJmGOVxcttSEfi8i4YHtAxq9107PncjLgc -jmgjutDzUNzPZY9zOjLHfP7KgiJPvo5iR2blzYfi6NUPGJ/lBHJLRjwQ8kTCZFZx -TnXonMkmdMV9WdEKWw9t/p51HBjGGjp82A0EzM23RWV6sY+4roRIPrN6TagD4uJ+ -ARZZaBhDM7DS3LAaQzXupdqpRlyuhoFBAUp0JuyfBr/CBTdkdXgpaP3F9ev+R/nk -hbDhezGdpn9yo7nELC7MmVcOIQxFAZRl62UJxmMiCzNJkkg8/M3OsD6Onov4/knF -NXJHAgMBAAGjgY4wgYswDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUqvyREBuH -kV8Wub9PS5FeAByxMoAwDgYDVR0PAQH/BAQDAgEGMEkGA1UdHwRCMEAwPqA8oDqG -OGh0dHA6Ly9jcmwuZC10cnVzdC5uZXQvY3JsL2QtdHJ1c3RfZXZfcm9vdF9jYV8y -XzIwMjMuY3JsMA0GCSqGSIb3DQEBDQUAA4ICAQCTy6UfmRHsmg1fLBWTxj++EI14 -QvBukEdHjqOSMo1wj/Zbjb6JzkcBahsgIIlbyIIQbODnmaprxiqgYzWRaoUlrRc4 -pZt+UPJ26oUFKidBK7GB0aL2QHWpDsvxVUjY7NHss+jOFKE17MJeNRqrphYBBo7q -3C+jisosketSjl8MmxfPy3MHGcRqwnNU73xDUmPBEcrCRbH0O1P1aa4846XerOhU -t7KR/aypH/KH5BfGSah82ApB9PI+53c0BFLd6IHyTS9URZ0V4U/M5d40VxDJI3IX -cI1QcB9WbMy5/zpaT2N6w25lBx2Eof+pDGOJbbJAiDnXH3dotfyc1dZnaVuodNv8 -ifYbMvekJKZ2t0dT741Jj6m2g1qllpBFYfXeA08mD6iL8AOWsKwV0HFaanuU5nCT -2vFp4LJiTZ6P/4mdm13NRemUAiKN4DV/6PEEeXFsVIP4M7kFMhtYVRFP0OUnR3Hs -7dpn1mKmS00PaaLJvOwiS5THaJQXfuKOKD62xur1NGyfN4gHONuGcfrNlUhDbqNP -gofXNJhuS5N5YHVpD/Aa1VP6IQzCP+k/HxiMkl14p3ZnGbuy6n/pcAlWVqOwDAst -Nl7F6cTVg8uGF5csbBNvh1qvSaYd2804BC5f4ko1Di1L+KIkBI3Y4WNeApI02phh -XBxvWHZks/wCuPWdCg== ------END CERTIFICATE----- - -# Issuer: CN=SwissSign RSA TLS Root CA 2022 - 1 O=SwissSign AG -# Subject: CN=SwissSign RSA TLS Root CA 2022 - 1 O=SwissSign AG -# Label: "SwissSign RSA TLS Root CA 2022 - 1" -# Serial: 388078645722908516278762308316089881486363258315 -# MD5 Fingerprint: 16:2e:e4:19:76:81:85:ba:8e:91:58:f1:15:ef:72:39 -# SHA1 Fingerprint: 81:34:0a:be:4c:cd:ce:cc:e7:7d:cc:8a:d4:57:e2:45:a0:77:5d:ce -# SHA256 Fingerprint: 19:31:44:f4:31:e0:fd:db:74:07:17:d4:de:92:6a:57:11:33:88:4b:43:60:d3:0e:27:29:13:cb:e6:60:ce:41 ------BEGIN CERTIFICATE----- -MIIFkzCCA3ugAwIBAgIUQ/oMX04bgBhE79G0TzUfRPSA7cswDQYJKoZIhvcNAQEL -BQAwUTELMAkGA1UEBhMCQ0gxFTATBgNVBAoTDFN3aXNzU2lnbiBBRzErMCkGA1UE -AxMiU3dpc3NTaWduIFJTQSBUTFMgUm9vdCBDQSAyMDIyIC0gMTAeFw0yMjA2MDgx -MTA4MjJaFw00NzA2MDgxMTA4MjJaMFExCzAJBgNVBAYTAkNIMRUwEwYDVQQKEwxT -d2lzc1NpZ24gQUcxKzApBgNVBAMTIlN3aXNzU2lnbiBSU0EgVExTIFJvb3QgQ0Eg -MjAyMiAtIDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDLKmjiC8NX -vDVjvHClO/OMPE5Xlm7DTjak9gLKHqquuN6orx122ro10JFwB9+zBvKK8i5VUXu7 -LCTLf5ImgKO0lPaCoaTo+nUdWfMHamFk4saMla+ju45vVs9xzF6BYQ1t8qsCLqSX -5XH8irCRIFucdFJtrhUnWXjyCcplDn/L9Ovn3KlMd/YrFgSVrpxxpT8q2kFC5zyE -EPThPYxr4iuRR1VPuFa+Rd4iUU1OKNlfGUEGjw5NBuBwQCMBauTLE5tzrE0USJIt -/m2n+IdreXXhvhCxqohAWVTXz8TQm0SzOGlkjIHRI36qOTw7D59Ke4LKa2/KIj4x -0LDQKhySio/YGZxH5D4MucLNvkEM+KRHBdvBFzA4OmnczcNpI/2aDwLOEGrOyvi5 -KaM2iYauC8BPY7kGWUleDsFpswrzd34unYyzJ5jSmY0lpx+Gs6ZUcDj8fV3oT4MM -0ZPlEuRU2j7yrTrePjxF8CgPBrnh25d7mUWe3f6VWQQvdT/TromZhqwUtKiE+shd -OxtYk8EXlFXIC+OCeYSf8wCENO7cMdWP8vpPlkwGqnj73mSiI80fPsWMvDdUDrta -clXvyFu1cvh43zcgTFeRc5JzrBh3Q4IgaezprClG5QtO+DdziZaKHG29777YtvTK -wP1H8K4LWCDFyB02rpeNUIMmJCn3nTsPBQIDAQABo2MwYTAPBgNVHRMBAf8EBTAD -AQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBRvjmKLk0Ow4UD2p8P98Q+4 -DxU4pTAdBgNVHQ4EFgQUb45ii5NDsOFA9qfD/fEPuA8VOKUwDQYJKoZIhvcNAQEL -BQADggIBAKwsKUF9+lz1GpUYvyypiqkkVHX1uECry6gkUSsYP2OprphWKwVDIqO3 -10aewCoSPY6WlkDfDDOLazeROpW7OSltwAJsipQLBwJNGD77+3v1dj2b9l4wBlgz -Hqp41eZUBDqyggmNzhYzWUUo8aWjlw5DI/0LIICQ/+Mmz7hkkeUFjxOgdg3XNwwQ -iJb0Pr6VvfHDffCjw3lHC1ySFWPtUnWK50Zpy1FVCypM9fJkT6lc/2cyjlUtMoIc -gC9qkfjLvH4YoiaoLqNTKIftV+Vlek4ASltOU8liNr3CjlvrzG4ngRhZi0Rjn9UM -ZfQpZX+RLOV/fuiJz48gy20HQhFRJjKKLjpHE7iNvUcNCfAWpO2Whi4Z2L6MOuhF -LhG6rlrnub+xzI/goP+4s9GFe3lmozm1O2bYQL7Pt2eLSMkZJVX8vY3PXtpOpvJp -zv1/THfQwUY1mFwjmwJFQ5Ra3bxHrSL+ul4vkSkphnsh3m5kt8sNjzdbowhq6/Td -Ao9QAwKxuDdollDruF/UKIqlIgyKhPBZLtU30WHlQnNYKoH3dtvi4k0NX/a3vgW0 -rk4N3hY9A4GzJl5LuEsAz/+MF7psYC0nhzck5npgL7XTgwSqT0N1osGDsieYK7EO -gLrAhV5Cud+xYJHT6xh+cHiudoO+cVrQkOPKwRYlZ0rwtnu64ZzZ ------END CERTIFICATE----- - -# Issuer: CN=OISTE Server Root ECC G1 O=OISTE Foundation -# Subject: CN=OISTE Server Root ECC G1 O=OISTE Foundation -# Label: "OISTE Server Root ECC G1" -# Serial: 47819833811561661340092227008453318557 -# MD5 Fingerprint: 42:a7:d2:35:ae:02:92:db:19:76:08:de:2f:05:b4:d4 -# SHA1 Fingerprint: 3b:f6:8b:09:ae:2a:92:7b:ba:e3:8d:3f:11:95:d9:e6:44:0c:45:e2 -# SHA256 Fingerprint: ee:c9:97:c0:c3:0f:21:6f:7e:3b:8b:30:7d:2b:ae:42:41:2d:75:3f:c8:21:9d:af:d1:52:0b:25:72:85:0f:49 ------BEGIN CERTIFICATE----- -MIICNTCCAbqgAwIBAgIQI/nD1jWvjyhLH/BU6n6XnTAKBggqhkjOPQQDAzBLMQsw -CQYDVQQGEwJDSDEZMBcGA1UECgwQT0lTVEUgRm91bmRhdGlvbjEhMB8GA1UEAwwY -T0lTVEUgU2VydmVyIFJvb3QgRUNDIEcxMB4XDTIzMDUzMTE0NDIyOFoXDTQ4MDUy -NDE0NDIyN1owSzELMAkGA1UEBhMCQ0gxGTAXBgNVBAoMEE9JU1RFIEZvdW5kYXRp -b24xITAfBgNVBAMMGE9JU1RFIFNlcnZlciBSb290IEVDQyBHMTB2MBAGByqGSM49 -AgEGBSuBBAAiA2IABBcv+hK8rBjzCvRE1nZCnrPoH7d5qVi2+GXROiFPqOujvqQy -cvO2Ackr/XeFblPdreqqLiWStukhEaivtUwL85Zgmjvn6hp4LrQ95SjeHIC6XG4N -2xml4z+cKrhAS93mT6NjMGEwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBQ3 -TYhlz/w9itWj8UnATgwQb0K0nDAdBgNVHQ4EFgQUN02IZc/8PYrVo/FJwE4MEG9C -tJwwDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMDA2kAMGYCMQCpKjAd0MKfkFFR -QD6VVCHNFmb3U2wIFjnQEnx/Yxvf4zgAOdktUyBFCxxgZzFDJe0CMQCSia7pXGKD -YmH5LVerVrkR3SW+ak5KGoJr3M/TvEqzPNcum9v4KGm8ay3sMaE641c= ------END CERTIFICATE----- - -# Issuer: CN=OISTE Server Root RSA G1 O=OISTE Foundation -# Subject: CN=OISTE Server Root RSA G1 O=OISTE Foundation -# Label: "OISTE Server Root RSA G1" -# Serial: 113845518112613905024960613408179309848 -# MD5 Fingerprint: 23:a7:9e:d4:70:b8:b9:14:57:41:8a:7e:44:59:e2:68 -# SHA1 Fingerprint: f7:00:34:25:94:88:68:31:e4:34:87:3f:70:fe:86:b3:86:9f:f0:6e -# SHA256 Fingerprint: 9a:e3:62:32:a5:18:9f:fd:db:35:3d:fd:26:52:0c:01:53:95:d2:27:77:da:c5:9d:b5:7b:98:c0:89:a6:51:e6 ------BEGIN CERTIFICATE----- -MIIFgzCCA2ugAwIBAgIQVaXZZ5Qoxu0M+ifdWwFNGDANBgkqhkiG9w0BAQwFADBL -MQswCQYDVQQGEwJDSDEZMBcGA1UECgwQT0lTVEUgRm91bmRhdGlvbjEhMB8GA1UE -AwwYT0lTVEUgU2VydmVyIFJvb3QgUlNBIEcxMB4XDTIzMDUzMTE0MzcxNloXDTQ4 -MDUyNDE0MzcxNVowSzELMAkGA1UEBhMCQ0gxGTAXBgNVBAoMEE9JU1RFIEZvdW5k -YXRpb24xITAfBgNVBAMMGE9JU1RFIFNlcnZlciBSb290IFJTQSBHMTCCAiIwDQYJ -KoZIhvcNAQEBBQADggIPADCCAgoCggIBAKqu9KuCz/vlNwvn1ZatkOhLKdxVYOPM -vLO8LZK55KN68YG0nnJyQ98/qwsmtO57Gmn7KNByXEptaZnwYx4M0rH/1ow00O7b -rEi56rAUjtgHqSSY3ekJvqgiG1k50SeH3BzN+Puz6+mTeO0Pzjd8JnduodgsIUzk -ik/HEzxux9UTl7Ko2yRpg1bTacuCErudG/L4NPKYKyqOBGf244ehHa1uzjZ0Dl4z -O8vbUZeUapU8zhhabkvG/AePLhq5SvdkNCncpo1Q4Y2LS+VIG24ugBA/5J8bZT8R -tOpXaZ+0AOuFJJkk9SGdl6r7NH8CaxWQrbueWhl/pIzY+m0o/DjH40ytas7ZTpOS -jswMZ78LS5bOZmdTaMsXEY5Z96ycG7mOaES3GK/m5Q9l3JUJsJMStR8+lKXHiHUh -sd4JJCpM4rzsTGdHwimIuQq6+cF0zowYJmXa92/GjHtoXAvuY8BeS/FOzJ8vD+Ho -mnqT8eDI278n5mUpezbgMxVz8p1rhAhoKzYHKyfMeNhqhw5HdPSqoBNdZH702xSu -+zrkL8Fl47l6QGzwBrd7KJvX4V84c5Ss2XCTLdyEr0YconosP4EmQufU2MVshGYR -i3drVByjtdgQ8K4p92cIiBdcuJd5z+orKu5YM+Vt6SmqZQENghPsJQtdLEByFSnT -kCz3GkPVavBpAgMBAAGjYzBhMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU -8snBDw1jALvsRQ5KH7WxszbNDo0wHQYDVR0OBBYEFPLJwQ8NYwC77EUOSh+1sbM2 -zQ6NMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQwFAAOCAgEANGd5sjrG5T33 -I3K5Ce+SrScfoE4KsvXaFwyihdJ+klH9FWXXXGtkFu6KRcoMQzZENdl//nk6HOjG -5D1rd9QhEOP28yBOqb6J8xycqd+8MDoX0TJD0KqKchxRKEzdNsjkLWd9kYccnbz8 -qyiWXmFcuCIzGEgWUOrKL+mlSdx/PKQZvDatkuK59EvV6wit53j+F8Bdh3foZ3dP -AGav9LEDOr4SfEE15fSmG0eLy3n31r8Xbk5l8PjaV8GUgeV6Vg27Rn9vkf195hfk -gSe7BYhW3SCl95gtkRlpMV+bMPKZrXJAlszYd2abtNUOshD+FKrDgHGdPY3ofRRs -YWSGRqbXVMW215AWRqWFyp464+YTFrYVI8ypKVL9AMb2kI5Wj4kI3Zaq5tNqqYY1 -9tVFeEJKRvwDyF7YZvZFZSS0vod7VSCd9521Kvy5YhnLbDuv0204bKt7ph6N/Ome -/msVuduCmsuY33OhkKCgxeDoAaijFJzIwZqsFVAzje18KotzlUBDJvyBpCpfOZC3 -J8tRd/iWkx7P8nd9H0aTolkelUTFLXVksNb54Dxp6gS1HAviRkRNQzuXSXERvSS2 -wq1yVAb+axj5d9spLFKebXd7Yv0PTY6YMjAwcRLWJTXjn/hvnLXrahut6hDTlhZy -BiElxky8j3C7DOReIoMt0r7+hVu05L0= ------END CERTIFICATE----- - -# Issuer: CN=e-Szigno TLS Root CA 2023 O=Microsec Ltd. -# Subject: CN=e-Szigno TLS Root CA 2023 O=Microsec Ltd. -# Label: "e-Szigno TLS Root CA 2023" -# Serial: 71934828665710877219916191754 -# MD5 Fingerprint: 6a:e9:99:74:a5:da:5e:f1:d9:2e:f2:c8:d1:86:8b:71 -# SHA1 Fingerprint: 6f:9a:d5:d5:df:e8:2c:eb:be:37:07:ee:4f:4f:52:58:29:41:d1:fe -# SHA256 Fingerprint: b4:91:41:50:2d:00:66:3d:74:0f:2e:7e:c3:40:c5:28:00:96:26:66:12:1a:36:d0:9c:f7:dd:2b:90:38:4f:b4 ------BEGIN CERTIFICATE----- -MIICzzCCAjGgAwIBAgINAOhvGHvWOWuYSkmYCjAKBggqhkjOPQQDBDB1MQswCQYD -VQQGEwJIVTERMA8GA1UEBwwIQnVkYXBlc3QxFjAUBgNVBAoMDU1pY3Jvc2VjIEx0 -ZC4xFzAVBgNVBGEMDlZBVEhVLTIzNTg0NDk3MSIwIAYDVQQDDBllLVN6aWdubyBU -TFMgUm9vdCBDQSAyMDIzMB4XDTIzMDcxNzE0MDAwMFoXDTM4MDcxNzE0MDAwMFow -dTELMAkGA1UEBhMCSFUxETAPBgNVBAcMCEJ1ZGFwZXN0MRYwFAYDVQQKDA1NaWNy -b3NlYyBMdGQuMRcwFQYDVQRhDA5WQVRIVS0yMzU4NDQ5NzEiMCAGA1UEAwwZZS1T -emlnbm8gVExTIFJvb3QgQ0EgMjAyMzCBmzAQBgcqhkjOPQIBBgUrgQQAIwOBhgAE -AGgP36J8PKp0iGEKjcJMpQEiFNT3YHdCnAo4YKGMZz6zY+n6kbCLS+Y53wLCMAFS -AL/fjO1ZrTJlqwlZULUZwmgcAOAFX9pQJhzDrAQixTpN7+lXWDajwRlTEArRzT/v -SzUaQ49CE0y5LBqcvjC2xN7cS53kpDzLLtmt3999Cd8ukv+ho2MwYTAPBgNVHRMB -Af8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUWYQCYlpGePVd3I8K -ECgj3NXW+0UwHwYDVR0jBBgwFoAUWYQCYlpGePVd3I8KECgj3NXW+0UwCgYIKoZI -zj0EAwQDgYsAMIGHAkIBLdqu9S54tma4n7Zwf2Z0z+yOfP7AAXmazlIC58PRDHpt -y7Ve7hekm9sEdu4pKeiv+62sUvTXK9Z3hBC9xdIoaDQCQTV2WnXzkoYI9bIeCvZl -C9p2x1L/Cx6AcCIwwzPbGO2E14vs7dOoY4G1VnxHx1YwlGhza9IuqbnZLBwpvQy6 -uWWL ------END CERTIFICATE----- - -# Issuer: CN=SECOM TLS RSA Root CA 2024 O=SECOM Trust Systems Co., Ltd. -# Subject: CN=SECOM TLS RSA Root CA 2024 O=SECOM Trust Systems Co., Ltd. -# Label: "SECOM TLS RSA Root CA 2024" -# Serial: 17188327524208271538 -# MD5 Fingerprint: d0:a4:db:32:eb:44:98:d2:62:0b:3e:bc:4d:7c:5c:e9 -# SHA1 Fingerprint: fb:97:96:7c:ef:8d:98:63:06:c0:3b:b6:11:f8:e0:13:97:a2:98:d3 -# SHA256 Fingerprint: 14:35:f2:25:c5:d2:52:d7:a2:19:48:cc:3c:e6:2a:ec:fa:88:00:1e:3d:d7:2d:1c:c3:55:51:00:eb:37:2f:93 ------BEGIN CERTIFICATE----- -MIIFmjCCA4KgAwIBAgIJAO6JNNDLgOCyMA0GCSqGSIb3DQEBDAUAMFoxCzAJBgNV -BAYTAkpQMSYwJAYDVQQKEx1TRUNPTSBUcnVzdCBTeXN0ZW1zIENvLiwgTHRkLjEj -MCEGA1UEAxMaU0VDT00gVExTIFJTQSBSb290IENBIDIwMjQwHhcNMjQwMTMxMDUx -MTU1WhcNNDkwMTE0MDUxMTU1WjBaMQswCQYDVQQGEwJKUDEmMCQGA1UEChMdU0VD -T00gVHJ1c3QgU3lzdGVtcyBDby4sIEx0ZC4xIzAhBgNVBAMTGlNFQ09NIFRMUyBS -U0EgUm9vdCBDQSAyMDI0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA -4TjizUwzxbInq8Tx11gaFYNk5fO+34y7TyM4neh0UgL5JIZbJNLTz2x//L/B71+5 -m6X6nGIr7d4lFJBGtjO677hXOz93zkcWaUTm3VbOAjBlt4YWxlcccBHXuZ7o3Q+4 -R+ormrBdHeJ1CTUEG8ttQbKIl3G7OZYbnH8/pP8cjPub/0kDVNuMzp7xsVRROOis -Qt53fMoJLlYgoebbuMphOqMCtjkJ7R6efEMfLp8UAVi9ZaLRn76ET/CJkk925ndu -uufC4BatS4mnXFmxN0vUXb0ij9B8O/D8gixQEsVSD4GK8FWRPh3bVd/6bzdkHGJj -y21XI0yejVomZUbRrOfNuz0boPGV1pt18fFC39IHQEth3OFqb5NDO3L+A9bNqTgA -yUgRmIn4ucgDc/Ri/Km3V51ueZjy1/yk0qwJVadAVVrCt56iNeXOyEvzJADGgDQ8 -E1Pdaqct8Cynz/47ReQM62vFYO08wcQkrjmX/tesiko1V1yyaf6EfPzUFzmaGy9x -vkCwdbm15EdTolOjE0H2Vb5/APDOyCFEokiYGmXTLdAUl0wKZ4IyjkHGzy0jhpaX -EXE/GJcEvI6VzEchjaBL03EJ0h9pG4OqeIOycKvAo3A+TbetyfsrgYyHzU0a7/qU -jGat1AAq1nVljMpKqpinPTsf/d9H39FTUeJL7TpzzjUCAwEAAaNjMGEwHQYDVR0O -BBYEFCzrchKOWHdkNRVWNQFXB6l9DTbmMB8GA1UdIwQYMBaAFCzrchKOWHdkNRVW -NQFXB6l9DTbmMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqG -SIb3DQEBDAUAA4ICAQAVwsvluSafaez5tFPR/hRTBzRxEyMMQF3XJXCVi3yegZyK -oec7hmE6jx2ZM8KgM1kn2yJRwFXHX8zUW9nBLEDWc4wuE8LrlZqhGZM9pJQXGmGz -ResDJV6JgRBna+j4sA1M7yIdlvL0sAfFXFCTRaWTD4E1V99RLrFzWfTcC+e180hD -uNMpqOEo46+lMeW/Wvh7ifOQs+kiK0O2gHxQDNxslSavnCs4V7l8HRDJ2La10o70 -Bo7VLzf1W8MBvv0VTnxB+NjT5qTAbhGFh9Gvp4BaJpmdUf0C5CEP6dbQlfgxWfzY -r69yVT6dPQB+GFEaY03IMY+AcBCs+om1fNxrQXt9zoofMBNFbLhvpNH/JsXWdGUz -fNbO12uswTa5wah8LB18FTQN2/zPHYmvBEoLuyUgZ09VNLJo5YA0kXItVYkLjMe2 -SixzK4scUHv81IK99I91DWx7FwMVKw2xgFp+ZLYB2dnpQQrqwlW64glHUcK2N9BD -snjLSxeZ+UPECh9RxH4WAcKiZW+cqaKMmhP2WBfR4IcR7NOL32ml11ds87hhV1CZ -WWFCJAcCidYZz6CZa8exzHojP9SB5RH0/v1KdHAisqhSjtJl/UIAHIQ48elOn8wr -TdFap4Yb5aHglmMeNx+fAIhDluWVfxTO7H4dTPU+SFVRMLAh+wwKZfqb94nMeQ== ------END CERTIFICATE----- - -# Issuer: CN=SECOM TLS ECC Root CA 2024 O=SECOM Trust Systems Co., Ltd. -# Subject: CN=SECOM TLS ECC Root CA 2024 O=SECOM Trust Systems Co., Ltd. -# Label: "SECOM TLS ECC Root CA 2024" -# Serial: 9329818985461676612 -# MD5 Fingerprint: 99:d3:9d:e4:d2:b1:2d:f0:2a:04:67:85:f3:df:46:d6 -# SHA1 Fingerprint: 7a:1f:22:2d:72:b2:c3:19:87:44:db:61:69:e8:a6:4b:d7:0d:44:0e -# SHA256 Fingerprint: 6a:b2:ab:75:f5:1c:b4:f4:f0:15:62:03:fb:f6:f6:46:23:2f:51:4b:e0:59:f6:28:33:30:8b:82:b4:d7:2d:b1 ------BEGIN CERTIFICATE----- -MIICTDCCAdGgAwIBAgIJAIF6LO+PI3pEMAoGCCqGSM49BAMDMFoxCzAJBgNVBAYT -AkpQMSYwJAYDVQQKEx1TRUNPTSBUcnVzdCBTeXN0ZW1zIENvLiwgTHRkLjEjMCEG -A1UEAxMaU0VDT00gVExTIEVDQyBSb290IENBIDIwMjQwHhcNMjQwMTMxMDU1MjM0 -WhcNNDkwMTE0MDU1MjM0WjBaMQswCQYDVQQGEwJKUDEmMCQGA1UEChMdU0VDT00g -VHJ1c3QgU3lzdGVtcyBDby4sIEx0ZC4xIzAhBgNVBAMTGlNFQ09NIFRMUyBFQ0Mg -Um9vdCBDQSAyMDI0MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE7NzFMtu9dzQXSNC1 -2fabk0+GlC5finB3R7XaZonRUd20aFiWObtuNBCLUZSfk6QXAE55BjEXsXQ/NG8y -UqicXjsu9ksDK3JZBgCwLOVh6+nwJXTvso/dEj/GUYH5mBdoo2MwYTAdBgNVHQ4E -FgQUO3YReyl04k4GTFaCQNAhL3qzydUwHwYDVR0jBBgwFoAUO3YReyl04k4GTFaC -QNAhL3qzydUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wCgYIKoZI -zj0EAwMDaQAwZgIxAN3ib8fi1pMYtAPjMilB5e5/H+t5CL0xPL+cZ5oTTZuSCjpA -n1v7F/VAr8bFxQXAowIxAKsBVO1ACFp7skwzPvdv1EUY5a897WGLT4lb+bjxFAWy -l8wDcZJdwGZ/pAHxt1AJ1g== ------END CERTIFICATE----- - -# Issuer: CN=Telia EC TLS Root CA v3 O=Telia Company AB -# Subject: CN=Telia EC TLS Root CA v3 O=Telia Company AB -# Label: "Telia EC TLS Root CA v3" -# Serial: 8028200332079443110287896228123317 -# MD5 Fingerprint: b6:fa:6a:5c:42:fb:c5:67:72:c0:e0:2f:72:fb:5a:44 -# SHA1 Fingerprint: b4:d6:07:c2:a5:95:bc:5b:f4:67:4d:c9:dc:6f:6f:0a:00:7a:a5:35 -# SHA256 Fingerprint: 09:8e:08:a9:1d:bb:f7:74:78:b9:6c:ce:b8:9b:14:13:a5:da:37:b7:c8:62:60:6a:95:5d:eb:07:17:9f:43:26 ------BEGIN CERTIFICATE----- -MIICMjCCAbegAwIBAgIPAYvSIlRjTQSLbOVHH9K1MAoGCCqGSM49BAMDMEoxCzAJ -BgNVBAYTAlNFMRkwFwYDVQQKDBBUZWxpYSBDb21wYW55IEFCMSAwHgYDVQQDDBdU -ZWxpYSBFQyBUTFMgUm9vdCBDQSB2MzAeFw0yMzExMTUwODU1MjZaFw00ODA1MjMx -MTAwMDBaMEoxCzAJBgNVBAYTAlNFMRkwFwYDVQQKDBBUZWxpYSBDb21wYW55IEFC -MSAwHgYDVQQDDBdUZWxpYSBFQyBUTFMgUm9vdCBDQSB2MzB2MBAGByqGSM49AgEG -BSuBBAAiA2IABMHIlhVDLbmFKUpW0iK4dpryT6emYOeS31JPwWnWPmkWRrAkTbPX -40sQfHI9mpR7Rbktu3ngg6W+BBSXSechtMCnBmWXj/EaVlmV5cY1jD2HoTfhBQ3A -acpCNMLJK4NpZaNjMGEwHwYDVR0jBBgwFoAU1GToQ4g6cy/QGnGCNgtehd7H3kMw -HQYDVR0OBBYEFNRk6EOIOnMv0BpxgjYLXoXex95DMA4GA1UdDwEB/wQEAwIBBjAP -BgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMDA2kAMGYCMQCXAUdS/9bbJ8A1JYaG -f/bWt/s7Ta0ot5Ulno8OjSNYRWQIlS4tVWldvTAVA7heOFgCMQCvKr8+Z2Rn+OBr -5UHzlgBObpad1LuwNTRcdNgUJxIWadcki+UBLEi1/AURKV5md2M= ------END CERTIFICATE----- - -# Issuer: CN=Telia RSA TLS Root CA v3 O=Telia Company AB -# Subject: CN=Telia RSA TLS Root CA v3 O=Telia Company AB -# Label: "Telia RSA TLS Root CA v3" -# Serial: 8028214673410753541795188766939845 -# MD5 Fingerprint: f4:8b:9c:f3:f8:63:cf:dc:25:8f:b4:bb:a2:e9:9d:e2 -# SHA1 Fingerprint: b5:2e:88:4e:40:c1:11:fb:50:c7:e2:4f:ac:18:2b:bd:68:15:d2:34 -# SHA256 Fingerprint: d1:3d:b1:29:4c:45:eb:c6:fc:86:c6:bb:f6:9f:a2:9b:df:e6:92:df:f7:c7:13:c2:43:c7:a9:56:c6:a2:28:4c ------BEGIN CERTIFICATE----- -MIIFgjCCA2qgAwIBAgIPAYvSUKtCVSxHWr2h3BrFMA0GCSqGSIb3DQEBDAUAMEsx -CzAJBgNVBAYTAlNFMRkwFwYDVQQKDBBUZWxpYSBDb21wYW55IEFCMSEwHwYDVQQD -DBhUZWxpYSBSU0EgVExTIFJvb3QgQ0EgdjMwHhcNMjMxMTE1MDk0NzQyWhcNNDgw -NTIzMTEwMDAwWjBLMQswCQYDVQQGEwJTRTEZMBcGA1UECgwQVGVsaWEgQ29tcGFu -eSBBQjEhMB8GA1UEAwwYVGVsaWEgUlNBIFRMUyBSb290IENBIHYzMIICIjANBgkq -hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAsV89KG19hCf4S1Fvk8D3TyDERhmcvx8F -7Kmb4WATx3ije1id3KHxRE0TKmcNCbAQ57bvHFEYa4hR2l20VjVadExqOW+2ld99 -MbEiO+jRVOz+BbxLxJnmGwCqI+BfuTjjVReDxsxjQvjgBsClaO/sm5i70nlZcWGR -tIkvWDK3NNkT5RtwXc/O8NTFVpbUqT6cRjIj3olAblR+lRf4Ffy5o+Q9fabjYn9Z -9S4itruElcEFf9Ljk7fwdTycT/rvJW9w/B3G2a3r0f/zXNOVruIBcqE6pkSospAC -U2bG42fYKrbM/GWnp7u+p9Frz4jaNwpb4YHuEeS8BratNcP8X62jXIvvKHxlsMDJ -Cnb4U8JzFOLsU6mohVY58BdZrvi0Gk9UOuqmgoG6dskHoksjZTlK61D/InzmEoA1 -yAYJFDVysjRxDUOu9cAwANbqmq77WIFL6BpnZgVqPtMfG6wN8BrTKdapvilVsYR5 -9BFgIsAVBMxrGh+W+QcvmJafUpASvlArKvVG2FI4i6PiLjSBT0+6F6EQLrYqefOQ -F/fBNEXb+njUQ0SUVrAqtH4Y+OjCI/a4/JJQppxeemZcQ0SUShgiI5AM5xHO5iya -UrTjYH4zxUz9j+1FEbDH/xpstr1gXBykspup+hRTaJcbA+UbpJqtWZndAPddJmt6 -YJQ+dU3pDu8CAwEAAaNjMGEwHwYDVR0jBBgwFoAUsMep0t2yKFZzBJSMFFxIbzdS -kqgwHQYDVR0OBBYEFLDHqdLdsihWcwSUjBRcSG83UpKoMA4GA1UdDwEB/wQEAwIB -BjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBDAUAA4ICAQBdYzFsNGDRk7bR -/AgRKq+5637YuOW+w6uhpoS0VnKMUpyHCwku86hEvqivakPtfmlm4bFwt++sb/8O -XsWBqtfbXMaBNDTZl8XRMJuLWOW2JrbKkRzgG0eBUcvsadG1rrhbmZqYvFXaAZO7 -o4TdOZzxhBB5GOAWWXB3IeraNP4J63zyo9n8Gqw3sJBG44em5hoYjBffP+npibys -lnslRi4L6xHsCYj/Pab+OlqbMCB6v+sTCLeEIukRVzoR9aQ45pEK7Z1QBnSsbAKQ -tss0JKD9d/mX143H1xePjPhTXlv5JCkhrcj+SShz0P9+EHoWe6m9lyUEOIVn0rp+ -yVJWNbmyDv3VkwFxHC1ApSQsgSimjGQ4wtr6cSmordYxkV+Ro8lOIIhRksXPyDk2 -7gW6IjUXCkZKpxFjkL3jiBSc8SkxnwCWtXg8xwNwdFVNBGLCCuJnsneYXjJNqzRq -UcoGwzsvF3Qi/ZnHUNvISdevlgIAXL4Wvrxaqvoa01wB+GCfs57RTGE4TvAGhKNK -us8K3hRT1BSpigzMIRzSxtAOrqPN6j//QSmW9f8Jcncri4j2ihSpVrFU0NdNkMhZ -eAKidTFPsxCVFuW4Aniz7jqiw5sWtjbQrlW035izIEU4sYwQoC1Nx0Svy+mMTRai -50LqFQ+A1/Hq6xHHDNx7CI83d23Erw== ------END CERTIFICATE----- diff --git a/bundle/python-cpu/Lib/site-packages/certifi/core.py b/bundle/python-cpu/Lib/site-packages/certifi/core.py deleted file mode 100644 index 1c9661cc7c2f6917c2506a30b2710002f81ab23a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/certifi/core.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -certifi.py -~~~~~~~~~~ - -This module returns the installation location of cacert.pem or its contents. -""" -import sys -import atexit - -def exit_cacert_ctx() -> None: - _CACERT_CTX.__exit__(None, None, None) # type: ignore[union-attr] - - -if sys.version_info >= (3, 11): - - from importlib.resources import as_file, files - - _CACERT_CTX = None - _CACERT_PATH = None - - def where() -> str: - # This is slightly terrible, but we want to delay extracting the file - # in cases where we're inside of a zipimport situation until someone - # actually calls where(), but we don't want to re-extract the file - # on every call of where(), so we'll do it once then store it in a - # global variable. - global _CACERT_CTX - global _CACERT_PATH - if _CACERT_PATH is None: - # This is slightly janky, the importlib.resources API wants you to - # manage the cleanup of this file, so it doesn't actually return a - # path, it returns a context manager that will give you the path - # when you enter it and will do any cleanup when you leave it. In - # the common case of not needing a temporary file, it will just - # return the file system location and the __exit__() is a no-op. - # - # We also have to hold onto the actual context manager, because - # it will do the cleanup whenever it gets garbage collected, so - # we will also store that at the global level as well. - _CACERT_CTX = as_file(files("certifi").joinpath("cacert.pem")) - _CACERT_PATH = str(_CACERT_CTX.__enter__()) - atexit.register(exit_cacert_ctx) - - return _CACERT_PATH - - def contents() -> str: - return files("certifi").joinpath("cacert.pem").read_text(encoding="ascii") - -else: - - from importlib.resources import path as get_path, read_text - - _CACERT_CTX = None - _CACERT_PATH = None - - def where() -> str: - # This is slightly terrible, but we want to delay extracting the - # file in cases where we're inside of a zipimport situation until - # someone actually calls where(), but we don't want to re-extract - # the file on every call of where(), so we'll do it once then store - # it in a global variable. - global _CACERT_CTX - global _CACERT_PATH - if _CACERT_PATH is None: - # This is slightly janky, the importlib.resources API wants you - # to manage the cleanup of this file, so it doesn't actually - # return a path, it returns a context manager that will give - # you the path when you enter it and will do any cleanup when - # you leave it. In the common case of not needing a temporary - # file, it will just return the file system location and the - # __exit__() is a no-op. - # - # We also have to hold onto the actual context manager, because - # it will do the cleanup whenever it gets garbage collected, so - # we will also store that at the global level as well. - _CACERT_CTX = get_path("certifi", "cacert.pem") - _CACERT_PATH = str(_CACERT_CTX.__enter__()) - atexit.register(exit_cacert_ctx) - - return _CACERT_PATH - - def contents() -> str: - return read_text("certifi", "cacert.pem", encoding="ascii") diff --git a/bundle/python-cpu/Lib/site-packages/certifi/py.typed b/bundle/python-cpu/Lib/site-packages/certifi/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/certifi/tests/__init__.py b/bundle/python-cpu/Lib/site-packages/certifi/tests/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/METADATA deleted file mode 100644 index cc9aeaa6845e2234a2bdd53138a9abba0a222869..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/METADATA +++ /dev/null @@ -1,827 +0,0 @@ -Metadata-Version: 2.4 -Name: charset-normalizer -Version: 3.4.9 -Summary: The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet. -Author-email: "Ahmed R. TAHRI" -Maintainer-email: "Ahmed R. TAHRI" -License: MIT -Project-URL: Changelog, https://github.com/jawah/charset_normalizer/blob/master/CHANGELOG.md -Project-URL: Documentation, https://charset-normalizer.readthedocs.io/ -Project-URL: Code, https://github.com/jawah/charset_normalizer -Project-URL: Issue tracker, https://github.com/jawah/charset_normalizer/issues -Keywords: encoding,charset,charset-detector,detector,normalization,unicode,chardet,detect -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Programming Language :: Python :: Free Threading :: 4 - Resilient -Classifier: Topic :: Text Processing :: Linguistic -Classifier: Topic :: Utilities -Classifier: Typing :: Typed -Requires-Python: >=3.7 -Description-Content-Type: text/markdown -License-File: LICENSE -Provides-Extra: unicode-backport -Dynamic: license-file - -

Charset Detection, for Everyone 👋

- -

- The Real First Universal Charset Detector
- - - - - Download Count Total - - - - -

-

- Featured Packages
- - Static Badge - - - Static Badge - -

-

- In other language (unofficial port - by the community)
- - Static Badge - -

- -> A library that helps you read text from an unknown charset encoding.
Motivated by `chardet`, -> I'm trying to resolve the issue by taking a new approach. -> All IANA character set names for which the Python core library provides codecs are supported. -> You can also register your own set of codecs, and yes, it would work as-is. - -This project offers you an alternative to **Universal Charset Encoding Detector**, also known as **Chardet**. - -| Feature | [Chardet](https://github.com/chardet/chardet) | Charset Normalizer | [cChardet](https://github.com/PyYoshi/cChardet) | -|--------------------------------------------------|:---------------------------------------------:|:-----------------------------------------------------------------------------------------------:|:-----------------------------------------------:| -| `Fast` | ✅ | ✅ | ✅ | -| `Universal`[^1] | ❌ | ✅ | ❌ | -| `Reliable` **without** distinguishable standards | ✅ | ✅ | ✅ | -| `Reliable` **with** distinguishable standards | ✅ | ✅ | ✅ | -| `License` | _Disputed_[^2]
_restrictive_ | MIT | MPL-1.1
_restrictive_ | -| `Native Python` | ✅ | ✅ | ❌ | -| `Detect spoken language` | ✅ | ✅ | N/A | -| `UnicodeDecodeError Safety` | ✅ | ✅ | ❌ | -| `Whl Size (min)` | 500 kB | 150 kB | ~200 kB | -| `Supported Encoding` | 99 | [99](https://charset-normalizer.readthedocs.io/en/latest/user/support.html#supported-encodings) | 40 | -| `Can register custom encoding` | ❌ | ✅ | ❌ | - -

-Reading Normalized TextCat Reading Text -

- -[^1]: They are clearly using specific code for a specific encoding even if covering most of used one. -[^2]: Chardet 7.0+ was relicensed from LGPL-2.1 to MIT following an AI-assisted rewrite. This relicensing is disputed on two independent grounds: **(a)** the original author [contests](https://github.com/chardet/chardet/issues/327) that the maintainer had the right to relicense, arguing the rewrite is a derivative work of the LGPL-licensed codebase since it was not a clean room implementation; **(b)** the copyright claim itself is [questionable](https://github.com/chardet/chardet/issues/334) given the code was primarily generated by an LLM, and AI-generated output may not be copyrightable under most jurisdictions. Either issue alone could undermine the MIT license. Beyond licensing, the rewrite raises questions about responsible use of AI in open source: key architectural ideas pioneered by charset-normalizer - notably decode-first validity filtering (our foundational approach since v1) and encoding pairwise similarity with the same algorithm and threshold — surfaced in chardet 7 without acknowledgment. The project also imported test files from charset-normalizer to train and benchmark against it, then claimed superior accuracy on those very files. Charset-normalizer has always been MIT-licensed, encoding-agnostic by design, and built on a verifiable human-authored history. - -## ⚡ Performance - -This package offer better performances against Chardet. Here are some numbers. - -| Package | Accuracy | Mean per file (ms) | File per sec (est) | -|---------------------------------------------------|:--------:|:------------------:|:------------------:| -| [chardet 7.4](https://github.com/chardet/chardet) | 89 % | 3 ms | 333 file/sec | -| charset-normalizer | **97 %** | 1 ms | 1000 file/sec | - -| Package | 99th percentile | 95th percentile | 50th percentile | -|---------------------------------------------------|:---------------:|:---------------:|:---------------:| -| [chardet 7.4](https://github.com/chardet/chardet) | 28 ms | 16 ms | < 1 ms | -| charset-normalizer | 8 ms | 5 ms | 1 ms | - -_updated as of July 2026 using CPython 3.12, Charset-Normalizer 3.4.8, and Chardet 7.4.3_ - -~Chardet's performance on larger file (1MB+) are very poor. Expect huge difference on large payload.~ No longer the case since Chardet 7.0+ - -> Stats are generated using 400+ files using default parameters. More details on used files, see GHA workflows. -> And yes, these results might change at any time. The dataset can be updated to include more files. -> The actual delays heavily depends on your CPU capabilities. The factors should remain the same. -> Chardet claims on his documentation to have a greater accuracy than us based on the dataset they trained Chardet on(...) -> Well, it's normal, the opposite would have been worrying. Whereas charset-normalizer don't train on anything, our solution -> is based on a completely different algorithm, still heuristic through, it does not need weights across every encoding tables. - -## ✨ Installation - -Using pip: - -```sh -pip install charset-normalizer -U -``` - -## 🚀 Basic Usage - -### CLI -This package comes with a CLI. - -``` -usage: normalizer [-h] [-v] [-a] [-n] [-m] [-r] [-f] [-t THRESHOLD] - file [file ...] - -The Real First Universal Charset Detector. Discover originating encoding used -on text file. Normalize text to unicode. - -positional arguments: - files File(s) to be analysed - -optional arguments: - -h, --help show this help message and exit - -v, --verbose Display complementary information about file if any. - Stdout will contain logs about the detection process. - -a, --with-alternative - Output complementary possibilities if any. Top-level - JSON WILL be a list. - -n, --normalize Permit to normalize input file. If not set, program - does not write anything. - -m, --minimal Only output the charset detected to STDOUT. Disabling - JSON output. - -r, --replace Replace file when trying to normalize it instead of - creating a new one. - -f, --force Replace file without asking if you are sure, use this - flag with caution. - -t THRESHOLD, --threshold THRESHOLD - Define a custom maximum amount of chaos allowed in - decoded content. 0. <= chaos <= 1. - --version Show version information and exit. -``` - -```bash -normalizer ./data/sample.1.fr.srt -``` - -or - -```bash -python -m charset_normalizer ./data/sample.1.fr.srt -``` - -🎉 Since version 1.4.0 the CLI produce easily usable stdout result in JSON format. - -```json -{ - "path": "/home/default/projects/charset_normalizer/data/sample.1.fr.srt", - "encoding": "cp1252", - "encoding_aliases": [ - "1252", - "windows_1252" - ], - "alternative_encodings": [ - "cp1254", - "cp1256", - "cp1258", - "iso8859_14", - "iso8859_15", - "iso8859_16", - "iso8859_3", - "iso8859_9", - "latin_1", - "mbcs" - ], - "language": "French", - "alphabets": [ - "Basic Latin", - "Latin-1 Supplement" - ], - "has_sig_or_bom": false, - "chaos": 0.149, - "coherence": 97.152, - "unicode_path": null, - "is_preferred": true -} -``` - -### Python -*Just print out normalized text* -```python -from charset_normalizer import from_path - -results = from_path('./my_subtitle.srt') - -print(str(results.best())) -``` - -*Upgrade your code without effort* -```python -from charset_normalizer import detect -``` - -The above code will behave the same as **chardet**. We ensure that we offer the best (reasonable) BC result possible. - -See the docs for advanced usage : [readthedocs.io](https://charset-normalizer.readthedocs.io/en/latest/) - -## 😇 Why - -When I started using Chardet, I noticed that it was not suited to my expectations, and I wanted to propose a -reliable alternative using a completely different method. Also! I never back down on a good challenge! - -I **don't care** about the **originating charset** encoding, because **two different tables** can -produce **two identical rendered string.** -What I want is to get readable text, the best I can. - -In a way, **I'm brute forcing text decoding.** How cool is that ? 😎 - -Don't confuse package **ftfy** with charset-normalizer or chardet. ftfy goal is to repair Unicode string whereas charset-normalizer to convert raw file in unknown encoding to unicode. - -## 🍰 How - - - Discard all charset encoding table that could not fit the binary content. - - Measure noise, or the mess once opened (by chunks) with a corresponding charset encoding. - - Extract matches with the lowest mess detected. - - Additionally, we measure coherence / probe for a language. - -**Wait a minute**, what is noise/mess and coherence according to **YOU ?** - -*Noise :* I opened hundred of text files, **written by humans**, with the wrong encoding table. **I observed**, then -**I established** some ground rules about **what is obvious** when **it seems like** a mess (aka. defining noise in rendered text). - I know that my interpretation of what is noise is probably incomplete, feel free to contribute in order to - improve or rewrite it. - -*Coherence :* For each language there is on earth, we have computed ranked letter appearance occurrences (the best we can). So I thought -that intel is worth something here. So I use those records against decoded text to check if I can detect intelligent design. - -## ⚡ Known limitations - - - Language detection is unreliable when text contains two or more languages sharing identical letters. (eg. HTML (english tags) + Turkish content (Sharing Latin characters)) - - Every charset detector heavily depends on sufficient content. In common cases, do not bother run detection on very tiny content. - -## ⚠️ About Python EOLs - -**If you are running:** - -- Python >=2.7,<3.5: Unsupported -- Python 3.5: charset-normalizer < 2.1 -- Python 3.6: charset-normalizer < 3.1 - -Upgrade your Python interpreter as soon as possible. - -## 👤 Contributing - -Contributions, issues and feature requests are very much welcome.
-Feel free to check [issues page](https://github.com/ousret/charset_normalizer/issues) if you want to contribute. - -## 📝 License - -Copyright © [Ahmed TAHRI @Ousret](https://github.com/Ousret).
-This project is [MIT](https://github.com/Ousret/charset_normalizer/blob/master/LICENSE) licensed. - -Characters frequencies used in this project © 2012 [Denny Vrandečić](http://simia.net/letters/) - -## 💼 For Enterprise - -Professional support for charset-normalizer is available as part of the [Tidelift -Subscription][1]. Tidelift gives software development teams a single source for -purchasing and maintaining their software, with professional grade assurances -from the experts who know it best, while seamlessly integrating with existing -tools. - -[1]: https://tidelift.com/subscription/pkg/pypi-charset-normalizer?utm_source=pypi-charset-normalizer&utm_medium=readme - -[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/7297/badge)](https://www.bestpractices.dev/projects/7297) - -# Changelog -All notable changes to charset-normalizer will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - -## [3.4.9](https://github.com/Ousret/charset_normalizer/compare/3.4.8...3.4.9) (2026-07-07) - -### Fixed -- Regression in our fallback path leading to a decode error. (#771) - We've yanked 3.4.8 as a result of that bug. - -## [3.4.8](https://github.com/Ousret/charset_normalizer/compare/3.4.7...3.4.8) (2026-07-06) - -### Fixed -- Wall import time due to cascade codec imports for our multibyte first sort of iana supported codecs (#742) -- Unnecessary json import at runtime (#753) -- Inverse capitalization not seen by noise detector (#731) - -### Changed -- No longer holding a global cache for our noise / coherence measurements. Relax RSS memory usage. -- Micro-optimizations in our noise / coherence measurements. -- No longer using regex search by default for our preemptive charset mark algorithm. -- Raised upperbound of setuptools to v83. -- Raised upperbound of mypy(c) to v2.1. - -### Removed -- Redundant UTF7 BOM marker (#730) - -## [3.4.7](https://github.com/Ousret/charset_normalizer/compare/3.4.6...3.4.7) (2026-04-02) - -### Changed -- Pre-built optimized version using mypy[c] v1.20. -- Relax `setuptools` constraint to `setuptools>=68,<82.1`. - -### Fixed -- Correctly remove SIG remnant in utf-7 decoded string. (#718) (#716) - -## [3.4.6](https://github.com/Ousret/charset_normalizer/compare/3.4.5...3.4.6) (2026-03-15) - -### Changed -- Flattened the logic in `charset_normalizer.md` for higher performance. Removed `eligible(..)` and `feed(...)` - in favor of `feed_info(...)`. -- Raised upper bound for mypy[c] to 1.20, for our optimized version. -- Updated `UNICODE_RANGES_COMBINED` using Unicode blocks v17. - -### Fixed -- Edge case where noise difference between two candidates can be almost insignificant. (#672) -- CLI `--normalize` writing to wrong path when passing multiple files in. (#702) - -### Misc -- Freethreaded pre-built wheels now shipped in PyPI starting with 3.14t. (#616) - -## [3.4.5](https://github.com/Ousret/charset_normalizer/compare/3.4.4...3.4.5) (2026-03-06) - -### Changed -- Update `setuptools` constraint to `setuptools>=68,<=82`. -- Raised upper bound of mypyc for the optional pre-built extension to v1.19.1 - -### Fixed -- Add explicit link to lib math in our optimized build. (#692) -- Logger level not restored correctly for empty byte sequences. (#701) -- TypeError when passing bytearray to from_bytes. (#703) - -### Misc -- Applied safe micro-optimizations in both our noise detector and language detector. -- Rewrote the `query_yes_no` function (inside CLI) to avoid using ambiguous licensed code. -- Added `cd.py` submodule into mypyc optional compilation to reduce further the performance impact. - -## [3.4.4](https://github.com/Ousret/charset_normalizer/compare/3.4.2...3.4.4) (2025-10-13) - -### Changed -- Bound `setuptools` to a specific constraint `setuptools>=68,<=81`. -- Raised upper bound of mypyc for the optional pre-built extension to v1.18.2 - -### Removed -- `setuptools-scm` as a build dependency. - -### Misc -- Enforced hashes in `dev-requirements.txt` and created `ci-requirements.txt` for security purposes. -- Additional pre-built wheels for riscv64, s390x, and armv7l architectures. -- Restore ` multiple.intoto.jsonl` in GitHub releases in addition to individual attestation file per wheel. - -## [3.4.3](https://github.com/Ousret/charset_normalizer/compare/3.4.2...3.4.3) (2025-08-09) - -### Changed -- mypy(c) is no longer a required dependency at build time if `CHARSET_NORMALIZER_USE_MYPYC` isn't set to `1`. (#595) (#583) -- automatically lower confidence on small bytes samples that are not Unicode in `detect` output legacy function. (#391) - -### Added -- Custom build backend to overcome inability to mark mypy as an optional dependency in the build phase. -- Support for Python 3.14 - -### Fixed -- sdist archive contained useless directories. -- automatically fallback on valid UTF-16 or UTF-32 even if the md says it's noisy. (#633) - -### Misc -- SBOM are automatically published to the relevant GitHub release to comply with regulatory changes. - Each published wheel comes with its SBOM. We choose CycloneDX as the format. -- Prebuilt optimized wheel are no longer distributed by default for CPython 3.7 due to a change in cibuildwheel. - -## [3.4.2](https://github.com/Ousret/charset_normalizer/compare/3.4.1...3.4.2) (2025-05-02) - -### Fixed -- Addressed the DeprecationWarning in our CLI regarding `argparse.FileType` by backporting the target class into the package. (#591) -- Improved the overall reliability of the detector with CJK Ideographs. (#605) (#587) - -### Changed -- Optional mypyc compilation upgraded to version 1.15 for Python >= 3.8 - -## [3.4.1](https://github.com/Ousret/charset_normalizer/compare/3.4.0...3.4.1) (2024-12-24) - -### Changed -- Project metadata are now stored using `pyproject.toml` instead of `setup.cfg` using setuptools as the build backend. -- Enforce annotation delayed loading for a simpler and consistent types in the project. -- Optional mypyc compilation upgraded to version 1.14 for Python >= 3.8 - -### Added -- pre-commit configuration. -- noxfile. - -### Removed -- `build-requirements.txt` as per using `pyproject.toml` native build configuration. -- `bin/integration.py` and `bin/serve.py` in favor of downstream integration test (see noxfile). -- `setup.cfg` in favor of `pyproject.toml` metadata configuration. -- Unused `utils.range_scan` function. - -### Fixed -- Converting content to Unicode bytes may insert `utf_8` instead of preferred `utf-8`. (#572) -- Deprecation warning "'count' is passed as positional argument" when converting to Unicode bytes on Python 3.13+ - -## [3.4.0](https://github.com/Ousret/charset_normalizer/compare/3.3.2...3.4.0) (2024-10-08) - -### Added -- Argument `--no-preemptive` in the CLI to prevent the detector to search for hints. -- Support for Python 3.13 (#512) - -### Fixed -- Relax the TypeError exception thrown when trying to compare a CharsetMatch with anything else than a CharsetMatch. -- Improved the general reliability of the detector based on user feedbacks. (#520) (#509) (#498) (#407) (#537) -- Declared charset in content (preemptive detection) not changed when converting to utf-8 bytes. (#381) - -## [3.3.2](https://github.com/Ousret/charset_normalizer/compare/3.3.1...3.3.2) (2023-10-31) - -### Fixed -- Unintentional memory usage regression when using large payload that match several encoding (#376) -- Regression on some detection case showcased in the documentation (#371) - -### Added -- Noise (md) probe that identify malformed arabic representation due to the presence of letters in isolated form (credit to my wife) - -## [3.3.1](https://github.com/Ousret/charset_normalizer/compare/3.3.0...3.3.1) (2023-10-22) - -### Changed -- Optional mypyc compilation upgraded to version 1.6.1 for Python >= 3.8 -- Improved the general detection reliability based on reports from the community - -## [3.3.0](https://github.com/Ousret/charset_normalizer/compare/3.2.0...3.3.0) (2023-09-30) - -### Added -- Allow to execute the CLI (e.g. normalizer) through `python -m charset_normalizer.cli` or `python -m charset_normalizer` -- Support for 9 forgotten encoding that are supported by Python but unlisted in `encoding.aliases` as they have no alias (#323) - -### Removed -- (internal) Redundant utils.is_ascii function and unused function is_private_use_only -- (internal) charset_normalizer.assets is moved inside charset_normalizer.constant - -### Changed -- (internal) Unicode code blocks in constants are updated using the latest v15.0.0 definition to improve detection -- Optional mypyc compilation upgraded to version 1.5.1 for Python >= 3.8 - -### Fixed -- Unable to properly sort CharsetMatch when both chaos/noise and coherence were close due to an unreachable condition in \_\_lt\_\_ (#350) - -## [3.2.0](https://github.com/Ousret/charset_normalizer/compare/3.1.0...3.2.0) (2023-06-07) - -### Changed -- Typehint for function `from_path` no longer enforce `PathLike` as its first argument -- Minor improvement over the global detection reliability - -### Added -- Introduce function `is_binary` that relies on main capabilities, and optimized to detect binaries -- Propagate `enable_fallback` argument throughout `from_bytes`, `from_path`, and `from_fp` that allow a deeper control over the detection (default True) -- Explicit support for Python 3.12 - -### Fixed -- Edge case detection failure where a file would contain 'very-long' camel cased word (Issue #289) - -## [3.1.0](https://github.com/Ousret/charset_normalizer/compare/3.0.1...3.1.0) (2023-03-06) - -### Added -- Argument `should_rename_legacy` for legacy function `detect` and disregard any new arguments without errors (PR #262) - -### Removed -- Support for Python 3.6 (PR #260) - -### Changed -- Optional speedup provided by mypy/c 1.0.1 - -## [3.0.1](https://github.com/Ousret/charset_normalizer/compare/3.0.0...3.0.1) (2022-11-18) - -### Fixed -- Multi-bytes cutter/chunk generator did not always cut correctly (PR #233) - -### Changed -- Speedup provided by mypy/c 0.990 on Python >= 3.7 - -## [3.0.0](https://github.com/Ousret/charset_normalizer/compare/2.1.1...3.0.0) (2022-10-20) - -### Added -- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results -- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES -- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio -- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl) - -### Changed -- Build with static metadata using 'build' frontend -- Make the language detection stricter -- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1 - -### Fixed -- CLI with opt --normalize fail when using full path for files -- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it -- Sphinx warnings when generating the documentation - -### Removed -- Coherence detector no longer return 'Simple English' instead return 'English' -- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese' -- Breaking: Method `first()` and `best()` from CharsetMatch -- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII) -- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches -- Breaking: Top-level function `normalize` -- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch -- Support for the backport `unicodedata2` - -## [3.0.0rc1](https://github.com/Ousret/charset_normalizer/compare/3.0.0b2...3.0.0rc1) (2022-10-18) - -### Added -- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results -- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES -- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio - -### Changed -- Build with static metadata using 'build' frontend -- Make the language detection stricter - -### Fixed -- CLI with opt --normalize fail when using full path for files -- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it - -### Removed -- Coherence detector no longer return 'Simple English' instead return 'English' -- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese' - -## [3.0.0b2](https://github.com/Ousret/charset_normalizer/compare/3.0.0b1...3.0.0b2) (2022-08-21) - -### Added -- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl) - -### Removed -- Breaking: Method `first()` and `best()` from CharsetMatch -- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII) - -### Fixed -- Sphinx warnings when generating the documentation - -## [3.0.0b1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...3.0.0b1) (2022-08-15) - -### Changed -- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1 - -### Removed -- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches -- Breaking: Top-level function `normalize` -- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch -- Support for the backport `unicodedata2` - -## [2.1.1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...2.1.1) (2022-08-19) - -### Deprecated -- Function `normalize` scheduled for removal in 3.0 - -### Changed -- Removed useless call to decode in fn is_unprintable (#206) - -### Fixed -- Third-party library (i18n xgettext) crashing not recognizing utf_8 (PEP 263) with underscore from [@aleksandernovikov](https://github.com/aleksandernovikov) (#204) - -## [2.1.0](https://github.com/Ousret/charset_normalizer/compare/2.0.12...2.1.0) (2022-06-19) - -### Added -- Output the Unicode table version when running the CLI with `--version` (PR #194) - -### Changed -- Reuse decoded buffer for single byte character sets from [@nijel](https://github.com/nijel) (PR #175) -- Fixing some performance bottlenecks from [@deedy5](https://github.com/deedy5) (PR #183) - -### Fixed -- Workaround potential bug in cpython with Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space (PR #175) -- CLI default threshold aligned with the API threshold from [@oleksandr-kuzmenko](https://github.com/oleksandr-kuzmenko) (PR #181) - -### Removed -- Support for Python 3.5 (PR #192) - -### Deprecated -- Use of backport unicodedata from `unicodedata2` as Python is quickly catching up, scheduled for removal in 3.0 (PR #194) - -## [2.0.12](https://github.com/Ousret/charset_normalizer/compare/2.0.11...2.0.12) (2022-02-12) - -### Fixed -- ASCII miss-detection on rare cases (PR #170) - -## [2.0.11](https://github.com/Ousret/charset_normalizer/compare/2.0.10...2.0.11) (2022-01-30) - -### Added -- Explicit support for Python 3.11 (PR #164) - -### Changed -- The logging behavior have been completely reviewed, now using only TRACE and DEBUG levels (PR #163 #165) - -## [2.0.10](https://github.com/Ousret/charset_normalizer/compare/2.0.9...2.0.10) (2022-01-04) - -### Fixed -- Fallback match entries might lead to UnicodeDecodeError for large bytes sequence (PR #154) - -### Changed -- Skipping the language-detection (CD) on ASCII (PR #155) - -## [2.0.9](https://github.com/Ousret/charset_normalizer/compare/2.0.8...2.0.9) (2021-12-03) - -### Changed -- Moderating the logging impact (since 2.0.8) for specific environments (PR #147) - -### Fixed -- Wrong logging level applied when setting kwarg `explain` to True (PR #146) - -## [2.0.8](https://github.com/Ousret/charset_normalizer/compare/2.0.7...2.0.8) (2021-11-24) -### Changed -- Improvement over Vietnamese detection (PR #126) -- MD improvement on trailing data and long foreign (non-pure latin) data (PR #124) -- Efficiency improvements in cd/alphabet_languages from [@adbar](https://github.com/adbar) (PR #122) -- call sum() without an intermediary list following PEP 289 recommendations from [@adbar](https://github.com/adbar) (PR #129) -- Code style as refactored by Sourcery-AI (PR #131) -- Minor adjustment on the MD around european words (PR #133) -- Remove and replace SRTs from assets / tests (PR #139) -- Initialize the library logger with a `NullHandler` by default from [@nmaynes](https://github.com/nmaynes) (PR #135) -- Setting kwarg `explain` to True will add provisionally (bounded to function lifespan) a specific stream handler (PR #135) - -### Fixed -- Fix large (misleading) sequence giving UnicodeDecodeError (PR #137) -- Avoid using too insignificant chunk (PR #137) - -### Added -- Add and expose function `set_logging_handler` to configure a specific StreamHandler from [@nmaynes](https://github.com/nmaynes) (PR #135) -- Add `CHANGELOG.md` entries, format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) (PR #141) - -## [2.0.7](https://github.com/Ousret/charset_normalizer/compare/2.0.6...2.0.7) (2021-10-11) -### Added -- Add support for Kazakh (Cyrillic) language detection (PR #109) - -### Changed -- Further, improve inferring the language from a given single-byte code page (PR #112) -- Vainly trying to leverage PEP263 when PEP3120 is not supported (PR #116) -- Refactoring for potential performance improvements in loops from [@adbar](https://github.com/adbar) (PR #113) -- Various detection improvement (MD+CD) (PR #117) - -### Removed -- Remove redundant logging entry about detected language(s) (PR #115) - -### Fixed -- Fix a minor inconsistency between Python 3.5 and other versions regarding language detection (PR #117 #102) - -## [2.0.6](https://github.com/Ousret/charset_normalizer/compare/2.0.5...2.0.6) (2021-09-18) -### Fixed -- Unforeseen regression with the loss of the backward-compatibility with some older minor of Python 3.5.x (PR #100) -- Fix CLI crash when using --minimal output in certain cases (PR #103) - -### Changed -- Minor improvement to the detection efficiency (less than 1%) (PR #106 #101) - -## [2.0.5](https://github.com/Ousret/charset_normalizer/compare/2.0.4...2.0.5) (2021-09-14) -### Changed -- The project now comply with: flake8, mypy, isort and black to ensure a better overall quality (PR #81) -- The BC-support with v1.x was improved, the old staticmethods are restored (PR #82) -- The Unicode detection is slightly improved (PR #93) -- Add syntax sugar \_\_bool\_\_ for results CharsetMatches list-container (PR #91) - -### Removed -- The project no longer raise warning on tiny content given for detection, will be simply logged as warning instead (PR #92) - -### Fixed -- In some rare case, the chunks extractor could cut in the middle of a multi-byte character and could mislead the mess detection (PR #95) -- Some rare 'space' characters could trip up the UnprintablePlugin/Mess detection (PR #96) -- The MANIFEST.in was not exhaustive (PR #78) - -## [2.0.4](https://github.com/Ousret/charset_normalizer/compare/2.0.3...2.0.4) (2021-07-30) -### Fixed -- The CLI no longer raise an unexpected exception when no encoding has been found (PR #70) -- Fix accessing the 'alphabets' property when the payload contains surrogate characters (PR #68) -- The logger could mislead (explain=True) on detected languages and the impact of one MBCS match (PR #72) -- Submatch factoring could be wrong in rare edge cases (PR #72) -- Multiple files given to the CLI were ignored when publishing results to STDOUT. (After the first path) (PR #72) -- Fix line endings from CRLF to LF for certain project files (PR #67) - -### Changed -- Adjust the MD to lower the sensitivity, thus improving the global detection reliability (PR #69 #76) -- Allow fallback on specified encoding if any (PR #71) - -## [2.0.3](https://github.com/Ousret/charset_normalizer/compare/2.0.2...2.0.3) (2021-07-16) -### Changed -- Part of the detection mechanism has been improved to be less sensitive, resulting in more accurate detection results. Especially ASCII. (PR #63) -- According to the community wishes, the detection will fall back on ASCII or UTF-8 in a last-resort case. (PR #64) - -## [2.0.2](https://github.com/Ousret/charset_normalizer/compare/2.0.1...2.0.2) (2021-07-15) -### Fixed -- Empty/Too small JSON payload miss-detection fixed. Report from [@tseaver](https://github.com/tseaver) (PR #59) - -### Changed -- Don't inject unicodedata2 into sys.modules from [@akx](https://github.com/akx) (PR #57) - -## [2.0.1](https://github.com/Ousret/charset_normalizer/compare/2.0.0...2.0.1) (2021-07-13) -### Fixed -- Make it work where there isn't a filesystem available, dropping assets frequencies.json. Report from [@sethmlarson](https://github.com/sethmlarson). (PR #55) -- Using explain=False permanently disable the verbose output in the current runtime (PR #47) -- One log entry (language target preemptive) was not show in logs when using explain=True (PR #47) -- Fix undesired exception (ValueError) on getitem of instance CharsetMatches (PR #52) - -### Changed -- Public function normalize default args values were not aligned with from_bytes (PR #53) - -### Added -- You may now use charset aliases in cp_isolation and cp_exclusion arguments (PR #47) - -## [2.0.0](https://github.com/Ousret/charset_normalizer/compare/1.4.1...2.0.0) (2021-07-02) -### Changed -- 4x to 5 times faster than the previous 1.4.0 release. At least 2x faster than Chardet. -- Accent has been made on UTF-8 detection, should perform rather instantaneous. -- The backward compatibility with Chardet has been greatly improved. The legacy detect function returns an identical charset name whenever possible. -- The detection mechanism has been slightly improved, now Turkish content is detected correctly (most of the time) -- The program has been rewritten to ease the readability and maintainability. (+Using static typing)+ -- utf_7 detection has been reinstated. - -### Removed -- This package no longer require anything when used with Python 3.5 (Dropped cached_property) -- Removed support for these languages: Catalan, Esperanto, Kazakh, Baque, Volapük, Azeri, Galician, Nynorsk, Macedonian, and Serbocroatian. -- The exception hook on UnicodeDecodeError has been removed. - -### Deprecated -- Methods coherence_non_latin, w_counter, chaos_secondary_pass of the class CharsetMatch are now deprecated and scheduled for removal in v3.0 - -### Fixed -- The CLI output used the relative path of the file(s). Should be absolute. - -## [1.4.1](https://github.com/Ousret/charset_normalizer/compare/1.4.0...1.4.1) (2021-05-28) -### Fixed -- Logger configuration/usage no longer conflict with others (PR #44) - -## [1.4.0](https://github.com/Ousret/charset_normalizer/compare/1.3.9...1.4.0) (2021-05-21) -### Removed -- Using standard logging instead of using the package loguru. -- Dropping nose test framework in favor of the maintained pytest. -- Choose to not use dragonmapper package to help with gibberish Chinese/CJK text. -- Require cached_property only for Python 3.5 due to constraint. Dropping for every other interpreter version. -- Stop support for UTF-7 that does not contain a SIG. -- Dropping PrettyTable, replaced with pure JSON output in CLI. - -### Fixed -- BOM marker in a CharsetNormalizerMatch instance could be False in rare cases even if obviously present. Due to the sub-match factoring process. -- Not searching properly for the BOM when trying utf32/16 parent codec. - -### Changed -- Improving the package final size by compressing frequencies.json. -- Huge improvement over the larges payload. - -### Added -- CLI now produces JSON consumable output. -- Return ASCII if given sequences fit. Given reasonable confidence. - -## [1.3.9](https://github.com/Ousret/charset_normalizer/compare/1.3.8...1.3.9) (2021-05-13) - -### Fixed -- In some very rare cases, you may end up getting encode/decode errors due to a bad bytes payload (PR #40) - -## [1.3.8](https://github.com/Ousret/charset_normalizer/compare/1.3.7...1.3.8) (2021-05-12) - -### Fixed -- Empty given payload for detection may cause an exception if trying to access the `alphabets` property. (PR #39) - -## [1.3.7](https://github.com/Ousret/charset_normalizer/compare/1.3.6...1.3.7) (2021-05-12) - -### Fixed -- The legacy detect function should return UTF-8-SIG if sig is present in the payload. (PR #38) - -## [1.3.6](https://github.com/Ousret/charset_normalizer/compare/1.3.5...1.3.6) (2021-02-09) - -### Changed -- Amend the previous release to allow prettytable 2.0 (PR #35) - -## [1.3.5](https://github.com/Ousret/charset_normalizer/compare/1.3.4...1.3.5) (2021-02-08) - -### Fixed -- Fix error while using the package with a python pre-release interpreter (PR #33) - -### Changed -- Dependencies refactoring, constraints revised. - -### Added -- Add python 3.9 and 3.10 to the supported interpreters - -MIT License - -Copyright (c) 2025 TAHRI Ahmed R. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/RECORD deleted file mode 100644 index 59db16fefd0be43e85ae516afb292a096f3f7a9b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/RECORD +++ /dev/null @@ -1,25 +0,0 @@ -../../Scripts/normalizer.exe,sha256=mIIva1RZVzc7kbk8N88Cy__9s4ZBLqA6_GztJ6Dhd7w,46080 -ada92cb5d92a588d1b93__mypyc.cp310-win_amd64.pyd,sha256=30Sfa3OX46g_hNCP8476wsRVJ5RCIdHW9ueCKKBcjOc,223232 -charset_normalizer-3.4.9.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -charset_normalizer-3.4.9.dist-info/METADATA,sha256=g6KZPHU8xgmJhfinUoYCWgHpZFNAMoTOJeP-92ZUcuE,42505 -charset_normalizer-3.4.9.dist-info/RECORD,, -charset_normalizer-3.4.9.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -charset_normalizer-3.4.9.dist-info/WHEEL,sha256=jYVzCt0n0i5gm3gbcoSI7PVCxfvVd-hxxM6nWgDRieA,101 -charset_normalizer-3.4.9.dist-info/entry_points.txt,sha256=ADSTKrkXZ3hhdOVFi6DcUEHQRS0xfxDIE_pEz4wLIXA,65 -charset_normalizer-3.4.9.dist-info/licenses/LICENSE,sha256=GFd0hdNwTxpHne2OVzwJds_tMV_S_ReYP6mI2kwvcNE,1092 -charset_normalizer-3.4.9.dist-info/top_level.txt,sha256=y65Zf_GLs5FHscGHzcUTGAldXcp5mEVXbh8Mo0ZlmWs,47 -charset_normalizer/__init__.py,sha256=0NT8MHi7SKq3juMqYfOdrkzjisK0L73lneNHH4qaUAs,1638 -charset_normalizer/__main__.py,sha256=2sj_BS6H0sU25C1bMqz9DVwa6kOK9lchSEbSU-_iu7M,115 -charset_normalizer/api.py,sha256=un21wfCPwDo7x5pvtFq5ShCFVxapabvdmNCVbpxc2EA,43390 -charset_normalizer/cd.cp310-win_amd64.pyd,sha256=kZa8CpNNf4mOKFUZGYRQSb-Moxcc8gUnzTEBdX9Fgss,10752 -charset_normalizer/cd.py,sha256=9bULt-ywKO0vdprAJ1e2Mgq91Kow4VUSXkgPFcKNuNs,16447 -charset_normalizer/cli/__init__.py,sha256=d9MUx-1V_qD3x9igIy4JT4oC5CU0yjulk7QyZWeRFhg,144 -charset_normalizer/cli/__main__.py,sha256=GinmtKDY11sW9P592ngc_g-MFZ52AISBZVNI9pNVGIc,12312 -charset_normalizer/constant.py,sha256=f6Vd8eiet_OIB3CeetsjGEAI-G5SYv_NJxzhJcRGK1o,46679 -charset_normalizer/legacy.py,sha256=jVoZFrn8FLiQ9LaNuacV5yGJqFSqBnQwpl8DUlJkm9s,2730 -charset_normalizer/md.cp310-win_amd64.pyd,sha256=kFUfwapt9qELmTNXCY6d25btS3tOwtG7yNX4oUlI_oc,10752 -charset_normalizer/md.py,sha256=TykAxWVfusMezkHADIuWJVrmcvw3_TNOO9pXQT5-IUk,33643 -charset_normalizer/models.py,sha256=oy3It8wkvmIUZKm2J74x0E6UpH3ubW-RAAxVX3XWmpE,13200 -charset_normalizer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -charset_normalizer/utils.py,sha256=0YiR_gH5O6z_Y2Lnp9k1_qyvMl7YmTD0RrXG8wsPspU,13992 -charset_normalizer/version.py,sha256=ALs65EaqikBmMtcLYOHCecHLggiOCtUAa_edoBLbP7s,123 diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/WHEEL deleted file mode 100644 index 2b9121abf2e6a98834c480171a2640a5926fb337..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/WHEEL +++ /dev/null @@ -1,5 +0,0 @@ -Wheel-Version: 1.0 -Generator: setuptools (83.0.0) -Root-Is-Purelib: false -Tag: cp310-cp310-win_amd64 - diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/entry_points.txt b/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/entry_points.txt deleted file mode 100644 index 65619e73ec06c20c2a70c9507b872ad624d1a85c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -normalizer = charset_normalizer.cli:cli_detect diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/licenses/LICENSE b/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/licenses/LICENSE deleted file mode 100644 index 8885be9624d2bf3fdeca3551d3b5f475bcb536db..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 TAHRI Ahmed R. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/top_level.txt b/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/top_level.txt deleted file mode 100644 index b7995f2d7d1b63b6bbde1fd0793288516055c68f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer-3.4.9.dist-info/top_level.txt +++ /dev/null @@ -1,2 +0,0 @@ -ada92cb5d92a588d1b93__mypyc -charset_normalizer diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/__init__.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/__init__.py deleted file mode 100644 index 1993f86a9d55ddc3f2b7c1dcd92a0ed55a5637ce..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -Charset-Normalizer -~~~~~~~~~~~~~~ -The Real First Universal Charset Detector. -A library that helps you read text from an unknown charset encoding. -Motivated by chardet, This package is trying to resolve the issue by taking a new approach. -All IANA character set names for which the Python core library provides codecs are supported. - -Basic usage: - >>> from charset_normalizer import from_bytes - >>> results = from_bytes('Bсеки човек има право на образование. Oбразованието!'.encode('utf_8')) - >>> best_guess = results.best() - >>> str(best_guess) - 'Bсеки човек има право на образование. Oбразованието!' - -Others methods and usages are available - see the full documentation -at . -:copyright: (c) 2021 by Ahmed TAHRI -:license: MIT, see LICENSE for more details. -""" - -from __future__ import annotations - -import logging - -from .api import from_bytes, from_fp, from_path, is_binary -from .legacy import detect -from .models import CharsetMatch, CharsetMatches -from .utils import set_logging_handler -from .version import VERSION, __version__ - -__all__ = ( - "from_fp", - "from_path", - "from_bytes", - "is_binary", - "detect", - "CharsetMatch", - "CharsetMatches", - "__version__", - "VERSION", - "set_logging_handler", -) - -# Attach a NullHandler to the top level logger by default -# https://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library - -logging.getLogger("charset_normalizer").addHandler(logging.NullHandler()) diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/__main__.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/__main__.py deleted file mode 100644 index 7b623edfde51fc72f586702ed8b5c2f742cfe059..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import annotations - -from .cli import cli_detect - -if __name__ == "__main__": - cli_detect() diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/api.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/api.py deleted file mode 100644 index 0f901c01b05617359045bce33c5806b5bcf4961c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/api.py +++ /dev/null @@ -1,1065 +0,0 @@ -from __future__ import annotations - -import logging -from functools import lru_cache -from os import PathLike -from typing import BinaryIO - -from .cd import ( - coherence_ratio, - encoding_languages, - mb_encoding_languages, - merge_coherence_ratios, -) -from .constant import ( - IANA_SUPPORTED, - IANA_SUPPORTED_SIMILAR, - TOO_BIG_SEQUENCE, - TOO_SMALL_SEQUENCE, - TRACE, -) -from .md import mess_ratio -from .models import CharsetMatch, CharsetMatches -from .utils import ( - any_specified_encoding, - cut_sequence_chunks, - iana_name, - identify_sig_or_bom, - is_multi_byte_encoding, - should_strip_sig_or_bom, -) - -logger = logging.getLogger("charset_normalizer") -explain_handler = logging.StreamHandler() -explain_handler.setFormatter( - logging.Formatter("%(asctime)s | %(levelname)s | %(message)s") -) - -# Pre-compute a reordered encoding list: multibyte first, then single-byte. -# This allows the mb_definitive_match optimization to fire earlier, skipping -# all single-byte encodings for genuine CJK content. Multibyte codecs -# hard-fail (UnicodeDecodeError) on single-byte data almost instantly, so -# testing them first costs negligible time for non-CJK files. -# Stable sort on a boolean key: multibyte (False) first, IANA order kept -# within each group. -IANA_SUPPORTED_MB_FIRST: list[str] = sorted( - IANA_SUPPORTED, key=lambda encoding: not is_multi_byte_encoding(encoding) -) - - -def from_bytes( - sequences: bytes | bytearray, - steps: int = 5, - chunk_size: int = 512, - threshold: float = 0.2, - cp_isolation: list[str] | None = None, - cp_exclusion: list[str] | None = None, - preemptive_behaviour: bool = True, - explain: bool = False, - language_threshold: float = 0.1, - enable_fallback: bool = True, -) -> CharsetMatches: - """ - Given a raw bytes sequence, return the best possibles charset usable to render str objects. - If there is no results, it is a strong indicator that the source is binary/not text. - By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence. - And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will. - - The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page - but never take it for granted. Can improve the performance. - - You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that - purpose. - - This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32. - By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain' - toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging. - Custom logging format and handler can be set manually. - """ - - if not isinstance(sequences, (bytearray, bytes)): - raise TypeError( - "Expected object of type bytes or bytearray, got: {}".format( - type(sequences) - ) - ) - - if explain: - previous_logger_level: int = logger.level - logger.addHandler(explain_handler) - logger.setLevel(TRACE) - - length: int = len(sequences) - - if length == 0: - logger.debug("Encoding detection on empty bytes, assuming utf_8 intention.") - if explain: # Defensive: ensure exit path clean handler - logger.removeHandler(explain_handler) - logger.setLevel(previous_logger_level) - return CharsetMatches([CharsetMatch(sequences, "utf_8", 0.0, False, [], "")]) - - if cp_isolation is not None: - logger.log( - TRACE, - "cp_isolation is set. use this flag for debugging purpose. " - "limited list of encoding allowed : %s.", - ", ".join(cp_isolation), - ) - cp_isolation = [iana_name(cp, False) for cp in cp_isolation] - else: - cp_isolation = [] - - if cp_exclusion is not None: - logger.log( - TRACE, - "cp_exclusion is set. use this flag for debugging purpose. " - "limited list of encoding excluded : %s.", - ", ".join(cp_exclusion), - ) - cp_exclusion = [iana_name(cp, False) for cp in cp_exclusion] - else: - cp_exclusion = [] - - if length <= (chunk_size * steps): - logger.log( - TRACE, - "override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.", - steps, - chunk_size, - length, - ) - steps = 1 - chunk_size = length - - if steps > 1 and length / steps < chunk_size: - chunk_size = int(length / steps) - - is_too_small_sequence: bool = len(sequences) < TOO_SMALL_SEQUENCE - is_too_large_sequence: bool = len(sequences) >= TOO_BIG_SEQUENCE - - if is_too_small_sequence: - logger.log( - TRACE, - "Trying to detect encoding from a tiny portion of ({}) byte(s).".format( - length - ), - ) - elif is_too_large_sequence: - logger.log( - TRACE, - "Using lazy str decoding because the payload is quite large, ({}) byte(s).".format( - length - ), - ) - - prioritized_encodings: list[str] = [] - - specified_encoding: str | None = ( - any_specified_encoding(sequences) if preemptive_behaviour else None - ) - - if specified_encoding is not None: - prioritized_encodings.append(specified_encoding) - logger.log( - TRACE, - "Detected declarative mark in sequence. Priority +1 given for %s.", - specified_encoding, - ) - - tested: set[str] = set() - tested_but_hard_failure: list[str] = [] - tested_but_soft_failure: list[str] = [] - soft_failure_skip: set[str] = set() - success_fast_tracked: set[str] = set() - - # Cache for decoded payload deduplication: hash(decoded_payload) -> (mean_mess_ratio, cd_ratios_merged, passed) - # When multiple encodings decode to the exact same string, we can skip the expensive - # mess_ratio and coherence_ratio analysis and reuse the results from the first encoding. - payload_result_cache: dict[int, tuple[float, list[tuple[str, float]], bool]] = {} - - # Avoid unoptimized RSS usage. - # this cache is mostly interesting for - # local usage. Garbage collected at the - # end. Like it should. - cached_mess_ratio = lru_cache(maxsize=None)(mess_ratio) - cached_coherence_ratio = lru_cache(maxsize=None)(coherence_ratio) - - # When a definitive result (chaos=0.0 and good coherence) is found after testing - # the prioritized encodings (ascii, utf_8), we can significantly reduce the remaining - # work. Encodings that target completely different language families (e.g., Cyrillic - # when the definitive match is Latin) are skipped entirely. - # Additionally, for same-family encodings that pass chaos probing, we reuse the - # definitive match's coherence ratios instead of recomputing them — a major savings - # since coherence_ratio accounts for ~30% of total time on slow Latin files. - definitive_match_found: bool = False - definitive_target_languages: set[str] = set() - # After the definitive match fires, we cap the number of additional same-family - # single-byte encodings that pass chaos probing. Once we've accumulated enough - # good candidates (N), further same-family SB encodings are unlikely to produce - # a better best() result and just waste mess_ratio + coherence_ratio time. - # The first encoding to trigger the definitive match is NOT counted (it's already in). - post_definitive_sb_success_count: int = 0 - POST_DEFINITIVE_SB_CAP: int = 7 - - # When a non-UTF multibyte encoding passes chaos probing with significant multibyte - # content (decoded length < 98% of raw length), skip all remaining single-byte encodings. - # Rationale: multi-byte decoders (CJK) have strict byte-sequence validation — if they - # decode without error AND pass chaos probing with substantial multibyte content, the - # data is genuinely multibyte encoded. Single-byte encodings will always decode (every - # byte maps to something) but waste time on mess_ratio before failing. - # The 98% threshold prevents false triggers on files that happen to have a few valid - # multibyte pairs (e.g., cp424/_ude_1.txt where big5 decodes with 99% ratio). - mb_definitive_match_found: bool = False - - fallback_ascii: CharsetMatch | None = None - fallback_u8: CharsetMatch | None = None - fallback_specified: CharsetMatch | None = None - - results: CharsetMatches = CharsetMatches() - - early_stop_results: CharsetMatches = CharsetMatches() - - sig_encoding, sig_payload = identify_sig_or_bom(sequences) - - if sig_encoding is not None: - prioritized_encodings.append(sig_encoding) - logger.log( - TRACE, - "Detected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.", - len(sig_payload), - sig_encoding, - ) - - prioritized_encodings.append("ascii") - - if "utf_8" not in prioritized_encodings: - prioritized_encodings.append("utf_8") - - for encoding_iana in prioritized_encodings + IANA_SUPPORTED_MB_FIRST: - if cp_isolation and encoding_iana not in cp_isolation: - continue - - if cp_exclusion and encoding_iana in cp_exclusion: - continue - - if encoding_iana in tested: - continue - - tested.add(encoding_iana) - - decoded_payload: str | None = None - bom_or_sig_available: bool = sig_encoding == encoding_iana - strip_sig_or_bom: bool = bom_or_sig_available and should_strip_sig_or_bom( - encoding_iana - ) - - if encoding_iana in {"utf_16", "utf_32"} and not bom_or_sig_available: - logger.log( - TRACE, - "Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.", - encoding_iana, - ) - continue - if encoding_iana in {"utf_7"} and not bom_or_sig_available: - logger.log( - TRACE, - "Encoding %s won't be tested as-is because detection is unreliable without BOM/SIG.", - encoding_iana, - ) - continue - - # Skip encodings similar to ones that already soft-failed (high mess ratio). - # Checked BEFORE the expensive decode attempt. - if encoding_iana in soft_failure_skip: - logger.log( - TRACE, - "%s is deemed too similar to a code page that was already considered unsuited. Continuing!", - encoding_iana, - ) - continue - - # Skip encodings that were already fast-tracked from a similar successful encoding. - if encoding_iana in success_fast_tracked: - logger.log( - TRACE, - "Skipping %s: already fast-tracked from a similar successful encoding.", - encoding_iana, - ) - continue - - try: - is_multi_byte_decoder: bool = is_multi_byte_encoding(encoding_iana) - except (ModuleNotFoundError, ImportError): # Defensive: - logger.log( - TRACE, - "Encoding %s does not provide an IncrementalDecoder", - encoding_iana, - ) - continue - - # When we've already found a definitive match (chaos=0.0 with good coherence) - # after testing the prioritized encodings, skip encodings that target - # completely different language families. This avoids running expensive - # mess_ratio + coherence_ratio on clearly unrelated candidates (e.g., Cyrillic - # when the definitive match is Latin-based). - if definitive_match_found: - if not is_multi_byte_decoder: - enc_languages = set(encoding_languages(encoding_iana)) - else: - enc_languages = set(mb_encoding_languages(encoding_iana)) - if not enc_languages.intersection(definitive_target_languages): - logger.log( - TRACE, - "Skipping %s: definitive match already found, this encoding targets different languages (%s vs %s).", - encoding_iana, - enc_languages, - definitive_target_languages, - ) - continue - - # After the definitive match, cap the number of additional same-family - # single-byte encodings that pass chaos probing. This avoids testing the - # tail of rare, low-value same-family encodings (mac_iceland, cp860, etc.) - # that almost never change best() but each cost ~1-2ms of mess_ratio + coherence. - if ( - definitive_match_found - and not is_multi_byte_decoder - and post_definitive_sb_success_count >= POST_DEFINITIVE_SB_CAP - ): - logger.log( - TRACE, - "Skipping %s: already accumulated %d same-family results after definitive match (cap=%d).", - encoding_iana, - post_definitive_sb_success_count, - POST_DEFINITIVE_SB_CAP, - ) - continue - - # When a multibyte encoding with significant multibyte content has already - # passed chaos probing, skip all single-byte encodings. They will either fail - # chaos probing (wasting mess_ratio time) or produce inferior results. - if mb_definitive_match_found and not is_multi_byte_decoder: - logger.log( - TRACE, - "Skipping single-byte %s: multi-byte definitive match already found.", - encoding_iana, - ) - continue - - # Single-byte candidates of regular size defer the expensive whole - # payload decode until after chunk probing: single-byte codecs are - # stateless (1 byte == 1 char) so decoding chunk slices is provably - # identical to slicing the decoded payload, and candidates rejected - # by chaos probing (the common case) never pay the full decode nor - # the payload hash. - deferred_decoding: bool = ( - not is_multi_byte_decoder and not is_too_large_sequence - ) - - try: - if is_too_large_sequence and not is_multi_byte_decoder: - str( - ( - sequences[: int(50e4)] - if not strip_sig_or_bom - else sequences[len(sig_payload) : int(50e4)] - ), - encoding=encoding_iana, - ) - elif not deferred_decoding: - # UTF-7 BOM is encoded in modified Base64 whose byte boundary - # can overlap with the next character. Stripping raw SIG bytes - # before decoding may leave stray bytes that decode as garbage. - # Decode the full sequence and remove the leading BOM char instead. - # see https://github.com/jawah/charset_normalizer/issues/718 - # and https://github.com/jawah/charset_normalizer/issues/716 - if encoding_iana == "utf_7" and bom_or_sig_available: - decoded_payload = str( - sequences, - encoding=encoding_iana, - ) - if decoded_payload and decoded_payload[0] == "\ufeff": - decoded_payload = decoded_payload[1:] - else: - decoded_payload = str( - ( - sequences - if not strip_sig_or_bom - else sequences[len(sig_payload) :] - ), - encoding=encoding_iana, - ) - except (UnicodeDecodeError, LookupError) as e: - if not isinstance(e, LookupError): - logger.log( - TRACE, - "Code page %s does not fit given bytes sequence at ALL. %s", - encoding_iana, - str(e), - ) - tested_but_hard_failure.append(encoding_iana) - continue - - r_ = range( - 0 if not bom_or_sig_available else len(sig_payload), - length, - int(length / steps), - ) - - multi_byte_bonus: bool = ( - is_multi_byte_decoder - and decoded_payload is not None - and len(decoded_payload) < length - ) - - if multi_byte_bonus: - logger.log( - TRACE, - "Code page %s is a multi byte encoding table and it appear that at least one character " - "was encoded using n-bytes.", - encoding_iana, - ) - - max_chunk_gave_up: int = int(len(r_) / 4) - - max_chunk_gave_up = max(max_chunk_gave_up, 2) - early_stop_count: int = 0 - lazy_str_hard_failure = False - - md_chunks: list[str] = [] - md_ratios = [] - - try: - for chunk in cut_sequence_chunks( - sequences, - encoding_iana, - r_, - chunk_size, - bom_or_sig_available, - strip_sig_or_bom, - sig_payload, - is_multi_byte_decoder, - decoded_payload, - deferred_decoding, - ): - md_chunks.append(chunk) - - md_ratios.append( - cached_mess_ratio( - chunk, - threshold, - explain and 1 <= len(cp_isolation) <= 2, - ) - ) - - if md_ratios[-1] >= threshold: - early_stop_count += 1 - - if (early_stop_count >= max_chunk_gave_up) or ( - bom_or_sig_available and not strip_sig_or_bom - ): - break - except ( - UnicodeDecodeError, - LookupError, - ) as e: # Lazy str loading may have missed something there - if deferred_decoding: - # Deferred single-byte validation failed on a chunk (or the - # codec is unavailable on this interpreter build): identical - # outcome and bookkeeping to the eager full-decode failure. - logger.log( - TRACE, - "Code page %s does not fit given bytes sequence at ALL. %s", - encoding_iana, - str(e), - ) - tested_but_hard_failure.append(encoding_iana) - continue - logger.log( - TRACE, - "LazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %s", - encoding_iana, - str(e), - ) - early_stop_count = max_chunk_gave_up - lazy_str_hard_failure = True - - # We might want to check the sequence again with the whole content - # Only if initial MD tests passes - if ( - not lazy_str_hard_failure - and is_too_large_sequence - and not is_multi_byte_decoder - ): - try: - sequences[int(50e3) :].decode(encoding_iana, errors="strict") - except UnicodeDecodeError as e: - logger.log( - TRACE, - "LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %s", - encoding_iana, - str(e), - ) - tested_but_hard_failure.append(encoding_iana) - continue - - mean_mess_ratio: float = sum(md_ratios) / len(md_ratios) if md_ratios else 0.0 - if mean_mess_ratio >= threshold or early_stop_count >= max_chunk_gave_up: - tested_but_soft_failure.append(encoding_iana) - if encoding_iana in IANA_SUPPORTED_SIMILAR: - soft_failure_skip.update(IANA_SUPPORTED_SIMILAR[encoding_iana]) - # Cache this soft-failure so identical decoding from other encodings - # can be skipped immediately. - if decoded_payload is not None and not is_multi_byte_decoder: - payload_result_cache.setdefault( - hash(decoded_payload), (mean_mess_ratio, [], False) - ) - logger.log( - TRACE, - "%s was excluded because of initial chaos probing. Gave up %i time(s). " - "Computed mean chaos is %f %%.", - encoding_iana, - early_stop_count, - round(mean_mess_ratio * 100, ndigits=3), - ) - # Preparing those fallbacks in case we got nothing. - if ( - enable_fallback - and encoding_iana - in ["ascii", "utf_8", specified_encoding, "utf_16", "utf_32"] - and not lazy_str_hard_failure - ): - # Always fully decode payload before. - # We've missed a UnicodeDecodeError proof - # while issuing release 3.4.8 - # see https://github.com/jawah/charset_normalizer/issues/771 - if decoded_payload is None: - try: - decoded_payload = str( - ( - sequences - if not strip_sig_or_bom - else sequences[len(sig_payload) :] - ), - encoding=encoding_iana, - ) - except (UnicodeDecodeError, LookupError): - logger.log( - TRACE, - "%s does not decode the whole payload: fallback entry withheld.", - encoding_iana, - ) - continue - if is_too_large_sequence: - # Don't retain huge payload in RAM. - decoded_payload = None - - fallback_entry = CharsetMatch( - sequences, - encoding_iana, - threshold, - bom_or_sig_available, - [], - decoded_payload, - preemptive_declaration=specified_encoding, - ) - if encoding_iana == specified_encoding: - fallback_specified = fallback_entry - elif encoding_iana == "ascii": - fallback_ascii = fallback_entry - else: - fallback_u8 = fallback_entry - continue - - if deferred_decoding: - # The candidate passed chaos probing: perform the whole payload - # decode (validation + payload reuse) that was deferred earlier. - try: - decoded_payload = str( - ( - sequences - if not strip_sig_or_bom - else sequences[len(sig_payload) :] - ), - encoding=encoding_iana, - ) - except (UnicodeDecodeError, LookupError) as e: - logger.log( - TRACE, - "Code page %s does not fit given bytes sequence at ALL. %s", - encoding_iana, - str(e), - ) - tested_but_hard_failure.append(encoding_iana) - continue - - # Payload-hash deduplication: if another encoding already decoded to the - # exact same string, reuse its mess_ratio and coherence results entirely. - # This is strictly more general than the old IANA_SUPPORTED_SIMILAR approach - # because it catches ALL identical decoding, not just pre-mapped ones. - if decoded_payload is not None and not is_multi_byte_decoder: - payload_hash: int = hash(decoded_payload) - cached = payload_result_cache.get(payload_hash) - if cached is not None: - cached_mess, cached_cd, cached_passed = cached - if cached_passed: - # The previous encoding with identical output passed chaos probing. - fast_match = CharsetMatch( - sequences, - encoding_iana, - cached_mess, - bom_or_sig_available, - cached_cd, - ( - decoded_payload - if ( - not is_too_large_sequence - or encoding_iana - in [specified_encoding, "ascii", "utf_8"] - ) - else None - ), - preemptive_declaration=specified_encoding, - ) - results.append(fast_match) - success_fast_tracked.add(encoding_iana) - logger.log( - TRACE, - "%s fast-tracked (identical decoded payload to a prior encoding, chaos=%f %%).", - encoding_iana, - round(cached_mess * 100, ndigits=3), - ) - - if ( - encoding_iana in [specified_encoding, "ascii", "utf_8"] - and cached_mess < 0.1 - ): - if cached_mess == 0.0: - logger.debug( - "Encoding detection: %s is most likely the one.", - fast_match.encoding, - ) - if explain: - logger.removeHandler(explain_handler) - logger.setLevel(previous_logger_level) - return CharsetMatches([fast_match]) - early_stop_results.append(fast_match) - - if ( - len(early_stop_results) - and (specified_encoding is None or specified_encoding in tested) - and "ascii" in tested - and "utf_8" in tested - ): - probable_result: CharsetMatch = early_stop_results.best() # type: ignore[assignment] - logger.debug( - "Encoding detection: %s is most likely the one.", - probable_result.encoding, - ) - if explain: - logger.removeHandler(explain_handler) - logger.setLevel(previous_logger_level) - return CharsetMatches([probable_result]) - - continue - else: - # The previous encoding with identical output failed chaos - # probing. Unreachable when the current candidate passed - # probing on the identical payload (deterministic ratios), - # kept for structural parity with the historic flow. - tested_but_soft_failure.append(encoding_iana) - logger.log( - TRACE, - "%s fast-skipped (identical decoded payload to a prior encoding that failed chaos probing).", - encoding_iana, - ) - # Prepare fallbacks for special encodings even when skipped. - if enable_fallback and encoding_iana in [ - "ascii", - "utf_8", - specified_encoding, - "utf_16", - "utf_32", - ]: - fallback_entry = CharsetMatch( - sequences, - encoding_iana, - threshold, - bom_or_sig_available, - [], - decoded_payload, - preemptive_declaration=specified_encoding, - ) - if encoding_iana == specified_encoding: - fallback_specified = fallback_entry - elif encoding_iana == "ascii": - fallback_ascii = fallback_entry - else: - fallback_u8 = fallback_entry - continue - - logger.log( - TRACE, - "%s passed initial chaos probing. Mean measured chaos is %f %%", - encoding_iana, - round(mean_mess_ratio * 100, ndigits=3), - ) - - if not is_multi_byte_decoder: - target_languages: list[str] = encoding_languages(encoding_iana) - else: - target_languages = mb_encoding_languages(encoding_iana) - - if target_languages: - logger.log( - TRACE, - "{} should target any language(s) of {}".format( - encoding_iana, str(target_languages) - ), - ) - - cd_ratios = [] - - # Run coherence detection on all chunks. We previously tried limiting to - # 1-2 chunks for post-definitive encodings to save time, but this caused - # coverage regressions by producing unrepresentative coherence scores. - # The SB cap and language-family skip optimizations provide sufficient - # speedup without sacrificing coherence accuracy. - if encoding_iana != "ascii": - # We shall skip the CD when its about ASCII - # Most of the time its not relevant to run "language-detection" on it. - lg_inclusion: str | None = ( - ",".join(target_languages) if target_languages else None - ) - - for chunk in md_chunks: - chunk_languages = cached_coherence_ratio( - chunk, - language_threshold, - lg_inclusion, - ) - - cd_ratios.append(chunk_languages) - - cd_ratios_merged = merge_coherence_ratios(cd_ratios) - - if cd_ratios_merged: - logger.log( - TRACE, - "We detected language {} using {}".format( - cd_ratios_merged, encoding_iana - ), - ) - - current_match = CharsetMatch( - sequences, - encoding_iana, - mean_mess_ratio, - bom_or_sig_available, - cd_ratios_merged, - ( - decoded_payload - if ( - not is_too_large_sequence - or encoding_iana in [specified_encoding, "ascii", "utf_8"] - ) - else None - ), - preemptive_declaration=specified_encoding, - ) - - results.append(current_match) - - # Cache the successful result for payload-hash deduplication. - if decoded_payload is not None and not is_multi_byte_decoder: - payload_result_cache.setdefault( - hash(decoded_payload), - (mean_mess_ratio, cd_ratios_merged, True), - ) - - # Count post-definitive same-family SB successes for the early termination cap. - # Only count low-mess encodings (< 2%) toward the cap. High-mess encodings are - # marginal results that shouldn't prevent better-quality candidates from being - # tested. For example, iso8859_4 (mess=0%) should not be skipped just because - # 7 high-mess Latin encodings (cp1252 at 8%, etc.) were tried first. - if ( - definitive_match_found - and not is_multi_byte_decoder - and mean_mess_ratio < 0.02 - ): - post_definitive_sb_success_count += 1 - - if ( - encoding_iana in [specified_encoding, "ascii", "utf_8"] - and mean_mess_ratio < 0.1 - ): - # If md says nothing to worry about, then... stop immediately! - if mean_mess_ratio == 0.0: - logger.debug( - "Encoding detection: %s is most likely the one.", - current_match.encoding, - ) - if explain: # Defensive: ensure exit path clean handler - logger.removeHandler(explain_handler) - logger.setLevel(previous_logger_level) - return CharsetMatches([current_match]) - - early_stop_results.append(current_match) - - if ( - len(early_stop_results) - and (specified_encoding is None or specified_encoding in tested) - and "ascii" in tested - and "utf_8" in tested - ): - probable_result = early_stop_results.best() # type: ignore[assignment] - logger.debug( - "Encoding detection: %s is most likely the one.", - probable_result.encoding, # type: ignore[union-attr] - ) - if explain: # Defensive: ensure exit path clean handler - logger.removeHandler(explain_handler) - logger.setLevel(previous_logger_level) - - return CharsetMatches([probable_result]) - - # Once we find a result with good coherence (>= 0.5) after testing the - # prioritized encodings (ascii, utf_8), activate "definitive mode": skip - # encodings that target completely different language families. This avoids - # running expensive mess_ratio + coherence_ratio on clearly unrelated - # candidates (e.g., Cyrillic encodings when the match is Latin-based). - # We require coherence >= 0.5 to avoid false positives (e.g., cp1251 decoding - # Hebrew text with 0.0 chaos but wrong language detection at coherence 0.33). - if not definitive_match_found and not is_multi_byte_decoder: - best_coherence = ( - max((v for _, v in cd_ratios_merged), default=0.0) - if cd_ratios_merged - else 0.0 - ) - if best_coherence >= 0.5 and "ascii" in tested and "utf_8" in tested: - definitive_match_found = True - definitive_target_languages.update(target_languages) - logger.log( - TRACE, - "Definitive match found: %s (chaos=%.3f, coherence=%.2f). Encodings targeting different language families will be skipped.", - encoding_iana, - mean_mess_ratio, - best_coherence, - ) - - # When a non-UTF multibyte encoding passes chaos probing with significant - # multibyte content (decoded < 98% of raw), activate mb_definitive_match. - # This skips all remaining single-byte encodings which would either soft-fail - # (running expensive mess_ratio for nothing) or produce inferior results. - if ( - not mb_definitive_match_found - and is_multi_byte_decoder - and multi_byte_bonus - and decoded_payload is not None - and len(decoded_payload) < length * 0.98 - and encoding_iana - not in { - "utf_8", - "utf_8_sig", - "utf_16", - "utf_16_be", - "utf_16_le", - "utf_32", - "utf_32_be", - "utf_32_le", - "utf_7", - } - and "ascii" in tested - and "utf_8" in tested - ): - mb_definitive_match_found = True - logger.log( - TRACE, - "Multi-byte definitive match: %s (chaos=%.3f, decoded=%d/%d=%.1f%%). Single-byte encodings will be skipped.", - encoding_iana, - mean_mess_ratio, - len(decoded_payload), - length, - len(decoded_payload) / length * 100, - ) - - if encoding_iana == sig_encoding: - logger.debug( - "Encoding detection: %s is most likely the one as we detected a BOM or SIG within " - "the beginning of the sequence.", - encoding_iana, - ) - if explain: # Defensive: ensure exit path clean handler - logger.removeHandler(explain_handler) - logger.setLevel(previous_logger_level) - return CharsetMatches([results[encoding_iana]]) - - if len(results) == 0: - if fallback_u8 or fallback_ascii or fallback_specified: - logger.log( - TRACE, - "Nothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.", - ) - - if fallback_specified: - logger.debug( - "Encoding detection: %s will be used as a fallback match", - fallback_specified.encoding, - ) - results.append(fallback_specified) - elif ( - (fallback_u8 and fallback_ascii is None) - or ( - fallback_u8 - and fallback_ascii - and fallback_u8.fingerprint != fallback_ascii.fingerprint - ) - or (fallback_u8 is not None) - ): - logger.debug("Encoding detection: utf_8 will be used as a fallback match") - results.append(fallback_u8) - elif fallback_ascii: - logger.debug("Encoding detection: ascii will be used as a fallback match") - results.append(fallback_ascii) - - if results: - logger.debug( - "Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.", - results.best().encoding, # type: ignore - len(results) - 1, - ) - else: - logger.debug("Encoding detection: Unable to determine any suitable charset.") - - if explain: - logger.removeHandler(explain_handler) - logger.setLevel(previous_logger_level) - - return results - - -def from_fp( - fp: BinaryIO, - steps: int = 5, - chunk_size: int = 512, - threshold: float = 0.20, - cp_isolation: list[str] | None = None, - cp_exclusion: list[str] | None = None, - preemptive_behaviour: bool = True, - explain: bool = False, - language_threshold: float = 0.1, - enable_fallback: bool = True, -) -> CharsetMatches: - """ - Same thing than the function from_bytes but using a file pointer that is already ready. - Will not close the file pointer. - """ - return from_bytes( - fp.read(), - steps, - chunk_size, - threshold, - cp_isolation, - cp_exclusion, - preemptive_behaviour, - explain, - language_threshold, - enable_fallback, - ) - - -def from_path( - path: str | bytes | PathLike, # type: ignore[type-arg] - steps: int = 5, - chunk_size: int = 512, - threshold: float = 0.20, - cp_isolation: list[str] | None = None, - cp_exclusion: list[str] | None = None, - preemptive_behaviour: bool = True, - explain: bool = False, - language_threshold: float = 0.1, - enable_fallback: bool = True, -) -> CharsetMatches: - """ - Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode. - Can raise IOError. - """ - with open(path, "rb") as fp: - return from_fp( - fp, - steps, - chunk_size, - threshold, - cp_isolation, - cp_exclusion, - preemptive_behaviour, - explain, - language_threshold, - enable_fallback, - ) - - -def is_binary( - fp_or_path_or_payload: PathLike | str | BinaryIO | bytes, # type: ignore[type-arg] - steps: int = 5, - chunk_size: int = 512, - threshold: float = 0.20, - cp_isolation: list[str] | None = None, - cp_exclusion: list[str] | None = None, - preemptive_behaviour: bool = True, - explain: bool = False, - language_threshold: float = 0.1, - enable_fallback: bool = False, -) -> bool: - """ - Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string. - Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match - are disabled to be stricter around ASCII-compatible but unlikely to be a string. - """ - if isinstance(fp_or_path_or_payload, (str, PathLike)): - guesses = from_path( - fp_or_path_or_payload, - steps=steps, - chunk_size=chunk_size, - threshold=threshold, - cp_isolation=cp_isolation, - cp_exclusion=cp_exclusion, - preemptive_behaviour=preemptive_behaviour, - explain=explain, - language_threshold=language_threshold, - enable_fallback=enable_fallback, - ) - elif isinstance( - fp_or_path_or_payload, - ( - bytes, - bytearray, - ), - ): - guesses = from_bytes( - fp_or_path_or_payload, - steps=steps, - chunk_size=chunk_size, - threshold=threshold, - cp_isolation=cp_isolation, - cp_exclusion=cp_exclusion, - preemptive_behaviour=preemptive_behaviour, - explain=explain, - language_threshold=language_threshold, - enable_fallback=enable_fallback, - ) - else: - guesses = from_fp( - fp_or_path_or_payload, - steps=steps, - chunk_size=chunk_size, - threshold=threshold, - cp_isolation=cp_isolation, - cp_exclusion=cp_exclusion, - preemptive_behaviour=preemptive_behaviour, - explain=explain, - language_threshold=language_threshold, - enable_fallback=enable_fallback, - ) - - return not guesses diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/cd.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/charset_normalizer/cd.cp310-win_amd64.pyd deleted file mode 100644 index f637f92469024746a7e78af86dee7ee70a66e777..0000000000000000000000000000000000000000 Binary files a/bundle/python-cpu/Lib/site-packages/charset_normalizer/cd.cp310-win_amd64.pyd and /dev/null differ diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/cd.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/cd.py deleted file mode 100644 index 45a01c656a2d3c6b45fb3e6ed2594331e6538c0f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/cd.py +++ /dev/null @@ -1,467 +0,0 @@ -from __future__ import annotations - -import importlib -from codecs import IncrementalDecoder -from functools import lru_cache - -from .constant import ( - FREQUENCIES, - KO_NAMES, - LANGUAGE_SUPPORTED_COUNT, - TOO_SMALL_SEQUENCE, - ZH_NAMES, - _FREQUENCIES_SET, - _FREQUENCIES_RANK, -) -from .md import _ASCII_CHAR_INFO, _char_info, is_suspiciously_successive_range -from .models import CoherenceMatches -from .utils import ( - is_multi_byte_encoding, - is_unicode_range_secondary, -) - - -def encoding_unicode_range(iana_name: str) -> list[str]: - """ - Return associated unicode ranges in a single byte code page. - """ - if is_multi_byte_encoding(iana_name): - raise OSError( # Defensive: - "Function not supported on multi-byte code page" - ) - - decoder = importlib.import_module(f"encodings.{iana_name}").IncrementalDecoder - - p: IncrementalDecoder = decoder(errors="ignore") - seen_ranges: dict[str, int] = {} - character_count: int = 0 - - for i in range(0x40, 0xFF): - chunk: str = p.decode(bytes([i])) - - if chunk: - chunk_codepoint = ord(chunk) - character_range: str | None = ( - _ASCII_CHAR_INFO[chunk_codepoint].range - if chunk_codepoint < 128 - else _char_info(chunk).range - ) - - if character_range is None: - continue - - if not is_unicode_range_secondary(character_range): - if character_range not in seen_ranges: - seen_ranges[character_range] = 0 - seen_ranges[character_range] += 1 - character_count += 1 - - return sorted( - [ - character_range - for character_range in seen_ranges - if seen_ranges[character_range] / character_count >= 0.15 - ] - ) - - -def unicode_range_languages(primary_range: str) -> list[str]: - """ - Return inferred languages used with a unicode range. - """ - languages: list[str] = [] - - for language, characters in FREQUENCIES.items(): - for character in characters: - codepoint = ord(character) - info = ( - _ASCII_CHAR_INFO[codepoint] - if codepoint < 128 - else _char_info(character) - ) - if info.range == primary_range: - languages.append(language) - break - - return languages - - -@lru_cache() -def encoding_languages(iana_name: str) -> list[str]: - """ - Single-byte encoding language association. Some code page are heavily linked to particular language(s). - This function does the correspondence. - """ - try: - unicode_ranges: list[str] = encoding_unicode_range(iana_name) - except ImportError: # Defensive: encoding unavailable on this build. - return [] - - primary_range: str | None = None - - for specified_range in unicode_ranges: - if "Latin" not in specified_range: - primary_range = specified_range - break - - if primary_range is None: - return ["Latin Based"] - - return unicode_range_languages(primary_range) - - -@lru_cache() -def mb_encoding_languages(iana_name: str) -> list[str]: - """ - Multi-byte encoding language association. Some code page are heavily linked to particular language(s). - This function does the correspondence. - """ - if ( - iana_name.startswith("shift_") - or iana_name.startswith("iso2022_jp") - or iana_name.startswith("euc_j") - or iana_name == "cp932" - ): - return ["Japanese"] - if iana_name.startswith("gb") or iana_name in ZH_NAMES: - return ["Chinese"] - if iana_name.startswith("iso2022_kr") or iana_name in KO_NAMES: - return ["Korean"] - - return [] - - -@lru_cache(maxsize=LANGUAGE_SUPPORTED_COUNT) -def get_target_features(language: str) -> tuple[bool, bool]: - """ - Determine main aspects from a supported language if it contains accents and if is pure Latin. - """ - target_have_accents: bool = False - target_pure_latin: bool = True - - for character in FREQUENCIES[language]: - codepoint = ord(character) - info = _ASCII_CHAR_INFO[codepoint] if codepoint < 128 else _char_info(character) - if not target_have_accents and info.accentuated: - target_have_accents = True - if target_pure_latin and not info.latin: - target_pure_latin = False - - return target_have_accents, target_pure_latin - - -def alphabet_languages( - characters: list[str], ignore_non_latin: bool = False -) -> list[str]: - """ - Return associated languages associated to given characters. - """ - languages: list[tuple[str, float]] = [] - - characters_set: frozenset[str] = frozenset(characters) - source_have_accents = False - for character in characters: - codepoint = ord(character) - info = _ASCII_CHAR_INFO[codepoint] if codepoint < 128 else _char_info(character) - if info.accentuated: - source_have_accents = True - break - - for language, language_characters in FREQUENCIES.items(): - target_have_accents, target_pure_latin = get_target_features(language) - - if ignore_non_latin and not target_pure_latin: - continue - - if not target_have_accents and source_have_accents: - continue - - character_count: int = len(language_characters) - - character_match_count: int = len(_FREQUENCIES_SET[language] & characters_set) - - ratio: float = character_match_count / character_count - - if ratio >= 0.2: - languages.append((language, ratio)) - - languages = sorted(languages, key=lambda x: x[1], reverse=True) - - return [compatible_language[0] for compatible_language in languages] - - -def characters_popularity_compare( - language: str, ordered_characters: list[str] -) -> float: - """ - Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language. - The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit). - Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.) - """ - if language not in FREQUENCIES: - raise ValueError(f"{language} not available") # Defensive: - - character_approved_count: int = 0 - lang_rank: dict[str, int] = _FREQUENCIES_RANK[language] - - ordered_characters_count: int = len(ordered_characters) - target_language_characters_count: int = len(FREQUENCIES[language]) - - large_alphabet: bool = target_language_characters_count > 26 - large_alphabet_threshold: float = target_language_characters_count / 3 - - expected_projection_ratio: float = ( - target_language_characters_count / ordered_characters_count - ) - - # Single pass: characters present in the language vocabulary, as - # (language rank, popularity rank) pairs. The scoring below only ever - # needs ranks, never the characters themselves. - common_lr: list[int] = [] - common_orr: list[int] = [] - for popularity_rank, character in enumerate(ordered_characters): - language_rank = lang_rank.get(character) - if language_rank is not None: - common_lr.append(language_rank) - common_orr.append(popularity_rank) - - for character_rank_in_language, character_rank in zip(common_lr, common_orr): - character_rank_projection: int = int(character_rank * expected_projection_ratio) - - if ( - not large_alphabet - and abs(character_rank_projection - character_rank_in_language) > 4 - ): - continue - - if ( - large_alphabet - and abs(character_rank_projection - character_rank_in_language) - < large_alphabet_threshold - ): - character_approved_count += 1 - continue - - if character_rank_in_language == 0: - # before_match_count is structurally 0 here (no pair can have a - # smaller language rank): the historic "before <= 4" acceptance - # always holds. (The symmetric "after_len == 0" case is - # impossible: language ranks are strictly below the language - # character count, hence after_len >= 1.) - character_approved_count += 1 - continue - - after_len: int = target_language_characters_count - character_rank_in_language - - # Count how many characters appear "before" in both orderings, and - # how many appear "at or after" in both orderings. Both counts grow - # monotonically and the approval thresholds - # (before / rank >= 0.4 or after / after_len >= 0.4) are known - # upfront, expressed below as exact integer comparisons: exit as - # soon as one is crossed. - before_match_count: int = 0 - after_match_count: int = 0 - - for lr_i, orr_i in zip(common_lr, common_orr): - if lr_i < character_rank_in_language: - if orr_i < character_rank: - before_match_count += 1 - if 5 * before_match_count >= 2 * character_rank_in_language: - character_approved_count += 1 - break - else: - if orr_i >= character_rank: - after_match_count += 1 - if 5 * after_match_count >= 2 * after_len: - character_approved_count += 1 - break - - return character_approved_count / len(ordered_characters) - - -def alpha_unicode_split(decoded_sequence: str) -> list[str]: - """ - Given a decoded text sequence, return a list of str. Unicode range / alphabet separation. - Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list; - One containing the latin letters and the other hebrew. - """ - layers: dict[str, list[str]] = {} - - # Fast path: track single-layer key to skip dict iteration for single-script text. - single_layer_key: str | None = None - multi_layer: bool = False - - # Cache the last character_range and its resolved layer to avoid repeated - # is_suspiciously_successive_range calls for consecutive same-range chars. - prev_character_range: str | None = None - prev_layer_target: str | None = None - - for character in decoded_sequence: - # Reuse the per-codepoint CharInfo cache: info.alpha and info.range - # are computed with the very same str.isalpha() / unicode_range() - # calls this loop historically made per character occurrence. - codepoint: int = ord(character) - if codepoint < 128: - info = _ASCII_CHAR_INFO[codepoint] - else: - info = _char_info(character) - - if not info.alpha: - continue - - character_range: str | None = info.range - - if character_range is None: - continue - - # Fast path: same range as previous character → reuse cached layer target. - if character_range == prev_character_range: - if prev_layer_target is not None: - layers[prev_layer_target].append(character) - continue - - layer_target_range: str | None = None - - if multi_layer: - for discovered_range in layers: - if not is_suspiciously_successive_range( - discovered_range, character_range - ): - layer_target_range = discovered_range - break - elif single_layer_key is not None: - if not is_suspiciously_successive_range(single_layer_key, character_range): - layer_target_range = single_layer_key - - if layer_target_range is None: - layer_target_range = character_range - - if layer_target_range not in layers: - layers[layer_target_range] = [] - if single_layer_key is None: - single_layer_key = layer_target_range - else: - multi_layer = True - - layers[layer_target_range].append(character) - - # Cache for next iteration - prev_character_range = character_range - prev_layer_target = layer_target_range - - return ["".join(chars).lower() for chars in layers.values()] - - -def merge_coherence_ratios(results: list[CoherenceMatches]) -> CoherenceMatches: - """ - This function merge results previously given by the function coherence_ratio. - The return type is the same as coherence_ratio. - """ - per_language_ratios: dict[str, list[float]] = {} - for result in results: - for sub_result in result: - language, ratio = sub_result - if language not in per_language_ratios: - per_language_ratios[language] = [ratio] - continue - per_language_ratios[language].append(ratio) - - merge = [ - ( - language, - round( - sum(per_language_ratios[language]) / len(per_language_ratios[language]), - 4, - ), - ) - for language in per_language_ratios - ] - - return sorted(merge, key=lambda x: x[1], reverse=True) - - -def filter_alt_coherence_matches(results: CoherenceMatches) -> CoherenceMatches: - """ - We shall NOT return "English—" in CoherenceMatches because it is an alternative - of "English". This function only keeps the best match and remove the em-dash in it. - """ - index_results: dict[str, list[float]] = dict() - - for result in results: - language, ratio = result - no_em_name: str = language.replace("—", "") - - if no_em_name not in index_results: - index_results[no_em_name] = [] - - index_results[no_em_name].append(ratio) - - if any(len(index_results[e]) > 1 for e in index_results): - filtered_results: CoherenceMatches = [] - - for language in index_results: - filtered_results.append((language, max(index_results[language]))) - - return filtered_results - - return results - - -def coherence_ratio( - decoded_sequence: str, threshold: float = 0.1, lg_inclusion: str | None = None -) -> CoherenceMatches: - """ - Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers. - A layer = Character extraction by alphabets/ranges. - """ - - results: list[tuple[str, float]] = [] - ignore_non_latin: bool = False - - sufficient_match_count: int = 0 - - lg_inclusion_list = lg_inclusion.split(",") if lg_inclusion is not None else [] - if "Latin Based" in lg_inclusion_list: - ignore_non_latin = True - lg_inclusion_list.remove("Latin Based") - - for layer in alpha_unicode_split(decoded_sequence): - # Native counting + stable sort reproduce Counter.most_common() - # ordering exactly (ties keep first-appearance order) without the - # interpreted Counter machinery in the compiled hot path. - char_counts: dict[str, int] = {} - for layer_character in layer: - char_counts[layer_character] = char_counts.get(layer_character, 0) + 1 - - character_count: int = len(layer) - - if character_count <= TOO_SMALL_SEQUENCE: - continue - - popular_character_ordered: list[str] = [ - item[0] - for item in sorted( - char_counts.items(), key=lambda item: item[1], reverse=True - ) - ] - - for language in lg_inclusion_list or alphabet_languages( - popular_character_ordered, ignore_non_latin - ): - ratio: float = characters_popularity_compare( - language, popular_character_ordered - ) - - if ratio < threshold: - continue - elif ratio >= 0.8: - sufficient_match_count += 1 - - results.append((language, round(ratio, 4))) - - if sufficient_match_count >= 3: - break - - return sorted( - filter_alt_coherence_matches(results), key=lambda x: x[1], reverse=True - ) diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/cli/__init__.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/cli/__init__.py deleted file mode 100644 index 562b64e01cb42d989f2ddd5a8651841f700888e4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/cli/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import annotations - -from .__main__ import cli_detect, query_yes_no - -__all__ = ( - "cli_detect", - "query_yes_no", -) diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/cli/__main__.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/cli/__main__.py deleted file mode 100644 index f1d6e53a47b5cd65eb0a969db75b05d387541662..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/cli/__main__.py +++ /dev/null @@ -1,363 +0,0 @@ -from __future__ import annotations - -import argparse -import sys -import typing -from os.path import abspath, basename, dirname, join, realpath -from platform import python_version -from unicodedata import unidata_version - -import charset_normalizer.md as md_module -from charset_normalizer import from_fp -from charset_normalizer.models import CliDetectionResult -from charset_normalizer.version import __version__ - - -def query_yes_no(question: str, default: str = "yes") -> bool: # Defensive: - """Ask a yes/no question via input() and return the answer as a bool.""" - prompt = " [Y/n] " if default == "yes" else " [y/N] " - - while True: - choice = input(question + prompt).strip().lower() - if not choice: - return default == "yes" - if choice in ("y", "yes"): - return True - if choice in ("n", "no"): - return False - print("Please respond with 'y' or 'n'.") - - -class FileType: - """Factory for creating file object types - - Instances of FileType are typically passed as type= arguments to the - ArgumentParser add_argument() method. - - Keyword Arguments: - - mode -- A string indicating how the file is to be opened. Accepts the - same values as the builtin open() function. - - bufsize -- The file's desired buffer size. Accepts the same values as - the builtin open() function. - - encoding -- The file's encoding. Accepts the same values as the - builtin open() function. - - errors -- A string indicating how encoding and decoding errors are to - be handled. Accepts the same value as the builtin open() function. - - Backported from CPython 3.12 - """ - - def __init__( - self, - mode: str = "r", - bufsize: int = -1, - encoding: str | None = None, - errors: str | None = None, - ): - self._mode = mode - self._bufsize = bufsize - self._encoding = encoding - self._errors = errors - - def __call__(self, string: str) -> typing.IO: # type: ignore[type-arg] - # the special argument "-" means sys.std{in,out} - if string == "-": - if "r" in self._mode: - return sys.stdin.buffer if "b" in self._mode else sys.stdin - elif any(c in self._mode for c in "wax"): - return sys.stdout.buffer if "b" in self._mode else sys.stdout - else: - msg = f'argument "-" with mode {self._mode}' - raise ValueError(msg) - - # all other arguments are used as file names - try: - return open(string, self._mode, self._bufsize, self._encoding, self._errors) - except OSError as e: - message = f"can't open '{string}': {e}" - raise argparse.ArgumentTypeError(message) - - def __repr__(self) -> str: - args = self._mode, self._bufsize - kwargs = [("encoding", self._encoding), ("errors", self._errors)] - args_str = ", ".join( - [repr(arg) for arg in args if arg != -1] - + [f"{kw}={arg!r}" for kw, arg in kwargs if arg is not None] - ) - return f"{type(self).__name__}({args_str})" - - -def cli_detect(argv: list[str] | None = None) -> int: - """ - CLI assistant using ARGV and ArgumentParser - :param argv: - :return: 0 if everything is fine, anything else equal trouble - """ - parser = argparse.ArgumentParser( - description="The Real First Universal Charset Detector. " - "Discover originating encoding used on text file. " - "Normalize text to unicode." - ) - - parser.add_argument( - "files", type=FileType("rb"), nargs="+", help="File(s) to be analysed" - ) - parser.add_argument( - "-v", - "--verbose", - action="store_true", - default=False, - dest="verbose", - help="Display complementary information about file if any. " - "Stdout will contain logs about the detection process.", - ) - parser.add_argument( - "-a", - "--with-alternative", - action="store_true", - default=False, - dest="alternatives", - help="Output complementary possibilities if any. Top-level JSON WILL be a list.", - ) - parser.add_argument( - "-n", - "--normalize", - action="store_true", - default=False, - dest="normalize", - help="Permit to normalize input file. If not set, program does not write anything.", - ) - parser.add_argument( - "-m", - "--minimal", - action="store_true", - default=False, - dest="minimal", - help="Only output the charset detected to STDOUT. Disabling JSON output.", - ) - parser.add_argument( - "-r", - "--replace", - action="store_true", - default=False, - dest="replace", - help="Replace file when trying to normalize it instead of creating a new one.", - ) - parser.add_argument( - "-f", - "--force", - action="store_true", - default=False, - dest="force", - help="Replace file without asking if you are sure, use this flag with caution.", - ) - parser.add_argument( - "-i", - "--no-preemptive", - action="store_true", - default=False, - dest="no_preemptive", - help="Disable looking at a charset declaration to hint the detector.", - ) - parser.add_argument( - "-t", - "--threshold", - action="store", - default=0.2, - type=float, - dest="threshold", - help="Define a custom maximum amount of noise allowed in decoded content. 0. <= noise <= 1.", - ) - parser.add_argument( - "--version", - action="version", - version="Charset-Normalizer {} - Python {} - Unicode {} - SpeedUp {}".format( - __version__, - python_version(), - unidata_version, - "OFF" if md_module.__file__.lower().endswith(".py") else "ON", - ), - help="Show version information and exit.", - ) - - args = parser.parse_args(argv) - - if args.replace is True and args.normalize is False: - if args.files: - for my_file in args.files: - my_file.close() - print("Use --replace in addition of --normalize only.", file=sys.stderr) - return 1 - - if args.force is True and args.replace is False: - if args.files: - for my_file in args.files: - my_file.close() - print("Use --force in addition of --replace only.", file=sys.stderr) - return 1 - - if args.threshold < 0.0 or args.threshold > 1.0: - if args.files: - for my_file in args.files: - my_file.close() - print("--threshold VALUE should be between 0. AND 1.", file=sys.stderr) - return 1 - - x_ = [] - - for my_file in args.files: - matches = from_fp( - my_file, - threshold=args.threshold, - explain=args.verbose, - preemptive_behaviour=args.no_preemptive is False, - ) - - best_guess = matches.best() - - if best_guess is None: - print( - 'Unable to identify originating encoding for "{}". {}'.format( - my_file.name, - ( - "Maybe try increasing maximum amount of chaos." - if args.threshold < 1.0 - else "" - ), - ), - file=sys.stderr, - ) - x_.append( - CliDetectionResult( - abspath(my_file.name), - None, - [], - [], - "Unknown", - [], - False, - 1.0, - 0.0, - None, - True, - ) - ) - else: - cli_result = CliDetectionResult( - abspath(my_file.name), - best_guess.encoding, - best_guess.encoding_aliases, - [ - cp - for cp in best_guess.could_be_from_charset - if cp != best_guess.encoding - ], - best_guess.language, - best_guess.alphabets, - best_guess.bom, - best_guess.percent_chaos, - best_guess.percent_coherence, - None, - True, - ) - x_.append(cli_result) - - if len(matches) > 1 and args.alternatives: - for el in matches: - if el != best_guess: - x_.append( - CliDetectionResult( - abspath(my_file.name), - el.encoding, - el.encoding_aliases, - [ - cp - for cp in el.could_be_from_charset - if cp != el.encoding - ], - el.language, - el.alphabets, - el.bom, - el.percent_chaos, - el.percent_coherence, - None, - False, - ) - ) - - if args.normalize is True: - if best_guess.encoding.startswith("utf") is True: - print( - '"{}" file does not need to be normalized, as it already came from unicode.'.format( - my_file.name - ), - file=sys.stderr, - ) - if my_file.closed is False: - my_file.close() - continue - - dir_path = dirname(realpath(my_file.name)) - file_name = basename(realpath(my_file.name)) - - o_: list[str] = file_name.split(".") - - if args.replace is False: - o_.insert(-1, best_guess.encoding) - if my_file.closed is False: - my_file.close() - elif ( - args.force is False - and query_yes_no( - 'Are you sure to normalize "{}" by replacing it ?'.format( - my_file.name - ), - "no", - ) - is False - ): - if my_file.closed is False: - my_file.close() - continue - - try: - cli_result.unicode_path = join(dir_path, ".".join(o_)) - - with open(cli_result.unicode_path, "wb") as fp: - fp.write(best_guess.output()) - except OSError as e: # Defensive: - print(str(e), file=sys.stderr) - if my_file.closed is False: - my_file.close() - return 2 - - if my_file.closed is False: - my_file.close() - - if args.minimal is False: - from json import dumps - - print( - dumps( - [el.__dict__ for el in x_] if len(x_) > 1 else x_[0].__dict__, - ensure_ascii=True, - indent=4, - ) - ) - else: - for my_file in args.files: - print( - ", ".join( - [ - el.encoding or "undefined" - for el in x_ - if el.path == abspath(my_file.name) - ] - ) - ) - - return 0 - - -if __name__ == "__main__": # Defensive: - cli_detect() diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/constant.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/constant.py deleted file mode 100644 index 32200b6fab36fd0e1992c9dfb9415a1bd755a788..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/constant.py +++ /dev/null @@ -1,2055 +0,0 @@ -from __future__ import annotations - -from codecs import BOM_UTF8, BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE -from encodings.aliases import aliases -from re import IGNORECASE -from re import compile as re_compile - -# Contain for each eligible encoding a list of/item bytes SIG/BOM -ENCODING_MARKS: dict[str, bytes | list[bytes]] = { - "utf_8": BOM_UTF8, - "utf_7": [ - b"\x2b\x2f\x76\x38", - b"\x2b\x2f\x76\x39", - b"\x2b\x2f\x76\x2b", - b"\x2b\x2f\x76\x2f", - ], - "gb18030": b"\x84\x31\x95\x33", - "utf_32": [BOM_UTF32_BE, BOM_UTF32_LE], - "utf_16": [BOM_UTF16_BE, BOM_UTF16_LE], -} - -TOO_SMALL_SEQUENCE: int = 32 -TOO_BIG_SEQUENCE: int = int(10e6) - -UTF8_MAXIMAL_ALLOCATION: int = 1_112_064 - -# Up-to-date Unicode ucd/17.0.0 -UNICODE_RANGES_COMBINED: dict[str, range] = { - "Control character": range(32), - "Basic Latin": range(32, 128), - "Latin-1 Supplement": range(128, 256), - "Latin Extended-A": range(256, 384), - "Latin Extended-B": range(384, 592), - "IPA Extensions": range(592, 688), - "Spacing Modifier Letters": range(688, 768), - "Combining Diacritical Marks": range(768, 880), - "Greek and Coptic": range(880, 1024), - "Cyrillic": range(1024, 1280), - "Cyrillic Supplement": range(1280, 1328), - "Armenian": range(1328, 1424), - "Hebrew": range(1424, 1536), - "Arabic": range(1536, 1792), - "Syriac": range(1792, 1872), - "Arabic Supplement": range(1872, 1920), - "Thaana": range(1920, 1984), - "NKo": range(1984, 2048), - "Samaritan": range(2048, 2112), - "Mandaic": range(2112, 2144), - "Syriac Supplement": range(2144, 2160), - "Arabic Extended-B": range(2160, 2208), - "Arabic Extended-A": range(2208, 2304), - "Devanagari": range(2304, 2432), - "Bengali": range(2432, 2560), - "Gurmukhi": range(2560, 2688), - "Gujarati": range(2688, 2816), - "Oriya": range(2816, 2944), - "Tamil": range(2944, 3072), - "Telugu": range(3072, 3200), - "Kannada": range(3200, 3328), - "Malayalam": range(3328, 3456), - "Sinhala": range(3456, 3584), - "Thai": range(3584, 3712), - "Lao": range(3712, 3840), - "Tibetan": range(3840, 4096), - "Myanmar": range(4096, 4256), - "Georgian": range(4256, 4352), - "Hangul Jamo": range(4352, 4608), - "Ethiopic": range(4608, 4992), - "Ethiopic Supplement": range(4992, 5024), - "Cherokee": range(5024, 5120), - "Unified Canadian Aboriginal Syllabics": range(5120, 5760), - "Ogham": range(5760, 5792), - "Runic": range(5792, 5888), - "Tagalog": range(5888, 5920), - "Hanunoo": range(5920, 5952), - "Buhid": range(5952, 5984), - "Tagbanwa": range(5984, 6016), - "Khmer": range(6016, 6144), - "Mongolian": range(6144, 6320), - "Unified Canadian Aboriginal Syllabics Extended": range(6320, 6400), - "Limbu": range(6400, 6480), - "Tai Le": range(6480, 6528), - "New Tai Lue": range(6528, 6624), - "Khmer Symbols": range(6624, 6656), - "Buginese": range(6656, 6688), - "Tai Tham": range(6688, 6832), - "Combining Diacritical Marks Extended": range(6832, 6912), - "Balinese": range(6912, 7040), - "Sundanese": range(7040, 7104), - "Batak": range(7104, 7168), - "Lepcha": range(7168, 7248), - "Ol Chiki": range(7248, 7296), - "Cyrillic Extended-C": range(7296, 7312), - "Georgian Extended": range(7312, 7360), - "Sundanese Supplement": range(7360, 7376), - "Vedic Extensions": range(7376, 7424), - "Phonetic Extensions": range(7424, 7552), - "Phonetic Extensions Supplement": range(7552, 7616), - "Combining Diacritical Marks Supplement": range(7616, 7680), - "Latin Extended Additional": range(7680, 7936), - "Greek Extended": range(7936, 8192), - "General Punctuation": range(8192, 8304), - "Superscripts and Subscripts": range(8304, 8352), - "Currency Symbols": range(8352, 8400), - "Combining Diacritical Marks for Symbols": range(8400, 8448), - "Letterlike Symbols": range(8448, 8528), - "Number Forms": range(8528, 8592), - "Arrows": range(8592, 8704), - "Mathematical Operators": range(8704, 8960), - "Miscellaneous Technical": range(8960, 9216), - "Control Pictures": range(9216, 9280), - "Optical Character Recognition": range(9280, 9312), - "Enclosed Alphanumerics": range(9312, 9472), - "Box Drawing": range(9472, 9600), - "Block Elements": range(9600, 9632), - "Geometric Shapes": range(9632, 9728), - "Miscellaneous Symbols": range(9728, 9984), - "Dingbats": range(9984, 10176), - "Miscellaneous Mathematical Symbols-A": range(10176, 10224), - "Supplemental Arrows-A": range(10224, 10240), - "Braille Patterns": range(10240, 10496), - "Supplemental Arrows-B": range(10496, 10624), - "Miscellaneous Mathematical Symbols-B": range(10624, 10752), - "Supplemental Mathematical Operators": range(10752, 11008), - "Miscellaneous Symbols and Arrows": range(11008, 11264), - "Glagolitic": range(11264, 11360), - "Latin Extended-C": range(11360, 11392), - "Coptic": range(11392, 11520), - "Georgian Supplement": range(11520, 11568), - "Tifinagh": range(11568, 11648), - "Ethiopic Extended": range(11648, 11744), - "Cyrillic Extended-A": range(11744, 11776), - "Supplemental Punctuation": range(11776, 11904), - "CJK Radicals Supplement": range(11904, 12032), - "Kangxi Radicals": range(12032, 12256), - "Ideographic Description Characters": range(12272, 12288), - "CJK Symbols and Punctuation": range(12288, 12352), - "Hiragana": range(12352, 12448), - "Katakana": range(12448, 12544), - "Bopomofo": range(12544, 12592), - "Hangul Compatibility Jamo": range(12592, 12688), - "Kanbun": range(12688, 12704), - "Bopomofo Extended": range(12704, 12736), - "CJK Strokes": range(12736, 12784), - "Katakana Phonetic Extensions": range(12784, 12800), - "Enclosed CJK Letters and Months": range(12800, 13056), - "CJK Compatibility": range(13056, 13312), - "CJK Unified Ideographs Extension A": range(13312, 19904), - "Yijing Hexagram Symbols": range(19904, 19968), - "CJK Unified Ideographs": range(19968, 40960), - "Yi Syllables": range(40960, 42128), - "Yi Radicals": range(42128, 42192), - "Lisu": range(42192, 42240), - "Vai": range(42240, 42560), - "Cyrillic Extended-B": range(42560, 42656), - "Bamum": range(42656, 42752), - "Modifier Tone Letters": range(42752, 42784), - "Latin Extended-D": range(42784, 43008), - "Syloti Nagri": range(43008, 43056), - "Common Indic Number Forms": range(43056, 43072), - "Phags-pa": range(43072, 43136), - "Saurashtra": range(43136, 43232), - "Devanagari Extended": range(43232, 43264), - "Kayah Li": range(43264, 43312), - "Rejang": range(43312, 43360), - "Hangul Jamo Extended-A": range(43360, 43392), - "Javanese": range(43392, 43488), - "Myanmar Extended-B": range(43488, 43520), - "Cham": range(43520, 43616), - "Myanmar Extended-A": range(43616, 43648), - "Tai Viet": range(43648, 43744), - "Meetei Mayek Extensions": range(43744, 43776), - "Ethiopic Extended-A": range(43776, 43824), - "Latin Extended-E": range(43824, 43888), - "Cherokee Supplement": range(43888, 43968), - "Meetei Mayek": range(43968, 44032), - "Hangul Syllables": range(44032, 55216), - "Hangul Jamo Extended-B": range(55216, 55296), - "High Surrogates": range(55296, 56192), - "High Private Use Surrogates": range(56192, 56320), - "Low Surrogates": range(56320, 57344), - "Private Use Area": range(57344, 63744), - "CJK Compatibility Ideographs": range(63744, 64256), - "Alphabetic Presentation Forms": range(64256, 64336), - "Arabic Presentation Forms-A": range(64336, 65024), - "Variation Selectors": range(65024, 65040), - "Vertical Forms": range(65040, 65056), - "Combining Half Marks": range(65056, 65072), - "CJK Compatibility Forms": range(65072, 65104), - "Small Form Variants": range(65104, 65136), - "Arabic Presentation Forms-B": range(65136, 65280), - "Halfwidth and Fullwidth Forms": range(65280, 65520), - "Specials": range(65520, 65536), - "Linear B Syllabary": range(65536, 65664), - "Linear B Ideograms": range(65664, 65792), - "Aegean Numbers": range(65792, 65856), - "Ancient Greek Numbers": range(65856, 65936), - "Ancient Symbols": range(65936, 66000), - "Phaistos Disc": range(66000, 66048), - "Lycian": range(66176, 66208), - "Carian": range(66208, 66272), - "Coptic Epact Numbers": range(66272, 66304), - "Old Italic": range(66304, 66352), - "Gothic": range(66352, 66384), - "Old Permic": range(66384, 66432), - "Ugaritic": range(66432, 66464), - "Old Persian": range(66464, 66528), - "Deseret": range(66560, 66640), - "Shavian": range(66640, 66688), - "Osmanya": range(66688, 66736), - "Osage": range(66736, 66816), - "Elbasan": range(66816, 66864), - "Caucasian Albanian": range(66864, 66928), - "Vithkuqi": range(66928, 67008), - "Todhri": range(67008, 67072), - "Linear A": range(67072, 67456), - "Latin Extended-F": range(67456, 67520), - "Cypriot Syllabary": range(67584, 67648), - "Imperial Aramaic": range(67648, 67680), - "Palmyrene": range(67680, 67712), - "Nabataean": range(67712, 67760), - "Hatran": range(67808, 67840), - "Phoenician": range(67840, 67872), - "Lydian": range(67872, 67904), - "Sidetic": range(67904, 67936), - "Meroitic Hieroglyphs": range(67968, 68000), - "Meroitic Cursive": range(68000, 68096), - "Kharoshthi": range(68096, 68192), - "Old South Arabian": range(68192, 68224), - "Old North Arabian": range(68224, 68256), - "Manichaean": range(68288, 68352), - "Avestan": range(68352, 68416), - "Inscriptional Parthian": range(68416, 68448), - "Inscriptional Pahlavi": range(68448, 68480), - "Psalter Pahlavi": range(68480, 68528), - "Old Turkic": range(68608, 68688), - "Old Hungarian": range(68736, 68864), - "Hanifi Rohingya": range(68864, 68928), - "Garay": range(68928, 69008), - "Rumi Numeral Symbols": range(69216, 69248), - "Yezidi": range(69248, 69312), - "Arabic Extended-C": range(69312, 69376), - "Old Sogdian": range(69376, 69424), - "Sogdian": range(69424, 69488), - "Old Uyghur": range(69488, 69552), - "Chorasmian": range(69552, 69600), - "Elymaic": range(69600, 69632), - "Brahmi": range(69632, 69760), - "Kaithi": range(69760, 69840), - "Sora Sompeng": range(69840, 69888), - "Chakma": range(69888, 69968), - "Mahajani": range(69968, 70016), - "Sharada": range(70016, 70112), - "Sinhala Archaic Numbers": range(70112, 70144), - "Khojki": range(70144, 70224), - "Multani": range(70272, 70320), - "Khudawadi": range(70320, 70400), - "Grantha": range(70400, 70528), - "Tulu-Tigalari": range(70528, 70656), - "Newa": range(70656, 70784), - "Tirhuta": range(70784, 70880), - "Siddham": range(71040, 71168), - "Modi": range(71168, 71264), - "Mongolian Supplement": range(71264, 71296), - "Takri": range(71296, 71376), - "Myanmar Extended-C": range(71376, 71424), - "Ahom": range(71424, 71504), - "Dogra": range(71680, 71760), - "Warang Citi": range(71840, 71936), - "Dives Akuru": range(71936, 72032), - "Nandinagari": range(72096, 72192), - "Zanabazar Square": range(72192, 72272), - "Soyombo": range(72272, 72368), - "Unified Canadian Aboriginal Syllabics Extended-A": range(72368, 72384), - "Pau Cin Hau": range(72384, 72448), - "Devanagari Extended-A": range(72448, 72544), - "Sharada Supplement": range(72544, 72576), - "Sunuwar": range(72640, 72704), - "Bhaiksuki": range(72704, 72816), - "Marchen": range(72816, 72896), - "Masaram Gondi": range(72960, 73056), - "Gunjala Gondi": range(73056, 73136), - "Tolong Siki": range(73136, 73200), - "Makasar": range(73440, 73472), - "Kawi": range(73472, 73568), - "Lisu Supplement": range(73648, 73664), - "Tamil Supplement": range(73664, 73728), - "Cuneiform": range(73728, 74752), - "Cuneiform Numbers and Punctuation": range(74752, 74880), - "Early Dynastic Cuneiform": range(74880, 75088), - "Cypro-Minoan": range(77712, 77824), - "Egyptian Hieroglyphs": range(77824, 78896), - "Egyptian Hieroglyph Format Controls": range(78896, 78944), - "Egyptian Hieroglyphs Extended-A": range(78944, 82944), - "Anatolian Hieroglyphs": range(82944, 83584), - "Gurung Khema": range(90368, 90432), - "Bamum Supplement": range(92160, 92736), - "Mro": range(92736, 92784), - "Tangsa": range(92784, 92880), - "Bassa Vah": range(92880, 92928), - "Pahawh Hmong": range(92928, 93072), - "Kirat Rai": range(93504, 93568), - "Medefaidrin": range(93760, 93856), - "Beria Erfe": range(93856, 93920), - "Miao": range(93952, 94112), - "Ideographic Symbols and Punctuation": range(94176, 94208), - "Tangut": range(94208, 100352), - "Tangut Components": range(100352, 101120), - "Khitan Small Script": range(101120, 101632), - "Tangut Supplement": range(101632, 101760), - "Tangut Components Supplement": range(101760, 101888), - "Kana Extended-B": range(110576, 110592), - "Kana Supplement": range(110592, 110848), - "Kana Extended-A": range(110848, 110896), - "Small Kana Extension": range(110896, 110960), - "Nushu": range(110960, 111360), - "Duployan": range(113664, 113824), - "Shorthand Format Controls": range(113824, 113840), - "Symbols for Legacy Computing Supplement": range(117760, 118464), - "Miscellaneous Symbols Supplement": range(118464, 118528), - "Znamenny Musical Notation": range(118528, 118736), - "Byzantine Musical Symbols": range(118784, 119040), - "Musical Symbols": range(119040, 119296), - "Ancient Greek Musical Notation": range(119296, 119376), - "Kaktovik Numerals": range(119488, 119520), - "Mayan Numerals": range(119520, 119552), - "Tai Xuan Jing Symbols": range(119552, 119648), - "Counting Rod Numerals": range(119648, 119680), - "Mathematical Alphanumeric Symbols": range(119808, 120832), - "Sutton SignWriting": range(120832, 121520), - "Latin Extended-G": range(122624, 122880), - "Glagolitic Supplement": range(122880, 122928), - "Cyrillic Extended-D": range(122928, 123024), - "Nyiakeng Puachue Hmong": range(123136, 123216), - "Toto": range(123536, 123584), - "Wancho": range(123584, 123648), - "Nag Mundari": range(124112, 124160), - "Ol Onal": range(124368, 124416), - "Tai Yo": range(124608, 124672), - "Ethiopic Extended-B": range(124896, 124928), - "Mende Kikakui": range(124928, 125152), - "Adlam": range(125184, 125280), - "Indic Siyaq Numbers": range(126064, 126144), - "Ottoman Siyaq Numbers": range(126208, 126288), - "Arabic Mathematical Alphabetic Symbols": range(126464, 126720), - "Mahjong Tiles": range(126976, 127024), - "Domino Tiles": range(127024, 127136), - "Playing Cards": range(127136, 127232), - "Enclosed Alphanumeric Supplement": range(127232, 127488), - "Enclosed Ideographic Supplement": range(127488, 127744), - "Miscellaneous Symbols and Pictographs": range(127744, 128512), - "Emoticons": range(128512, 128592), - "Ornamental Dingbats": range(128592, 128640), - "Transport and Map Symbols": range(128640, 128768), - "Alchemical Symbols": range(128768, 128896), - "Geometric Shapes Extended": range(128896, 129024), - "Supplemental Arrows-C": range(129024, 129280), - "Supplemental Symbols and Pictographs": range(129280, 129536), - "Chess Symbols": range(129536, 129648), - "Symbols and Pictographs Extended-A": range(129648, 129792), - "Symbols for Legacy Computing": range(129792, 130048), - "CJK Unified Ideographs Extension B": range(131072, 173792), - "CJK Unified Ideographs Extension C": range(173824, 177984), - "CJK Unified Ideographs Extension D": range(177984, 178208), - "CJK Unified Ideographs Extension E": range(178208, 183984), - "CJK Unified Ideographs Extension F": range(183984, 191472), - "CJK Unified Ideographs Extension I": range(191472, 192096), - "CJK Compatibility Ideographs Supplement": range(194560, 195104), - "CJK Unified Ideographs Extension G": range(196608, 201552), - "CJK Unified Ideographs Extension H": range(201552, 205744), - "CJK Unified Ideographs Extension J": range(205744, 210048), - "Tags": range(917504, 917632), - "Variation Selectors Supplement": range(917760, 918000), - "Supplementary Private Use Area-A": range(983040, 1048576), - "Supplementary Private Use Area-B": range(1048576, 1114112), -} - - -UNICODE_SECONDARY_RANGE_KEYWORD: list[str] = [ - "Supplement", - "Extended", - "Extensions", - "Modifier", - "Marks", - "Punctuation", - "Symbols", - "Forms", - "Operators", - "Miscellaneous", - "Drawing", - "Block", - "Shapes", - "Supplemental", - "Tags", -] - -RE_POSSIBLE_ENCODING_INDICATION = re_compile( - r"(?:(?:encoding)|(?:charset)|(?:coding))(?:[\:= ]{1,10})(?:[\"\']?)([a-zA-Z0-9\-_]+)(?:[\"\']?)", - IGNORECASE, -) - -IANA_NO_ALIASES = [ - "cp720", - "cp737", - "cp856", - "cp874", - "cp875", - "cp1006", - "koi8_r", - "koi8_t", - "koi8_u", -] - -IANA_SUPPORTED: list[str] = sorted( - filter( - lambda x: not x.endswith("_codec") and x not in {"rot_13", "tactis", "mbcs"}, - list(set(aliases.values())) + IANA_NO_ALIASES, - ) -) - -IANA_SUPPORTED_COUNT: int = len(IANA_SUPPORTED) - -# pre-computed code page that are similar using the function cp_similarity. -IANA_SUPPORTED_SIMILAR: dict[str, list[str]] = { - "cp037": ["cp1026", "cp1140", "cp273", "cp500"], - "cp1026": ["cp037", "cp1140", "cp273", "cp500"], - "cp1125": ["cp866"], - "cp1140": ["cp037", "cp1026", "cp273", "cp500"], - "cp1250": ["iso8859_2"], - "cp1251": ["kz1048", "ptcp154"], - "cp1252": ["iso8859_15", "iso8859_9", "latin_1"], - "cp1253": ["iso8859_7"], - "cp1254": ["iso8859_15", "iso8859_9", "latin_1"], - "cp1257": ["iso8859_13"], - "cp273": ["cp037", "cp1026", "cp1140", "cp500"], - "cp437": ["cp850", "cp858", "cp860", "cp861", "cp862", "cp863", "cp865"], - "cp500": ["cp037", "cp1026", "cp1140", "cp273"], - "cp850": ["cp437", "cp857", "cp858", "cp865"], - "cp857": ["cp850", "cp858", "cp865"], - "cp858": ["cp437", "cp850", "cp857", "cp865"], - "cp860": ["cp437", "cp861", "cp862", "cp863", "cp865"], - "cp861": ["cp437", "cp860", "cp862", "cp863", "cp865"], - "cp862": ["cp437", "cp860", "cp861", "cp863", "cp865"], - "cp863": ["cp437", "cp860", "cp861", "cp862", "cp865"], - "cp865": ["cp437", "cp850", "cp857", "cp858", "cp860", "cp861", "cp862", "cp863"], - "cp866": ["cp1125"], - "iso8859_10": ["iso8859_14", "iso8859_15", "iso8859_4", "iso8859_9", "latin_1"], - "iso8859_11": ["tis_620"], - "iso8859_13": ["cp1257"], - "iso8859_14": [ - "iso8859_10", - "iso8859_15", - "iso8859_16", - "iso8859_3", - "iso8859_9", - "latin_1", - ], - "iso8859_15": [ - "cp1252", - "cp1254", - "iso8859_10", - "iso8859_14", - "iso8859_16", - "iso8859_3", - "iso8859_9", - "latin_1", - ], - "iso8859_16": [ - "iso8859_14", - "iso8859_15", - "iso8859_2", - "iso8859_3", - "iso8859_9", - "latin_1", - ], - "iso8859_2": ["cp1250", "iso8859_16", "iso8859_4"], - "iso8859_3": ["iso8859_14", "iso8859_15", "iso8859_16", "iso8859_9", "latin_1"], - "iso8859_4": ["iso8859_10", "iso8859_2", "iso8859_9", "latin_1"], - "iso8859_7": ["cp1253"], - "iso8859_9": [ - "cp1252", - "cp1254", - "cp1258", - "iso8859_10", - "iso8859_14", - "iso8859_15", - "iso8859_16", - "iso8859_3", - "iso8859_4", - "latin_1", - ], - "kz1048": ["cp1251", "ptcp154"], - "latin_1": [ - "cp1252", - "cp1254", - "cp1258", - "iso8859_10", - "iso8859_14", - "iso8859_15", - "iso8859_16", - "iso8859_3", - "iso8859_4", - "iso8859_9", - ], - "mac_iceland": ["mac_roman", "mac_turkish"], - "mac_roman": ["mac_iceland", "mac_turkish"], - "mac_turkish": ["mac_iceland", "mac_roman"], - "ptcp154": ["cp1251", "kz1048"], - "tis_620": ["iso8859_11"], -} - - -CHARDET_CORRESPONDENCE: dict[str, str] = { - "iso2022_kr": "ISO-2022-KR", - "iso2022_jp": "ISO-2022-JP", - "euc_kr": "EUC-KR", - "tis_620": "TIS-620", - "utf_32": "UTF-32", - "euc_jp": "EUC-JP", - "koi8_r": "KOI8-R", - "iso8859_1": "ISO-8859-1", - "iso8859_2": "ISO-8859-2", - "iso8859_5": "ISO-8859-5", - "iso8859_6": "ISO-8859-6", - "iso8859_7": "ISO-8859-7", - "iso8859_8": "ISO-8859-8", - "utf_16": "UTF-16", - "cp855": "IBM855", - "mac_cyrillic": "MacCyrillic", - "gb2312": "GB2312", - "gb18030": "GB18030", - "cp932": "CP932", - "cp866": "IBM866", - "utf_8": "utf-8", - "utf_8_sig": "UTF-8-SIG", - "shift_jis": "SHIFT_JIS", - "big5": "Big5", - "cp1250": "windows-1250", - "cp1251": "windows-1251", - "cp1252": "Windows-1252", - "cp1253": "windows-1253", - "cp1255": "windows-1255", - "cp1256": "windows-1256", - "cp1254": "Windows-1254", - "cp949": "CP949", -} - - -COMMON_SAFE_ASCII_CHARACTERS: frozenset[str] = frozenset( - { - "<", - ">", - "=", - ":", - "/", - "&", - ";", - "{", - "}", - "[", - "]", - ",", - "|", - '"', - "-", - "(", - ")", - } -) - -# Sample character sets — replace with full lists if needed -COMMON_CHINESE_CHARACTERS = "的一是在不了有和人这中大为上个国我以要他时来用们生到作地于出就分对成会可主发年动同工也能下过子说产种面而方后多定行学法所民得经十三之进着等部度家电力里如水化高自二理起小物现实加量都两体制机当使点从业本去把性好应开它合还因由其些然前外天政四日那社义事平形相全表间样与关各重新线内数正心反你明看原又么利比或但质气第向道命此变条只没结解问意建月公无系军很情者最立代想已通并提直题党程展五果料象员革位入常文总次品式活设及管特件长求老头基资边流路级少图山统接知较将组见计别她手角期根论运农指几九区强放决西被干做必战先回则任取据处队南给色光门即保治北造百规热领七海口东导器压志世金增争济阶油思术极交受联什认六共权收证改清己美再采转更单风切打白教速花带安场身车例真务具万每目至达走积示议声报斗完类八离华名确才科张信马节话米整空元况今集温传土许步群广石记需段研界拉林律叫且究观越织装影算低持音众书布复容儿须际商非验连断深难近矿千周委素技备半办青省列习响约支般史感劳便团往酸历市克何除消构府太准精值号率族维划选标写存候毛亲快效斯院查江型眼王按格养易置派层片始却专状育厂京识适属圆包火住调满县局照参红细引听该铁价严龙飞" - -COMMON_JAPANESE_CHARACTERS = "日一国年大十二本中長出三時行見月分後前生五間上東四今金九入学高円子外八六下来気小七山話女北午百書先名川千水半男西電校語土木聞食車何南万毎白天母火右読友左休父雨" - -COMMON_KOREAN_CHARACTERS = "一二三四五六七八九十百千萬上下左右中人女子大小山川日月火水木金土父母天地國名年時文校學生" - -# Combine all into a frozenset -COMMON_CJK_CHARACTERS = frozenset( - "".join( - [ - COMMON_CHINESE_CHARACTERS, - COMMON_JAPANESE_CHARACTERS, - COMMON_KOREAN_CHARACTERS, - ] - ) -) - -KO_NAMES: frozenset[str] = frozenset({"johab", "cp949", "euc_kr"}) -ZH_NAMES: frozenset[str] = frozenset({"big5", "cp950", "big5hkscs", "hz"}) - -# Logging LEVEL below DEBUG -TRACE: int = 5 - - -# Language label that contain the em dash "—" -# character are to be considered alternative seq to origin -FREQUENCIES: dict[str, list[str]] = { - "English": [ - "e", - "a", - "t", - "i", - "o", - "n", - "s", - "r", - "h", - "l", - "d", - "c", - "u", - "m", - "f", - "p", - "g", - "w", - "y", - "b", - "v", - "k", - "x", - "j", - "z", - "q", - ], - "English—": [ - "e", - "a", - "t", - "i", - "o", - "n", - "s", - "r", - "h", - "l", - "d", - "c", - "m", - "u", - "f", - "p", - "g", - "w", - "b", - "y", - "v", - "k", - "j", - "x", - "z", - "q", - ], - "German": [ - "e", - "n", - "i", - "r", - "s", - "t", - "a", - "d", - "h", - "u", - "l", - "g", - "o", - "c", - "m", - "b", - "f", - "k", - "w", - "z", - "p", - "v", - "ü", - "ä", - "ö", - "j", - ], - "French": [ - "e", - "a", - "s", - "n", - "i", - "t", - "r", - "l", - "u", - "o", - "d", - "c", - "p", - "m", - "é", - "v", - "g", - "f", - "b", - "h", - "q", - "à", - "x", - "è", - "y", - "j", - ], - "Dutch": [ - "e", - "n", - "a", - "i", - "r", - "t", - "o", - "d", - "s", - "l", - "g", - "h", - "v", - "m", - "u", - "k", - "c", - "p", - "b", - "w", - "j", - "z", - "f", - "y", - "x", - "ë", - ], - "Italian": [ - "e", - "i", - "a", - "o", - "n", - "l", - "t", - "r", - "s", - "c", - "d", - "u", - "p", - "m", - "g", - "v", - "f", - "b", - "z", - "h", - "q", - "è", - "à", - "k", - "y", - "ò", - ], - "Polish": [ - "a", - "i", - "o", - "e", - "n", - "r", - "z", - "w", - "s", - "c", - "t", - "k", - "y", - "d", - "p", - "m", - "u", - "l", - "j", - "ł", - "g", - "b", - "h", - "ą", - "ę", - "ó", - ], - "Spanish": [ - "e", - "a", - "o", - "n", - "s", - "r", - "i", - "l", - "d", - "t", - "c", - "u", - "m", - "p", - "b", - "g", - "v", - "f", - "y", - "ó", - "h", - "q", - "í", - "j", - "z", - "á", - ], - "Russian": [ - "о", - "е", - "а", - "и", - "н", - "т", - "с", - "р", - "в", - "л", - "к", - "м", - "д", - "п", - "у", - "г", - "я", - "ы", - "з", - "б", - "й", - "ь", - "ч", - "х", - "ж", - "ц", - ], - # Jap-Kanji - "Japanese": [ - "日", - "一", - "人", - "年", - "大", - "十", - "二", - "本", - "中", - "長", - "出", - "三", - "時", - "行", - "見", - "月", - "分", - "後", - "前", - "生", - "五", - "間", - "上", - "東", - "四", - "今", - "金", - "九", - "入", - "学", - "高", - "円", - "子", - "外", - "八", - "六", - "下", - "来", - "気", - "小", - "七", - "山", - "話", - "女", - "北", - "午", - "百", - "書", - "先", - "名", - "川", - "千", - "水", - "半", - "男", - "西", - "電", - "校", - "語", - "土", - "木", - "聞", - "食", - "車", - "何", - "南", - "万", - "毎", - "白", - "天", - "母", - "火", - "右", - "読", - "友", - "左", - "休", - "父", - "雨", - ], - # Jap-Katakana - "Japanese—": [ - "ー", - "ン", - "ス", - "・", - "ル", - "ト", - "リ", - "イ", - "ア", - "ラ", - "ッ", - "ク", - "ド", - "シ", - "レ", - "ジ", - "タ", - "フ", - "ロ", - "カ", - "テ", - "マ", - "ィ", - "グ", - "バ", - "ム", - "プ", - "オ", - "コ", - "デ", - "ニ", - "ウ", - "メ", - "サ", - "ビ", - "ナ", - "ブ", - "ャ", - "エ", - "ュ", - "チ", - "キ", - "ズ", - "ダ", - "パ", - "ミ", - "ェ", - "ョ", - "ハ", - "セ", - "ベ", - "ガ", - "モ", - "ツ", - "ネ", - "ボ", - "ソ", - "ノ", - "ァ", - "ヴ", - "ワ", - "ポ", - "ペ", - "ピ", - "ケ", - "ゴ", - "ギ", - "ザ", - "ホ", - "ゲ", - "ォ", - "ヤ", - "ヒ", - "ユ", - "ヨ", - "ヘ", - "ゼ", - "ヌ", - "ゥ", - "ゾ", - "ヶ", - "ヂ", - "ヲ", - "ヅ", - "ヵ", - "ヱ", - "ヰ", - "ヮ", - "ヽ", - "゠", - "ヾ", - "ヷ", - "ヿ", - "ヸ", - "ヹ", - "ヺ", - ], - # Jap-Hiragana - "Japanese——": [ - "の", - "に", - "る", - "た", - "と", - "は", - "し", - "い", - "を", - "で", - "て", - "が", - "な", - "れ", - "か", - "ら", - "さ", - "っ", - "り", - "す", - "あ", - "も", - "こ", - "ま", - "う", - "く", - "よ", - "き", - "ん", - "め", - "お", - "け", - "そ", - "つ", - "だ", - "や", - "え", - "ど", - "わ", - "ち", - "み", - "せ", - "じ", - "ば", - "へ", - "び", - "ず", - "ろ", - "ほ", - "げ", - "む", - "べ", - "ひ", - "ょ", - "ゆ", - "ぶ", - "ご", - "ゃ", - "ね", - "ふ", - "ぐ", - "ぎ", - "ぼ", - "ゅ", - "づ", - "ざ", - "ぞ", - "ぬ", - "ぜ", - "ぱ", - "ぽ", - "ぷ", - "ぴ", - "ぃ", - "ぁ", - "ぇ", - "ぺ", - "ゞ", - "ぢ", - "ぉ", - "ぅ", - "ゐ", - "ゝ", - "ゑ", - "゛", - "゜", - "ゎ", - "ゔ", - "゚", - "ゟ", - "゙", - "ゕ", - "ゖ", - ], - "Portuguese": [ - "a", - "e", - "o", - "s", - "i", - "r", - "d", - "n", - "t", - "m", - "u", - "c", - "l", - "p", - "g", - "v", - "b", - "f", - "h", - "ã", - "q", - "é", - "ç", - "á", - "z", - "í", - ], - "Swedish": [ - "e", - "a", - "n", - "r", - "t", - "s", - "i", - "l", - "d", - "o", - "m", - "k", - "g", - "v", - "h", - "f", - "u", - "p", - "ä", - "c", - "b", - "ö", - "å", - "y", - "j", - "x", - ], - "Chinese": [ - "的", - "一", - "是", - "不", - "了", - "在", - "人", - "有", - "我", - "他", - "这", - "个", - "们", - "中", - "来", - "上", - "大", - "为", - "和", - "国", - "地", - "到", - "以", - "说", - "时", - "要", - "就", - "出", - "会", - "可", - "也", - "你", - "对", - "生", - "能", - "而", - "子", - "那", - "得", - "于", - "着", - "下", - "自", - "之", - "年", - "过", - "发", - "后", - "作", - "里", - "用", - "道", - "行", - "所", - "然", - "家", - "种", - "事", - "成", - "方", - "多", - "经", - "么", - "去", - "法", - "学", - "如", - "都", - "同", - "现", - "当", - "没", - "动", - "面", - "起", - "看", - "定", - "天", - "分", - "还", - "进", - "好", - "小", - "部", - "其", - "些", - "主", - "样", - "理", - "心", - "她", - "本", - "前", - "开", - "但", - "因", - "只", - "从", - "想", - "实", - ], - "Ukrainian": [ - "о", - "а", - "н", - "і", - "и", - "р", - "в", - "т", - "е", - "с", - "к", - "л", - "у", - "д", - "м", - "п", - "з", - "я", - "ь", - "б", - "г", - "й", - "ч", - "х", - "ц", - "ї", - ], - "Norwegian": [ - "e", - "r", - "n", - "t", - "a", - "s", - "i", - "o", - "l", - "d", - "g", - "k", - "m", - "v", - "f", - "p", - "u", - "b", - "h", - "å", - "y", - "j", - "ø", - "c", - "æ", - "w", - ], - "Finnish": [ - "a", - "i", - "n", - "t", - "e", - "s", - "l", - "o", - "u", - "k", - "ä", - "m", - "r", - "v", - "j", - "h", - "p", - "y", - "d", - "ö", - "g", - "c", - "b", - "f", - "w", - "z", - ], - "Vietnamese": [ - "n", - "h", - "t", - "i", - "c", - "g", - "a", - "o", - "u", - "m", - "l", - "r", - "à", - "đ", - "s", - "e", - "v", - "p", - "b", - "y", - "ư", - "d", - "á", - "k", - "ộ", - "ế", - ], - "Czech": [ - "o", - "e", - "a", - "n", - "t", - "s", - "i", - "l", - "v", - "r", - "k", - "d", - "u", - "m", - "p", - "í", - "c", - "h", - "z", - "á", - "y", - "j", - "b", - "ě", - "é", - "ř", - ], - "Hungarian": [ - "e", - "a", - "t", - "l", - "s", - "n", - "k", - "r", - "i", - "o", - "z", - "á", - "é", - "g", - "m", - "b", - "y", - "v", - "d", - "h", - "u", - "p", - "j", - "ö", - "f", - "c", - ], - "Korean": [ - "이", - "다", - "에", - "의", - "는", - "로", - "하", - "을", - "가", - "고", - "지", - "서", - "한", - "은", - "기", - "으", - "년", - "대", - "사", - "시", - "를", - "리", - "도", - "인", - "스", - "일", - ], - "Indonesian": [ - "a", - "n", - "e", - "i", - "r", - "t", - "u", - "s", - "d", - "k", - "m", - "l", - "g", - "p", - "b", - "o", - "h", - "y", - "j", - "c", - "w", - "f", - "v", - "z", - "x", - "q", - ], - "Turkish": [ - "a", - "e", - "i", - "n", - "r", - "l", - "ı", - "k", - "d", - "t", - "s", - "m", - "y", - "u", - "o", - "b", - "ü", - "ş", - "v", - "g", - "z", - "h", - "c", - "p", - "ç", - "ğ", - ], - "Romanian": [ - "e", - "i", - "a", - "r", - "n", - "t", - "u", - "l", - "o", - "c", - "s", - "d", - "p", - "m", - "ă", - "f", - "v", - "î", - "g", - "b", - "ș", - "ț", - "z", - "h", - "â", - "j", - ], - "Farsi": [ - "ا", - "ی", - "ر", - "د", - "ن", - "ه", - "و", - "م", - "ت", - "ب", - "س", - "ل", - "ک", - "ش", - "ز", - "ف", - "گ", - "ع", - "خ", - "ق", - "ج", - "آ", - "پ", - "ح", - "ط", - "ص", - ], - "Arabic": [ - "ا", - "ل", - "ي", - "م", - "و", - "ن", - "ر", - "ت", - "ب", - "ة", - "ع", - "د", - "س", - "ف", - "ه", - "ك", - "ق", - "أ", - "ح", - "ج", - "ش", - "ط", - "ص", - "ى", - "خ", - "إ", - ], - "Danish": [ - "e", - "r", - "n", - "t", - "a", - "i", - "s", - "d", - "l", - "o", - "g", - "m", - "k", - "f", - "v", - "u", - "b", - "h", - "p", - "å", - "y", - "ø", - "æ", - "c", - "j", - "w", - ], - "Serbian": [ - "а", - "и", - "о", - "е", - "н", - "р", - "с", - "у", - "т", - "к", - "ј", - "в", - "д", - "м", - "п", - "л", - "г", - "з", - "б", - "a", - "i", - "e", - "o", - "n", - "ц", - "ш", - ], - "Lithuanian": [ - "i", - "a", - "s", - "o", - "r", - "e", - "t", - "n", - "u", - "k", - "m", - "l", - "p", - "v", - "d", - "j", - "g", - "ė", - "b", - "y", - "ų", - "š", - "ž", - "c", - "ą", - "į", - ], - "Slovene": [ - "e", - "a", - "i", - "o", - "n", - "r", - "s", - "l", - "t", - "j", - "v", - "k", - "d", - "p", - "m", - "u", - "z", - "b", - "g", - "h", - "č", - "c", - "š", - "ž", - "f", - "y", - ], - "Slovak": [ - "o", - "a", - "e", - "n", - "i", - "r", - "v", - "t", - "s", - "l", - "k", - "d", - "m", - "p", - "u", - "c", - "h", - "j", - "b", - "z", - "á", - "y", - "ý", - "í", - "č", - "é", - ], - "Hebrew": [ - "י", - "ו", - "ה", - "ל", - "ר", - "ב", - "ת", - "מ", - "א", - "ש", - "נ", - "ע", - "ם", - "ד", - "ק", - "ח", - "פ", - "ס", - "כ", - "ג", - "ט", - "צ", - "ן", - "ז", - "ך", - ], - "Bulgarian": [ - "а", - "и", - "о", - "е", - "н", - "т", - "р", - "с", - "в", - "л", - "к", - "д", - "п", - "м", - "з", - "г", - "я", - "ъ", - "у", - "б", - "ч", - "ц", - "й", - "ж", - "щ", - "х", - ], - "Croatian": [ - "a", - "i", - "o", - "e", - "n", - "r", - "j", - "s", - "t", - "u", - "k", - "l", - "v", - "d", - "m", - "p", - "g", - "z", - "b", - "c", - "č", - "h", - "š", - "ž", - "ć", - "f", - ], - "Hindi": [ - "क", - "र", - "स", - "न", - "त", - "म", - "ह", - "प", - "य", - "ल", - "व", - "ज", - "द", - "ग", - "ब", - "श", - "ट", - "अ", - "ए", - "थ", - "भ", - "ड", - "च", - "ध", - "ष", - "इ", - ], - "Estonian": [ - "a", - "i", - "e", - "s", - "t", - "l", - "u", - "n", - "o", - "k", - "r", - "d", - "m", - "v", - "g", - "p", - "j", - "h", - "ä", - "b", - "õ", - "ü", - "f", - "c", - "ö", - "y", - ], - "Thai": [ - "า", - "น", - "ร", - "อ", - "ก", - "เ", - "ง", - "ม", - "ย", - "ล", - "ว", - "ด", - "ท", - "ส", - "ต", - "ะ", - "ป", - "บ", - "ค", - "ห", - "แ", - "จ", - "พ", - "ช", - "ข", - "ใ", - ], - "Greek": [ - "α", - "τ", - "ο", - "ι", - "ε", - "ν", - "ρ", - "σ", - "κ", - "η", - "π", - "ς", - "υ", - "μ", - "λ", - "ί", - "ό", - "ά", - "γ", - "έ", - "δ", - "ή", - "ω", - "χ", - "θ", - "ύ", - ], - "Tamil": [ - "க", - "த", - "ப", - "ட", - "ர", - "ம", - "ல", - "ன", - "வ", - "ற", - "ய", - "ள", - "ச", - "ந", - "இ", - "ண", - "அ", - "ஆ", - "ழ", - "ங", - "எ", - "உ", - "ஒ", - "ஸ", - ], - "Kazakh": [ - "а", - "ы", - "е", - "н", - "т", - "р", - "л", - "і", - "д", - "с", - "м", - "қ", - "к", - "о", - "б", - "и", - "у", - "ғ", - "ж", - "ң", - "з", - "ш", - "й", - "п", - "г", - "ө", - ], -} - -LANGUAGE_SUPPORTED_COUNT: int = len(FREQUENCIES) - -# Bit flags for unified character classification. -# A single unicodedata.name() call sets all relevant flags at once. -_LATIN: int = 1 -_ACCENTUATED: int = 1 << 1 -_CJK: int = 1 << 2 -_HANGUL: int = 1 << 3 -_KATAKANA: int = 1 << 4 -_HIRAGANA: int = 1 << 5 -_THAI: int = 1 << 6 -_ARABIC: int = 1 << 7 -_ARABIC_ISOLATED_FORM: int = 1 << 8 - -_ACCENT_KEYWORDS: tuple[str, ...] = ( - "WITH GRAVE", - "WITH ACUTE", - "WITH CEDILLA", - "WITH DIAERESIS", - "WITH CIRCUMFLEX", - "WITH TILDE", - "WITH MACRON", - "WITH RING ABOVE", -) - -# Pre-built lookup structures for FREQUENCIES (computed once at import time). -# character -> rank mapping per language (replaces list .index() calls). -_FREQUENCIES_RANK: dict[str, dict[str, int]] = { - lang: {char: rank for rank, char in enumerate(chars)} - for lang, chars in FREQUENCIES.items() -} - -# frozenset per language (avoids rebuilding set() per call). -_FREQUENCIES_SET: dict[str, frozenset[str]] = { - lang: frozenset(chars) for lang, chars in FREQUENCIES.items() -} - -# prebuilt list of secondary range names. -_SECONDARY_RANGE_NAMES: frozenset[str] = frozenset( - range_name - for range_name in UNICODE_RANGES_COMBINED - if any(keyword in range_name for keyword in UNICODE_SECONDARY_RANGE_KEYWORD) -) diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/legacy.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/legacy.py deleted file mode 100644 index 3af232cfd2cfba3f50d40f440572ed2ce5215224..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/legacy.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Any -from warnings import warn - -from .api import from_bytes -from .constant import CHARDET_CORRESPONDENCE, TOO_SMALL_SEQUENCE - -if TYPE_CHECKING: - from typing import TypedDict - - class ResultDict(TypedDict): - encoding: str | None - language: str - confidence: float | None - - -def detect( - byte_str: bytes, should_rename_legacy: bool = False, **kwargs: Any -) -> ResultDict: - """ - chardet legacy method - Detect the encoding of the given byte string. It should be mostly backward-compatible. - Encoding name will match Chardet own writing whenever possible. (Not on encoding name unsupported by it) - This function is deprecated and should be used to migrate your project easily, consult the documentation for - further information. Not planned for removal. - - :param byte_str: The byte sequence to examine. - :param should_rename_legacy: Should we rename legacy encodings - to their more modern equivalents? - """ - if len(kwargs): - warn( - f"charset-normalizer disregard arguments '{','.join(list(kwargs.keys()))}' in legacy function detect()" - ) - - if not isinstance(byte_str, (bytearray, bytes)): - raise TypeError( # pragma: nocover - f"Expected object of type bytes or bytearray, got: {type(byte_str)}" - ) - - if isinstance(byte_str, bytearray): - byte_str = bytes(byte_str) - - r = from_bytes(byte_str).best() - - encoding = r.encoding if r is not None else None - language = r.language if r is not None and r.language != "Unknown" else "" - confidence = 1.0 - r.chaos if r is not None else None - - # automatically lower confidence - # on small bytes samples. - # https://github.com/jawah/charset_normalizer/issues/391 - if ( - confidence is not None - and confidence >= 0.9 - and encoding - not in { - "utf_8", - "ascii", - } - and not r.bom # type: ignore[union-attr] - and len(byte_str) < TOO_SMALL_SEQUENCE - ): - confidence -= 0.2 - - # Note: CharsetNormalizer does not return 'UTF-8-SIG' as the sig get stripped in the detection/normalization process - # but chardet does return 'utf-8-sig' and it is a valid codec name. - if r is not None and encoding == "utf_8" and r.bom: - encoding += "_sig" - - if not should_rename_legacy and encoding in CHARDET_CORRESPONDENCE: - encoding = CHARDET_CORRESPONDENCE[encoding] - - return { - "encoding": encoding, - "language": language, - "confidence": confidence, - } diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/md.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/charset_normalizer/md.cp310-win_amd64.pyd deleted file mode 100644 index 97f093cedd64ad0f749c93f4b219c22b7ab95246..0000000000000000000000000000000000000000 Binary files a/bundle/python-cpu/Lib/site-packages/charset_normalizer/md.cp310-win_amd64.pyd and /dev/null differ diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/md.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/md.py deleted file mode 100644 index 873402a3b82b313aa91690e4041542ba819f88e9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/md.py +++ /dev/null @@ -1,1002 +0,0 @@ -from __future__ import annotations - -import sys -from functools import lru_cache -from logging import getLogger - -if sys.version_info >= (3, 8): - from typing import final -else: - try: - from typing_extensions import final - except ImportError: - - def final(cls): # type: ignore[misc,no-untyped-def] - return cls - - -from .constant import ( - COMMON_CJK_CHARACTERS, - COMMON_SAFE_ASCII_CHARACTERS, - TRACE, - UNICODE_SECONDARY_RANGE_KEYWORD, - _ACCENTUATED, - _ARABIC, - _ARABIC_ISOLATED_FORM, - _CJK, - _HANGUL, - _HIRAGANA, - _KATAKANA, - _LATIN, - _THAI, -) -from .utils import ( - _character_flags, - is_emoticon, - is_punctuation, - is_separator, - is_symbol, - remove_accent, - unicode_range, -) - -# Combined bitmask for CJK/Hangul/Katakana/Hiragana/Thai glyph detection. -_GLYPH_MASK: int = _CJK | _HANGUL | _KATAKANA | _HIRAGANA | _THAI - - -@final -class CharInfo: - """Pre-computed character properties shared across all detectors.""" - - __slots__ = ( - "character", - "printable", - "alpha", - "upper", - "lower", - "space", - "digit", - "is_ascii", - "case_variable", - "flags", - "accentuated", - "latin", - "is_cjk", - "is_arabic", - "is_glyph", - "punct", - "sym", - "range", - "sep", - "emoticon", - "safe", - "common_cjk", - ) - - character: str - printable: bool - alpha: bool - upper: bool - lower: bool - space: bool - digit: bool - is_ascii: bool - case_variable: bool - flags: int - accentuated: bool - latin: bool - is_cjk: bool - is_arabic: bool - is_glyph: bool - punct: bool - sym: bool - range: str | None - sep: bool - emoticon: bool - safe: bool - common_cjk: bool - - def __init__(self, character: str) -> None: - """Compute all properties for *character* (built once per codepoint, - every branch assigns every slot).""" - self.character = character - - # ASCII fast-path: for characters with ord < 128, we can skip - # _character_flags() entirely and derive most properties from ord. - o: int = ord(character) - if o < 128: - self.is_ascii = True - self.accentuated = False - self.is_cjk = False - self.is_arabic = False - self.is_glyph = False - # ASCII alpha: a-z (97-122) or A-Z (65-90) - if 65 <= o <= 90: - # Uppercase ASCII letter - self.alpha = True - self.upper = True - self.lower = False - self.space = False - self.digit = False - self.printable = True - self.case_variable = True - self.flags = _LATIN - self.latin = True - self.punct = False - self.sym = False - elif 97 <= o <= 122: - # Lowercase ASCII letter - self.alpha = True - self.upper = False - self.lower = True - self.space = False - self.digit = False - self.printable = True - self.case_variable = True - self.flags = _LATIN - self.latin = True - self.punct = False - self.sym = False - elif 48 <= o <= 57: - # ASCII digit 0-9 - self.alpha = False - self.upper = False - self.lower = False - self.space = False - self.digit = True - self.printable = True - self.case_variable = False - self.flags = 0 - self.latin = False - self.punct = False - self.sym = False - elif o == 32 or (9 <= o <= 13): - # Space, tab, newline, etc. - self.alpha = False - self.upper = False - self.lower = False - self.space = True - self.digit = False - self.printable = o == 32 - self.case_variable = False - self.flags = 0 - self.latin = False - self.punct = False - self.sym = False - else: - # Other ASCII (punctuation, symbols, control chars) - self.printable = character.isprintable() - self.alpha = False - self.upper = False - self.lower = False - self.space = False - self.digit = False - self.case_variable = False - self.flags = 0 - self.latin = False - self.punct = is_punctuation(character) if self.printable else False - self.sym = is_symbol(character) if self.printable else False - else: - # Non-ASCII path - self.is_ascii = False - self.printable = character.isprintable() - self.alpha = character.isalpha() - self.upper = character.isupper() - self.lower = character.islower() - self.space = character.isspace() - self.digit = character.isdigit() - self.case_variable = self.lower != self.upper - - # Flag-based classification (single unicodedata.name() call, lru-cached) - flags: int - if self.alpha: - flags = _character_flags(character) - else: - flags = 0 - self.flags = flags - self.accentuated = bool(flags & _ACCENTUATED) - self.latin = bool(flags & _LATIN) - self.is_cjk = bool(flags & _CJK) - self.is_arabic = bool(flags & _ARABIC) - self.is_glyph = bool(flags & _GLYPH_MASK) - - # Eagerly compute punct and sym (avoids property dispatch overhead - # on 300K+ accesses in the hot loop). - self.punct = is_punctuation(character) if self.printable else False - self.sym = is_symbol(character) if self.printable else False - - self.range = unicode_range(character) - self.sep = is_separator(character) - self.emoticon = is_emoticon(character) - self.safe = character in COMMON_SAFE_ASCII_CHARACTERS - self.common_cjk = character in COMMON_CJK_CHARACTERS - - -# Per-codepoint cache of CharInfo instances -# At most UTF-8 size allocated. -@lru_cache(maxsize=None) -def _char_info(character: str) -> CharInfo: - """Build (once per codepoint) and cache the CharInfo for *character*.""" - return CharInfo(character) - - -# ASCII table indexed by codepoint. -_ASCII_CHAR_INFO: list[CharInfo] = [ - CharInfo(chr(_codepoint)) for _codepoint in range(128) -] - - -class MessDetectorPlugin: - """ - Base abstract class used for mess detection plugins. - All detectors MUST extend and implement given methods. - """ - - __slots__ = () - - def feed_info(self, character: str, info: CharInfo) -> None: - """ - The main routine to be executed upon character. - Insert the logic in witch the text would be considered chaotic. - """ - raise NotImplementedError # Defensive: - - def reset(self) -> None: # Defensive: - """ - Permit to reset the plugin to the initial state. - """ - raise NotImplementedError - - @property - def ratio(self) -> float: - """ - Compute the chaos ratio based on what your feed() has seen. - Must NOT be lower than 0.; No restriction gt 0. - """ - raise NotImplementedError # Defensive: - - -@final -class TooManySymbolOrPunctuationPlugin(MessDetectorPlugin): - __slots__ = ( - "_punctuation_count", - "_symbol_count", - "_character_count", - "_last_printable_char", - "_frenzy_symbol_in_word", - ) - - def __init__(self) -> None: - self._punctuation_count: int = 0 - self._symbol_count: int = 0 - self._character_count: int = 0 - - self._last_printable_char: str | None = None - self._frenzy_symbol_in_word: bool = False - - def feed_info(self, character: str, info: CharInfo) -> None: - """Optimized feed using pre-computed character info.""" - self._character_count += 1 - - if character != self._last_printable_char and not info.safe: - if info.punct: - self._punctuation_count += 1 - elif not info.digit and info.sym and not info.emoticon: - self._symbol_count += 2 - - self._last_printable_char = character - - def reset(self) -> None: # Abstract - self._punctuation_count = 0 - self._character_count = 0 - self._symbol_count = 0 - - @property - def ratio(self) -> float: - if self._character_count == 0: - return 0.0 - - ratio_of_punctuation: float = ( - self._punctuation_count + self._symbol_count - ) / self._character_count - - return ratio_of_punctuation if ratio_of_punctuation >= 0.3 else 0.0 - - -@final -class TooManyAccentuatedPlugin(MessDetectorPlugin): - __slots__ = ("_character_count", "_accentuated_count") - - def __init__(self) -> None: - self._character_count: int = 0 - self._accentuated_count: int = 0 - - def feed_info(self, character: str, info: CharInfo) -> None: - """Optimized feed using pre-computed character info.""" - self._character_count += 1 - - if info.accentuated: - self._accentuated_count += 1 - - def reset(self) -> None: # Abstract - self._character_count = 0 - self._accentuated_count = 0 - - @property - def ratio(self) -> float: - if self._character_count < 8: - return 0.0 - - ratio_of_accentuation: float = self._accentuated_count / self._character_count - return ratio_of_accentuation if ratio_of_accentuation >= 0.35 else 0.0 - - -@final -class UnprintablePlugin(MessDetectorPlugin): - __slots__ = ("_unprintable_count", "_character_count") - - def __init__(self) -> None: - self._unprintable_count: int = 0 - self._character_count: int = 0 - - def feed_info(self, character: str, info: CharInfo) -> None: - """Optimized feed using pre-computed character info.""" - if ( - not info.space - and not info.printable - and character != "\x1a" - and character != "\ufeff" - ): - self._unprintable_count += 1 - self._character_count += 1 - - def reset(self) -> None: # Abstract - self._unprintable_count = 0 - - @property - def ratio(self) -> float: - if self._character_count == 0: # Defensive: - return 0.0 - - return (self._unprintable_count * 8) / self._character_count - - -@final -class SuspiciousDuplicateAccentPlugin(MessDetectorPlugin): - __slots__ = ( - "_successive_count", - "_character_count", - "_last_latin_character", - "_last_was_accentuated", - ) - - def __init__(self) -> None: - self._successive_count: int = 0 - self._character_count: int = 0 - - self._last_latin_character: str | None = None - self._last_was_accentuated: bool = False - - def feed_info(self, character: str, info: CharInfo) -> None: - """Optimized feed using pre-computed character info.""" - self._character_count += 1 - if ( - self._last_latin_character is not None - and info.accentuated - and self._last_was_accentuated - ): - if info.upper and self._last_latin_character.isupper(): - self._successive_count += 1 - if remove_accent(character) == remove_accent(self._last_latin_character): - self._successive_count += 1 - self._last_latin_character = character - self._last_was_accentuated = info.accentuated - - def reset(self) -> None: # Abstract - self._successive_count = 0 - self._character_count = 0 - self._last_latin_character = None - self._last_was_accentuated = False - - @property - def ratio(self) -> float: - if self._character_count == 0: - return 0.0 - - return (self._successive_count * 2) / self._character_count - - -@final -class SuspiciousRange(MessDetectorPlugin): - __slots__ = ( - "_suspicious_successive_range_count", - "_character_count", - "_last_printable_seen", - "_last_printable_range", - ) - - def __init__(self) -> None: - self._suspicious_successive_range_count: int = 0 - self._character_count: int = 0 - self._last_printable_seen: str | None = None - self._last_printable_range: str | None = None - - def feed_info(self, character: str, info: CharInfo) -> None: - """Optimized feed using pre-computed character info.""" - self._character_count += 1 - - if info.space or info.punct or info.safe: - self._last_printable_seen = None - self._last_printable_range = None - return - - if self._last_printable_seen is None: - self._last_printable_seen = character - self._last_printable_range = info.range - return - - unicode_range_a: str | None = self._last_printable_range - unicode_range_b: str | None = info.range - - # Identical non-None ranges can never be suspicious. - if unicode_range_a != unicode_range_b or unicode_range_a is None: - if is_suspiciously_successive_range(unicode_range_a, unicode_range_b): - self._suspicious_successive_range_count += 1 - - self._last_printable_seen = character - self._last_printable_range = unicode_range_b - - def reset(self) -> None: # Abstract - self._character_count = 0 - self._suspicious_successive_range_count = 0 - self._last_printable_seen = None - self._last_printable_range = None - - @property - def ratio(self) -> float: - if self._character_count <= 13: - return 0.0 - - ratio_of_suspicious_range_usage: float = ( - self._suspicious_successive_range_count * 2 - ) / self._character_count - - return ratio_of_suspicious_range_usage - - -@final -class SuperWeirdWordPlugin(MessDetectorPlugin): - __slots__ = ( - "_word_count", - "_bad_word_count", - "_foreign_long_count", - "_is_current_word_bad", - "_foreign_long_watch", - "_character_count", - "_bad_character_count", - "_buffer_length", - "_buffer_last_char", - "_buffer_last_char_accentuated", - "_buffer_accent_count", - "_buffer_glyph_count", - "_buffer_upper_count", - "_buffer_first_lower", - "_buffer_has_non_ascii", - ) - - def __init__(self) -> None: - self._word_count: int = 0 - self._bad_word_count: int = 0 - self._foreign_long_count: int = 0 - - self._is_current_word_bad: bool = False - self._foreign_long_watch: bool = False - - self._character_count: int = 0 - self._bad_character_count: int = 0 - - self._buffer_length: int = 0 - self._buffer_last_char: str | None = None - self._buffer_last_char_accentuated: bool = False - self._buffer_accent_count: int = 0 - self._buffer_glyph_count: int = 0 - self._buffer_upper_count: int = 0 - self._buffer_first_lower: bool = False - self._buffer_has_non_ascii: bool = False - - def feed_info(self, character: str, info: CharInfo) -> None: - """Optimized feed using pre-computed character info.""" - if info.alpha: - if self._buffer_length == 0: - self._buffer_first_lower = info.lower - self._buffer_length += 1 - self._buffer_last_char = character - - if info.upper: - self._buffer_upper_count += 1 - if not info.is_ascii: - self._buffer_has_non_ascii = True - - self._buffer_last_char_accentuated = info.accentuated - - if info.accentuated: - self._buffer_accent_count += 1 - if ( - not self._foreign_long_watch - and (not info.latin or info.accentuated) - and not info.is_glyph - ): - self._foreign_long_watch = True - if info.is_glyph: - self._buffer_glyph_count += 1 - return - if not self._buffer_length: - return - if info.space or info.punct or info.sep: - self._word_count += 1 - buffer_length: int = self._buffer_length - - self._character_count += buffer_length - - if buffer_length >= 4: - if self._buffer_accent_count / buffer_length >= 0.5: - self._is_current_word_bad = True - elif ( - self._buffer_last_char_accentuated - and self._buffer_last_char.isupper() # type: ignore[union-attr] - and self._buffer_upper_count != buffer_length - ): - self._foreign_long_count += 1 - self._is_current_word_bad = True - elif self._buffer_glyph_count == 1: - self._is_current_word_bad = True - self._foreign_long_count += 1 - elif ( - self._buffer_has_non_ascii - and self._buffer_first_lower - and self._buffer_upper_count == buffer_length - 1 - ): - # Inverse capitalization detector. - # No natural writing produces such words. - # see https://github.com/jawah/charset_normalizer/issues/731 - self._foreign_long_count += 1 - self._is_current_word_bad = True - if buffer_length >= 24 and self._foreign_long_watch: - probable_camel_cased: bool = ( - self._buffer_upper_count > 0 - and self._buffer_upper_count / buffer_length <= 0.3 - ) - - if not probable_camel_cased: - self._foreign_long_count += 1 - self._is_current_word_bad = True - - if self._is_current_word_bad: - self._bad_word_count += 1 - self._bad_character_count += buffer_length - self._is_current_word_bad = False - - self._foreign_long_watch = False - self._buffer_length = 0 - self._buffer_last_char = None - self._buffer_last_char_accentuated = False - self._buffer_accent_count = 0 - self._buffer_glyph_count = 0 - self._buffer_upper_count = 0 - self._buffer_first_lower = False - self._buffer_has_non_ascii = False - elif ( - character not in {"<", ">", "-", "=", "~", "|", "_"} - and not info.digit - and info.sym - ): - self._is_current_word_bad = True - self._buffer_length += 1 - self._buffer_last_char = character - self._buffer_last_char_accentuated = False - - def reset(self) -> None: # Abstract - self._buffer_length = 0 - self._buffer_last_char = None - self._buffer_last_char_accentuated = False - self._is_current_word_bad = False - self._foreign_long_watch = False - self._bad_word_count = 0 - self._word_count = 0 - self._character_count = 0 - self._bad_character_count = 0 - self._foreign_long_count = 0 - self._buffer_accent_count = 0 - self._buffer_glyph_count = 0 - self._buffer_upper_count = 0 - self._buffer_first_lower = False - self._buffer_has_non_ascii = False - - @property - def ratio(self) -> float: - if self._word_count <= 10 and self._foreign_long_count == 0: - return 0.0 - - return self._bad_character_count / self._character_count - - -@final -class CjkUncommonPlugin(MessDetectorPlugin): - """ - Detect messy CJK text that probably means nothing. - """ - - __slots__ = ("_character_count", "_uncommon_count") - - def __init__(self) -> None: - self._character_count: int = 0 - self._uncommon_count: int = 0 - - def feed_info(self, character: str, info: CharInfo) -> None: - """Optimized feed using pre-computed character info.""" - self._character_count += 1 - - if not info.common_cjk: - self._uncommon_count += 1 - - def reset(self) -> None: # Abstract - self._character_count = 0 - self._uncommon_count = 0 - - @property - def ratio(self) -> float: - if self._character_count < 8: - return 0.0 - - uncommon_form_usage: float = self._uncommon_count / self._character_count - - # we can be pretty sure it's garbage when uncommon characters are widely - # used. otherwise it could just be traditional chinese for example. - return uncommon_form_usage / 10 if uncommon_form_usage > 0.5 else 0.0 - - -@final -class ArchaicUpperLowerPlugin(MessDetectorPlugin): - __slots__ = ( - "_buf", - "_character_count_since_last_sep", - "_successive_upper_lower_count", - "_successive_upper_lower_count_final", - "_character_count", - "_last_alpha_seen", - "_last_alpha_seen_upper", - "_last_alpha_seen_lower", - "_current_ascii_only", - ) - - def __init__(self) -> None: - self._buf: bool = False - - self._character_count_since_last_sep: int = 0 - - self._successive_upper_lower_count: int = 0 - self._successive_upper_lower_count_final: int = 0 - - self._character_count: int = 0 - - self._last_alpha_seen: str | None = None - self._last_alpha_seen_upper: bool = False - self._last_alpha_seen_lower: bool = False - self._current_ascii_only: bool = True - - def feed_info(self, character: str, info: CharInfo) -> None: - """Optimized feed using pre-computed character info.""" - is_concerned: bool = info.alpha and info.case_variable - chunk_sep: bool = not is_concerned - - if chunk_sep and self._character_count_since_last_sep > 0: - if ( - self._character_count_since_last_sep <= 64 - and not info.digit - and not self._current_ascii_only - ): - self._successive_upper_lower_count_final += ( - self._successive_upper_lower_count - ) - - self._successive_upper_lower_count = 0 - self._character_count_since_last_sep = 0 - self._last_alpha_seen = None - self._buf = False - self._character_count += 1 - self._current_ascii_only = True - - return - - if self._current_ascii_only and not info.is_ascii: - self._current_ascii_only = False - - if self._last_alpha_seen is not None: - if (info.upper and self._last_alpha_seen_lower) or ( - info.lower and self._last_alpha_seen_upper - ): - if self._buf: - self._successive_upper_lower_count += 2 - self._buf = False - else: - self._buf = True - else: - self._buf = False - - self._character_count += 1 - self._character_count_since_last_sep += 1 - self._last_alpha_seen = character - self._last_alpha_seen_upper = info.upper - self._last_alpha_seen_lower = info.lower - - def reset(self) -> None: # Abstract - self._character_count = 0 - self._character_count_since_last_sep = 0 - self._successive_upper_lower_count = 0 - self._successive_upper_lower_count_final = 0 - self._last_alpha_seen = None - self._last_alpha_seen_upper = False - self._last_alpha_seen_lower = False - self._buf = False - self._current_ascii_only = True - - @property - def ratio(self) -> float: - if self._character_count == 0: # Defensive: - return 0.0 - - return self._successive_upper_lower_count_final / self._character_count - - -@final -class ArabicIsolatedFormPlugin(MessDetectorPlugin): - __slots__ = ("_character_count", "_isolated_form_count") - - def __init__(self) -> None: - self._character_count: int = 0 - self._isolated_form_count: int = 0 - - def reset(self) -> None: # Abstract - self._character_count = 0 - self._isolated_form_count = 0 - - def feed_info(self, character: str, info: CharInfo) -> None: - """Optimized feed using pre-computed character info.""" - self._character_count += 1 - - if info.flags & _ARABIC_ISOLATED_FORM: - self._isolated_form_count += 1 - - @property - def ratio(self) -> float: - if self._character_count < 8: - return 0.0 - - isolated_form_usage: float = self._isolated_form_count / self._character_count - - return isolated_form_usage - - -@lru_cache(maxsize=1024) -def is_suspiciously_successive_range( - unicode_range_a: str | None, unicode_range_b: str | None -) -> bool: - """ - Determine if two Unicode range seen next to each other can be considered as suspicious. - """ - if unicode_range_a is None or unicode_range_b is None: - return True - - if unicode_range_a == unicode_range_b: - return False - - if "Latin" in unicode_range_a and "Latin" in unicode_range_b: - return False - - if "Emoticons" in unicode_range_a or "Emoticons" in unicode_range_b: - return False - - # Latin characters can be accompanied with a combining diacritical mark - # eg. Vietnamese. - if ("Latin" in unicode_range_a or "Latin" in unicode_range_b) and ( - "Combining" in unicode_range_a or "Combining" in unicode_range_b - ): - return False - - keywords_range_a, keywords_range_b = ( - unicode_range_a.split(" "), - unicode_range_b.split(" "), - ) - - for el in keywords_range_a: - if el in UNICODE_SECONDARY_RANGE_KEYWORD: - continue - if el in keywords_range_b: - return False - - # Japanese Exception - range_a_jp_chars, range_b_jp_chars = ( - unicode_range_a - in ( - "Hiragana", - "Katakana", - ), - unicode_range_b in ("Hiragana", "Katakana"), - ) - if (range_a_jp_chars or range_b_jp_chars) and ( - "CJK" in unicode_range_a or "CJK" in unicode_range_b - ): - return False - if range_a_jp_chars and range_b_jp_chars: - return False - - if "Hangul" in unicode_range_a or "Hangul" in unicode_range_b: - if "CJK" in unicode_range_a or "CJK" in unicode_range_b: - return False - if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin": - return False - - # Chinese/Japanese use dedicated range for punctuation and/or separators. - if ("CJK" in unicode_range_a or "CJK" in unicode_range_b) or ( - unicode_range_a in ["Katakana", "Hiragana"] - and unicode_range_b in ["Katakana", "Hiragana"] - ): - if "Punctuation" in unicode_range_a or "Punctuation" in unicode_range_b: - return False - if "Forms" in unicode_range_a or "Forms" in unicode_range_b: - return False - if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin": - return False - - return True - - -def mess_ratio( - decoded_sequence: str, maximum_threshold: float = 0.2, debug: bool = False -) -> float: - """ - Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier. - """ - - seq_len: int = len(decoded_sequence) - - if seq_len < 511: - step: int = 32 - elif seq_len < 1024: - step = 64 - else: - step = 128 - - # str.isascii() is O(1) (the flag lives in the str header). Six of the - # nine detectors provably keep a 0.0 ratio on ASCII-only input and are - # therefore not fed at all. - is_pure_ascii: bool = decoded_sequence.isascii() - - # Cached per-codepoint character properties (see CharInfo). ASCII - # characters resolve through the immutable import-time table; anything - # else goes through the lru_cache-backed slow path. - ascii_info = _ASCII_CHAR_INFO - char_info = _char_info - - mean_mess_ratio: float - info: CharInfo - - # Create each detector as a named local variable (unrolled from the generic loop). - # This eliminates per-character iteration over the detector list and - # per-character eligible() virtual dispatch, while keeping every plugin class - # intact and fully readable. - d_sp: TooManySymbolOrPunctuationPlugin = TooManySymbolOrPunctuationPlugin() - d_ta: TooManyAccentuatedPlugin = TooManyAccentuatedPlugin() - d_up: UnprintablePlugin = UnprintablePlugin() - d_sda: SuspiciousDuplicateAccentPlugin = SuspiciousDuplicateAccentPlugin() - d_sr: SuspiciousRange = SuspiciousRange() - d_sw: SuperWeirdWordPlugin = SuperWeirdWordPlugin() - d_cu: CjkUncommonPlugin = CjkUncommonPlugin() - d_au: ArchaicUpperLowerPlugin = ArchaicUpperLowerPlugin() - d_ai: ArabicIsolatedFormPlugin = ArabicIsolatedFormPlugin() - - # Local references for feed_info methods called in the hot loop. - d_sp_feed = d_sp.feed_info - d_ta_feed = d_ta.feed_info - d_up_feed = d_up.feed_info - d_sda_feed = d_sda.feed_info - d_sr_feed = d_sr.feed_info - d_sw_feed = d_sw.feed_info - d_cu_feed = d_cu.feed_info - d_au_feed = d_au.feed_info - d_ai_feed = d_ai.feed_info - - for block_start in range(0, seq_len, step): - for character in decoded_sequence[block_start : block_start + step]: - # Character properties computed once per distinct codepoint - # (shared across all plugins and all mess_ratio calls). - # ord() doubles as the ASCII table index and, unlike - # str.isascii(), lowers to a mypyc primitive. - codepoint: int = ord(character) - if codepoint < 128: - info = ascii_info[codepoint] - else: - info = char_info(character) - - # Detectors with eligible() == always True - d_up_feed(character, info) - d_sw_feed(character, info) - - if is_pure_ascii: - # The six remaining detectors provably stay at 0.0 (see above). - if info.printable: - d_sp_feed(character, info) - continue - - d_au_feed(character, info) - - # Detectors with eligible() == isprintable - if info.printable: - d_sp_feed(character, info) - d_sr_feed(character, info) - - # Detectors with eligible() == isalpha - if info.alpha: - d_ta_feed(character, info) - # SuspiciousDuplicateAccent: isalpha() and is_latin() - if info.latin: - d_sda_feed(character, info) - # CjkUncommon: is_cjk() - if info.is_cjk: - d_cu_feed(character, info) - # ArabicIsolatedForm: is_arabic() - if info.is_arabic: - d_ai_feed(character, info) - - mean_mess_ratio = ( - d_sp.ratio - + d_ta.ratio - + d_up.ratio - + d_sda.ratio - + d_sr.ratio - + d_sw.ratio - + d_cu.ratio - + d_au.ratio - + d_ai.ratio - ) - - if mean_mess_ratio >= maximum_threshold: - break - else: - # Flush last word buffer in SuperWeirdWordPlugin via trailing newline. - nl_info = ascii_info[10] # "\n" - d_sw_feed("\n", nl_info) - if not is_pure_ascii: - d_au_feed("\n", nl_info) - d_up_feed("\n", nl_info) - - mean_mess_ratio = ( - d_sp.ratio - + d_ta.ratio - + d_up.ratio - + d_sda.ratio - + d_sr.ratio - + d_sw.ratio - + d_cu.ratio - + d_au.ratio - + d_ai.ratio - ) - - if debug: # Defensive: - logger = getLogger("charset_normalizer") - - logger.log( - TRACE, - "Mess-detector extended-analysis start. " - f"intermediary_mean_mess_ratio_calc={step} mean_mess_ratio={mean_mess_ratio} " - f"maximum_threshold={maximum_threshold}", - ) - - if seq_len > 16: - logger.log(TRACE, f"Starting with: {decoded_sequence[:16]}") - logger.log(TRACE, f"Ending with: {decoded_sequence[-16::]}") - - for dt in [d_sp, d_ta, d_up, d_sda, d_sr, d_sw, d_cu, d_au, d_ai]: - logger.log(TRACE, f"{dt.__class__}: {dt.ratio}") - - return round(mean_mess_ratio, 3) diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/models.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/models.py deleted file mode 100644 index 3ac4853431a7d15456b41f816c7b1e6e6c1e55e5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/models.py +++ /dev/null @@ -1,370 +0,0 @@ -from __future__ import annotations - -from encodings.aliases import aliases -from re import sub -from typing import Any, Iterator, List, Tuple - -from .constant import RE_POSSIBLE_ENCODING_INDICATION, TOO_BIG_SEQUENCE -from .utils import iana_name, is_multi_byte_encoding, unicode_range - - -class CharsetMatch: - def __init__( - self, - payload: bytes | bytearray, - guessed_encoding: str, - mean_mess_ratio: float, - has_sig_or_bom: bool, - languages: CoherenceMatches, - decoded_payload: str | None = None, - preemptive_declaration: str | None = None, - ): - self._payload: bytes | bytearray = payload - - self._encoding: str = guessed_encoding - self._mean_mess_ratio: float = mean_mess_ratio - self._languages: CoherenceMatches = languages - self._has_sig_or_bom: bool = has_sig_or_bom - self._unicode_ranges: list[str] | None = None - - self._leaves: list[CharsetMatch] = [] - self._mean_coherence_ratio: float = 0.0 - - self._output_payload: bytes | None = None - self._output_encoding: str | None = None - - self._string: str | None = decoded_payload - - self._preemptive_declaration: str | None = preemptive_declaration - - def __eq__(self, other: object) -> bool: - if not isinstance(other, CharsetMatch): - if isinstance(other, str): - return iana_name(other) == self.encoding - return False - return self.encoding == other.encoding and self.fingerprint == other.fingerprint - - def __lt__(self, other: object) -> bool: - """ - Implemented to make sorted available upon CharsetMatches items. - """ - if not isinstance(other, CharsetMatch): - raise ValueError - - chaos_difference: float = abs(self.chaos - other.chaos) - coherence_difference: float = abs(self.coherence - other.coherence) - - # Below 0.5% difference --> Use Coherence - if chaos_difference < 0.005 and coherence_difference > 0.02: - return self.coherence > other.coherence - elif chaos_difference < 0.005 and coherence_difference <= 0.02: - # When having a difficult decision, use the result that decoded as many multi-byte as possible. - # preserve RAM usage! - if len(self._payload) >= TOO_BIG_SEQUENCE: - return self.chaos < other.chaos - return self.multi_byte_usage > other.multi_byte_usage - - return self.chaos < other.chaos - - @property - def multi_byte_usage(self) -> float: - return 1.0 - (len(str(self)) / len(self.raw)) - - def __str__(self) -> str: - # Lazy Str Loading - if self._string is None: - self._string = str(self._payload, self._encoding, "strict") - # UTF-7 BOM is encoded in modified Base64 whose byte boundary - # can overlap with the next character, so raw-byte stripping - # is unreliable. Strip the decoded BOM character instead. - if ( - self._has_sig_or_bom - and self._encoding == "utf_7" - and self._string - and self._string[0] == "\ufeff" - ): - self._string = self._string[1:] - return self._string - - def __repr__(self) -> str: - return f"" - - def add_submatch(self, other: CharsetMatch) -> None: - if not isinstance(other, CharsetMatch) or other == self: - raise ValueError( - "Unable to add instance <{}> as a submatch of a CharsetMatch".format( - other.__class__ - ) - ) - - other._string = None # Unload RAM usage; dirty trick. - self._leaves.append(other) - - @property - def encoding(self) -> str: - return self._encoding - - @property - def encoding_aliases(self) -> list[str]: - """ - Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855. - """ - also_known_as: list[str] = [] - for u, p in aliases.items(): - if self.encoding == u: - also_known_as.append(p) - elif self.encoding == p: - also_known_as.append(u) - return also_known_as - - @property - def bom(self) -> bool: - return self._has_sig_or_bom - - @property - def byte_order_mark(self) -> bool: - return self._has_sig_or_bom - - @property - def languages(self) -> list[str]: - """ - Return the complete list of possible languages found in decoded sequence. - Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'. - """ - return [e[0] for e in self._languages] - - @property - def language(self) -> str: - """ - Most probable language found in decoded sequence. If none were detected or inferred, the property will return - "Unknown". - """ - if not self._languages: - # Trying to infer the language based on the given encoding - # Its either English or we should not pronounce ourselves in certain cases. - if "ascii" in self.could_be_from_charset: - return "English" - - # doing it there to avoid circular import - from charset_normalizer.cd import encoding_languages, mb_encoding_languages - - languages = ( - mb_encoding_languages(self.encoding) - if is_multi_byte_encoding(self.encoding) - else encoding_languages(self.encoding) - ) - - if len(languages) == 0 or "Latin Based" in languages: - return "Unknown" - - return languages[0] - - return self._languages[0][0] - - @property - def chaos(self) -> float: - return self._mean_mess_ratio - - @property - def coherence(self) -> float: - if not self._languages: - return 0.0 - return self._languages[0][1] - - @property - def percent_chaos(self) -> float: - return round(self.chaos * 100, ndigits=3) - - @property - def percent_coherence(self) -> float: - return round(self.coherence * 100, ndigits=3) - - @property - def raw(self) -> bytes | bytearray: - """ - Original untouched bytes. - """ - return self._payload - - @property - def submatch(self) -> list[CharsetMatch]: - return self._leaves - - @property - def has_submatch(self) -> bool: - return len(self._leaves) > 0 - - @property - def alphabets(self) -> list[str]: - if self._unicode_ranges is not None: - return self._unicode_ranges - # list detected ranges - detected_ranges: list[str | None] = [unicode_range(char) for char in str(self)] - # filter and sort - self._unicode_ranges = sorted(list({r for r in detected_ranges if r})) - return self._unicode_ranges - - @property - def could_be_from_charset(self) -> list[str]: - """ - The complete list of encoding that output the exact SAME str result and therefore could be the originating - encoding. - This list does include the encoding available in property 'encoding'. - """ - return [self._encoding] + [m.encoding for m in self._leaves] - - def output(self, encoding: str = "utf_8") -> bytes: - """ - Method to get re-encoded bytes payload using given target encoding. Default to UTF-8. - Any errors will be simply ignored by the encoder NOT replaced. - """ - if self._output_encoding is None or self._output_encoding != encoding: - self._output_encoding = encoding - decoded_string = str(self) - if ( - self._preemptive_declaration is not None - and self._preemptive_declaration.lower() - not in ["utf-8", "utf8", "utf_8"] - ): - patched_header = sub( - RE_POSSIBLE_ENCODING_INDICATION, - lambda m: m.string[m.span()[0] : m.span()[1]].replace( - m.groups()[0], - iana_name(self._output_encoding).replace("_", "-"), # type: ignore[arg-type] - ), - decoded_string[:8192], - count=1, - ) - - decoded_string = patched_header + decoded_string[8192:] - - self._output_payload = decoded_string.encode(encoding, "replace") - - return self._output_payload # type: ignore - - @property - def fingerprint(self) -> int: - """ - Retrieve a hash fingerprint of the decoded payload, used for deduplication. - """ - return hash(str(self)) - - -class CharsetMatches: - """ - Container with every CharsetMatch items ordered by default from most probable to the less one. - Act like a list(iterable) but does not implements all related methods. - """ - - def __init__(self, results: list[CharsetMatch] | None = None): - self._results: list[CharsetMatch] = sorted(results) if results else [] - - def __iter__(self) -> Iterator[CharsetMatch]: - yield from self._results - - def __getitem__(self, item: int | str) -> CharsetMatch: - """ - Retrieve a single item either by its position or encoding name (alias may be used here). - Raise KeyError upon invalid index or encoding not present in results. - """ - if isinstance(item, int): - return self._results[item] - if isinstance(item, str): - item = iana_name(item, False) - for result in self._results: - if item in result.could_be_from_charset: - return result - raise KeyError - - def __len__(self) -> int: - return len(self._results) - - def __bool__(self) -> bool: - return len(self._results) > 0 - - def append(self, item: CharsetMatch) -> None: - """ - Insert a single match. Will be inserted accordingly to preserve sort. - Can be inserted as a submatch. - """ - if not isinstance(item, CharsetMatch): - raise ValueError( - "Cannot append instance '{}' to CharsetMatches".format( - str(item.__class__) - ) - ) - # We should disable the submatch factoring when the input file is too heavy (conserve RAM usage) - if len(item.raw) < TOO_BIG_SEQUENCE: - for match in self._results: - if match.fingerprint == item.fingerprint and match.chaos == item.chaos: - match.add_submatch(item) - return - self._results.append(item) - self._results = sorted(self._results) - - def best(self) -> CharsetMatch | None: - """ - Simply return the first match. Strict equivalent to matches[0]. - """ - if not self._results: - return None - return self._results[0] - - def first(self) -> CharsetMatch | None: - """ - Redundant method, call the method best(). Kept for BC reasons. - """ - return self.best() - - -CoherenceMatch = Tuple[str, float] -CoherenceMatches = List[CoherenceMatch] - - -class CliDetectionResult: - def __init__( - self, - path: str, - encoding: str | None, - encoding_aliases: list[str], - alternative_encodings: list[str], - language: str, - alphabets: list[str], - has_sig_or_bom: bool, - chaos: float, - coherence: float, - unicode_path: str | None, - is_preferred: bool, - ): - self.path: str = path - self.unicode_path: str | None = unicode_path - self.encoding: str | None = encoding - self.encoding_aliases: list[str] = encoding_aliases - self.alternative_encodings: list[str] = alternative_encodings - self.language: str = language - self.alphabets: list[str] = alphabets - self.has_sig_or_bom: bool = has_sig_or_bom - self.chaos: float = chaos - self.coherence: float = coherence - self.is_preferred: bool = is_preferred - - @property - def __dict__(self) -> dict[str, Any]: # type: ignore - return { - "path": self.path, - "encoding": self.encoding, - "encoding_aliases": self.encoding_aliases, - "alternative_encodings": self.alternative_encodings, - "language": self.language, - "alphabets": self.alphabets, - "has_sig_or_bom": self.has_sig_or_bom, - "chaos": self.chaos, - "coherence": self.coherence, - "unicode_path": self.unicode_path, - "is_preferred": self.is_preferred, - } - - def to_json(self) -> str: - from json import dumps - - return dumps(self.__dict__, ensure_ascii=True, indent=4) diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/py.typed b/bundle/python-cpu/Lib/site-packages/charset_normalizer/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/utils.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/utils.py deleted file mode 100644 index 1fcd480c3f440a3373daac50859fb55d96de804d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/utils.py +++ /dev/null @@ -1,455 +0,0 @@ -from __future__ import annotations - -import importlib -import logging -import unicodedata -from bisect import bisect_right -from codecs import IncrementalDecoder -from encodings.aliases import aliases -from functools import lru_cache -from re import findall -from typing import Generator - -from .constant import ( - ENCODING_MARKS, - IANA_SUPPORTED_SIMILAR, - RE_POSSIBLE_ENCODING_INDICATION, - UNICODE_RANGES_COMBINED, - _SECONDARY_RANGE_NAMES, - UTF8_MAXIMAL_ALLOCATION, - COMMON_CJK_CHARACTERS, - _LATIN, - _CJK, - _HANGUL, - _KATAKANA, - _HIRAGANA, - _THAI, - _ARABIC, - _ARABIC_ISOLATED_FORM, - _ACCENT_KEYWORDS, - _ACCENTUATED, -) - - -def _character_flags(character: str) -> int: - """Compute all name-based classification flags with a single unicodedata.name() call.""" - try: - desc: str = unicodedata.name(character) - except ValueError: - return 0 - - flags: int = 0 - - if "LATIN" in desc: - flags |= _LATIN - if "CJK" in desc: - flags |= _CJK - if "HANGUL" in desc: - flags |= _HANGUL - if "KATAKANA" in desc: - flags |= _KATAKANA - if "HIRAGANA" in desc: - flags |= _HIRAGANA - if "THAI" in desc: - flags |= _THAI - if "ARABIC" in desc: - flags |= _ARABIC - if "ISOLATED FORM" in desc: - flags |= _ARABIC_ISOLATED_FORM - - for kw in _ACCENT_KEYWORDS: - if kw in desc: - flags |= _ACCENTUATED - break - - return flags - - -def is_accentuated(character: str) -> bool: - return bool(_character_flags(character) & _ACCENTUATED) - - -@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) -def remove_accent(character: str) -> str: - decomposed: str = unicodedata.decomposition(character) - if not decomposed: - return character - - codes: list[str] = decomposed.split(" ") - - return chr(int(codes[0], 16)) - - -# Pre-built sorted lookup table for O(log n) binary search in unicode_range(). -# Each entry is (range_start, range_end_exclusive, range_name). -_UNICODE_RANGES_SORTED: list[tuple[int, int, str]] = sorted( - (ord_range.start, ord_range.stop, name) - for name, ord_range in UNICODE_RANGES_COMBINED.items() -) -_UNICODE_RANGE_STARTS: list[int] = [e[0] for e in _UNICODE_RANGES_SORTED] - - -def unicode_range(character: str) -> str | None: - """ - Retrieve the Unicode range official name from a single character. - """ - character_ord: int = ord(character) - - # Binary search: find the rightmost range whose start <= character_ord - idx = bisect_right(_UNICODE_RANGE_STARTS, character_ord) - 1 - if idx >= 0: - start, stop, name = _UNICODE_RANGES_SORTED[idx] - if character_ord < stop: - return name - - return None - - -def is_latin(character: str) -> bool: - return bool(_character_flags(character) & _LATIN) - - -def is_punctuation(character: str) -> bool: - character_category: str = unicodedata.category(character) - - if "P" in character_category: - return True - - character_range: str | None = unicode_range(character) - - if character_range is None: - return False - - return "Punctuation" in character_range - - -def is_symbol(character: str) -> bool: - character_category: str = unicodedata.category(character) - - if "S" in character_category or "N" in character_category: - return True - - character_range: str | None = unicode_range(character) - - if character_range is None: - return False - - return "Forms" in character_range and character_category != "Lo" - - -def is_emoticon(character: str) -> bool: - character_range: str | None = unicode_range(character) - - if character_range is None: - return False - - return "Emoticons" in character_range or "Pictographs" in character_range - - -def is_separator(character: str) -> bool: - if character.isspace() or character in {"|", "+", "<", ">"}: - return True - - character_category: str = unicodedata.category(character) - - return "Z" in character_category or character_category in {"Po", "Pd", "Pc"} - - -@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) -def is_case_variable(character: str) -> bool: - return character.islower() != character.isupper() - - -@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) -def is_cjk(character: str) -> bool: - return bool(_character_flags(character) & _CJK) - - -@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) -def is_hiragana(character: str) -> bool: - return bool(_character_flags(character) & _HIRAGANA) - - -@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) -def is_katakana(character: str) -> bool: - return bool(_character_flags(character) & _KATAKANA) - - -@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) -def is_hangul(character: str) -> bool: - return bool(_character_flags(character) & _HANGUL) - - -@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) -def is_thai(character: str) -> bool: - return bool(_character_flags(character) & _THAI) - - -@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) -def is_arabic(character: str) -> bool: - return bool(_character_flags(character) & _ARABIC) - - -@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) -def is_arabic_isolated_form(character: str) -> bool: - return bool(_character_flags(character) & _ARABIC_ISOLATED_FORM) - - -@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) -def is_cjk_uncommon(character: str) -> bool: - return character not in COMMON_CJK_CHARACTERS - - -def is_unicode_range_secondary(range_name: str) -> bool: - return range_name in _SECONDARY_RANGE_NAMES - - -@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) -def is_unprintable(character: str) -> bool: - return ( - not character.isspace() # includes \n \t \r \v - and not character.isprintable() - and character != "\x1a" # Why? Its the ASCII substitute character. - and character != "\ufeff" # bug discovered in Python, - # Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space. - ) - - -def any_specified_encoding( - sequence: bytes | bytearray, search_zone: int = 8192 -) -> str | None: - """ - Extract using ASCII-only decoder any specified encoding in the first n-bytes. - """ - if not isinstance(sequence, (bytes, bytearray)): - raise TypeError - - seq_len: int = len(sequence) - - decoded_zone: str = sequence[: min(seq_len, search_zone)].decode( - "ascii", errors="ignore" - ) - - # Cheap literal pre-filter. - lowered_zone: str = decoded_zone.lower() - if "coding" not in lowered_zone and "charset" not in lowered_zone: - return None - - results: list[str] = findall( - RE_POSSIBLE_ENCODING_INDICATION, - decoded_zone, - ) - - if len(results) == 0: - return None - - for specified_encoding in results: - specified_encoding = specified_encoding.lower().replace("-", "_") - - encoding_alias: str - encoding_iana: str - - for encoding_alias, encoding_iana in aliases.items(): - if encoding_alias == specified_encoding: - return encoding_iana - if encoding_iana == specified_encoding: - return encoding_iana - - return None - - -@lru_cache(maxsize=128) -def is_multi_byte_encoding(name: str) -> bool: - """ - Verify is a specific encoding is a multi byte one based on it IANA name - """ - if name in { - "utf_8", - "utf_8_sig", - "utf_16", - "utf_16_be", - "utf_16_le", - "utf_32", - "utf_32_le", - "utf_32_be", - "utf_7", - }: - return True - - # Besides the Unicode family above, every multibyte codec shipped with - # Python is implemented by _multibytecodec through exactly one of the six - # cjkcodecs providers below. Probing those providers directly (getcodec) - # classifies a name without importing its "encodings." module: - # classifying the whole IANA_SUPPORTED list would otherwise import many - # modules and dominate "import charset_normalizer" wall time. - # see https://github.com/jawah/charset_normalizer/issues/742 - for provider in ( - "_codecs_cn", - "_codecs_hk", - "_codecs_iso2022", - "_codecs_jp", - "_codecs_kr", - "_codecs_tw", - ): - try: - importlib.import_module(provider).getcodec(name) # type: ignore[attr-defined] - except (ImportError, AttributeError, LookupError): # Defensive: edge cases - continue - return True - - return False - - -def identify_sig_or_bom(sequence: bytes | bytearray) -> tuple[str | None, bytes]: - """ - Identify and extract SIG/BOM in given sequence. - """ - - for iana_encoding in ENCODING_MARKS: - marks: bytes | list[bytes] = ENCODING_MARKS[iana_encoding] - - if isinstance(marks, bytes): - marks = [marks] - - for mark in marks: - if sequence.startswith(mark): - return iana_encoding, mark - - return None, b"" - - -def should_strip_sig_or_bom(iana_encoding: str) -> bool: - return iana_encoding not in {"utf_16", "utf_32"} - - -def iana_name(cp_name: str, strict: bool = True) -> str: - """Returns the Python normalized encoding name (Not the IANA official name).""" - cp_name = cp_name.lower().replace("-", "_") - - encoding_alias: str - encoding_iana: str - - for encoding_alias, encoding_iana in aliases.items(): - if cp_name in [encoding_alias, encoding_iana]: - return encoding_iana - - if strict: - raise ValueError(f"Unable to retrieve IANA for '{cp_name}'") - - return cp_name - - -def cp_similarity(iana_name_a: str, iana_name_b: str) -> float: - if is_multi_byte_encoding(iana_name_a) or is_multi_byte_encoding(iana_name_b): - return 0.0 - - decoder_a = importlib.import_module(f"encodings.{iana_name_a}").IncrementalDecoder - decoder_b = importlib.import_module(f"encodings.{iana_name_b}").IncrementalDecoder - - id_a: IncrementalDecoder = decoder_a(errors="ignore") - id_b: IncrementalDecoder = decoder_b(errors="ignore") - - character_match_count: int = 0 - - for i in range(256): - to_be_decoded: bytes = bytes([i]) - if id_a.decode(to_be_decoded) == id_b.decode(to_be_decoded): - character_match_count += 1 - - return character_match_count / 256 - - -def is_cp_similar(iana_name_a: str, iana_name_b: str) -> bool: - """ - Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using - the function cp_similarity. - """ - return ( - iana_name_a in IANA_SUPPORTED_SIMILAR - and iana_name_b in IANA_SUPPORTED_SIMILAR[iana_name_a] - ) - - -def set_logging_handler( - name: str = "charset_normalizer", - level: int = logging.INFO, - format_string: str = "%(asctime)s | %(levelname)s | %(message)s", -) -> None: - logger = logging.getLogger(name) - logger.setLevel(level) - - handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter(format_string)) - logger.addHandler(handler) - - -def cut_sequence_chunks( - sequences: bytes | bytearray, - encoding_iana: str, - offsets: range, - chunk_size: int, - bom_or_sig_available: bool, - strip_sig_or_bom: bool, - sig_payload: bytes, - is_multi_byte_decoder: bool, - decoded_payload: str | None = None, - deferred_decoding: bool = False, -) -> Generator[str, None, None]: - if decoded_payload and not is_multi_byte_decoder: - for i in offsets: - chunk = decoded_payload[i : i + chunk_size] - if not chunk: - break - yield chunk - elif deferred_decoding: - # Deferred single-byte probing: the whole payload is not decoded - # yet. Single-byte codecs are stateless (1 byte == 1 char), hence - # decode(base)[i:j] == decode(base[i:j]): slicing the raw bytes - # yields exactly the chunks the branch above would have produced, - # short trailing chunks included, and raises UnicodeDecodeError on - # invalid bytes just like the whole-payload decode would. - base_bytes = ( - sequences if not strip_sig_or_bom else sequences[len(sig_payload) :] - ) - for i in offsets: - cut_sequence = base_bytes[i : i + chunk_size] - if not cut_sequence: - break - yield str(cut_sequence, encoding_iana) - else: - for i in offsets: - chunk_end = i + chunk_size - if chunk_end > len(sequences) + 8: - continue - - cut_sequence = sequences[i : i + chunk_size] - - if bom_or_sig_available and not strip_sig_or_bom: - cut_sequence = sig_payload + cut_sequence - - chunk = cut_sequence.decode( - encoding_iana, - errors="ignore" if is_multi_byte_decoder else "strict", - ) - - # multi-byte bad cutting detector and adjustment - # not the cleanest way to perform that fix but clever enough for now. - if is_multi_byte_decoder and i > 0: - chunk_partial_size_chk: int = min(chunk_size, 16) - - if ( - decoded_payload - and chunk[:chunk_partial_size_chk] not in decoded_payload - ): - for j in range(i, i - 4, -1): - cut_sequence = sequences[j:chunk_end] - - if bom_or_sig_available and not strip_sig_or_bom: - cut_sequence = sig_payload + cut_sequence - - chunk = cut_sequence.decode(encoding_iana, errors="ignore") - - if chunk[:chunk_partial_size_chk] in decoded_payload: - break - - yield chunk diff --git a/bundle/python-cpu/Lib/site-packages/charset_normalizer/version.py b/bundle/python-cpu/Lib/site-packages/charset_normalizer/version.py deleted file mode 100644 index 21327766328685e1c7ef61bc21644e204c9b5f8f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/charset_normalizer/version.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -Expose version -""" - -from __future__ import annotations - -__version__ = "3.4.9" -VERSION = __version__.split(".") diff --git a/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/METADATA deleted file mode 100644 index 1fb06f0478b071761108832c2c0099cc5967ee6f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/METADATA +++ /dev/null @@ -1,84 +0,0 @@ -Metadata-Version: 2.4 -Name: click -Version: 8.4.2 -Summary: Composable command line interface toolkit -Maintainer-email: Pallets -Requires-Python: >=3.10 -Description-Content-Type: text/markdown -License-Expression: BSD-3-Clause -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Typing :: Typed -License-File: LICENSE.txt -Requires-Dist: colorama; platform_system == 'Windows' -Project-URL: Changes, https://click.palletsprojects.com/page/changes/ -Project-URL: Chat, https://discord.gg/pallets -Project-URL: Documentation, https://click.palletsprojects.com/ -Project-URL: Donate, https://palletsprojects.com/donate -Project-URL: Source, https://github.com/pallets/click/ - -
- -# Click - -Click is a Python package for creating beautiful command line interfaces -in a composable way with as little code as necessary. It's the "Command -Line Interface Creation Kit". It's highly configurable but comes with -sensible defaults out of the box. - -It aims to make the process of writing command line tools quick and fun -while also preventing any frustration caused by the inability to -implement an intended CLI API. - -Click in three points: - -- Arbitrary nesting of commands -- Automatic help page generation -- Supports lazy loading of subcommands at runtime - - -## A Simple Example - -```python -import click - -@click.command() -@click.option("--count", default=1, help="Number of greetings.") -@click.option("--name", prompt="Your name", help="The person to greet.") -def hello(count, name): - """Simple program that greets NAME for a total of COUNT times.""" - for _ in range(count): - click.echo(f"Hello, {name}!") - -if __name__ == '__main__': - hello() -``` - -``` -$ python hello.py --count=3 -Your name: Click -Hello, Click! -Hello, Click! -Hello, Click! -``` - - -## Donate - -The Pallets organization develops and supports Click and other popular -packages. In order to grow the community of contributors and users, and -allow the maintainers to devote more time to the projects, [please -donate today][]. - -[please donate today]: https://palletsprojects.com/donate - -## Contributing - -See our [detailed contributing documentation][contrib] for many ways to -contribute, including reporting issues, requesting features, asking or answering -questions, and making PRs. - -[contrib]: https://palletsprojects.com/contributing/ - diff --git a/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/RECORD deleted file mode 100644 index 3746bb142f54b7904709240ce243bb08747dc3c3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/RECORD +++ /dev/null @@ -1,24 +0,0 @@ -click-8.4.2.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -click-8.4.2.dist-info/METADATA,sha256=GUyd2B1Wf5CB8CbH5AEGD7r6e8FHyOClizZotApkwDE,2621 -click-8.4.2.dist-info/RECORD,, -click-8.4.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -click-8.4.2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 -click-8.4.2.dist-info/licenses/LICENSE.txt,sha256=morRBqOU6FO_4h9C9OctWSgZoigF2ZG18ydQKSkrZY0,1475 -click/__init__.py,sha256=FId2fXCSJB3yeWD-e2uON-mBhFa2Yc9MvXGmHu8OXG0,4634 -click/_compat.py,sha256=gPNtXQ9q-G6Qil2b-MC5CsHsGGcQ4u6YSWy9_tlmuhc,18879 -click/_termui_impl.py,sha256=CGdg24AeXijeGSzbu0Z7x3c4aaahVFjVBpEbbjhQ5K4,31730 -click/_textwrap.py,sha256=7Z0N7Vmn-66TNSTUwp6OXJbcUXRmYET9h9c2ucD8oQQ,6270 -click/_utils.py,sha256=eCZCtwJtsYD5QYkkNWJ8MY_8ABIjy8MczgMMyVY32rQ,996 -click/_winconsole.py,sha256=KSxfNbMlYRa6GOJuCLgsg2Pb3dVkgJNPqLJPae-Pa10,8543 -click/core.py,sha256=rZz76ihNTFV4Y2sxp3H-m93GxL2acD5Pqs0IobEvmuk,140616 -click/decorators.py,sha256=9e1Ndu4jhGAcP6RGdNPAwAWtuP9hEs4ETp1u3lKmH1o,19709 -click/exceptions.py,sha256=HvSY34G4auj_bYRR8-T8CU8Jwq_1-OcsRU4ezfozeEk,11862 -click/formatting.py,sha256=8SW2KGkvjfz9Q1NbeojMHuZBN0cfnQJDs4mqDP6oXms,10444 -click/globals.py,sha256=gM-Nh6A4M0HB_SgkaF5M4ncGGMDHc_flHXu9_oh4GEU,1923 -click/parser.py,sha256=oJ-fU_3mvxugIuNtHaCATZ56lgEmHRggjJiSqEgYrjA,19052 -click/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -click/shell_completion.py,sha256=5tGGY5pV3mAZ17xT23OnuKrWqzEyyLVtrJ30npUxjkU,22618 -click/termui.py,sha256=Vn9ehmrQl92z2_6R4bVZOsHUI6j8LrT8u0RzNZUpCvY,33213 -click/testing.py,sha256=S9I-pspAlJH3RvZJWDQoJXb-M0nrAEJzXcUzrVXsT34,26458 -click/types.py,sha256=9G4DB-nBj-omA_XWsYwbQ3H9BkpH82wJj-kxIPScKmA,44788 -click/utils.py,sha256=XwrDxOzU__rnHn-rvJmJcD7ecbypUKMeDJQRjN2F-OA,20942 diff --git a/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/WHEEL deleted file mode 100644 index d8b9936dad9ab2513fa6979f411560d3b6b57e37..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: flit 3.12.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/licenses/LICENSE.txt b/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/licenses/LICENSE.txt deleted file mode 100644 index d12a849186982399c537c5b9a8fd77bf2edd5eab..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click-8.4.2.dist-info/licenses/LICENSE.txt +++ /dev/null @@ -1,28 +0,0 @@ -Copyright 2014 Pallets - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bundle/python-cpu/Lib/site-packages/click/__init__.py b/bundle/python-cpu/Lib/site-packages/click/__init__.py deleted file mode 100644 index 64be7e0c3c942191ed1e2259b96c6225679ae317..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/__init__.py +++ /dev/null @@ -1,126 +0,0 @@ -""" -Click is a simple Python module inspired by the stdlib optparse to make -writing command line scripts fun. Unlike other modules, it's based -around a simple API that does not come with too much magic and is -composable. -""" - -from __future__ import annotations - -from .core import Argument as Argument -from .core import Command as Command -from .core import CommandCollection as CommandCollection -from .core import Context as Context -from .core import Group as Group -from .core import Option as Option -from .core import Parameter as Parameter -from .core import ParameterSource as ParameterSource -from .decorators import argument as argument -from .decorators import command as command -from .decorators import confirmation_option as confirmation_option -from .decorators import group as group -from .decorators import help_option as help_option -from .decorators import make_pass_decorator as make_pass_decorator -from .decorators import option as option -from .decorators import pass_context as pass_context -from .decorators import pass_obj as pass_obj -from .decorators import password_option as password_option -from .decorators import version_option as version_option -from .exceptions import Abort as Abort -from .exceptions import BadArgumentUsage as BadArgumentUsage -from .exceptions import BadOptionUsage as BadOptionUsage -from .exceptions import BadParameter as BadParameter -from .exceptions import ClickException as ClickException -from .exceptions import FileError as FileError -from .exceptions import MissingParameter as MissingParameter -from .exceptions import NoSuchCommand as NoSuchCommand -from .exceptions import NoSuchOption as NoSuchOption -from .exceptions import UsageError as UsageError -from .formatting import HelpFormatter as HelpFormatter -from .formatting import wrap_text as wrap_text -from .globals import get_current_context as get_current_context -from .termui import clear as clear -from .termui import confirm as confirm -from .termui import echo_via_pager as echo_via_pager -from .termui import edit as edit -from .termui import get_pager_file as get_pager_file -from .termui import getchar as getchar -from .termui import launch as launch -from .termui import pause as pause -from .termui import progressbar as progressbar -from .termui import prompt as prompt -from .termui import secho as secho -from .termui import style as style -from .termui import unstyle as unstyle -from .types import BOOL as BOOL -from .types import Choice as Choice -from .types import DateTime as DateTime -from .types import File as File -from .types import FLOAT as FLOAT -from .types import FloatRange as FloatRange -from .types import INT as INT -from .types import IntRange as IntRange -from .types import ParamType as ParamType -from .types import Path as Path -from .types import STRING as STRING -from .types import Tuple as Tuple -from .types import UNPROCESSED as UNPROCESSED -from .types import UUID as UUID -from .utils import echo as echo -from .utils import format_filename as format_filename -from .utils import get_app_dir as get_app_dir -from .utils import get_binary_stream as get_binary_stream -from .utils import get_text_stream as get_text_stream -from .utils import open_file as open_file - - -def __getattr__(name: str) -> object: - import warnings - - if name == "BaseCommand": - from .core import _BaseCommand - - warnings.warn( - "'BaseCommand' is deprecated and will be removed in Click 9.0. Use" - " 'Command' instead.", - DeprecationWarning, - stacklevel=2, - ) - return _BaseCommand - - if name == "MultiCommand": - from .core import _MultiCommand - - warnings.warn( - "'MultiCommand' is deprecated and will be removed in Click 9.0. Use" - " 'Group' instead.", - DeprecationWarning, - stacklevel=2, - ) - return _MultiCommand - - if name == "OptionParser": - from .parser import _OptionParser - - warnings.warn( - "'OptionParser' is deprecated and will be removed in Click 9.0. The" - " old parser is available in 'optparse'.", - DeprecationWarning, - stacklevel=2, - ) - return _OptionParser - - if name == "__version__": - import importlib.metadata - import warnings - - warnings.warn( - "The '__version__' attribute is deprecated and will be removed in" - " Click 9.1. Use feature detection or" - " 'importlib.metadata.version(\"click\")' instead.", - DeprecationWarning, - stacklevel=2, - ) - return importlib.metadata.version("click") - - raise AttributeError(name) diff --git a/bundle/python-cpu/Lib/site-packages/click/_compat.py b/bundle/python-cpu/Lib/site-packages/click/_compat.py deleted file mode 100644 index 134c4f38934be0a0be78a6d2a3ab100fe60389b2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/_compat.py +++ /dev/null @@ -1,626 +0,0 @@ -from __future__ import annotations - -import codecs -import collections.abc as cabc -import io -import os -import re -import sys -import typing as t -from types import TracebackType -from weakref import WeakKeyDictionary - -CYGWIN = sys.platform.startswith("cygwin") -WIN = sys.platform.startswith("win") -auto_wrap_for_ansi: t.Callable[[t.TextIO], t.TextIO] | None = None -_ansi_re = re.compile(r"\033\[[;?0-9]*[a-zA-Z]") - - -def _make_text_stream( - stream: t.BinaryIO, - encoding: str | None, - errors: str | None, - force_readable: bool = False, - force_writable: bool = False, -) -> t.TextIO: - if encoding is None: - encoding = get_best_encoding(stream) - if errors is None: - errors = "replace" - return _NonClosingTextIOWrapper( - stream, - encoding, - errors, - line_buffering=True, - force_readable=force_readable, - force_writable=force_writable, - ) - - -def is_ascii_encoding(encoding: str) -> bool: - """Checks if a given encoding is ascii.""" - try: - return codecs.lookup(encoding).name == "ascii" - except LookupError: - return False - - -def get_best_encoding(stream: t.IO[t.Any]) -> str: - """Returns the default stream encoding if not found.""" - rv = getattr(stream, "encoding", None) or sys.getdefaultencoding() - if is_ascii_encoding(rv): - return "utf-8" - return rv - - -class _NonClosingTextIOWrapper(io.TextIOWrapper): - def __init__( - self, - stream: t.BinaryIO, - encoding: str | None, - errors: str | None, - force_readable: bool = False, - force_writable: bool = False, - **extra: t.Any, - ) -> None: - self._stream = stream = t.cast( - t.BinaryIO, _FixupStream(stream, force_readable, force_writable) - ) - super().__init__(stream, encoding, errors, **extra) - - def __del__(self) -> None: - try: - self.detach() - except Exception: - pass - - def isatty(self) -> bool: - # https://bitbucket.org/pypy/pypy/issue/1803 - return self._stream.isatty() - - -class _FixupStream: - """The new io interface needs more from streams than streams - traditionally implement. As such, this fix-up code is necessary in - some circumstances. - - The forcing of readable and writable flags are there because some tools - put badly patched objects on sys (one such offender are certain version - of jupyter notebook). - """ - - def __init__( - self, - stream: t.BinaryIO, - force_readable: bool = False, - force_writable: bool = False, - ): - self._stream = stream - self._force_readable = force_readable - self._force_writable = force_writable - - def __getattr__(self, name: str) -> t.Any: - return getattr(self._stream, name) - - def read1(self, size: int) -> bytes: - f = getattr(self._stream, "read1", None) - - if f is not None: - return t.cast(bytes, f(size)) - - return self._stream.read(size) - - def readable(self) -> bool: - if self._force_readable: - return True - x = getattr(self._stream, "readable", None) - if x is not None: - return t.cast(bool, x()) - try: - self._stream.read(0) - except Exception: - return False - return True - - def writable(self) -> bool: - if self._force_writable: - return True - x = getattr(self._stream, "writable", None) - if x is not None: - return t.cast(bool, x()) - try: - self._stream.write(b"") - except Exception: - try: - self._stream.write(b"") - except Exception: - return False - return True - - def seekable(self) -> bool: - x = getattr(self._stream, "seekable", None) - if x is not None: - return t.cast(bool, x()) - try: - self._stream.seek(self._stream.tell()) - except Exception: - return False - return True - - -def _is_binary_reader(stream: t.IO[t.Any], default: bool = False) -> bool: - try: - return isinstance(stream.read(0), bytes) - except Exception: - return default - # This happens in some cases where the stream was already - # closed. In this case, we assume the default. - - -def _is_binary_writer(stream: t.IO[t.Any], default: bool = False) -> bool: - try: - stream.write(b"") - except Exception: - try: - stream.write("") - return False - except Exception: - pass - return default - return True - - -def _find_binary_reader(stream: t.IO[t.Any]) -> t.BinaryIO | None: - # We need to figure out if the given stream is already binary. - # This can happen because the official docs recommend detaching - # the streams to get binary streams. Some code might do this, so - # we need to deal with this case explicitly. - if _is_binary_reader(stream, False): - return t.cast(t.BinaryIO, stream) - - buf = getattr(stream, "buffer", None) - - # Same situation here; this time we assume that the buffer is - # actually binary in case it's closed. - if buf is not None and _is_binary_reader(buf, True): - return t.cast(t.BinaryIO, buf) - - return None - - -def _find_binary_writer(stream: t.IO[t.Any]) -> t.BinaryIO | None: - # We need to figure out if the given stream is already binary. - # This can happen because the official docs recommend detaching - # the streams to get binary streams. Some code might do this, so - # we need to deal with this case explicitly. - if _is_binary_writer(stream, False): - return t.cast(t.BinaryIO, stream) - - buf = getattr(stream, "buffer", None) - - # Same situation here; this time we assume that the buffer is - # actually binary in case it's closed. - if buf is not None and _is_binary_writer(buf, True): - return t.cast(t.BinaryIO, buf) - - return None - - -def _stream_is_misconfigured(stream: t.TextIO) -> bool: - """A stream is misconfigured if its encoding is ASCII.""" - # If the stream does not have an encoding set, we assume it's set - # to ASCII. This appears to happen in certain unittest - # environments. It's not quite clear what the correct behavior is - # but this at least will force Click to recover somehow. - return is_ascii_encoding(getattr(stream, "encoding", None) or "ascii") - - -def _is_compat_stream_attr(stream: t.TextIO, attr: str, value: str | None) -> bool: - """A stream attribute is compatible if it is equal to the - desired value or the desired value is unset and the attribute - has a value. - """ - stream_value = getattr(stream, attr, None) - return stream_value == value or (value is None and stream_value is not None) - - -def _is_compatible_text_stream( - stream: t.TextIO, encoding: str | None, errors: str | None -) -> bool: - """Check if a stream's encoding and errors attributes are - compatible with the desired values. - """ - return _is_compat_stream_attr( - stream, "encoding", encoding - ) and _is_compat_stream_attr(stream, "errors", errors) - - -def _force_correct_text_stream( - text_stream: t.IO[t.Any], - encoding: str | None, - errors: str | None, - is_binary: t.Callable[[t.IO[t.Any], bool], bool], - find_binary: t.Callable[[t.IO[t.Any]], t.BinaryIO | None], - force_readable: bool = False, - force_writable: bool = False, -) -> t.TextIO: - if is_binary(text_stream, False): - binary_reader = t.cast(t.BinaryIO, text_stream) - else: - text_stream = t.cast(t.TextIO, text_stream) - # If the stream looks compatible, and won't default to a - # misconfigured ascii encoding, return it as-is. - if _is_compatible_text_stream(text_stream, encoding, errors) and not ( - encoding is None and _stream_is_misconfigured(text_stream) - ): - return text_stream - - # Otherwise, get the underlying binary reader. - possible_binary_reader = find_binary(text_stream) - - # If that's not possible, silently use the original reader - # and get mojibake instead of exceptions. - if possible_binary_reader is None: - return text_stream - - binary_reader = possible_binary_reader - - # Default errors to replace instead of strict in order to get - # something that works. - if errors is None: - errors = "replace" - - # Wrap the binary stream in a text stream with the correct - # encoding parameters. - return _make_text_stream( - binary_reader, - encoding, - errors, - force_readable=force_readable, - force_writable=force_writable, - ) - - -def _force_correct_text_reader( - text_reader: t.IO[t.Any], - encoding: str | None, - errors: str | None, - force_readable: bool = False, -) -> t.TextIO: - return _force_correct_text_stream( - text_reader, - encoding, - errors, - _is_binary_reader, - _find_binary_reader, - force_readable=force_readable, - ) - - -def _force_correct_text_writer( - text_writer: t.IO[t.Any], - encoding: str | None, - errors: str | None, - force_writable: bool = False, -) -> t.TextIO: - return _force_correct_text_stream( - text_writer, - encoding, - errors, - _is_binary_writer, - _find_binary_writer, - force_writable=force_writable, - ) - - -def get_binary_stdin() -> t.BinaryIO: - reader = _find_binary_reader(sys.stdin) - if reader is None: - raise RuntimeError("Was not able to determine binary stream for sys.stdin.") - return reader - - -def get_binary_stdout() -> t.BinaryIO: - writer = _find_binary_writer(sys.stdout) - if writer is None: - raise RuntimeError("Was not able to determine binary stream for sys.stdout.") - return writer - - -def get_binary_stderr() -> t.BinaryIO: - writer = _find_binary_writer(sys.stderr) - if writer is None: - raise RuntimeError("Was not able to determine binary stream for sys.stderr.") - return writer - - -def get_text_stdin(encoding: str | None = None, errors: str | None = None) -> t.TextIO: - rv = _get_windows_console_stream(sys.stdin, encoding, errors) - if rv is not None: - return rv - return _force_correct_text_reader(sys.stdin, encoding, errors, force_readable=True) - - -def get_text_stdout(encoding: str | None = None, errors: str | None = None) -> t.TextIO: - rv = _get_windows_console_stream(sys.stdout, encoding, errors) - if rv is not None: - return rv - return _force_correct_text_writer(sys.stdout, encoding, errors, force_writable=True) - - -def get_text_stderr(encoding: str | None = None, errors: str | None = None) -> t.TextIO: - rv = _get_windows_console_stream(sys.stderr, encoding, errors) - if rv is not None: - return rv - return _force_correct_text_writer(sys.stderr, encoding, errors, force_writable=True) - - -def _wrap_io_open( - file: str | os.PathLike[str] | int, - mode: str, - encoding: str | None, - errors: str | None, -) -> t.IO[t.Any]: - """Handles not passing ``encoding`` and ``errors`` in binary mode.""" - if "b" in mode: - return open(file, mode) - - return open(file, mode, encoding=encoding, errors=errors) - - -def open_stream( - filename: str | os.PathLike[str], - mode: str = "r", - encoding: str | None = None, - errors: str | None = "strict", - atomic: bool = False, -) -> tuple[t.IO[t.Any], bool]: - binary = "b" in mode - filename = os.fspath(filename) - - # Standard streams first. These are simple because they ignore the - # atomic flag. Use fsdecode to handle Path("-"). - if os.fsdecode(filename) == "-": - if any(m in mode for m in ["w", "a", "x"]): - if binary: - return get_binary_stdout(), False - return get_text_stdout(encoding=encoding, errors=errors), False - if binary: - return get_binary_stdin(), False - return get_text_stdin(encoding=encoding, errors=errors), False - - # Non-atomic writes directly go out through the regular open functions. - if not atomic: - return _wrap_io_open(filename, mode, encoding, errors), True - - # Some usability stuff for atomic writes - if "a" in mode: - raise ValueError( - "Appending to an existing file is not supported, because that" - " would involve an expensive `copy`-operation to a temporary" - " file. Open the file in normal `w`-mode and copy explicitly" - " if that's what you're after." - ) - if "x" in mode: - raise ValueError("Use the `overwrite`-parameter instead.") - if "w" not in mode: - raise ValueError("Atomic writes only make sense with `w`-mode.") - - # Atomic writes are more complicated. They work by opening a file - # as a proxy in the same folder and then using the fdopen - # functionality to wrap it in a Python file. Then we wrap it in an - # atomic file that moves the file over on close. - import errno - import random - - try: - perm: int | None = os.stat(filename).st_mode - except OSError: - perm = None - - flags = os.O_RDWR | os.O_CREAT | os.O_EXCL - - if binary: - flags |= getattr(os, "O_BINARY", 0) - - while True: - tmp_filename = os.path.join( - os.path.dirname(filename), - f".__atomic-write{random.randrange(1 << 32):08x}", - ) - try: - fd = os.open(tmp_filename, flags, 0o666 if perm is None else perm) - break - except OSError as e: - if e.errno == errno.EEXIST or ( - os.name == "nt" - and e.errno == errno.EACCES - and os.path.isdir(e.filename) - and os.access(e.filename, os.W_OK) - ): - continue - raise - - if perm is not None: - os.chmod(tmp_filename, perm) # in case perm includes bits in umask - - f = _wrap_io_open(fd, mode, encoding, errors) - af = _AtomicFile(f, tmp_filename, os.path.realpath(filename)) - return t.cast(t.IO[t.Any], af), True - - -class _AtomicFile: - def __init__(self, f: t.IO[t.Any], tmp_filename: str, real_filename: str) -> None: - self._f = f - self._tmp_filename = tmp_filename - self._real_filename = real_filename - self.closed = False - - @property - def name(self) -> str: - return self._real_filename - - def close(self, delete: bool = False) -> None: - if self.closed: - return - self._f.close() - os.replace(self._tmp_filename, self._real_filename) - self.closed = True - - def __getattr__(self, name: str) -> t.Any: - return getattr(self._f, name) - - def __enter__(self) -> _AtomicFile: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> None: - self.close(delete=exc_type is not None) - - def __repr__(self) -> str: - return repr(self._f) - - -def strip_ansi(value: str) -> str: - return _ansi_re.sub("", value) - - -def _is_jupyter_kernel_output(stream: t.IO[t.Any]) -> bool: - while isinstance(stream, (_FixupStream, _NonClosingTextIOWrapper)): - stream = stream._stream - - return stream.__class__.__module__.startswith("ipykernel.") - - -def should_strip_ansi( - stream: t.IO[t.Any] | None = None, color: bool | None = None -) -> bool: - if color is None: - if stream is None: - stream = sys.stdin - elif hasattr(stream, "color"): - # ._termui_impl.MaybeStripAnsi handles stripping ansi itself, - # so we don't need to strip it here - return False - return not isatty(stream) and not _is_jupyter_kernel_output(stream) - return not color - - -# On Windows, wrap the output streams with colorama to support ANSI -# color codes. -# NOTE: double check is needed so mypy does not analyze this on Linux -if sys.platform.startswith("win") and WIN: - from ._winconsole import _get_windows_console_stream - - def _get_argv_encoding() -> str: - import locale - - return locale.getpreferredencoding() - - _ansi_stream_wrappers: cabc.MutableMapping[t.TextIO, t.TextIO] = WeakKeyDictionary() - - def auto_wrap_for_ansi(stream: t.TextIO, color: bool | None = None) -> t.TextIO: - """Support ANSI color and style codes on Windows by wrapping a - stream with colorama. - """ - try: - cached = _ansi_stream_wrappers.get(stream) - except Exception: - cached = None - - if cached is not None: - return cached - - import colorama - - strip = should_strip_ansi(stream, color) - ansi_wrapper = colorama.AnsiToWin32(stream, strip=strip) - rv = t.cast(t.TextIO, ansi_wrapper.stream) - _write = rv.write - - def _safe_write(s: str) -> int: - try: - return _write(s) - except BaseException: - ansi_wrapper.reset_all() - raise - - rv.write = _safe_write # type: ignore[method-assign] - - try: - _ansi_stream_wrappers[stream] = rv - except Exception: - pass - - return rv - -else: - - def _get_argv_encoding() -> str: - return getattr(sys.stdin, "encoding", None) or sys.getfilesystemencoding() - - def _get_windows_console_stream( - f: t.TextIO, encoding: str | None, errors: str | None - ) -> t.TextIO | None: - return None - - -def term_len(x: str) -> int: - return len(strip_ansi(x)) - - -def isatty(stream: t.IO[t.Any]) -> bool: - try: - return stream.isatty() - except Exception: - return False - - -def _make_cached_stream_func( - src_func: t.Callable[[], t.TextIO | None], - wrapper_func: t.Callable[[], t.TextIO], -) -> t.Callable[[], t.TextIO | None]: - cache: cabc.MutableMapping[t.TextIO, t.TextIO] = WeakKeyDictionary() - - def func() -> t.TextIO | None: - stream = src_func() - - if stream is None: - return None - - try: - rv = cache.get(stream) - except Exception: - rv = None - if rv is not None: - return rv - rv = wrapper_func() - try: - cache[stream] = rv - except Exception: - pass - return rv - - return func - - -_default_text_stdin = _make_cached_stream_func(lambda: sys.stdin, get_text_stdin) -_default_text_stdout = _make_cached_stream_func(lambda: sys.stdout, get_text_stdout) -_default_text_stderr = _make_cached_stream_func(lambda: sys.stderr, get_text_stderr) - - -binary_streams: cabc.Mapping[str, t.Callable[[], t.BinaryIO]] = { - "stdin": get_binary_stdin, - "stdout": get_binary_stdout, - "stderr": get_binary_stderr, -} - -text_streams: cabc.Mapping[str, t.Callable[[str | None, str | None], t.TextIO]] = { - "stdin": get_text_stdin, - "stdout": get_text_stdout, - "stderr": get_text_stderr, -} diff --git a/bundle/python-cpu/Lib/site-packages/click/_termui_impl.py b/bundle/python-cpu/Lib/site-packages/click/_termui_impl.py deleted file mode 100644 index fadae940625f8275fed17833d527c19e2422ef4e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/_termui_impl.py +++ /dev/null @@ -1,945 +0,0 @@ -""" -This module contains implementations for the termui module. To keep the -import time of Click down, some infrequently used functionality is -placed in this module and only imported as needed. -""" - -from __future__ import annotations - -import collections.abc as cabc -import contextlib -import io -import math -import os -import shlex -import sys -import time -import typing as t -from gettext import gettext as _ -from io import StringIO -from pathlib import Path -from types import TracebackType - -from ._compat import _default_text_stdout -from ._compat import CYGWIN -from ._compat import get_best_encoding -from ._compat import isatty -from ._compat import strip_ansi -from ._compat import term_len -from ._compat import WIN -from .exceptions import ClickException -from .utils import echo -from .utils import KeepOpenFile - -V = t.TypeVar("V") - - -class _BufferedTextPagerStream(t.Protocol): - buffer: t.BinaryIO - - -def _has_binary_buffer( - stream: t.BinaryIO | t.TextIO, -) -> t.TypeGuard[_BufferedTextPagerStream]: - # TextIO is wider than TextIOWrapper; text-only streams such as StringIO - # are valid TextIO values but do not expose a binary buffer to wrap. - return getattr(stream, "buffer", None) is not None - - -if os.name == "nt": - BEFORE_BAR = "\r" - AFTER_BAR = "\n" -else: - BEFORE_BAR = "\r\033[?25l" - AFTER_BAR = "\033[?25h\n" - - -class ProgressBar(t.Generic[V]): - def __init__( - self, - iterable: cabc.Iterable[V] | None, - length: int | None = None, - fill_char: str = "#", - empty_char: str = " ", - bar_template: str = "%(bar)s", - info_sep: str = " ", - hidden: bool = False, - show_eta: bool = True, - show_percent: bool | None = None, - show_pos: bool = False, - item_show_func: t.Callable[[V | None], str | None] | None = None, - label: str | None = None, - file: t.TextIO | None = None, - color: bool | None = None, - update_min_steps: int = 1, - width: int = 30, - ) -> None: - self.fill_char = fill_char - self.empty_char = empty_char - self.bar_template = bar_template - self.info_sep = info_sep - self.hidden = hidden - self.show_eta = show_eta - self.show_percent = show_percent - self.show_pos = show_pos - self.item_show_func = item_show_func - self.label: str = label or "" - - if file is None: - file = _default_text_stdout() - - # There are no standard streams attached to write to. For example, - # pythonw on Windows. - if file is None: - file = StringIO() - - self.file = file - self.color = color - self.update_min_steps = update_min_steps - self._completed_intervals = 0 - self.width: int = width - self.autowidth: bool = width == 0 - - if length is None: - from operator import length_hint - - length = length_hint(iterable, -1) - - if length == -1: - length = None - if iterable is None: - if length is None: - raise TypeError("iterable or length is required") - iterable = t.cast("cabc.Iterable[V]", range(length)) - self.iter: cabc.Iterable[V] = iter(iterable) - self.length = length - self.pos: int = 0 - self.avg: list[float] = [] - self.last_eta: float - self.start: float - self.start = self.last_eta = time.time() - self.eta_known: bool = False - self.finished: bool = False - self.max_width: int | None = None - self.entered: bool = False - self.current_item: V | None = None - self._is_atty = isatty(self.file) - self._last_line: str | None = None - - def __enter__(self) -> ProgressBar[V]: - self.entered = True - self.render_progress() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> None: - self.render_finish() - - def __iter__(self) -> cabc.Iterator[V]: - if not self.entered: - raise RuntimeError("You need to use progress bars in a with block.") - self.render_progress() - return self.generator() - - def __next__(self) -> V: - # Iteration is defined in terms of a generator function, - # returned by iter(self); use that to define next(). This works - # because `self.iter` is an iterable consumed by that generator, - # so it is re-entry safe. Calling `next(self.generator())` - # twice works and does "what you want". - return next(iter(self)) - - def render_finish(self) -> None: - if self.hidden or not self._is_atty: - return - self.file.write(AFTER_BAR) - self.file.flush() - - @property - def pct(self) -> float: - if self.finished: - return 1.0 - return min(self.pos / (float(self.length or 1) or 1), 1.0) - - @property - def time_per_iteration(self) -> float: - if not self.avg: - return 0.0 - return sum(self.avg) / float(len(self.avg)) - - @property - def eta(self) -> float: - if self.length is not None and not self.finished: - return self.time_per_iteration * (self.length - self.pos) - return 0.0 - - def format_eta(self) -> str: - if self.eta_known: - t = int(self.eta) - seconds = t % 60 - t //= 60 - minutes = t % 60 - t //= 60 - hours = t % 24 - t //= 24 - if t > 0: - return "{d}{day_label} {h:02}:{m:02}:{s:02}".format( - d=t, - day_label=_("d"), - h=hours, - m=minutes, - s=seconds, - ) - else: - return f"{hours:02}:{minutes:02}:{seconds:02}" - return "" - - def format_pos(self) -> str: - pos = str(self.pos) - if self.length is not None: - pos += f"/{self.length}" - return pos - - def format_pct(self) -> str: - return f"{int(self.pct * 100): 4}%"[1:] - - def format_bar(self) -> str: - if self.length is not None: - bar_length = int(self.pct * self.width) - bar = self.fill_char * bar_length - bar += self.empty_char * (self.width - bar_length) - elif self.finished: - bar = self.fill_char * self.width - else: - chars = list(self.empty_char * (self.width or 1)) - if self.time_per_iteration != 0: - chars[ - int( - (math.cos(self.pos * self.time_per_iteration) / 2.0 + 0.5) - * self.width - ) - ] = self.fill_char - bar = "".join(chars) - return bar - - def format_progress_line(self) -> str: - show_percent = self.show_percent - - info_bits = [] - if self.length is not None and show_percent is None: - show_percent = not self.show_pos - - if self.show_pos: - info_bits.append(self.format_pos()) - if show_percent: - info_bits.append(self.format_pct()) - if self.show_eta and self.eta_known and not self.finished: - info_bits.append(self.format_eta()) - if self.item_show_func is not None: - item_info = self.item_show_func(self.current_item) - if item_info is not None: - info_bits.append(item_info) - - return ( - self.bar_template - % { - "label": self.label, - "bar": self.format_bar(), - "info": self.info_sep.join(info_bits), - } - ).rstrip() - - def render_progress(self) -> None: - if self.hidden: - return - - if not self._is_atty: - # Only output the label once if the output is not a TTY. - if self._last_line != self.label: - self._last_line = self.label - echo(self.label, file=self.file, color=self.color) - return - - buf = [] - # Update width in case the terminal has been resized - if self.autowidth: - import shutil - - old_width = self.width - self.width = 0 - clutter_length = term_len(self.format_progress_line()) - new_width = max(0, shutil.get_terminal_size().columns - clutter_length) - if new_width < old_width and self.max_width is not None: - buf.append(BEFORE_BAR) - buf.append(" " * self.max_width) - self.max_width = new_width - self.width = new_width - - clear_width = self.width - if self.max_width is not None: - clear_width = self.max_width - - buf.append(BEFORE_BAR) - line = self.format_progress_line() - line_len = term_len(line) - if self.max_width is None or self.max_width < line_len: - self.max_width = line_len - - buf.append(line) - buf.append(" " * (clear_width - line_len)) - line = "".join(buf) - # Render the line only if it changed. - - if line != self._last_line: - self._last_line = line - echo(line, file=self.file, color=self.color, nl=False) - self.file.flush() - - def make_step(self, n_steps: int) -> None: - self.pos += n_steps - if self.length is not None and self.pos >= self.length: - self.finished = True - - if (time.time() - self.last_eta) < 1.0: - return - - self.last_eta = time.time() - - # self.avg is a rolling list of length <= 7 of steps where steps are - # defined as time elapsed divided by the total progress through - # self.length. - if self.pos: - step = (time.time() - self.start) / self.pos - else: - step = time.time() - self.start - - self.avg = self.avg[-6:] + [step] - - self.eta_known = self.length is not None - - def update(self, n_steps: int, current_item: V | None = None) -> None: - """Update the progress bar by advancing a specified number of - steps, and optionally set the ``current_item`` for this new - position. - - :param n_steps: Number of steps to advance. - :param current_item: Optional item to set as ``current_item`` - for the updated position. - - .. versionchanged:: 8.0 - Added the ``current_item`` optional parameter. - - .. versionchanged:: 8.0 - Only render when the number of steps meets the - ``update_min_steps`` threshold. - """ - if current_item is not None: - self.current_item = current_item - - self._completed_intervals += n_steps - - if self._completed_intervals >= self.update_min_steps: - self.make_step(self._completed_intervals) - self.render_progress() - self._completed_intervals = 0 - - def finish(self) -> None: - self.eta_known = False - self.current_item = None - self.finished = True - - def generator(self) -> cabc.Iterator[V]: - """Return a generator which yields the items added to the bar - during construction, and updates the progress bar *after* the - yielded block returns. - """ - # WARNING: the iterator interface for `ProgressBar` relies on - # this and only works because this is a simple generator which - # doesn't create or manage additional state. If this function - # changes, the impact should be evaluated both against - # `iter(bar)` and `next(bar)`. `next()` in particular may call - # `self.generator()` repeatedly, and this must remain safe in - # order for that interface to work. - if not self.entered: - raise RuntimeError("You need to use progress bars in a with block.") - - if not self._is_atty: - yield from self.iter - else: - for rv in self.iter: - self.current_item = rv - - # This allows show_item_func to be updated before the - # item is processed. Only trigger at the beginning of - # the update interval. - if self._completed_intervals == 0: - self.render_progress() - - yield rv - self.update(1) - - self.finish() - self.render_progress() - - -class MaybeStripAnsi(io.TextIOWrapper): - def __init__(self, stream: t.IO[bytes], *, color: bool, **kwargs: t.Any): - super().__init__(stream, **kwargs) - self.color = color - - def write(self, text: str) -> int: - if not self.color: - text = strip_ansi(text) - return super().write(text) - - -def _pager_contextmanager( - color: bool | None = None, -) -> t.ContextManager[tuple[t.BinaryIO | t.TextIO, str, bool]]: - """Decide what method to use for paging through text.""" - stdout = _default_text_stdout() - - # There are no standard streams attached to write to. For example, - # pythonw on Windows. - if stdout is None: - stdout = StringIO() - - if not isatty(sys.stdin) or not isatty(stdout): - return _nullpager(stdout, color) - - # Split using POSIX mode (the default) so that quote characters are - # stripped from tokens and quoted Windows paths are preserved. - # Non-POSIX mode retains quotes in tokens, and wrapping tokens - # with shlex.quote re-introduces quoting issues on Windows. - pager_cmd_parts = shlex.split(os.environ.get("PAGER", "")) - if pager_cmd_parts: - if WIN: - return _tempfilepager(pager_cmd_parts, color) - return _pipepager(pager_cmd_parts, color) - - if os.environ.get("TERM") in ("dumb", "emacs"): - return _nullpager(stdout, color) - if WIN or sys.platform.startswith("os2"): - return _tempfilepager(["more"], color) - return _pipepager(["less"], color) - - -@contextlib.contextmanager -def get_pager_file(color: bool | None = None) -> t.Generator[t.TextIO, None, None]: - """Context manager. - - Yields a writable file-like object which can be used as an output pager. - - .. versionadded:: 8.4.0 - - :param color: controls if the pager supports ANSI colors or not. The - default is autodetection. - """ - with _pager_contextmanager(color=color) as (stream, encoding, color): - # Split streams by capabilities rather than the abstract TextIO / - # BinaryIO annotations: buffered text streams can be unwrapped to bytes, - # while other streams are yielded as-is. - wrapper: MaybeStripAnsi | None = None - if _has_binary_buffer(stream): - # Text stream backed by a binary buffer. - wrapper = MaybeStripAnsi(stream.buffer, color=color, encoding=encoding) - stream = wrapper - try: - # Narrow the BinaryIO | TextIO union that _pager_contextmanager - # yields; the caller writes text to the pager. - yield t.cast(t.TextIO, stream) - finally: - try: - stream.flush() - finally: - # Hand the binary buffer back to the pager that produced it - # rather than letting this TextIOWrapper close it on garbage - # collection. The pager owns the buffer's lifecycle: subprocess - # pipes and temp files are closed by their own helpers, while a - # borrowed stdout must stay open for the caller. detach() runs - # even if flush() raised, so the buffer is never closed here. - if wrapper is not None: - wrapper.detach() - - -@contextlib.contextmanager -def _pipepager( - cmd_parts: list[str], color: bool | None = None -) -> t.Iterator[tuple[t.BinaryIO | t.TextIO, str, bool]]: - """Page through text by feeding it to another program. - - Invokes the pager via :class:`subprocess.Popen` with an ``argv`` list - produced by :func:`shlex.split`. The command is resolved to an absolute - path with :func:`shutil.which` as recommended by the - :mod:`subprocess` docs for Windows compatibility. - - Invoking a pager through this might support colors: if piping to - ``less`` and the user hasn't decided on colors, ``LESS=-R`` is set - automatically. - """ - # Split the command into the invoked CLI and its parameters. - if not cmd_parts: - # No usable pager: fall back to stdout through _nullpager so it gets the - # same borrowed-stream handling and the caller's stream is not closed. - stdout = _default_text_stdout() or StringIO() - with _nullpager(stdout, color) as rv: - yield rv - return - - import shutil - - cmd = cmd_parts[0] - cmd_params = cmd_parts[1:] - - cmd_filepath = shutil.which(cmd) - if not cmd_filepath: - # No usable pager: fall back to stdout through _nullpager so it gets the - # same borrowed-stream handling and the caller's stream is not closed. - stdout = _default_text_stdout() or StringIO() - with _nullpager(stdout, color) as rv: - yield rv - return - - # Produces a normalized absolute path string. - # multi-call binaries such as busybox derive their identity from the symlink - # less -> busybox. resolve() causes them to misbehave. (eg. less becomes busybox) - cmd_path = Path(cmd_filepath).absolute() - cmd_name = cmd_path.name - - import subprocess - - # Make a local copy of the environment to not affect the global one. - env = dict(os.environ) - - # If we're piping to less and the user hasn't decided on colors, we enable - # them by default we find the -R flag in the command line arguments. - if color is None and cmd_name == "less": - less_flags = f"{os.environ.get('LESS', '')}{' '.join(cmd_params)}" - if not less_flags: - env["LESS"] = "-R" - color = True - elif "r" in less_flags or "R" in less_flags: - color = True - - if color is None: - color = False - - c = subprocess.Popen( - [str(cmd_path)] + cmd_params, - shell=False, - stdin=subprocess.PIPE, - env=env, - errors="replace", - text=True, - ) - stdin = t.cast(t.BinaryIO, c.stdin) - encoding = get_best_encoding(stdin) - try: - yield stdin, encoding, color - except BrokenPipeError: - # In case the pager exited unexpectedly, ignore the broken pipe error. - pass - except Exception as e: - # In case there is an exception we want to close the pager immediately - # and let the caller handle it. - # Otherwise the pager will keep running, and the user may not notice - # the error message, or worse yet it may leave the terminal in a broken state. - c.terminate() - raise e - finally: - # We must close stdin and wait for the pager to exit before we continue - try: - stdin.close() - # Close implies flush, so it might throw a BrokenPipeError if the pager - # process exited already. - except BrokenPipeError: - pass - - # Less doesn't respect ^C, but catches it for its own UI purposes (aborting - # search or other commands inside less). - # - # That means when the user hits ^C, the parent process (click) terminates, - # but less is still alive, paging the output and messing up the terminal. - # - # If the user wants to make the pager exit on ^C, they should set - # `LESS='-K'`. It's not our decision to make. - while True: - try: - c.wait() - except KeyboardInterrupt: - pass - else: - break - - -@contextlib.contextmanager -def _tempfilepager( - cmd_parts: list[str], color: bool | None = None -) -> t.Iterator[tuple[t.BinaryIO | t.TextIO, str, bool]]: - """Page through text by invoking a program on a temporary file. - - Used as the primary pager strategy on Windows (where piping to - ``more`` adds spurious ``\\r\\n``), and as a fallback on other - platforms. The command is resolved to an absolute path with - :func:`shutil.which`. - """ - # Split the command into the invoked CLI and its parameters. - if not cmd_parts: - # No usable pager: fall back to stdout through _nullpager so it gets the - # same borrowed-stream handling and the caller's stream is not closed. - stdout = _default_text_stdout() or StringIO() - with _nullpager(stdout, color) as rv: - yield rv - return - - import shutil - import subprocess - - cmd = cmd_parts[0] - - cmd_filepath = shutil.which(cmd) - if not cmd_filepath: - # No usable pager: fall back to stdout through _nullpager so it gets the - # same borrowed-stream handling and the caller's stream is not closed. - stdout = _default_text_stdout() or StringIO() - with _nullpager(stdout, color) as rv: - yield rv - return - - # Produces a normalized absolute path string. - # multi-call binaries such as busybox derive their identity from the symlink - # less -> busybox. resolve() causes them to misbehave. (eg. less becomes busybox) - cmd_path = Path(cmd_filepath).absolute() - - import tempfile - - encoding = get_best_encoding(sys.stdout) - if color is None: - color = False - # On Windows, NamedTemporaryFile cannot be opened by another process - # while Python still has it open, so we use delete=False and clean up manually - # rather than using a contextmanager here. - f = tempfile.NamedTemporaryFile(mode="wb", delete=False) - try: - yield t.cast(t.BinaryIO, f), encoding, color - f.flush() - f.close() - subprocess.call([str(cmd_path), f.name]) - finally: - os.unlink(f.name) - - -@contextlib.contextmanager -def _nullpager( - stream: t.TextIO, color: bool | None = None -) -> t.Iterator[tuple[t.TextIO, str, bool]]: - """Simply print unformatted text. This is the ultimate fallback. Don't close the - output stream in this case, since it's coming from elsewhere rather than our - internal helpers. - - The stream is wrapped in :class:`~click.utils.KeepOpenFile` so that, as a - borrowed stream, it is not closed by a ``with`` block. The wrapper that - :func:`get_pager_file` builds around it is detached rather than closed. - """ - encoding = get_best_encoding(stream) - - if color is None: - color = False - - yield KeepOpenFile(stream), encoding, color # type: ignore[misc] - - -class Editor: - def __init__( - self, - editor: str | None = None, - env: cabc.Mapping[str, str] | None = None, - require_save: bool = True, - extension: str = ".txt", - ) -> None: - self.editor = editor - self.env = env - self.require_save = require_save - self.extension = extension - - def get_editor(self) -> str: - if self.editor is not None: - return self.editor - for key in "VISUAL", "EDITOR": - rv = os.environ.get(key) - if rv: - return rv - if WIN: - return "notepad" - - from shutil import which - - for editor in "sensible-editor", "vim", "nano": - if which(editor) is not None: - return editor - return "vi" - - def edit_files(self, filenames: cabc.Iterable[str]) -> None: - """Open files in the user's editor.""" - import shlex - import subprocess - - editor = self.get_editor() - environ: dict[str, str] | None = None - - if self.env: - environ = os.environ.copy() - environ.update(self.env) - - try: - # Split in POSIX mode (the default) for the same reasons as - # in pager(): strips quotes from tokens and preserves quoted - # Windows paths. - c = subprocess.Popen( - args=shlex.split(editor) + list(filenames), - env=environ, - ) - exit_code = c.wait() - if exit_code != 0: - raise ClickException( - _("{editor}: Editing failed").format(editor=editor) - ) - except OSError as e: - raise ClickException( - _("{editor}: Editing failed: {e}").format(editor=editor, e=e) - ) from e - - @t.overload - def edit(self, text: bytes | bytearray) -> bytes | None: ... - - # We cannot know whether or not the type expected is str or bytes when None - # is passed, so str is returned as that was what was done before. - @t.overload - def edit(self, text: str | None) -> str | None: ... - - def edit(self, text: str | bytes | bytearray | None) -> str | bytes | None: - import tempfile - - if text is None: - data: bytes | bytearray = b"" - elif isinstance(text, (bytes, bytearray)): - data = text - else: - if text and not text.endswith("\n"): - text += "\n" - - if WIN: - data = text.replace("\n", "\r\n").encode("utf-8-sig") - else: - data = text.encode("utf-8") - - fd, name = tempfile.mkstemp(prefix="editor-", suffix=self.extension) - f: t.BinaryIO - - try: - with os.fdopen(fd, "wb") as f: - f.write(data) - - # If the filesystem resolution is 1 second, like Mac OS - # 10.12 Extended, or 2 seconds, like FAT32, and the editor - # closes very fast, require_save can fail. Set the modified - # time to be 2 seconds in the past to work around this. - os.utime(name, (os.path.getatime(name), os.path.getmtime(name) - 2)) - # Depending on the resolution, the exact value might not be - # recorded, so get the new recorded value. - timestamp = os.path.getmtime(name) - - self.edit_files((name,)) - - if self.require_save and os.path.getmtime(name) == timestamp: - return None - - with open(name, "rb") as f: - rv = f.read() - - if isinstance(text, (bytes, bytearray)): - return rv - - return rv.decode("utf-8-sig").replace("\r\n", "\n") - finally: - os.unlink(name) - - -def open_url(url: str, wait: bool = False, locate: bool = False) -> int: - import subprocess - - def _unquote_file(url: str) -> str: - from urllib.parse import unquote - - if url.startswith("file://"): - url = unquote(url[7:]) - - return url - - if sys.platform == "darwin": - args = ["open"] - if wait: - args.append("-W") - if locate: - args.append("-R") - args.append(_unquote_file(url)) - null = open("/dev/null", "w") - try: - return subprocess.Popen(args, stderr=null).wait() - finally: - null.close() - elif WIN: - if locate: - url = _unquote_file(url) - args = ["explorer", "/select,", url] - try: - return subprocess.call(args) - except OSError: - return 127 - else: - try: - os.startfile(url) # type: ignore[attr-defined] - except OSError: - return 127 - return 0 - elif CYGWIN: - if locate: - url = _unquote_file(url) - args = ["cygstart", os.path.dirname(url)] - else: - args = ["cygstart"] - if wait: - args.append("-w") - args.append(url) - try: - return subprocess.call(args) - except OSError: - # Command not found - return 127 - - try: - if locate: - url = os.path.dirname(_unquote_file(url)) or "." - else: - url = _unquote_file(url) - c = subprocess.Popen(["xdg-open", url]) - if wait: - return c.wait() - return 0 - except OSError: - if url.startswith(("http://", "https://")) and not locate and not wait: - import webbrowser - - webbrowser.open(url) - return 0 - return 1 - - -def _translate_ch_to_exc(ch: str) -> None: - if ch == "\x03": - raise KeyboardInterrupt() - - if ch == "\x04" and not WIN: # Unix-like, Ctrl+D - raise EOFError() - - if ch == "\x1a" and WIN: # Windows, Ctrl+Z - raise EOFError() - - -if sys.platform == "win32": - import msvcrt - - @contextlib.contextmanager - def raw_terminal() -> cabc.Iterator[int]: - yield -1 - - def getchar(echo: bool) -> str: - # The function `getch` will return a bytes object corresponding to - # the pressed character. Since Windows 10 build 1803, it will also - # return \x00 when called a second time after pressing a regular key. - # - # `getwch` does not share this probably-bugged behavior. Moreover, it - # returns a Unicode object by default, which is what we want. - # - # Either of these functions will return \x00 or \xe0 to indicate - # a special key, and you need to call the same function again to get - # the "rest" of the code. The fun part is that \u00e0 is - # "latin small letter a with grave", so if you type that on a French - # keyboard, you _also_ get a \xe0. - # E.g., consider the Up arrow. This returns \xe0 and then \x48. The - # resulting Unicode string reads as "a with grave" + "capital H". - # This is indistinguishable from when the user actually types - # "a with grave" and then "capital H". - # - # When \xe0 is returned, we assume it's part of a special-key sequence - # and call `getwch` again, but that means that when the user types - # the \u00e0 character, `getchar` doesn't return until a second - # character is typed. - # The alternative is returning immediately, but that would mess up - # cross-platform handling of arrow keys and others that start with - # \xe0. Another option is using `getch`, but then we can't reliably - # read non-ASCII characters, because return values of `getch` are - # limited to the current 8-bit codepage. - # - # Anyway, Click doesn't claim to do this Right(tm), and using `getwch` - # is doing the right thing in more situations than with `getch`. - - if echo: - func = t.cast(t.Callable[[], str], msvcrt.getwche) - else: - func = t.cast(t.Callable[[], str], msvcrt.getwch) - - rv = func() - - if rv in ("\x00", "\xe0"): - # \x00 and \xe0 are control characters that indicate special key, - # see above. - rv += func() - - _translate_ch_to_exc(rv) - return rv - -else: - import termios - import tty - - @contextlib.contextmanager - def raw_terminal() -> cabc.Iterator[int]: - f: t.TextIO | None - fd: int - - if not isatty(sys.stdin): - f = open("/dev/tty") - fd = f.fileno() - else: - fd = sys.stdin.fileno() - f = None - - try: - old_settings = termios.tcgetattr(fd) - - try: - tty.setraw(fd) - yield fd - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) - sys.stdout.flush() - - if f is not None: - f.close() - except termios.error: - pass - - def getchar(echo: bool) -> str: - with raw_terminal() as fd: - ch = os.read(fd, 32).decode(get_best_encoding(sys.stdin), "replace") - - if echo and isatty(sys.stdout): - sys.stdout.write(ch) - - _translate_ch_to_exc(ch) - return ch diff --git a/bundle/python-cpu/Lib/site-packages/click/_textwrap.py b/bundle/python-cpu/Lib/site-packages/click/_textwrap.py deleted file mode 100644 index 82840f2dff3ce627712c0ece2752382a0f7dab8b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/_textwrap.py +++ /dev/null @@ -1,188 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import textwrap -from contextlib import contextmanager - -from ._compat import _ansi_re -from ._compat import term_len - - -def _truncate_visible(text: str, n: int) -> str: - """Return the longest prefix of ``text`` containing at most ``n`` visible - characters. - - ANSI escape sequences inside the prefix are kept intact and do not count - toward the visible width. A cut is never placed inside an escape sequence. - """ - if n <= 0: - return "" - - visible = 0 - i = 0 - cut = 0 - end = len(text) - while i < end: - m = _ansi_re.match(text, i) - if m is not None: - i = m.end() - continue - visible += 1 - i += 1 - cut = i - if visible >= n: - break - return text[:cut] - - -class TextWrapper(textwrap.TextWrapper): - """``textwrap.TextWrapper`` variant that measures widths by visible - character count. - - ANSI escape sequences embedded in chunks, indents, or the placeholder are - excluded from the width budget. Without this, styled help text (a styled - ``Usage:`` prefix, a colorized option name, ...) would be wrapped earlier - than its visible length warrants and tokens would split mid-word. - """ - - def _handle_long_word( - self, - reversed_chunks: list[str], - cur_line: list[str], - cur_len: int, - width: int, - ) -> None: - space_left = max(width - cur_len, 1) - - if self.break_long_words: - last = reversed_chunks[-1] - cut = _truncate_visible(last, space_left) - res = last[len(cut) :] - cur_line.append(cut) - reversed_chunks[-1] = res - elif not cur_line: - cur_line.append(reversed_chunks.pop()) - - def _wrap_chunks(self, chunks: list[str]) -> list[str]: - """Wrap chunks counting widths in visible characters. - - Mirrors the algorithm of :meth:`textwrap.TextWrapper._wrap_chunks` - with every width measurement routed through - :func:`click._compat.term_len` instead of :func:`len`, so ANSI escape - bytes in chunks, indents, or the placeholder do not inflate the count. - - .. seealso:: - :class:`textwrap.TextWrapper` in the Python standard library documentation: - https://docs.python.org/3/library/textwrap.html#textwrap.TextWrapper - - Reference implementation in CPython: - https://github.com/python/cpython/blob/main/Lib/textwrap.py - """ - lines: list[str] = [] - if self.width <= 0: - raise ValueError(f"invalid width {self.width!r} (must be > 0)") - if self.max_lines is not None: - if self.max_lines > 1: - indent = self.subsequent_indent - else: - indent = self.initial_indent - if term_len(indent) + term_len(self.placeholder.lstrip()) > self.width: - raise ValueError("placeholder too large for max width") - - chunks.reverse() - - while chunks: - cur_line: list[str] = [] - cur_len = 0 - - if lines: - indent = self.subsequent_indent - else: - indent = self.initial_indent - - width = self.width - term_len(indent) - - if self.drop_whitespace and chunks[-1].strip() == "" and lines: - del chunks[-1] - - while chunks: - n = term_len(chunks[-1]) - - if cur_len + n <= width: - cur_line.append(chunks.pop()) - cur_len += n - - else: - break - - if chunks and term_len(chunks[-1]) > width: - self._handle_long_word(chunks, cur_line, cur_len, width) - cur_len = sum(map(term_len, cur_line)) - - if self.drop_whitespace and cur_line and cur_line[-1].strip() == "": - cur_len -= term_len(cur_line[-1]) - del cur_line[-1] - - if cur_line: - if ( - self.max_lines is None - or len(lines) + 1 < self.max_lines - or ( - not chunks - or self.drop_whitespace - and len(chunks) == 1 - and not chunks[0].strip() - ) - and cur_len <= width - ): - lines.append(indent + "".join(cur_line)) - else: - while cur_line: - if ( - cur_line[-1].strip() - and cur_len + term_len(self.placeholder) <= width - ): - cur_line.append(self.placeholder) - lines.append(indent + "".join(cur_line)) - break - cur_len -= term_len(cur_line[-1]) - del cur_line[-1] - else: - if lines: - prev_line = lines[-1].rstrip() - if ( - term_len(prev_line) + term_len(self.placeholder) - <= self.width - ): - lines[-1] = prev_line + self.placeholder - break - lines.append(indent + self.placeholder.lstrip()) - break - - return lines - - @contextmanager - def extra_indent(self, indent: str) -> cabc.Iterator[None]: - old_initial_indent = self.initial_indent - old_subsequent_indent = self.subsequent_indent - self.initial_indent += indent - self.subsequent_indent += indent - - try: - yield - finally: - self.initial_indent = old_initial_indent - self.subsequent_indent = old_subsequent_indent - - def indent_only(self, text: str) -> str: - rv = [] - - for idx, line in enumerate(text.splitlines()): - indent = self.initial_indent - - if idx > 0: - indent = self.subsequent_indent - - rv.append(f"{indent}{line}") - - return "\n".join(rv) diff --git a/bundle/python-cpu/Lib/site-packages/click/_utils.py b/bundle/python-cpu/Lib/site-packages/click/_utils.py deleted file mode 100644 index 05ee2e99757adb19882664e1fc0377ac29c92f80..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/_utils.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -import enum -import typing as t - - -class Sentinel(enum.Enum): - """Enum used to define sentinel values. - - .. seealso:: - - `PEP 661 - Sentinel Values `_. - """ - - UNSET = object() - FLAG_NEEDS_VALUE = object() - - def __repr__(self) -> str: - return f"{self.__class__.__name__}.{self.name}" - - -UNSET: t.Literal[Sentinel.UNSET] = Sentinel.UNSET -"""Sentinel used to indicate that a value is not set.""" - -FLAG_NEEDS_VALUE: t.Literal[Sentinel.FLAG_NEEDS_VALUE] = Sentinel.FLAG_NEEDS_VALUE -"""Sentinel used to indicate an option was passed as a flag without a -value but is not a flag option. - -``Option.consume_value`` uses this to prompt or use the ``flag_value``. -""" - -T_UNSET: t.TypeAlias = t.Literal[Sentinel.UNSET] -"""Type hint for the :data:`UNSET` sentinel value.""" - -T_FLAG_NEEDS_VALUE: t.TypeAlias = t.Literal[Sentinel.FLAG_NEEDS_VALUE] -"""Type hint for the :data:`FLAG_NEEDS_VALUE` sentinel value.""" diff --git a/bundle/python-cpu/Lib/site-packages/click/_winconsole.py b/bundle/python-cpu/Lib/site-packages/click/_winconsole.py deleted file mode 100644 index d25178d66ff7a9c8a1da52e61379da240be625b6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/_winconsole.py +++ /dev/null @@ -1,297 +0,0 @@ -# This module is based on the excellent work by Adam Bartoš who -# provided a lot of what went into the implementation here in -# the discussion to issue1602 in the Python bug tracker. -# -# There are some general differences in regards to how this works -# compared to the original patches as we do not need to patch -# the entire interpreter but just work in our little world of -# echo and prompt. -from __future__ import annotations - -import collections.abc as cabc -import io -import sys -import time -import typing as t -from ctypes import Array -from ctypes import byref -from ctypes import c_char -from ctypes import c_char_p -from ctypes import c_int -from ctypes import c_ssize_t -from ctypes import c_ulong -from ctypes import c_void_p -from ctypes import POINTER -from ctypes import py_object -from ctypes import Structure -from ctypes.wintypes import DWORD -from ctypes.wintypes import HANDLE -from ctypes.wintypes import LPCWSTR -from ctypes.wintypes import LPWSTR -from gettext import gettext as _ - -from ._compat import _NonClosingTextIOWrapper - -assert sys.platform == "win32" -import msvcrt # noqa: E402 -from ctypes import windll # noqa: E402 -from ctypes import WINFUNCTYPE # noqa: E402 - -c_ssize_p = POINTER(c_ssize_t) - -kernel32 = windll.kernel32 -GetStdHandle = kernel32.GetStdHandle -ReadConsoleW = kernel32.ReadConsoleW -WriteConsoleW = kernel32.WriteConsoleW -GetConsoleMode = kernel32.GetConsoleMode -GetLastError = kernel32.GetLastError -GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) -CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( - ("CommandLineToArgvW", windll.shell32) -) -LocalFree = WINFUNCTYPE(c_void_p, c_void_p)(("LocalFree", windll.kernel32)) - -STDIN_HANDLE = GetStdHandle(-10) -STDOUT_HANDLE = GetStdHandle(-11) -STDERR_HANDLE = GetStdHandle(-12) - -PyBUF_SIMPLE = 0 -PyBUF_WRITABLE = 1 - -ERROR_SUCCESS = 0 -ERROR_NOT_ENOUGH_MEMORY = 8 -ERROR_OPERATION_ABORTED = 995 - -STDIN_FILENO = 0 -STDOUT_FILENO = 1 -STDERR_FILENO = 2 - -EOF = b"\x1a" -MAX_BYTES_WRITTEN = 32767 - -if t.TYPE_CHECKING: - try: - # Using `typing_extensions.Buffer` instead of `collections.abc` - # on Windows for some reason does not have `Sized` implemented. - from collections.abc import Buffer # type: ignore - except ImportError: - from typing_extensions import Buffer - -try: - from ctypes import pythonapi -except ImportError: - # On PyPy we cannot get buffers so our ability to operate here is - # severely limited. - get_buffer = None -else: - - class Py_buffer(Structure): - _fields_ = [ # noqa: RUF012 - ("buf", c_void_p), - ("obj", py_object), - ("len", c_ssize_t), - ("itemsize", c_ssize_t), - ("readonly", c_int), - ("ndim", c_int), - ("format", c_char_p), - ("shape", c_ssize_p), - ("strides", c_ssize_p), - ("suboffsets", c_ssize_p), - ("internal", c_void_p), - ] - - PyObject_GetBuffer = pythonapi.PyObject_GetBuffer - PyBuffer_Release = pythonapi.PyBuffer_Release - - def get_buffer(obj: Buffer, writable: bool = False) -> Array[c_char]: - buf = Py_buffer() - flags: int = PyBUF_WRITABLE if writable else PyBUF_SIMPLE - PyObject_GetBuffer(py_object(obj), byref(buf), flags) - - try: - buffer_type = c_char * buf.len - out: Array[c_char] = buffer_type.from_address(buf.buf) - return out - finally: - PyBuffer_Release(byref(buf)) - - -class _WindowsConsoleRawIOBase(io.RawIOBase): - def __init__(self, handle: int | None) -> None: - self.handle = handle - - def isatty(self) -> t.Literal[True]: - super().isatty() - return True - - -class _WindowsConsoleReader(_WindowsConsoleRawIOBase): - def readable(self) -> t.Literal[True]: - return True - - def readinto(self, b: Buffer) -> int: - bytes_to_be_read = len(b) - if not bytes_to_be_read: - return 0 - elif bytes_to_be_read % 2: - raise ValueError( - "cannot read odd number of bytes from UTF-16-LE encoded console" - ) - - buffer = get_buffer(b, writable=True) - code_units_to_be_read = bytes_to_be_read // 2 - code_units_read = c_ulong() - - rv = ReadConsoleW( - HANDLE(self.handle), - buffer, - code_units_to_be_read, - byref(code_units_read), - None, - ) - if GetLastError() == ERROR_OPERATION_ABORTED: - # wait for KeyboardInterrupt - time.sleep(0.1) - if not rv: - raise OSError(_("Windows error: {error}").format(error=GetLastError())) - - if buffer[0] == EOF: - return 0 - return 2 * code_units_read.value - - -class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): - def writable(self) -> t.Literal[True]: - return True - - @staticmethod - def _get_error_message(errno: int) -> str: - if errno == ERROR_SUCCESS: - return "ERROR_SUCCESS" - elif errno == ERROR_NOT_ENOUGH_MEMORY: - return "ERROR_NOT_ENOUGH_MEMORY" - return _("Windows error: {error}").format(error=errno) - - def write(self, b: Buffer) -> int: - bytes_to_be_written = len(b) - buf = get_buffer(b) - code_units_to_be_written = min(bytes_to_be_written, MAX_BYTES_WRITTEN) // 2 - code_units_written = c_ulong() - - WriteConsoleW( - HANDLE(self.handle), - buf, - code_units_to_be_written, - byref(code_units_written), - None, - ) - bytes_written = 2 * code_units_written.value - - if bytes_written == 0 and bytes_to_be_written > 0: - raise OSError(self._get_error_message(GetLastError())) - return bytes_written - - -class ConsoleStream: - def __init__(self, text_stream: t.TextIO, byte_stream: t.BinaryIO) -> None: - self._text_stream = text_stream - self.buffer = byte_stream - - @property - def name(self) -> str: - return self.buffer.name - - def write(self, x: t.AnyStr) -> int: - if isinstance(x, str): - return self._text_stream.write(x) - try: - self.flush() - except Exception: - pass - return self.buffer.write(x) - - def writelines(self, lines: cabc.Iterable[t.AnyStr]) -> None: - for line in lines: - self.write(line) - - def __getattr__(self, name: str) -> t.Any: - return getattr(self._text_stream, name) - - def isatty(self) -> bool: - return self.buffer.isatty() - - def __repr__(self) -> str: - return f"" - - -def _get_text_stdin(buffer_stream: t.BinaryIO) -> t.TextIO: - text_stream = _NonClosingTextIOWrapper( - io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)), - "utf-16-le", - "strict", - line_buffering=True, - ) - return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) - - -def _get_text_stdout(buffer_stream: t.BinaryIO) -> t.TextIO: - text_stream = _NonClosingTextIOWrapper( - io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)), - "utf-16-le", - "strict", - line_buffering=True, - ) - return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) - - -def _get_text_stderr(buffer_stream: t.BinaryIO) -> t.TextIO: - text_stream = _NonClosingTextIOWrapper( - io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)), - "utf-16-le", - "strict", - line_buffering=True, - ) - return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) - - -_stream_factories: cabc.Mapping[int, t.Callable[[t.BinaryIO], t.TextIO]] = { - 0: _get_text_stdin, - 1: _get_text_stdout, - 2: _get_text_stderr, -} - - -def _is_console(f: t.TextIO) -> bool: - if not hasattr(f, "fileno"): - return False - - try: - fileno = f.fileno() - except (OSError, io.UnsupportedOperation): - return False - - handle = msvcrt.get_osfhandle(fileno) - return bool(GetConsoleMode(handle, byref(DWORD()))) - - -def _get_windows_console_stream( - f: t.TextIO, encoding: str | None, errors: str | None -) -> t.TextIO | None: - if ( - get_buffer is None - or encoding not in {"utf-16-le", None} - or errors not in {"strict", None} - or not _is_console(f) - ): - return None - - func = _stream_factories.get(f.fileno()) - if func is None: - return None - - b = getattr(f, "buffer", None) - - if b is None: - return None - - return func(b) diff --git a/bundle/python-cpu/Lib/site-packages/click/core.py b/bundle/python-cpu/Lib/site-packages/click/core.py deleted file mode 100644 index d7ecbefbc491a9582e1a47385f2922c10302b58c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/core.py +++ /dev/null @@ -1,3639 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import enum -import errno -import inspect -import os -import sys -import typing as t -from abc import ABC -from abc import abstractmethod -from collections import abc -from collections import Counter -from contextlib import AbstractContextManager -from contextlib import contextmanager -from contextlib import ExitStack -from functools import update_wrapper -from gettext import gettext as _ -from gettext import ngettext -from itertools import repeat -from types import TracebackType - -from . import types -from ._utils import FLAG_NEEDS_VALUE -from ._utils import UNSET -from .exceptions import Abort -from .exceptions import BadParameter -from .exceptions import ClickException -from .exceptions import Exit -from .exceptions import MissingParameter -from .exceptions import NoArgsIsHelpError -from .exceptions import NoSuchCommand -from .exceptions import UsageError -from .formatting import HelpFormatter -from .formatting import join_options -from .globals import pop_context -from .globals import push_context -from .parser import _OptionParser -from .parser import _split_opt -from .termui import confirm -from .termui import prompt -from .termui import style -from .utils import _detect_program_name -from .utils import _expand_args -from .utils import echo -from .utils import make_default_short_help -from .utils import make_str -from .utils import PacifyFlushWrapper - -if t.TYPE_CHECKING: - from typing_extensions import Self - - from .shell_completion import CompletionItem - -F = t.TypeVar("F", bound="t.Callable[..., t.Any]") -V = t.TypeVar("V") - - -def _complete_visible_commands( - ctx: Context, incomplete: str -) -> cabc.Iterator[tuple[str, Command]]: - """List all the subcommands of a group that start with the - incomplete value and aren't hidden. - - :param ctx: Invocation context for the group. - :param incomplete: Value being completed. May be empty. - """ - multi = t.cast(Group, ctx.command) - - for name in multi.list_commands(ctx): - if name.startswith(incomplete): - command = multi.get_command(ctx, name) - - if command is not None and not command.hidden: - yield name, command - - -def _check_nested_chain( - base_command: Group, cmd_name: str, cmd: Command, register: bool = False -) -> None: - if not base_command.chain or not isinstance(cmd, Group): - return - - if register: - message = ( - f"It is not possible to add the group {cmd_name!r} to another" - f" group {base_command.name!r} that is in chain mode." - ) - else: - message = ( - f"Found the group {cmd_name!r} as subcommand to another group " - f" {base_command.name!r} that is in chain mode. This is not supported." - ) - - raise RuntimeError(message) - - -def _format_deprecated_label(deprecated: bool | str) -> str: - """Return the parenthesized deprecation label shown in help text.""" - label = _("deprecated").upper() - if isinstance(deprecated, str): - return f"({label}: {deprecated})" - return f"({label})" - - -def _format_deprecated_suffix(deprecated: bool | str) -> str: - """Return the trailing reason for a ``DeprecationWarning`` message, - prefixed with a space, or an empty string when no reason was given. - """ - if isinstance(deprecated, str): - return f" {deprecated}" - return "" - - -def batch(iterable: cabc.Iterable[V], batch_size: int) -> list[tuple[V, ...]]: - return list(zip(*repeat(iter(iterable), batch_size), strict=False)) - - -@contextmanager -def augment_usage_errors( - ctx: Context, param: Parameter | None = None -) -> cabc.Generator[None]: - """Context manager that attaches extra information to exceptions.""" - try: - yield - except BadParameter as e: - if e.ctx is None: - e.ctx = ctx - if param is not None and e.param is None: - e.param = param - raise - except UsageError as e: - if e.ctx is None: - e.ctx = ctx - raise - - -def iter_params_for_processing( - invocation_order: cabc.Sequence[Parameter], - declaration_order: cabc.Sequence[Parameter], -) -> list[Parameter]: - """Returns all declared parameters in the order they should be processed. - - The declared parameters are re-shuffled depending on the order in which - they were invoked, as well as the eagerness of each parameters. - - The invocation order takes precedence over the declaration order. I.e. the - order in which the user provided them to the CLI is respected. - - This behavior and its effect on callback evaluation is detailed at: - https://click.palletsprojects.com/en/stable/advanced/#callback-evaluation-order - """ - - def sort_key(item: Parameter) -> tuple[bool, float]: - try: - idx: float = invocation_order.index(item) - except ValueError: - idx = float("inf") - - return not item.is_eager, idx - - return sorted(declaration_order, key=sort_key) - - -class ParameterSource(enum.IntEnum): - """This is an :class:`~enum.IntEnum` that indicates the source of a - parameter's value. - - Use :meth:`click.Context.get_parameter_source` to get the - source for a parameter by name. - - Members are ordered from most explicit to least explicit source. - This allows comparison to check if a value was explicitly provided: - - .. code-block:: python - - source = ctx.get_parameter_source("port") - if source < click.ParameterSource.DEFAULT_MAP: - ... # value was explicitly set - - .. versionchanged:: 8.3.3 - Use :class:`~enum.IntEnum` and reorder members from most to - least explicit. Supports comparison operators. - - .. versionchanged:: 8.0 - Use :class:`~enum.Enum` and drop the ``validate`` method. - - .. versionchanged:: 8.0 - Added the ``PROMPT`` value. - """ - - PROMPT = enum.auto() - """Used a prompt to confirm a default or provide a value.""" - COMMANDLINE = enum.auto() - """The value was provided by the command line args.""" - ENVIRONMENT = enum.auto() - """The value was provided with an environment variable.""" - DEFAULT_MAP = enum.auto() - """Used a default provided by :attr:`Context.default_map`.""" - DEFAULT = enum.auto() - """Used the default specified by the parameter.""" - - -class Context: - """The context is a special internal object that holds state relevant - for the script execution at every single level. It's normally invisible - to commands unless they opt-in to getting access to it. - - The context is useful as it can pass internal objects around and can - control special execution features such as reading data from - environment variables. - - A context can be used as context manager in which case it will call - :meth:`close` on teardown. - - :param command: the command class for this context. - :param parent: the parent context. - :param info_name: the info name for this invocation. Generally this - is the most descriptive name for the script or - command. For the toplevel script it is usually - the name of the script, for commands below that it's - the name of the script. - :param obj: an arbitrary object of user data. - :param auto_envvar_prefix: the prefix to use for automatic environment - variables. If this is `None` then reading - from environment variables is disabled. This - does not affect manually set environment - variables which are always read. - :param default_map: a dictionary (like object) with default values - for parameters. - :param terminal_width: the width of the terminal. The default is - inherit from parent context. If no context - defines the terminal width then auto - detection will be applied. - :param max_content_width: the maximum width for content rendered by - Click (this currently only affects help - pages). This defaults to 80 characters if - not overridden. In other words: even if the - terminal is larger than that, Click will not - format things wider than 80 characters by - default. In addition to that, formatters might - add some safety mapping on the right. - :param resilient_parsing: if this flag is enabled then Click will - parse without any interactivity or callback - invocation. Default values will also be - ignored. This is useful for implementing - things such as completion support. - :param allow_extra_args: if this is set to `True` then extra arguments - at the end will not raise an error and will be - kept on the context. The default is to inherit - from the command. - :param allow_interspersed_args: if this is set to `False` then options - and arguments cannot be mixed. The - default is to inherit from the command. - :param ignore_unknown_options: instructs click to ignore options it does - not know and keeps them for later - processing. - :param help_option_names: optionally a list of strings that define how - the default help parameter is named. The - default is ``['--help']``. - :param token_normalize_func: an optional function that is used to - normalize tokens (options, choices, - etc.). This for instance can be used to - implement case insensitive behavior. - :param color: controls if the terminal supports ANSI colors or not. The - default is autodetection. This is only needed if ANSI - codes are used in texts that Click prints which is by - default not the case. This for instance would affect - help output. - :param show_default: Show the default value for commands. If this - value is not set, it defaults to the value from the parent - context. ``Command.show_default`` overrides this default for the - specific command. - - .. versionchanged:: 8.2 - The ``protected_args`` attribute is deprecated and will be removed in - Click 9.0. ``args`` will contain remaining unparsed tokens. - - .. versionchanged:: 8.1 - The ``show_default`` parameter is overridden by - ``Command.show_default``, instead of the other way around. - - .. versionchanged:: 8.0 - The ``show_default`` parameter defaults to the value from the - parent context. - - .. versionchanged:: 7.1 - Added the ``show_default`` parameter. - - .. versionchanged:: 4.0 - Added the ``color``, ``ignore_unknown_options``, and - ``max_content_width`` parameters. - - .. versionchanged:: 3.0 - Added the ``allow_extra_args`` and ``allow_interspersed_args`` - parameters. - - .. versionchanged:: 2.0 - Added the ``resilient_parsing``, ``help_option_names``, and - ``token_normalize_func`` parameters. - """ - - #: The formatter class to create with :meth:`make_formatter`. - #: - #: .. versionadded:: 8.0 - formatter_class: type[HelpFormatter] = HelpFormatter - - parent: Context | None - command: Command - info_name: str | None - params: dict[str, t.Any] - args: list[str] - _protected_args: list[str] - _opt_prefixes: set[str] - obj: t.Any - _meta: dict[str, t.Any] - default_map: cabc.MutableMapping[str, t.Any] | None - invoked_subcommand: str | None - terminal_width: int | None - max_content_width: int | None - allow_extra_args: bool - allow_interspersed_args: bool - ignore_unknown_options: bool - help_option_names: list[str] - token_normalize_func: t.Callable[[str], str] | None - resilient_parsing: bool - auto_envvar_prefix: str | None - color: bool | None - show_default: bool | None - _close_callbacks: list[t.Callable[[], t.Any]] - _depth: int - _parameter_source: dict[str, ParameterSource] - _param_default_explicit: dict[str, bool] - _exit_stack: ExitStack - - def __init__( - self, - command: Command, - parent: Context | None = None, - info_name: str | None = None, - obj: t.Any | None = None, - auto_envvar_prefix: str | None = None, - default_map: cabc.MutableMapping[str, t.Any] | None = None, - terminal_width: int | None = None, - max_content_width: int | None = None, - resilient_parsing: bool = False, - allow_extra_args: bool | None = None, - allow_interspersed_args: bool | None = None, - ignore_unknown_options: bool | None = None, - help_option_names: list[str] | None = None, - token_normalize_func: t.Callable[[str], str] | None = None, - color: bool | None = None, - show_default: bool | None = None, - ) -> None: - #: the parent context or `None` if none exists. - self.parent = parent - #: the :class:`Command` for this context. - self.command = command - #: the descriptive information name - self.info_name = info_name - #: Map of parameter names to their parsed values. Parameters - #: with ``expose_value=False`` are not stored. - self.params = {} - #: the leftover arguments. - self.args = [] - #: protected arguments. These are arguments that are prepended - #: to `args` when certain parsing scenarios are encountered but - #: must be never propagated to another arguments. This is used - #: to implement nested parsing. - self._protected_args = [] - #: the collected prefixes of the command's options. - self._opt_prefixes = set(parent._opt_prefixes) if parent else set() - - if obj is None and parent is not None: - obj = parent.obj - - #: the user object stored. - self.obj = obj - self._meta = getattr(parent, "meta", {}) - - #: A dictionary (-like object) with defaults for parameters. - if ( - default_map is None - and info_name is not None - and parent is not None - and parent.default_map is not None - ): - default_map = parent.default_map.get(info_name) - - self.default_map = default_map - - #: This flag indicates if a subcommand is going to be executed. A - #: group callback can use this information to figure out if it's - #: being executed directly or because the execution flow passes - #: onwards to a subcommand. By default it's None, but it can be - #: the name of the subcommand to execute. - #: - #: If chaining is enabled this will be set to ``'*'`` in case - #: any commands are executed. It is however not possible to - #: figure out which ones. If you require this knowledge you - #: should use a :func:`result_callback`. - self.invoked_subcommand = None - - if terminal_width is None and parent is not None: - terminal_width = parent.terminal_width - - #: The width of the terminal (None is autodetection). - self.terminal_width = terminal_width - - if max_content_width is None and parent is not None: - max_content_width = parent.max_content_width - - #: The maximum width of formatted content (None implies a sensible - #: default which is 80 for most things). - self.max_content_width = max_content_width - - if allow_extra_args is None: - allow_extra_args = command.allow_extra_args - - #: Indicates if the context allows extra args or if it should - #: fail on parsing. - #: - #: .. versionadded:: 3.0 - self.allow_extra_args = allow_extra_args - - if allow_interspersed_args is None: - allow_interspersed_args = command.allow_interspersed_args - - #: Indicates if the context allows mixing of arguments and - #: options or not. - #: - #: .. versionadded:: 3.0 - self.allow_interspersed_args = allow_interspersed_args - - if ignore_unknown_options is None: - ignore_unknown_options = command.ignore_unknown_options - - #: Instructs click to ignore options that a command does not - #: understand and will store it on the context for later - #: processing. This is primarily useful for situations where you - #: want to call into external programs. Generally this pattern is - #: strongly discouraged because it's not possibly to losslessly - #: forward all arguments. - #: - #: .. versionadded:: 4.0 - self.ignore_unknown_options = ignore_unknown_options - - if help_option_names is None: - if parent is not None: - help_option_names = parent.help_option_names - else: - help_option_names = ["--help"] - - #: The names for the help options. - self.help_option_names = help_option_names - - if token_normalize_func is None and parent is not None: - token_normalize_func = parent.token_normalize_func - - #: An optional normalization function for tokens. This is - #: options, choices, commands etc. - self.token_normalize_func = token_normalize_func - - #: Indicates if resilient parsing is enabled. In that case Click - #: will do its best to not cause any failures and default values - #: will be ignored. Useful for completion. - self.resilient_parsing = resilient_parsing - - # If there is no envvar prefix yet, but the parent has one and - # the command on this level has a name, we can expand the envvar - # prefix automatically. - if auto_envvar_prefix is None: - if ( - parent is not None - and parent.auto_envvar_prefix is not None - and self.info_name is not None - ): - auto_envvar_prefix = ( - f"{parent.auto_envvar_prefix}_{self.info_name.upper()}" - ) - else: - auto_envvar_prefix = auto_envvar_prefix.upper() - - if auto_envvar_prefix is not None: - auto_envvar_prefix = auto_envvar_prefix.replace("-", "_") - - self.auto_envvar_prefix = auto_envvar_prefix - - if color is None and parent is not None: - color = parent.color - - #: Controls if styling output is wanted or not. - self.color = color - - if show_default is None and parent is not None: - show_default = parent.show_default - - #: Show option default values when formatting help text. - self.show_default = show_default - - self._close_callbacks = [] - self._depth = 0 - self._parameter_source = {} - # Tracks whether the option that currently owns each parameter slot in - # :attr:`params` had its ``default`` set explicitly by the user. Used - # to tie-break feature-switch groups where multiple options share a - # parameter name and both fall back to their default value. - # Refs: https://github.com/pallets/click/issues/3403 - self._param_default_explicit = {} - self._exit_stack = ExitStack() - - @property - def protected_args(self) -> list[str]: - import warnings - - warnings.warn( - "'protected_args' is deprecated and will be removed in Click 9.0." - " 'args' will contain remaining unparsed tokens.", - DeprecationWarning, - stacklevel=2, - ) - return self._protected_args - - def to_info_dict(self) -> dict[str, t.Any]: - """Gather information that could be useful for a tool generating - user-facing documentation. This traverses the entire CLI - structure. - - .. code-block:: python - - with Context(cli) as ctx: - info = ctx.to_info_dict() - - .. versionadded:: 8.0 - """ - return { - "command": self.command.to_info_dict(self), - "info_name": self.info_name, - "allow_extra_args": self.allow_extra_args, - "allow_interspersed_args": self.allow_interspersed_args, - "ignore_unknown_options": self.ignore_unknown_options, - "auto_envvar_prefix": self.auto_envvar_prefix, - } - - def __enter__(self) -> Self: - self._depth += 1 - push_context(self) - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> bool | None: - self._depth -= 1 - exit_result: bool | None = None - if self._depth == 0: - exit_result = self._close_with_exception_info(exc_type, exc_value, tb) - pop_context() - - return exit_result - - @contextmanager - def scope(self, cleanup: bool = True) -> cabc.Generator[Context]: - """This helper method can be used with the context object to promote - it to the current thread local (see :func:`get_current_context`). - The default behavior of this is to invoke the cleanup functions which - can be disabled by setting `cleanup` to `False`. The cleanup - functions are typically used for things such as closing file handles. - - If the cleanup is intended the context object can also be directly - used as a context manager. - - Example usage:: - - with ctx.scope(): - assert get_current_context() is ctx - - This is equivalent:: - - with ctx: - assert get_current_context() is ctx - - .. versionadded:: 5.0 - - :param cleanup: controls if the cleanup functions should be run or - not. The default is to run these functions. In - some situations the context only wants to be - temporarily pushed in which case this can be disabled. - Nested pushes automatically defer the cleanup. - """ - if not cleanup: - self._depth += 1 - try: - with self as rv: - yield rv - finally: - if not cleanup: - self._depth -= 1 - - @property - def meta(self) -> dict[str, t.Any]: - """This is a dictionary which is shared with all the contexts - that are nested. It exists so that click utilities can store some - state here if they need to. It is however the responsibility of - that code to manage this dictionary well. - - The keys are supposed to be unique dotted strings. For instance - module paths are a good choice for it. What is stored in there is - irrelevant for the operation of click. However what is important is - that code that places data here adheres to the general semantics of - the system. - - Example usage:: - - LANG_KEY = f'{__name__}.lang' - - def set_language(value): - ctx = get_current_context() - ctx.meta[LANG_KEY] = value - - def get_language(): - return get_current_context().meta.get(LANG_KEY, 'en_US') - - .. versionadded:: 5.0 - """ - return self._meta - - def make_formatter(self) -> HelpFormatter: - """Creates the :class:`~click.HelpFormatter` for the help and - usage output. - - To quickly customize the formatter class used without overriding - this method, set the :attr:`formatter_class` attribute. - - .. versionchanged:: 8.0 - Added the :attr:`formatter_class` attribute. - """ - return self.formatter_class( - width=self.terminal_width, max_width=self.max_content_width - ) - - def with_resource(self, context_manager: AbstractContextManager[V]) -> V: - """Register a resource as if it were used in a ``with`` - statement. The resource will be cleaned up when the context is - popped. - - Uses :meth:`contextlib.ExitStack.enter_context`. It calls the - resource's ``__enter__()`` method and returns the result. When - the context is popped, it closes the stack, which calls the - resource's ``__exit__()`` method. - - To register a cleanup function for something that isn't a - context manager, use :meth:`call_on_close`. Or use something - from :mod:`contextlib` to turn it into a context manager first. - - .. code-block:: python - - @click.group() - @click.option("--name") - @click.pass_context - def cli(ctx): - ctx.obj = ctx.with_resource(connect_db(name)) - - :param context_manager: The context manager to enter. - :return: Whatever ``context_manager.__enter__()`` returns. - - .. versionadded:: 8.0 - """ - return self._exit_stack.enter_context(context_manager) - - def call_on_close(self, f: t.Callable[..., t.Any]) -> t.Callable[..., t.Any]: - """Register a function to be called when the context tears down. - - This can be used to close resources opened during the script - execution. Resources that support Python's context manager - protocol which would be used in a ``with`` statement should be - registered with :meth:`with_resource` instead. - - :param f: The function to execute on teardown. - """ - return self._exit_stack.callback(f) - - def close(self) -> None: - """Invoke all close callbacks registered with - :meth:`call_on_close`, and exit all context managers entered - with :meth:`with_resource`. - """ - self._close_with_exception_info(None, None, None) - - def _close_with_exception_info( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> bool | None: - """Unwind the exit stack by calling its :meth:`__exit__` providing the exception - information to allow for exception handling by the various resources registered - using :meth;`with_resource` - - :return: Whatever ``exit_stack.__exit__()`` returns. - """ - exit_result = self._exit_stack.__exit__(exc_type, exc_value, tb) - # In case the context is reused, create a new exit stack. - self._exit_stack = ExitStack() - - return exit_result - - @property - def command_path(self) -> str: - """The computed command path. This is used for the ``usage`` - information on the help page. It's automatically created by - combining the info names of the chain of contexts to the root. - """ - rv = "" - if self.info_name is not None: - rv = self.info_name - if self.parent is not None: - parent_command_path = [self.parent.command_path] - - if isinstance(self.parent.command, Command): - for param in self.parent.command.get_params(self): - parent_command_path.extend(param.get_usage_pieces(self)) - - rv = f"{' '.join(parent_command_path)} {rv}" - return rv.lstrip() - - def find_root(self) -> Context: - """Finds the outermost context.""" - node = self - while node.parent is not None: - node = node.parent - return node - - def find_object(self, object_type: type[V]) -> V | None: - """Finds the closest object of a given type.""" - node: Context | None = self - - while node is not None: - if isinstance(node.obj, object_type): - return node.obj - - node = node.parent - - return None - - def ensure_object(self, object_type: type[V]) -> V: - """Like :meth:`find_object` but sets the innermost object to a - new instance of `object_type` if it does not exist. - """ - rv = self.find_object(object_type) - if rv is None: - self.obj = rv = object_type() - return rv - - def _default_map_has(self, name: str | None) -> bool: - """Check if :attr:`default_map` contains a real value for ``name``. - - Returns ``False`` when the key is absent, the map is ``None``, - ``name`` is ``None``, or the stored value is the internal - :data:`UNSET` sentinel. - """ - return ( - name is not None - and self.default_map is not None - and name in self.default_map - and self.default_map[name] is not UNSET - ) - - @t.overload - def lookup_default( - self, name: str, call: t.Literal[True] = True - ) -> t.Any | None: ... - - @t.overload - def lookup_default( - self, name: str, call: t.Literal[False] = ... - ) -> t.Any | t.Callable[[], t.Any] | None: ... - - def lookup_default(self, name: str, call: bool = True) -> t.Any | None: - """Get the default for a parameter from :attr:`default_map`. - - :param name: Name of the parameter. - :param call: If the default is a callable, call it. Disable to - return the callable instead. - - .. versionchanged:: 8.0 - Added the ``call`` parameter. - """ - if not self._default_map_has(name): - return None - - # Assert to make the type checker happy. - assert self.default_map is not None - value = self.default_map[name] - - if call and callable(value): - return value() - - return value - - def fail(self, message: str) -> t.NoReturn: - """Aborts the execution of the program with a specific error - message. - - :param message: the error message to fail with. - """ - raise UsageError(message, self) - - def abort(self) -> t.NoReturn: - """Aborts the script.""" - raise Abort() - - def exit(self, code: int = 0) -> t.NoReturn: - """Exits the application with a given exit code. - - .. versionchanged:: 8.2 - Callbacks and context managers registered with :meth:`call_on_close` - and :meth:`with_resource` are closed before exiting. - """ - self.close() - raise Exit(code) - - def get_usage(self) -> str: - """Helper method to get formatted usage string for the current - context and command. - """ - return self.command.get_usage(self) - - def get_help(self) -> str: - """Helper method to get formatted help page for the current - context and command. - """ - return self.command.get_help(self) - - def _make_sub_context(self, command: Command) -> Context: - """Create a new context of the same type as this context, but - for a new command. - - :meta private: - """ - return type(self)(command, info_name=command.name, parent=self) - - @t.overload - def invoke( - self, callback: t.Callable[..., V], /, *args: t.Any, **kwargs: t.Any - ) -> V: ... - - @t.overload - def invoke(self, callback: Command, /, *args: t.Any, **kwargs: t.Any) -> t.Any: ... - - def invoke( - self, callback: Command | t.Callable[..., V], /, *args: t.Any, **kwargs: t.Any - ) -> t.Any | V: - """Invokes a command callback in exactly the way it expects. There - are two ways to invoke this method: - - 1. the first argument can be a callback and all other arguments and - keyword arguments are forwarded directly to the function. - 2. the first argument is a click command object. In that case all - arguments are forwarded as well but proper click parameters - (options and click arguments) must be keyword arguments and Click - will fill in defaults. - - .. versionchanged:: 8.0 - All ``kwargs`` are tracked in :attr:`params` so they will be - passed if :meth:`forward` is called at multiple levels. - - .. versionchanged:: 3.2 - A new context is created, and missing arguments use default values. - """ - if isinstance(callback, Command): - other_cmd = callback - - if other_cmd.callback is None: - raise TypeError( - "The given command does not have a callback that can be invoked." - ) - else: - callback = t.cast("t.Callable[..., V]", other_cmd.callback) - - ctx = self._make_sub_context(other_cmd) - - for param in other_cmd.params: - if param.name not in kwargs and param.expose_value: - default_value = param.get_default(ctx) - # We explicitly hide the :attr:`UNSET` value to the user, as we - # choose to make it an implementation detail. And because ``invoke`` - # has been designed as part of Click public API, we return ``None`` - # instead. Refs: - # https://github.com/pallets/click/issues/3066 - # https://github.com/pallets/click/issues/3065 - # https://github.com/pallets/click/pull/3068 - if default_value is UNSET: - default_value = None - kwargs[param.name] = param.type_cast_value(ctx, default_value) - - # Track all kwargs as params, so that forward() will pass - # them on in subsequent calls. - ctx.params.update(kwargs) - else: - ctx = self - - with augment_usage_errors(self): - with ctx: - return callback(*args, **kwargs) - - def forward(self, cmd: Command, /, *args: t.Any, **kwargs: t.Any) -> t.Any: - """Similar to :meth:`invoke` but fills in default keyword - arguments from the current context if the other command expects - it. This cannot invoke callbacks directly, only other commands. - - .. versionchanged:: 8.0 - All ``kwargs`` are tracked in :attr:`params` so they will be - passed if ``forward`` is called at multiple levels. - """ - # Can only forward to other commands, not direct callbacks. - if not isinstance(cmd, Command): - raise TypeError("Callback is not a command.") - - for param in self.params: - if param not in kwargs: - kwargs[param] = self.params[param] - - return self.invoke(cmd, *args, **kwargs) - - def set_parameter_source(self, name: str, source: ParameterSource) -> None: - """Set the source of a parameter. This indicates the location - from which the value of the parameter was obtained. - - :param name: The name of the parameter. - :param source: A member of :class:`~click.core.ParameterSource`. - """ - self._parameter_source[name] = source - - def get_parameter_source(self, name: str) -> ParameterSource | None: - """Get the source of a parameter. This indicates the location - from which the value of the parameter was obtained. - - This can be useful for determining when a user specified a value - on the command line that is the same as the default value. It - will be :attr:`~click.core.ParameterSource.DEFAULT` only if the - value was actually taken from the default. - - :param name: The name of the parameter. - :rtype: ParameterSource - - .. versionchanged:: 8.0 - Returns ``None`` if the parameter was not provided from any - source. - """ - return self._parameter_source.get(name) - - -class Command: - """Commands are the basic building block of command line interfaces in - Click. A basic command handles command line parsing and might dispatch - more parsing to commands nested below it. - - :param name: the name of the command to use unless a group overrides it. - :param context_settings: an optional dictionary with defaults that are - passed to the context object. - :param callback: the callback to invoke. This is optional. - :param params: the parameters to register with this command. This can - be either :class:`Option` or :class:`Argument` objects. - :param help: the help string to use for this command. - :param epilog: like the help string but it's printed at the end of the - help page after everything else. - :param short_help: the short help to use for this command. This is - shown on the command listing of the parent command. - :param add_help_option: by default each command registers a ``--help`` - option. This can be disabled by this parameter. - :param no_args_is_help: this controls what happens if no arguments are - provided. This option is disabled by default. - If enabled this will add ``--help`` as argument - if no arguments are passed - :param hidden: hide this command from help outputs. - :param deprecated: If ``True`` or non-empty string, issues a message - indicating that the command is deprecated and highlights - its deprecation in --help. The message can be customized - by using a string as the value. - - .. versionchanged:: 8.2 - This is the base class for all commands, not ``BaseCommand``. - ``deprecated`` can be set to a string as well to customize the - deprecation message. - - .. versionchanged:: 8.1 - ``help``, ``epilog``, and ``short_help`` are stored unprocessed, - all formatting is done when outputting help text, not at init, - and is done even if not using the ``@command`` decorator. - - .. versionchanged:: 8.0 - Added a ``repr`` showing the command name. - - .. versionchanged:: 7.1 - Added the ``no_args_is_help`` parameter. - - .. versionchanged:: 2.0 - Added the ``context_settings`` parameter. - """ - - #: The context class to create with :meth:`make_context`. - #: - #: .. versionadded:: 8.0 - context_class: type[Context] = Context - - #: the default for the :attr:`Context.allow_extra_args` flag. - allow_extra_args = False - - #: the default for the :attr:`Context.allow_interspersed_args` flag. - allow_interspersed_args = True - - #: the default for the :attr:`Context.ignore_unknown_options` flag. - ignore_unknown_options = False - - name: str | None - context_settings: cabc.MutableMapping[str, t.Any] - callback: t.Callable[..., t.Any] | None - params: list[Parameter] - help: str | None - epilog: str | None - options_metavar: str | None - short_help: str | None - add_help_option: bool - _help_option: Option | None - no_args_is_help: bool - hidden: bool - deprecated: bool | str - - def __init__( - self, - name: str | None, - context_settings: cabc.MutableMapping[str, t.Any] | None = None, - callback: t.Callable[..., t.Any] | None = None, - params: list[Parameter] | None = None, - help: str | None = None, - epilog: str | None = None, - short_help: str | None = None, - options_metavar: str | None = "[OPTIONS]", - add_help_option: bool = True, - no_args_is_help: bool = False, - hidden: bool = False, - deprecated: bool | str = False, - ) -> None: - #: the name the command thinks it has. Upon registering a command - #: on a :class:`Group` the group will default the command name - #: with this information. You should instead use the - #: :class:`Context`\'s :attr:`~Context.info_name` attribute. - self.name = name - - if context_settings is None: - context_settings = {} - - #: an optional dictionary with defaults passed to the context. - self.context_settings = context_settings - - #: the callback to execute when the command fires. This might be - #: `None` in which case nothing happens. - self.callback = callback - #: the list of parameters for this command in the order they - #: should show up in the help page and execute. Eager parameters - #: will automatically be handled before non eager ones. - self.params = params or [] - self.help = help - self.epilog = epilog - self.options_metavar = options_metavar - self.short_help = short_help - self.add_help_option = add_help_option - self._help_option = None - self.no_args_is_help = no_args_is_help - self.hidden = hidden - self.deprecated = deprecated - - def to_info_dict(self, ctx: Context) -> dict[str, t.Any]: - return { - "name": self.name, - "params": [param.to_info_dict() for param in self.get_params(ctx)], - "help": self.help, - "epilog": self.epilog, - "short_help": self.short_help, - "hidden": self.hidden, - "deprecated": self.deprecated, - } - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} {self.name}>" - - def get_usage(self, ctx: Context) -> str: - """Formats the usage line into a string and returns it. - - Calls :meth:`format_usage` internally. - """ - formatter = ctx.make_formatter() - self.format_usage(ctx, formatter) - return formatter.getvalue().rstrip("\n") - - def get_params(self, ctx: Context) -> list[Parameter]: - params = self.params - help_option = self.get_help_option(ctx) - - if help_option is not None: - params = [*params, help_option] - - if __debug__: - import warnings - - opts = [opt for param in params for opt in param.opts] - opts_counter = Counter(opts) - duplicate_opts = (opt for opt, count in opts_counter.items() if count > 1) - - for duplicate_opt in duplicate_opts: - warnings.warn( - ( - f"The parameter {duplicate_opt} is used more than once. " - "Remove its duplicate as parameters should be unique." - ), - stacklevel=3, - ) - - return params - - def format_usage(self, ctx: Context, formatter: HelpFormatter) -> None: - """Writes the usage line into the formatter. - - This is a low-level method called by :meth:`get_usage`. - """ - pieces = self.collect_usage_pieces(ctx) - formatter.write_usage(ctx.command_path, " ".join(pieces)) - - def collect_usage_pieces(self, ctx: Context) -> list[str]: - """Returns all the pieces that go into the usage line and returns - it as a list of strings. - """ - rv = [self.options_metavar] if self.options_metavar else [] - - for param in self.get_params(ctx): - rv.extend(param.get_usage_pieces(ctx)) - - return rv - - def get_help_option_names(self, ctx: Context) -> list[str]: - """Returns the names for the help option.""" - all_names = set(ctx.help_option_names) - for param in self.params: - all_names.difference_update(param.opts) - all_names.difference_update(param.secondary_opts) - return list(all_names) - - def get_help_option(self, ctx: Context) -> Option | None: - """Returns the help option object. - - Skipped if :attr:`add_help_option` is ``False``. - - .. versionchanged:: 8.1.8 - The help option is now cached to avoid creating it multiple times. - """ - help_option_names = self.get_help_option_names(ctx) - - if not help_option_names or not self.add_help_option: - return None - - # Cache the help option object in private _help_option attribute to - # avoid creating it multiple times. Not doing this will break the - # callback ordering by iter_params_for_processing(), which relies on - # object comparison. - if self._help_option is None: - # Avoid circular import. - from .decorators import help_option - - # Apply help_option decorator and pop resulting option - help_option(*help_option_names)(self) - self._help_option = self.params.pop() # type: ignore[assignment] - - return self._help_option - - def make_parser(self, ctx: Context) -> _OptionParser: - """Creates the underlying option parser for this command.""" - parser = _OptionParser(ctx) - for param in self.get_params(ctx): - param.add_to_parser(parser, ctx) - return parser - - def get_help(self, ctx: Context) -> str: - """Formats the help into a string and returns it. - - Calls :meth:`format_help` internally. - """ - formatter = ctx.make_formatter() - self.format_help(ctx, formatter) - return formatter.getvalue().rstrip("\n") - - def get_short_help_str(self, limit: int = 45) -> str: - """Gets short help for the command or makes it by shortening the - long help string. - """ - if self.short_help: - text = inspect.cleandoc(self.short_help) - elif self.help: - text = make_default_short_help(self.help, limit) - else: - text = "" - - if self.deprecated: - text = f"{_(text)} {_format_deprecated_label(self.deprecated)}" - - return text.strip() - - def format_help(self, ctx: Context, formatter: HelpFormatter) -> None: - """Writes the help into the formatter if it exists. - - This is a low-level method called by :meth:`get_help`. - - This calls the following methods: - - - :meth:`format_usage` - - :meth:`format_help_text` - - :meth:`format_options` - - :meth:`format_epilog` - """ - self.format_usage(ctx, formatter) - self.format_help_text(ctx, formatter) - self.format_options(ctx, formatter) - self.format_epilog(ctx, formatter) - - def format_help_text(self, ctx: Context, formatter: HelpFormatter) -> None: - """Writes the help text to the formatter if it exists.""" - if self.help is not None: - # truncate the help text to the first form feed - text = inspect.cleandoc(self.help).partition("\f")[0] - else: - text = "" - - if self.deprecated: - label = _format_deprecated_label(self.deprecated) - text = f"{_(text)} {label}" if text else label - - if text: - formatter.write_paragraph() - - with formatter.indentation(): - formatter.write_text(text) - - def format_options(self, ctx: Context, formatter: HelpFormatter) -> None: - """Writes all the options into the formatter if they exist.""" - opts = [] - for param in self.get_params(ctx): - rv = param.get_help_record(ctx) - if rv is not None: - opts.append(rv) - - if opts: - with formatter.section(_("Options")): - formatter.write_dl(opts) - - def format_epilog(self, ctx: Context, formatter: HelpFormatter) -> None: - """Writes the epilog into the formatter if it exists.""" - if self.epilog: - epilog = inspect.cleandoc(self.epilog) - formatter.write_paragraph() - - with formatter.indentation(): - formatter.write_text(epilog) - - def make_context( - self, - info_name: str | None, - args: list[str], - parent: Context | None = None, - **extra: t.Any, - ) -> Context: - """This function when given an info name and arguments will kick - off the parsing and create a new :class:`Context`. It does not - invoke the actual command callback though. - - To quickly customize the context class used without overriding - this method, set the :attr:`context_class` attribute. - - :param info_name: the info name for this invocation. Generally this - is the most descriptive name for the script or - command. For the toplevel script it's usually - the name of the script, for commands below it's - the name of the command. - :param args: the arguments to parse as list of strings. - :param parent: the parent context if available. - :param extra: extra keyword arguments forwarded to the context - constructor. - - .. versionchanged:: 8.0 - Added the :attr:`context_class` attribute. - """ - for key, value in self.context_settings.items(): - if key not in extra: - extra[key] = value - - ctx = self.context_class(self, info_name=info_name, parent=parent, **extra) - - with ctx.scope(cleanup=False): - self.parse_args(ctx, args) - return ctx - - def parse_args(self, ctx: Context, args: list[str]) -> list[str]: - if not args and self.no_args_is_help and not ctx.resilient_parsing: - raise NoArgsIsHelpError(ctx) - - parser = self.make_parser(ctx) - opts, args, param_order = parser.parse_args(args=args) - - for param in iter_params_for_processing(param_order, self.get_params(ctx)): - _, args = param.handle_parse_result(ctx, opts, args) - - # We now have all parameters' values into `ctx.params`, but the data may contain - # the `UNSET` sentinel. - # Convert `UNSET` to `None` to ensure that the user doesn't see `UNSET`. - # - # Waiting until after the initial parse to convert allows us to treat `UNSET` - # more like a missing value when multiple params use the same name. - # Refs: - # https://github.com/pallets/click/issues/3071 - # https://github.com/pallets/click/pull/3079 - for name, value in ctx.params.items(): - if value is UNSET: - ctx.params[name] = None - - if args and not ctx.allow_extra_args and not ctx.resilient_parsing: - ctx.fail( - ngettext( - "Got unexpected extra argument ({args})", - "Got unexpected extra arguments ({args})", - len(args), - ).format(args=" ".join(map(str, args))) - ) - - ctx.args = args - ctx._opt_prefixes.update(parser._opt_prefixes) - return args - - def invoke(self, ctx: Context) -> t.Any: - """Given a context, this invokes the attached callback (if it exists) - in the right way. - """ - if self.deprecated: - message = _( - "DeprecationWarning: The command {name!r} is deprecated.{extra_message}" - ).format( - name=self.name, - extra_message=_format_deprecated_suffix(self.deprecated), - ) - echo(style(message, fg="red"), err=True) - - if self.callback is not None: - return ctx.invoke(self.callback, **ctx.params) - - def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]: - """Return a list of completions for the incomplete value. Looks - at the names of options and chained multi-commands. - - Any command could be part of a chained multi-command, so sibling - commands are valid at any point during command completion. - - :param ctx: Invocation context for this command. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - from click.shell_completion import CompletionItem - - results: list[CompletionItem] = [] - - if incomplete and not incomplete[0].isalnum(): - for param in self.get_params(ctx): - if ( - not isinstance(param, Option) - or param.hidden - or ( - not param.multiple - and ctx.get_parameter_source(param.name) - is ParameterSource.COMMANDLINE - ) - ): - continue - - results.extend( - CompletionItem(name, help=param.help) - for name in [*param.opts, *param.secondary_opts] - if name.startswith(incomplete) - ) - - while ctx.parent is not None: - ctx = ctx.parent - - if isinstance(ctx.command, Group) and ctx.command.chain: - results.extend( - CompletionItem(name, help=command.get_short_help_str()) - for name, command in _complete_visible_commands(ctx, incomplete) - if name not in ctx._protected_args - ) - - return results - - @t.overload - def main( - self, - args: cabc.Sequence[str] | None = None, - prog_name: str | None = None, - complete_var: str | None = None, - standalone_mode: t.Literal[True] = True, - **extra: t.Any, - ) -> t.NoReturn: ... - - @t.overload - def main( - self, - args: cabc.Sequence[str] | None = None, - prog_name: str | None = None, - complete_var: str | None = None, - standalone_mode: bool = ..., - **extra: t.Any, - ) -> t.Any: ... - - def main( - self, - args: cabc.Sequence[str] | None = None, - prog_name: str | None = None, - complete_var: str | None = None, - standalone_mode: bool = True, - windows_expand_args: bool = True, - **extra: t.Any, - ) -> t.Any: - """This is the way to invoke a script with all the bells and - whistles as a command line application. This will always terminate - the application after a call. If this is not wanted, ``SystemExit`` - needs to be caught. - - This method is also available by directly calling the instance of - a :class:`Command`. - - :param args: the arguments that should be used for parsing. If not - provided, ``sys.argv[1:]`` is used. - :param prog_name: the program name that should be used. By default - the program name is constructed by taking the file - name from ``sys.argv[0]``. - :param complete_var: the environment variable that controls the - bash completion support. The default is - ``"__COMPLETE"`` with prog_name in - uppercase. - :param standalone_mode: the default behavior is to invoke the script - in standalone mode. Click will then - handle exceptions and convert them into - error messages and the function will never - return but shut down the interpreter. If - this is set to `False` they will be - propagated to the caller and the return - value of this function is the return value - of :meth:`invoke`. - :param windows_expand_args: Expand glob patterns, user dir, and - env vars in command line args on Windows. - :param extra: extra keyword arguments are forwarded to the context - constructor. See :class:`Context` for more information. - - .. versionchanged:: 8.0.1 - Added the ``windows_expand_args`` parameter to allow - disabling command line arg expansion on Windows. - - .. versionchanged:: 8.0 - When taking arguments from ``sys.argv`` on Windows, glob - patterns, user dir, and env vars are expanded. - - .. versionchanged:: 3.0 - Added the ``standalone_mode`` parameter. - """ - if args is None: - args = sys.argv[1:] - - if os.name == "nt" and windows_expand_args: - args = _expand_args(args) - else: - args = list(args) - - if prog_name is None: - prog_name = _detect_program_name() - - # Process shell completion requests and exit early. - self._main_shell_completion(extra, prog_name, complete_var) - - try: - try: - with self.make_context(prog_name, args, **extra) as ctx: - rv = self.invoke(ctx) - if not standalone_mode: - return rv - # it's not safe to `ctx.exit(rv)` here! - # note that `rv` may actually contain data like "1" which - # has obvious effects - # more subtle case: `rv=[None, None]` can come out of - # chained commands which all returned `None` -- so it's not - # even always obvious that `rv` indicates success/failure - # by its truthiness/falsiness - ctx.exit() - except (EOFError, KeyboardInterrupt) as e: - echo(file=sys.stderr) - raise Abort() from e - except ClickException as e: - if not standalone_mode: - raise - e.show() - sys.exit(e.exit_code) - except OSError as e: - if e.errno == errno.EPIPE: - sys.stdout = t.cast(t.TextIO, PacifyFlushWrapper(sys.stdout)) - sys.stderr = t.cast(t.TextIO, PacifyFlushWrapper(sys.stderr)) - sys.exit(1) - else: - raise - except Exit as e: - if standalone_mode: - sys.exit(e.exit_code) - else: - # in non-standalone mode, return the exit code - # note that this is only reached if `self.invoke` above raises - # an Exit explicitly -- thus bypassing the check there which - # would return its result - # the results of non-standalone execution may therefore be - # somewhat ambiguous: if there are codepaths which lead to - # `ctx.exit(1)` and to `return 1`, the caller won't be able to - # tell the difference between the two - return e.exit_code - except Abort: - if not standalone_mode: - raise - echo(_("Aborted!"), file=sys.stderr) - sys.exit(1) - - def _main_shell_completion( - self, - ctx_args: cabc.MutableMapping[str, t.Any], - prog_name: str, - complete_var: str | None = None, - ) -> None: - """Check if the shell is asking for tab completion, process - that, then exit early. Called from :meth:`main` before the - program is invoked. - - :param prog_name: Name of the executable in the shell. - :param complete_var: Name of the environment variable that holds - the completion instruction. Defaults to - ``_{PROG_NAME}_COMPLETE``. - - .. versionchanged:: 8.2.0 - Dots (``.``) in ``prog_name`` are replaced with underscores (``_``). - """ - if complete_var is None: - complete_name = prog_name.replace("-", "_").replace(".", "_") - complete_var = f"_{complete_name}_COMPLETE".upper() - - instruction = os.environ.get(complete_var) - - if not instruction: - return - - from .shell_completion import shell_complete - - rv = shell_complete(self, ctx_args, prog_name, complete_var, instruction) - sys.exit(rv) - - def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any: - """Alias for :meth:`main`.""" - return self.main(*args, **kwargs) - - -class _FakeSubclassCheck(type): - def __subclasscheck__(cls, subclass: type) -> bool: - return issubclass(subclass, cls.__bases__[0]) - - def __instancecheck__(cls, instance: t.Any) -> bool: - return isinstance(instance, cls.__bases__[0]) - - -class _BaseCommand(Command, metaclass=_FakeSubclassCheck): - """ - .. deprecated:: 8.2 - Will be removed in Click 9.0. Use ``Command`` instead. - """ - - -class Group(Command): - """A group is a command that nests other commands (or more groups). - - :param name: The name of the group command. - :param commands: Map names to :class:`Command` objects. Can be a list, which - will use :attr:`Command.name` as the keys. - :param invoke_without_command: Invoke the group's callback even if a - subcommand is not given. - :param no_args_is_help: If no arguments are given, show the group's help and - exit. Defaults to the opposite of ``invoke_without_command``. - :param subcommand_metavar: How to represent the subcommand argument in help. - The default will represent whether ``chain`` is set or not. - :param chain: Allow passing more than one subcommand argument. After parsing - a command's arguments, if any arguments remain another command will be - matched, and so on. - :param result_callback: A function to call after the group's and - subcommand's callbacks. The value returned by the subcommand is passed. - If ``chain`` is enabled, the value will be a list of values returned by - all the commands. If ``invoke_without_command`` is enabled, the value - will be the value returned by the group's callback, or an empty list if - ``chain`` is enabled. - :param kwargs: Other arguments passed to :class:`Command`. - - .. versionchanged:: 8.0 - The ``commands`` argument can be a list of command objects. - - .. versionchanged:: 8.2 - Merged with and replaces the ``MultiCommand`` base class. - """ - - allow_extra_args = True - allow_interspersed_args = False - - #: If set, this is used by the group's :meth:`command` decorator - #: as the default :class:`Command` class. This is useful to make all - #: subcommands use a custom command class. - #: - #: .. versionadded:: 8.0 - command_class: type[Command] | None = None - - #: If set, this is used by the group's :meth:`group` decorator - #: as the default :class:`Group` class. This is useful to make all - #: subgroups use a custom group class. - #: - #: If set to the special value :class:`type` (literally - #: ``group_class = type``), this group's class will be used as the - #: default class. This makes a custom group class continue to make - #: custom groups. - #: - #: .. versionadded:: 8.0 - group_class: type[Group] | type[type] | None = None - # Literal[type] isn't valid, so use Type[type] - - commands: cabc.MutableMapping[str, Command] - invoke_without_command: bool - subcommand_metavar: str - chain: bool - _result_callback: t.Callable[..., t.Any] | None - - def __init__( - self, - name: str | None = None, - commands: cabc.MutableMapping[str, Command] - | cabc.Sequence[Command] - | None = None, - invoke_without_command: bool = False, - no_args_is_help: bool | None = None, - subcommand_metavar: str | None = None, - chain: bool = False, - result_callback: t.Callable[..., t.Any] | None = None, - **kwargs: t.Any, - ) -> None: - super().__init__(name, **kwargs) - - if commands is None: - commands = {} - elif isinstance(commands, abc.Sequence): - commands = {c.name: c for c in commands if c.name is not None} - - #: The registered subcommands by their exported names. - self.commands = commands - - if no_args_is_help is None: - no_args_is_help = not invoke_without_command - - self.no_args_is_help = no_args_is_help - self.invoke_without_command = invoke_without_command - - if subcommand_metavar is None: - # When the group can run without a subcommand, the leading command - # token is optional, so wrap it in brackets to reflect that. - if chain: - if invoke_without_command: - subcommand_metavar = "[COMMAND1] [ARGS]... [COMMAND2 [ARGS]...]..." - else: - subcommand_metavar = "COMMAND1 [ARGS]... [COMMAND2 [ARGS]...]..." - elif invoke_without_command: - subcommand_metavar = "[COMMAND] [ARGS]..." - else: - subcommand_metavar = "COMMAND [ARGS]..." - - self.subcommand_metavar = subcommand_metavar - self.chain = chain - # The result callback that is stored. This can be set or - # overridden with the :func:`result_callback` decorator. - self._result_callback = result_callback - - if self.chain: - for param in self.params: - if isinstance(param, Argument) and not param.required: - raise RuntimeError( - "A group in chain mode cannot have optional arguments." - ) - - def to_info_dict(self, ctx: Context) -> dict[str, t.Any]: - info_dict = super().to_info_dict(ctx) - commands = {} - - for name in self.list_commands(ctx): - command = self.get_command(ctx, name) - - if command is None: - continue - - sub_ctx = ctx._make_sub_context(command) - - with sub_ctx.scope(cleanup=False): - commands[name] = command.to_info_dict(sub_ctx) - - info_dict.update(commands=commands, chain=self.chain) - return info_dict - - def add_command(self, cmd: Command, name: str | None = None) -> None: - """Registers another :class:`Command` with this group. If the name - is not provided, the name of the command is used. - """ - name = name or cmd.name - if name is None: - raise TypeError("Command has no name.") - _check_nested_chain(self, name, cmd, register=True) - self.commands[name] = cmd - - @t.overload - def command(self, __func: t.Callable[..., t.Any]) -> Command: ... - - @t.overload - def command( - self, *args: t.Any, **kwargs: t.Any - ) -> t.Callable[[t.Callable[..., t.Any]], Command]: ... - - def command( - self, *args: t.Any, **kwargs: t.Any - ) -> t.Callable[[t.Callable[..., t.Any]], Command] | Command: - """A shortcut decorator for declaring and attaching a command to - the group. This takes the same arguments as :func:`command` and - immediately registers the created command with this group by - calling :meth:`add_command`. - - To customize the command class used, set the - :attr:`command_class` attribute. - - .. versionchanged:: 8.1 - This decorator can be applied without parentheses. - - .. versionchanged:: 8.0 - Added the :attr:`command_class` attribute. - """ - from .decorators import command - - func: t.Callable[..., t.Any] | None = None - - if args and callable(args[0]): - assert len(args) == 1 and not kwargs, ( - "Use 'command(**kwargs)(callable)' to provide arguments." - ) - (func,) = args - args = () - - if self.command_class and kwargs.get("cls") is None: - kwargs["cls"] = self.command_class - - def decorator(f: t.Callable[..., t.Any]) -> Command: - cmd: Command = command(*args, **kwargs)(f) - self.add_command(cmd) - return cmd - - if func is not None: - return decorator(func) - - return decorator - - @t.overload - def group(self, __func: t.Callable[..., t.Any]) -> Group: ... - - @t.overload - def group( - self, *args: t.Any, **kwargs: t.Any - ) -> t.Callable[[t.Callable[..., t.Any]], Group]: ... - - def group( - self, *args: t.Any, **kwargs: t.Any - ) -> t.Callable[[t.Callable[..., t.Any]], Group] | Group: - """A shortcut decorator for declaring and attaching a group to - the group. This takes the same arguments as :func:`group` and - immediately registers the created group with this group by - calling :meth:`add_command`. - - To customize the group class used, set the :attr:`group_class` - attribute. - - .. versionchanged:: 8.1 - This decorator can be applied without parentheses. - - .. versionchanged:: 8.0 - Added the :attr:`group_class` attribute. - """ - from .decorators import group - - func: t.Callable[..., t.Any] | None = None - - if args and callable(args[0]): - assert len(args) == 1 and not kwargs, ( - "Use 'group(**kwargs)(callable)' to provide arguments." - ) - (func,) = args - args = () - - if self.group_class is not None and kwargs.get("cls") is None: - if self.group_class is type: - kwargs["cls"] = type(self) - else: - kwargs["cls"] = self.group_class - - def decorator(f: t.Callable[..., t.Any]) -> Group: - cmd: Group = group(*args, **kwargs)(f) - self.add_command(cmd) - return cmd - - if func is not None: - return decorator(func) - - return decorator - - def result_callback(self, replace: bool = False) -> t.Callable[[F], F]: - """Adds a result callback to the command. By default if a - result callback is already registered this will chain them but - this can be disabled with the `replace` parameter. The result - callback is invoked with the return value of the subcommand - (or the list of return values from all subcommands if chaining - is enabled) as well as the parameters as they would be passed - to the main callback. - - Example:: - - @click.group() - @click.option('-i', '--input', default=23) - def cli(input): - return 42 - - @cli.result_callback() - def process_result(result, input): - return result + input - - :param replace: if set to `True` an already existing result - callback will be removed. - - .. versionchanged:: 8.0 - Renamed from ``resultcallback``. - - .. versionadded:: 3.0 - """ - - def decorator(f: F) -> F: - old_callback = self._result_callback - - if old_callback is None or replace: - self._result_callback = f - return f - - def function(value: t.Any, /, *args: t.Any, **kwargs: t.Any) -> t.Any: - inner = old_callback(value, *args, **kwargs) - return f(inner, *args, **kwargs) - - self._result_callback = rv = update_wrapper(t.cast(F, function), f) - return rv # type: ignore[return-value] - - return decorator - - def get_command(self, ctx: Context, cmd_name: str) -> Command | None: - """Given a context and a command name, this returns a :class:`Command` - object if it exists or returns ``None``. - """ - return self.commands.get(cmd_name) - - def list_commands(self, ctx: Context) -> list[str]: - """Returns a list of subcommand names in the order they should appear.""" - return sorted(self.commands) - - def collect_usage_pieces(self, ctx: Context) -> list[str]: - rv = super().collect_usage_pieces(ctx) - rv.append(self.subcommand_metavar) - return rv - - def format_options(self, ctx: Context, formatter: HelpFormatter) -> None: - super().format_options(ctx, formatter) - self.format_commands(ctx, formatter) - - def format_commands(self, ctx: Context, formatter: HelpFormatter) -> None: - """Extra format methods for multi methods that adds all the commands - after the options. - """ - commands = [] - for subcommand in self.list_commands(ctx): - cmd = self.get_command(ctx, subcommand) - # What is this, the tool lied about a command. Ignore it - if cmd is None: - continue - if cmd.hidden: - continue - - commands.append((subcommand, cmd)) - - # allow for 3 times the default spacing - if len(commands): - limit = formatter.width - 6 - max(len(cmd[0]) for cmd in commands) - - rows = [] - for subcommand, cmd in commands: - help = cmd.get_short_help_str(limit) - rows.append((subcommand, help)) - - if rows: - with formatter.section(_("Commands")): - formatter.write_dl(rows) - - def parse_args(self, ctx: Context, args: list[str]) -> list[str]: - if not args and self.no_args_is_help and not ctx.resilient_parsing: - raise NoArgsIsHelpError(ctx) - - rest = super().parse_args(ctx, args) - - if self.chain: - ctx._protected_args = rest - ctx.args = [] - elif rest: - ctx._protected_args, ctx.args = rest[:1], rest[1:] - - return ctx.args - - def invoke(self, ctx: Context) -> t.Any: - def _process_result(value: t.Any) -> t.Any: - if self._result_callback is not None: - value = ctx.invoke(self._result_callback, value, **ctx.params) - return value - - if not ctx._protected_args: - if self.invoke_without_command: - # No subcommand was invoked, so the result callback is - # invoked with the group return value for regular - # groups, or an empty list for chained groups. - with ctx: - rv = super().invoke(ctx) - return _process_result([] if self.chain else rv) - ctx.fail(_("Missing command.")) - - # Fetch args back out - args = [*ctx._protected_args, *ctx.args] - ctx.args = [] - ctx._protected_args = [] - - # If we're not in chain mode, we only allow the invocation of a - # single command but we also inform the current context about the - # name of the command to invoke. - if not self.chain: - # Make sure the context is entered so we do not clean up - # resources until the result processor has worked. - with ctx: - cmd_name, cmd, args = self.resolve_command(ctx, args) - assert cmd is not None - ctx.invoked_subcommand = cmd_name - super().invoke(ctx) - sub_ctx = cmd.make_context(cmd_name, args, parent=ctx) - with sub_ctx: - return _process_result(sub_ctx.command.invoke(sub_ctx)) - - # In chain mode we create the contexts step by step, but after the - # base command has been invoked. Because at that point we do not - # know the subcommands yet, the invoked subcommand attribute is - # set to ``*`` to inform the command that subcommands are executed - # but nothing else. - with ctx: - ctx.invoked_subcommand = "*" if args else None - super().invoke(ctx) - - # Otherwise we make every single context and invoke them in a - # chain. In that case the return value to the result processor - # is the list of all invoked subcommand's results. - contexts = [] - while args: - cmd_name, cmd, args = self.resolve_command(ctx, args) - assert cmd is not None - sub_ctx = cmd.make_context( - cmd_name, - args, - parent=ctx, - allow_extra_args=True, - allow_interspersed_args=False, - ) - contexts.append(sub_ctx) - args, sub_ctx.args = sub_ctx.args, [] - - rv = [] - for sub_ctx in contexts: - with sub_ctx: - rv.append(sub_ctx.command.invoke(sub_ctx)) - return _process_result(rv) - - def resolve_command( - self, ctx: Context, args: list[str] - ) -> tuple[str | None, Command | None, list[str]]: - cmd_name = make_str(args[0]) - - # Get the command - cmd = self.get_command(ctx, cmd_name) - - # If we can't find the command but there is a normalization - # function available, we try with that one. - if cmd is None and ctx.token_normalize_func is not None: - cmd_name = ctx.token_normalize_func(cmd_name) - cmd = self.get_command(ctx, cmd_name) - - # If we don't find the command we want to show an error message - # to the user that it was not provided. However, there is - # something else we should do: if the first argument looks like - # an option we want to kick off parsing again for arguments to - # resolve things like --help which now should go to the main - # place. - if cmd is None and not ctx.resilient_parsing: - if _split_opt(cmd_name)[0]: - self.parse_args(ctx, args) - raise NoSuchCommand(cmd_name, possibilities=self.commands, ctx=ctx) - return cmd_name if cmd else None, cmd, args[1:] - - def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]: - """Return a list of completions for the incomplete value. Looks - at the names of options, subcommands, and chained - multi-commands. - - :param ctx: Invocation context for this command. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - from click.shell_completion import CompletionItem - - results = [ - CompletionItem(name, help=command.get_short_help_str()) - for name, command in _complete_visible_commands(ctx, incomplete) - ] - results.extend(super().shell_complete(ctx, incomplete)) - return results - - -class _MultiCommand(Group, metaclass=_FakeSubclassCheck): - """ - .. deprecated:: 8.2 - Will be removed in Click 9.0. Use ``Group`` instead. - """ - - -class CommandCollection(Group): - """A :class:`Group` that looks up subcommands on other groups. If a command - is not found on this group, each registered source is checked in order. - Parameters on a source are not added to this group, and a source's callback - is not invoked when invoking its commands. In other words, this "flattens" - commands in many groups into this one group. - - :param name: The name of the group command. - :param sources: A list of :class:`Group` objects to look up commands from. - :param kwargs: Other arguments passed to :class:`Group`. - - .. versionchanged:: 8.2 - This is a subclass of ``Group``. Commands are looked up first on this - group, then each of its sources. - """ - - sources: list[Group] - - def __init__( - self, - name: str | None = None, - sources: list[Group] | None = None, - **kwargs: t.Any, - ) -> None: - super().__init__(name, **kwargs) - #: The list of registered groups. - self.sources = sources or [] - - def add_source(self, group: Group) -> None: - """Add a group as a source of commands.""" - self.sources.append(group) - - def get_command(self, ctx: Context, cmd_name: str) -> Command | None: - rv = super().get_command(ctx, cmd_name) - - if rv is not None: - return rv - - for source in self.sources: - rv = source.get_command(ctx, cmd_name) - - if rv is not None: - if self.chain: - _check_nested_chain(self, cmd_name, rv) - - return rv - - return None - - def list_commands(self, ctx: Context) -> list[str]: - rv: set[str] = set(super().list_commands(ctx)) - - for source in self.sources: - rv.update(source.list_commands(ctx)) - - return sorted(rv) - - -def _check_iter(value: cabc.Iterable[V]) -> cabc.Iterator[V]: - """Check if the value is iterable but not a string. Raises a type - error, or return an iterator over the value. - """ - if isinstance(value, str): - raise TypeError - - return iter(value) - - -class Parameter(ABC): - r"""A parameter to a command comes in two versions: they are either - :class:`Option`\s or :class:`Argument`\s. Other subclasses are currently - not supported by design as some of the internals for parsing are - intentionally not finalized. - - Some settings are supported by both options and arguments. - - :param param_decls: the parameter declarations for this option or - argument. This is a list of flags or argument - names. - :param type: the type that should be used. Either a :class:`ParamType` - or a Python type. The latter is converted into the former - automatically if supported. - :param required: controls if this is optional or not. - :param default: the default value if omitted. This can also be a callable, - in which case it's invoked when the default is needed - without any arguments. - :param callback: A function to further process or validate the value - after type conversion. It is called as ``f(ctx, param, value)`` - and must return the value. It is called for all sources, - including prompts. - :param nargs: the number of arguments to match. If not ``1`` the return - value is a tuple instead of single value. The default for - nargs is ``1`` (except if the type is a tuple, then it's - the arity of the tuple). If ``nargs=-1``, all remaining - parameters are collected. - :param metavar: how the value is represented in the help page. - :param expose_value: if this is `True` then the value is passed onwards - to the command callback and stored on the context, - otherwise it's skipped. - :param is_eager: eager values are processed before non eager ones. This - should not be set for arguments or it will inverse the - order of processing. - :param envvar: environment variable(s) that are used to provide a default value for - this parameter. This can be a string or a sequence of strings. If a sequence is - given, only the first non-empty environment variable is used for the parameter. - :param shell_complete: A function that returns custom shell - completions. Used instead of the param's type completion if - given. Takes ``ctx, param, incomplete`` and must return a list - of :class:`~click.shell_completion.CompletionItem` or a list of - strings. - :param deprecated: If ``True`` or non-empty string, issues a message - indicating that the argument is deprecated and highlights - its deprecation in --help. The message can be customized - by using a string as the value. A deprecated parameter - cannot be required, a ValueError will be raised otherwise. - - .. versionchanged:: 8.2.0 - Introduction of ``deprecated``. - - .. versionchanged:: 8.2 - Adding duplicate parameter names to a :class:`~click.core.Command` will - result in a ``UserWarning`` being shown. - - .. versionchanged:: 8.2 - Adding duplicate parameter names to a :class:`~click.core.Command` will - result in a ``UserWarning`` being shown. - - .. versionchanged:: 8.0 - ``process_value`` validates required parameters and bounded - ``nargs``, and invokes the parameter callback before returning - the value. This allows the callback to validate prompts. - ``full_process_value`` is removed. - - .. versionchanged:: 8.0 - ``autocompletion`` is renamed to ``shell_complete`` and has new - semantics described above. The old name is deprecated and will - be removed in 8.1, until then it will be wrapped to match the - new requirements. - - .. versionchanged:: 8.0 - For ``multiple=True, nargs>1``, the default must be a list of - tuples. - - .. versionchanged:: 8.0 - Setting a default is no longer required for ``nargs>1``, it will - default to ``None``. ``multiple=True`` or ``nargs=-1`` will - default to ``()``. - - .. versionchanged:: 7.1 - Empty environment variables are ignored rather than taking the - empty string value. This makes it possible for scripts to clear - variables if they can't unset them. - - .. versionchanged:: 2.0 - Changed signature for parameter callback to also be passed the - parameter. The old callback format will still work, but it will - raise a warning to give you a chance to migrate the code easier. - """ - - param_type_name = "parameter" - - name: str - opts: list[str] - secondary_opts: list[str] - # `Parameter.type` is annotated in `__init__` to avoid confusing mypy - required: bool - callback: t.Callable[[Context, Parameter, t.Any], t.Any] | None - nargs: int - multiple: bool - expose_value: bool - default: t.Any | t.Callable[[], t.Any] | None - _default_explicit: bool - is_eager: bool - metavar: str | None - envvar: str | cabc.Sequence[str] | None - _custom_shell_complete: ( - t.Callable[[Context, Parameter, str], list[CompletionItem] | list[str]] | None - ) - deprecated: bool | str - - def __init__( - self, - param_decls: cabc.Sequence[str] | None = None, - type: types.ParamType[t.Any] | t.Any | None = None, - required: bool = False, - # XXX The default historically embed two concepts: - # - the declaration of a Parameter object carrying the default (handy to - # arbitrage the default value of coupled Parameters sharing the same - # self.name, like flag options), - # - and the actual value of the default. - # It is confusing and is the source of many issues discussed in: - # https://github.com/pallets/click/pull/3030 - # In the future, we might think of splitting it in two, not unlike - # Option.is_flag and Option.flag_value: we could have something like - # Parameter.is_default and Parameter.default_value. - default: t.Any | t.Callable[[], t.Any] | None = UNSET, - callback: t.Callable[[Context, Parameter, t.Any], t.Any] | None = None, - nargs: int | None = None, - multiple: bool = False, - metavar: str | None = None, - expose_value: bool = True, - is_eager: bool = False, - envvar: str | cabc.Sequence[str] | None = None, - shell_complete: t.Callable[ - [Context, Parameter, str], list[CompletionItem] | list[str] - ] - | None = None, - deprecated: bool | str = False, - ) -> None: - self.name, self.opts, self.secondary_opts = self._parse_decls( - param_decls or (), expose_value - ) - self.type: types.ParamType[t.Any] = types.convert_type(type, default) - - # Default nargs to what the type tells us if we have that - # information available. - if nargs is None: - if self.type.is_composite: - nargs = self.type.arity - else: - nargs = 1 - - self.required = required - self.callback = callback - self.nargs = nargs - self.multiple = multiple - self.expose_value = expose_value - self.default = default - # Whether the user passed ``default`` explicitly to the constructor. - # Captured before any auto-derived default (like ``False`` for boolean - # flags in :class:`Option`) replaces the :data:`UNSET` sentinel, so it - # remains ``False`` when the default was inferred rather than chosen. - # Refs: https://github.com/pallets/click/issues/3403 - self._default_explicit = default is not UNSET - self.is_eager = is_eager - self.metavar = metavar - self.envvar = envvar - self._custom_shell_complete = shell_complete - self.deprecated = deprecated - - if __debug__: - if self.type.is_composite and nargs != self.type.arity: - raise ValueError( - f"'nargs' must be {self.type.arity} (or None) for" - f" type {self.type!r}, but it was {nargs}." - ) - - if required and deprecated: - raise ValueError( - f"The {self.param_type_name} '{self.human_readable_name}' " - "is deprecated and still required. A deprecated " - f"{self.param_type_name} cannot be required." - ) - - def to_info_dict(self) -> dict[str, t.Any]: - """Gather information that could be useful for a tool generating - user-facing documentation. - - Use :meth:`click.Context.to_info_dict` to traverse the entire - CLI structure. - - .. versionchanged:: 8.3.0 - Returns ``None`` for the :attr:`default` if it was not set. - - .. versionadded:: 8.0 - """ - return { - "name": self.name, - "param_type_name": self.param_type_name, - "opts": self.opts, - "secondary_opts": self.secondary_opts, - "type": self.type.to_info_dict(), - "required": self.required, - "nargs": self.nargs, - "multiple": self.multiple, - # We explicitly hide the :attr:`UNSET` value to the user, as we choose to - # make it an implementation detail. And because ``to_info_dict`` has been - # designed for documentation purposes, we return ``None`` instead. - "default": self.default if self.default is not UNSET else None, - "envvar": self.envvar, - } - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} {self.name}>" - - @abstractmethod - def _parse_decls( - self, decls: cabc.Sequence[str], expose_value: bool - ) -> tuple[str, list[str], list[str]]: ... - - @property - def human_readable_name(self) -> str: - """Returns the human readable name of this parameter. This is the - same as the name for options, but the metavar for arguments. - """ - return self.name - - def make_metavar(self, ctx: Context) -> str: - if self.metavar is not None: - return self.metavar - - metavar = self.type.get_metavar(param=self, ctx=ctx) - - if metavar is None: - metavar = self.type.name.upper() - - if self.nargs != 1: - metavar += "..." - - return metavar - - @t.overload - def get_default( - self, ctx: Context, call: t.Literal[True] = True - ) -> t.Any | None: ... - - @t.overload - def get_default( - self, ctx: Context, call: bool = ... - ) -> t.Any | t.Callable[[], t.Any] | None: ... - - def get_default( - self, ctx: Context, call: bool = True - ) -> t.Any | t.Callable[[], t.Any] | None: - """Get the default for the parameter. Tries - :meth:`Context.lookup_default` first, then the local default. - - :param ctx: Current context. - :param call: If the default is a callable, call it. Disable to - return the callable instead. - - .. versionchanged:: 8.0.2 - Type casting is no longer performed when getting a default. - - .. versionchanged:: 8.0.1 - Type casting can fail in resilient parsing mode. Invalid - defaults will not prevent showing help text. - - .. versionchanged:: 8.0 - Looks at ``ctx.default_map`` first. - - .. versionchanged:: 8.0 - Added the ``call`` parameter. - """ - value = ctx.lookup_default(self.name, call=False) - - if value is None and not ctx._default_map_has(self.name): - value = self.default - - if call and callable(value): - value = value() - - return value - - @abstractmethod - def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None: ... - - def consume_value( - self, ctx: Context, opts: cabc.Mapping[str, t.Any] - ) -> tuple[t.Any, ParameterSource]: - """Returns the parameter value produced by the parser. - - If the parser did not produce a value from user input, the value is either - sourced from the environment variable, the default map, or the parameter's - default value. In that order of precedence. - - If no value is found, an internal sentinel value is returned. - - :meta private: - """ - # Collect from the parse the value passed by the user to the CLI. - value = opts.get(self.name, UNSET) - # If the value is set, it means it was sourced from the command line by the - # parser, otherwise it left unset by default. - source = ( - ParameterSource.COMMANDLINE - if value is not UNSET - else ParameterSource.DEFAULT - ) - - if value is UNSET: - envvar_value = self.value_from_envvar(ctx) - if envvar_value is not None: - value = envvar_value - source = ParameterSource.ENVIRONMENT - - if value is UNSET: - default_map_value = ctx.lookup_default(self.name) - if default_map_value is not None or ctx._default_map_has(self.name): - value = default_map_value - source = ParameterSource.DEFAULT_MAP - - # A string from default_map must be split for multi-value - # parameters, matching value_from_envvar behavior. - if isinstance(value, str) and self.nargs != 1: - value = self.type.split_envvar_value(value) - - if value is UNSET: - default_value = self.get_default(ctx) - if default_value is not UNSET: - value = default_value - source = ParameterSource.DEFAULT - - return value, source - - def type_cast_value(self, ctx: Context, value: t.Any) -> t.Any: - """Convert and validate a value against the parameter's - :attr:`type`, :attr:`multiple`, and :attr:`nargs`. - """ - if value is None: - if self.multiple or self.nargs == -1: - return () - else: - return value - - def check_iter(value: t.Any) -> cabc.Iterator[t.Any]: - try: - return _check_iter(value) - except TypeError: - # This should only happen when passing in args manually, - # the parser should construct an iterable when parsing - # the command line. - raise BadParameter( - _("Value must be an iterable."), ctx=ctx, param=self - ) from None - - # Define the conversion function based on nargs and type. - - if self.nargs == 1 or self.type.is_composite: - - def convert(value: t.Any) -> t.Any: - return self.type(value, param=self, ctx=ctx) - - elif self.nargs == -1: - - def convert(value: t.Any) -> t.Any: # tuple[t.Any, ...] - return tuple(self.type(x, self, ctx) for x in check_iter(value)) - - else: # nargs > 1 - - def convert(value: t.Any) -> t.Any: # tuple[t.Any, ...] - value = tuple(check_iter(value)) - - if len(value) != self.nargs: - raise BadParameter( - ngettext( - "Takes {nargs} values but 1 was given.", - "Takes {nargs} values but {len} were given.", - len(value), - ).format(nargs=self.nargs, len=len(value)), - ctx=ctx, - param=self, - ) - - return tuple(self.type(x, self, ctx) for x in value) - - if self.multiple: - return tuple(convert(x) for x in check_iter(value)) - - return convert(value) - - def value_is_missing(self, value: t.Any) -> bool: - """A value is considered missing if: - - - it is :attr:`UNSET`, - - or if it is an empty sequence while the parameter is suppose to have - non-single value (i.e. :attr:`nargs` is not ``1`` or :attr:`multiple` is - set). - - :meta private: - """ - if value is UNSET: - return True - - if (self.nargs != 1 or self.multiple) and value == (): - return True - - return False - - def process_value(self, ctx: Context, value: t.Any) -> t.Any: - """Process the value of this parameter: - - 1. Type cast the value using :meth:`type_cast_value`. - 2. Check if the value is missing (see: :meth:`value_is_missing`), and raise - :exc:`MissingParameter` if it is required. - 3. If a :attr:`callback` is set, call it to have the value replaced by the - result of the callback. If the value was not set, the callback receive - ``None``. This keep the legacy behavior as it was before the introduction of - the :attr:`UNSET` sentinel. - - :meta private: - """ - # shelter `type_cast_value` from ever seeing an `UNSET` value by handling the - # cases in which `UNSET` gets special treatment explicitly at this layer - # - # Refs: - # https://github.com/pallets/click/issues/3069 - if value is UNSET: - if self.multiple or self.nargs == -1: - value = () - else: - value = self.type_cast_value(ctx, value) - - if self.required and self.value_is_missing(value): - raise MissingParameter(ctx=ctx, param=self) - - if self.callback is not None: - # Legacy case: UNSET is not exposed directly to the callback, but converted - # to None. - if value is UNSET: - value = None - - # Search for parameters with UNSET values in the context. - unset_keys = {k: None for k, v in ctx.params.items() if v is UNSET} - # No UNSET values, call the callback as usual. - if not unset_keys: - value = self.callback(ctx, self, value) - - # Legacy case: provide a temporarily manipulated context to the callback - # to hide UNSET values as None. - # - # Refs: - # https://github.com/pallets/click/issues/3136 - # https://github.com/pallets/click/pull/3137 - else: - # Add another layer to the context stack to clearly hint that the - # context is temporarily modified. - with ctx: - # Update the context parameters to replace UNSET with None. - ctx.params.update(unset_keys) - # Feed these fake context parameters to the callback. - value = self.callback(ctx, self, value) - # Restore the UNSET values in the context parameters. - ctx.params.update( - { - k: UNSET - for k in unset_keys - # Only restore keys that are present and still None, in case - # the callback modified other parameters. - if k in ctx.params and ctx.params[k] is None - } - ) - - return value - - def resolve_envvar_value(self, ctx: Context) -> str | None: - """Returns the value found in the environment variable(s) attached to this - parameter. - - Environment variables values are `always returned as strings - `_. - - This method returns ``None`` if: - - - the :attr:`envvar` property is not set on the :class:`Parameter`, - - the environment variable is not found in the environment, - - the variable is found in the environment but its value is empty (i.e. the - environment variable is present but has an empty string). - - If :attr:`envvar` is setup with multiple environment variables, - then only the first non-empty value is returned. - - .. caution:: - - The raw value extracted from the environment is not normalized and is - returned as-is. Any normalization or reconciliation is performed later by - the :class:`Parameter`'s :attr:`type`. - - :meta private: - """ - if not self.envvar: - return None - - if isinstance(self.envvar, str): - rv = os.environ.get(self.envvar) - - if rv: - return rv - else: - for envvar in self.envvar: - rv = os.environ.get(envvar) - - # Return the first non-empty value of the list of environment variables. - if rv: - return rv - # Else, absence of value is interpreted as an environment variable that - # is not set, so proceed to the next one. - - return None - - def value_from_envvar(self, ctx: Context) -> str | cabc.Sequence[str] | None: - """Process the raw environment variable string for this parameter. - - Returns the string as-is or splits it into a sequence of strings if the - parameter is expecting multiple values (i.e. its :attr:`nargs` property is set - to a value other than ``1``). - - :meta private: - """ - rv = self.resolve_envvar_value(ctx) - - if rv is not None and self.nargs != 1: - return self.type.split_envvar_value(rv) - - return rv - - def handle_parse_result( - self, ctx: Context, opts: cabc.Mapping[str, t.Any], args: list[str] - ) -> tuple[t.Any, list[str]]: - """Process the value produced by the parser from user input. - - Always process the value through the Parameter's :attr:`type`, wherever it - comes from. - - If the parameter is deprecated, this method warn the user about it. But only if - the value has been explicitly set by the user (and as such, is not coming from - a default). - - :meta private: - """ - # Capture the slot's existing state before we mutate - # ``_parameter_source`` so the write decision below can compare our - # incoming source against the source of the option that already wrote - # the slot (if any). - existing_value = ctx.params.get(self.name, UNSET) - existing_source = ctx.get_parameter_source(self.name) - existing_default_explicit = ctx._param_default_explicit.get(self.name, False) - - with augment_usage_errors(ctx, param=self): - value, source = self.consume_value(ctx, opts) - - # Record the source before processing so eager callbacks and type - # conversion can inspect it. Restored after arbitration if this - # option loses a feature-switch group. - ctx.set_parameter_source(self.name, source) - - # Display a deprecation warning if necessary. - if ( - self.deprecated - and value is not UNSET - and source < ParameterSource.DEFAULT_MAP - ): - message = _( - "DeprecationWarning: The {param_type} {name!r} is deprecated." - "{extra_message}" - ).format( - param_type=self.param_type_name, - name=self.human_readable_name, - extra_message=_format_deprecated_suffix(self.deprecated), - ) - echo(style(message, fg="red"), err=True) - - # Process the value through the parameter's type. - try: - value = self.process_value(ctx, value) - except Exception: - if not ctx.resilient_parsing: - raise - # In resilient parsing mode, we do not want to fail the command if the - # value is incompatible with the parameter type, so we reset the value - # to UNSET, which will be interpreted as a missing value. - value = UNSET - - # Arbitrate the slot when several parameters target the same variable - # name (feature-switch groups). See: https://github.com/pallets/click/issues/3403 - slot_empty = existing_value is UNSET - more_explicit = existing_source is not None and source < existing_source - same_source = existing_source is not None and source == existing_source - auto_would_downgrade_explicit = ( - same_source - and source == ParameterSource.DEFAULT - and existing_default_explicit - and not self._default_explicit - ) - is_winner = ( - slot_empty - or more_explicit - or (same_source and not auto_would_downgrade_explicit) - ) - - if is_winner: - if self.expose_value: - ctx.params[self.name] = value - ctx._param_default_explicit[self.name] = self._default_explicit - elif existing_source is not None: - # Lost arbitration; restore the winning option's source. - ctx.set_parameter_source(self.name, existing_source) - # else: ctx.params[self.name] was populated by code that bypassed - # handle_parse_result (from another option's callback for example). Keep - # the provisional source recorded before process_value so downstream - # lookups don't return ``None``. - - return value, args - - def get_help_record(self, ctx: Context) -> tuple[str, str] | None: - return None - - def get_usage_pieces(self, ctx: Context) -> list[str]: - return [] - - def get_error_hint(self, ctx: Context | None) -> str: - """Get a stringified version of the param for use in error messages to - indicate which param caused the error. - - .. versionchanged:: 8.4.0 - ``ctx`` can be ``None``. - """ - hint_list = self.opts or [self.human_readable_name] - return " / ".join(f"'{x}'" for x in hint_list) - - def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]: - """Return a list of completions for the incomplete value. If a - ``shell_complete`` function was given during init, it is used. - Otherwise, the :attr:`type` - :meth:`~click.types.ParamType[t.Any].shell_complete` function is used. - - :param ctx: Invocation context for this command. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - if self._custom_shell_complete is not None: - results = self._custom_shell_complete(ctx, self, incomplete) - - if results and isinstance(results[0], str): - from click.shell_completion import CompletionItem - - results = [CompletionItem(c) for c in results] - - return t.cast("list[CompletionItem]", results) - - return self.type.shell_complete(ctx, self, incomplete) - - -class Option(Parameter): - """Options are usually optional values on the command line and - have some extra features that arguments don't have. - - All other parameters are passed onwards to the parameter constructor. - - :param show_default: Show the default value for this option in its - help text. Values are not shown by default, unless - :attr:`Context.show_default` is ``True``. If this value is a - string, it shows that string in parentheses instead of the - actual value. This is particularly useful for dynamic options. - For single option boolean flags, the default remains hidden if - its value is ``False``. - :param show_envvar: Controls if an environment variable should be - shown on the help page and error messages. - Normally, environment variables are not shown. - :param prompt: If set to ``True`` or a non empty string then the - user will be prompted for input. If set to ``True`` the prompt - will be the option name capitalized. A deprecated option cannot be - prompted. - :param confirmation_prompt: Prompt a second time to confirm the - value if it was prompted for. Can be set to a string instead of - ``True`` to customize the message. - :param prompt_required: If set to ``False``, the user will be - prompted for input only when the option was specified as a flag - without a value. - :param hide_input: If this is ``True`` then the input on the prompt - will be hidden from the user. This is useful for password input. - :param is_flag: forces this option to act as a flag. The default is - auto detection. - :param flag_value: which value should be used for this flag if it's - enabled. This is set to a boolean automatically if - the option string contains a slash to mark two options. - :param multiple: if this is set to `True` then the argument is accepted - multiple times and recorded. This is similar to ``nargs`` - in how it works but supports arbitrary number of - arguments. - :param count: this flag makes an option increment an integer. - :param allow_from_autoenv: if this is enabled then the value of this - parameter will be pulled from an environment - variable in case a prefix is defined on the - context. - :param help: the help string. - :param hidden: hide this option from help outputs. - :param attrs: Other command arguments described in :class:`Parameter`. - - .. versionchanged:: 8.4.0 - Non-basic ``flag_value`` types (not ``str``, ``int``, ``float``, or - ``bool``) are passed through unchanged instead of being stringified. - Previously, ``type=click.UNPROCESSED`` was required to preserve them. - - .. versionchanged:: 8.2 - ``envvar`` used with ``flag_value`` will always use the ``flag_value``, - previously it would use the value of the environment variable. - - .. versionchanged:: 8.1 - Help text indentation is cleaned here instead of only in the - ``@option`` decorator. - - .. versionchanged:: 8.1 - The ``show_default`` parameter overrides - ``Context.show_default``. - - .. versionchanged:: 8.1 - The default of a single option boolean flag is not shown if the - default value is ``False``. - - .. versionchanged:: 8.0.1 - ``type`` is detected from ``flag_value`` if given, for basic Python - types (``str``, ``int``, ``float``, ``bool``). - """ - - param_type_name = "option" - - prompt: str | None - confirmation_prompt: bool | str - prompt_required: bool - hide_input: bool - hidden: bool - - _flag_needs_value: bool - is_flag: bool - is_bool_flag: bool - flag_value: t.Any - - count: bool - allow_from_autoenv: bool - help: str | None - show_default: bool | str | None - show_choices: bool - show_envvar: bool - - def __init__( - self, - param_decls: cabc.Sequence[str] | None = None, - show_default: bool | str | None = None, - prompt: bool | str = False, - confirmation_prompt: bool | str = False, - prompt_required: bool = True, - hide_input: bool = False, - is_flag: bool | None = None, - flag_value: t.Any = UNSET, - multiple: bool = False, - count: bool = False, - allow_from_autoenv: bool = True, - type: types.ParamType[t.Any] | t.Any | None = None, - help: str | None = None, - hidden: bool = False, - show_choices: bool = True, - show_envvar: bool = False, - deprecated: bool | str = False, - **attrs: t.Any, - ) -> None: - if help: - help = inspect.cleandoc(help) - - super().__init__( - param_decls, type=type, multiple=multiple, deprecated=deprecated, **attrs - ) - - if prompt is True: - if not self.name: - raise TypeError("'name' is required with 'prompt=True'.") - - prompt_text = self.name.replace("_", " ").capitalize() - elif prompt is False: - prompt_text = None - else: - prompt_text = prompt - - if deprecated: - label = _format_deprecated_label(deprecated) - help = f"{help} {label}" if help else label - - self.prompt = prompt_text - self.confirmation_prompt = confirmation_prompt - self.prompt_required = prompt_required - self.hide_input = hide_input - self.hidden = hidden - - # The _flag_needs_value property tells the parser that this option is a flag - # that cannot be used standalone and needs a value. With this information, the - # parser can determine whether to consider the next user-provided argument in - # the CLI as a value for this flag or as a new option. - # If prompt is enabled but not required, then it opens the possibility for the - # option to gets its value from the user. - self._flag_needs_value = self.prompt is not None and not self.prompt_required - - # Auto-detect if this is a flag or not. - if is_flag is None: - # Implicitly a flag because flag_value was set. - if flag_value is not UNSET: - is_flag = True - # Not a flag, but when used as a flag it shows a prompt. - elif self._flag_needs_value: - is_flag = False - # Implicitly a flag because secondary options names were given. - elif self.secondary_opts: - is_flag = True - - # The option is explicitly not a flag, but to determine whether or not it needs - # value, we need to check if `flag_value` or `default` was set. Either one is - # sufficient. - # Ref: https://github.com/pallets/click/issues/3084 - elif is_flag is False and not self._flag_needs_value: - self._flag_needs_value = flag_value is not UNSET or self.default is UNSET - - if is_flag: - # Set missing default for flags if not explicitly required or prompted. - if self.default is UNSET and not self.required and not self.prompt: - if multiple: - self.default = () - - # Auto-detect the type of the flag based on the flag_value. - if type is None: - # A flag without a flag_value is a boolean flag. - if flag_value is UNSET: - self.type: types.ParamType[t.Any] = types.BoolParamType() - # If the flag value is a boolean, use BoolParamType. - elif isinstance(flag_value, bool): - self.type = types.BoolParamType() - # Otherwise, guess the type from the flag value. - else: - guessed = types.convert_type(None, flag_value) - if ( - isinstance(guessed, types.StringParamType) - and not isinstance(flag_value, str) - and flag_value is not None - ): - # The flag_value type couldn't be auto-detected - # (not str, int, float, or bool). Since flag_value - # is a programmer-provided Python object, not CLI - # input, pass it through unchanged instead of - # stringifying it. - self.type = types.UNPROCESSED - else: - self.type = guessed - - self.is_flag = bool(is_flag) - self.is_bool_flag = self.is_flag and isinstance(self.type, types.BoolParamType) - self.flag_value = flag_value - - # Set boolean flag default to False if unset and not required. - if self.is_bool_flag: - if self.default is UNSET and not self.required: - self.default = False - - # The alignment of default to the flag_value is resolved lazily in - # get_default() to prevent callable flag_values (like classes) from - # being instantiated. Refs: - # https://github.com/pallets/click/issues/3121 - # https://github.com/pallets/click/issues/3024#issuecomment-3146199461 - # https://github.com/pallets/click/pull/3030/commits/06847da - - # Set the default flag_value if it is not set. - if self.flag_value is UNSET: - if self.is_flag: - self.flag_value = True - else: - self.flag_value = None - - # Counting. - self.count = count - if count: - if type is None: - self.type = types.IntRange(min=0) - if self.default is UNSET: - self.default = 0 - - self.allow_from_autoenv = allow_from_autoenv - self.help = help - self.show_default = show_default - self.show_choices = show_choices - self.show_envvar = show_envvar - - if __debug__: - if deprecated and prompt: - raise ValueError("`deprecated` options cannot use `prompt`.") - - if self.nargs == -1: - raise TypeError("nargs=-1 is not supported for options.") - - if not self.is_bool_flag and self.secondary_opts: - raise TypeError("Secondary flag is not valid for non-boolean flag.") - - if self.is_bool_flag and self.hide_input and self.prompt is not None: - raise TypeError( - "'prompt' with 'hide_input' is not valid for boolean flag." - ) - - if self.count: - if self.multiple: - raise TypeError("'count' is not valid with 'multiple'.") - - if self.is_flag: - raise TypeError("'count' is not valid with 'is_flag'.") - - def to_info_dict(self) -> dict[str, t.Any]: - """ - .. versionchanged:: 8.3.0 - Returns ``None`` for the :attr:`flag_value` if it was not set. - """ - info_dict = super().to_info_dict() - info_dict.update( - help=self.help, - prompt=self.prompt, - is_flag=self.is_flag, - # We explicitly hide the :attr:`UNSET` value to the user, as we choose to - # make it an implementation detail. And because ``to_info_dict`` has been - # designed for documentation purposes, we return ``None`` instead. - flag_value=self.flag_value if self.flag_value is not UNSET else None, - count=self.count, - hidden=self.hidden, - ) - return info_dict - - def get_default( - self, ctx: Context, call: bool = True - ) -> t.Any | t.Callable[[], t.Any] | None: - """Return the default value for this option. - - For non-boolean flag options, ``default=True`` is treated as a sentinel - meaning "activate this flag by default" and is resolved to - :attr:`flag_value`. For example, with ``--upper/--lower`` feature - switches where ``flag_value="upper"`` and ``default=True``, the default - resolves to ``"upper"``. - - .. caution:: - This substitution only applies to non-boolean flags - (:attr:`is_bool_flag` is ``False``). For boolean flags, ``True`` is - a legitimate Python value and ``default=True`` is returned as-is. - - .. versionchanged:: 8.3.3 - ``default=True`` is no longer substituted with ``flag_value`` for - boolean flags, fixing negative boolean flags like - ``flag_value=False, default=True``. - """ - value = super().get_default(ctx, call=False) - - # Resolve default=True to flag_value lazily (here instead of - # __init__) to prevent callable flag_values (like classes) from - # being instantiated by the callable check below. - if value is True and self.is_flag and not self.is_bool_flag: - value = self.flag_value - elif call and callable(value): - value = value() - - return value - - def get_error_hint(self, ctx: Context | None) -> str: - result = super().get_error_hint(ctx) - if self.show_envvar and self.envvar is not None: - result += f" (env var: '{self.envvar}')" - return result - - def _parse_decls( - self, decls: cabc.Sequence[str], expose_value: bool - ) -> tuple[str, list[str], list[str]]: - opts = [] - secondary_opts = [] - name = None - possible_names = [] - - for decl in decls: - if decl.isidentifier(): - if name is not None: - raise TypeError(_("Name '{name}' defined twice").format(name=name)) - name = decl - else: - split_char = ";" if decl[:1] == "/" else "/" - if split_char in decl: - first, second = decl.split(split_char, 1) - first = first.rstrip() - if first: - possible_names.append(_split_opt(first)) - opts.append(first) - second = second.lstrip() - if second: - secondary_opts.append(second.lstrip()) - if first == second: - raise ValueError( - _( - "Boolean option {decl!r} cannot use the" - " same flag for true/false." - ).format(decl=decl) - ) - else: - possible_names.append(_split_opt(decl)) - opts.append(decl) - - if name is None and possible_names: - possible_names.sort(key=lambda x: -len(x[0])) # group long options first - name = possible_names[0][1].replace("-", "_").lower() - if not name.isidentifier(): - name = None - - if name is None: - if not expose_value: - return "", opts, secondary_opts - raise TypeError( - _( - "Could not determine name for option with declarations {decls!r}" - ).format(decls=decls) - ) - - if not opts and not secondary_opts: - raise TypeError( - _( - "No options defined but a name was passed ({name})." - " Did you mean to declare an argument instead? Did" - " you mean to pass '--{name}'?" - ).format(name=name) - ) - - return name, opts, secondary_opts - - def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None: - if self.multiple: - action = "append" - elif self.count: - action = "count" - else: - action = "store" - - if self.is_flag: - action = f"{action}_const" - - if self.is_bool_flag and self.secondary_opts: - parser.add_option( - obj=self, opts=self.opts, dest=self.name, action=action, const=True - ) - parser.add_option( - obj=self, - opts=self.secondary_opts, - dest=self.name, - action=action, - const=False, - ) - else: - parser.add_option( - obj=self, - opts=self.opts, - dest=self.name, - action=action, - const=self.flag_value, - ) - else: - parser.add_option( - obj=self, - opts=self.opts, - dest=self.name, - action=action, - nargs=self.nargs, - ) - - def get_help_record(self, ctx: Context) -> tuple[str, str] | None: - if self.hidden: - return None - - any_prefix_is_slash = False - - def _write_opts(opts: cabc.Sequence[str]) -> str: - nonlocal any_prefix_is_slash - - rv, any_slashes = join_options(opts) - - if any_slashes: - any_prefix_is_slash = True - - if not self.is_flag and not self.count: - rv += f" {self.make_metavar(ctx=ctx)}" - - return rv - - rv = [_write_opts(self.opts)] - - if self.secondary_opts: - rv.append(_write_opts(self.secondary_opts)) - - help = self.help or "" - - extra = self.get_help_extra(ctx) - extra_items = [] - if "envvars" in extra: - extra_items.append( - _("env var: {var}").format(var=", ".join(extra["envvars"])) - ) - if "default" in extra: - extra_items.append(_("default: {default}").format(default=extra["default"])) - if "range" in extra: - extra_items.append(extra["range"]) - if "required" in extra: - extra_items.append(_(extra["required"])) - - if extra_items: - extra_str = "; ".join(extra_items) - help = f"{help} [{extra_str}]" if help else f"[{extra_str}]" - - return ("; " if any_prefix_is_slash else " / ").join(rv), help - - def get_help_extra(self, ctx: Context) -> types.OptionHelpExtra: - extra: types.OptionHelpExtra = {} - - if self.show_envvar: - envvar = self.envvar - - if envvar is None: - if ( - self.allow_from_autoenv - and ctx.auto_envvar_prefix is not None - and self.name - ): - envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}" - - if envvar is not None: - if isinstance(envvar, str): - extra["envvars"] = (envvar,) - else: - extra["envvars"] = tuple(str(d) for d in envvar) - - # Temporarily enable resilient parsing to avoid type casting - # failing for the default. Might be possible to extend this to - # help formatting in general. - resilient = ctx.resilient_parsing - ctx.resilient_parsing = True - - try: - default_value = self.get_default(ctx, call=False) - finally: - ctx.resilient_parsing = resilient - - show_default = False - show_default_is_str = False - - if self.show_default is not None: - if isinstance(self.show_default, str): - show_default_is_str = show_default = True - else: - show_default = self.show_default - elif ctx.show_default is not None: - show_default = ctx.show_default - - if show_default_is_str or ( - show_default and (default_value not in (None, UNSET)) - ): - if show_default_is_str: - default_string = f"({self.show_default})" - elif isinstance(default_value, (list, tuple)): - default_string = ", ".join(str(d) for d in default_value) - elif isinstance(default_value, enum.Enum): - default_string = default_value.name - elif inspect.isfunction(default_value): - default_string = _("(dynamic)") - elif self.is_bool_flag and self.secondary_opts: - # For boolean flags that have distinct True/False opts, - # use the opt without prefix instead of the value. - default_string = _split_opt( - (self.opts if default_value else self.secondary_opts)[0] - )[1] - elif self.is_bool_flag and not self.secondary_opts and not default_value: - default_string = "" - elif isinstance(default_value, str) and default_value == "": - default_string = '""' - else: - default_string = str(default_value) - - if default_string: - extra["default"] = default_string - - if ( - isinstance(self.type, types._NumberRangeBase) - # skip count with default range type - and not (self.count and self.type.min == 0 and self.type.max is None) - ): - range_str = self.type._describe_range() - - if range_str: - extra["range"] = range_str - - if self.required: - extra["required"] = "required" - - return extra - - def prompt_for_value(self, ctx: Context) -> t.Any: - """This is an alternative flow that can be activated in the full - value processing if a value does not exist. It will prompt the - user until a valid value exists and then returns the processed - value as result. - """ - assert self.prompt is not None - - # Calculate the default before prompting anything to lock in the value before - # attempting any user interaction. - default = self.get_default(ctx) - - # A boolean flag can use a simplified [y/n] confirmation prompt. - if self.is_bool_flag: - # If we have no boolean default, we force the user to explicitly provide - # one. - if default in (UNSET, None): - default = None - # Nothing prevent you to declare an option that is simultaneously: - # 1) auto-detected as a boolean flag, - # 2) allowed to prompt, and - # 3) still declare a non-boolean default. - # This forced casting into a boolean is necessary to align any non-boolean - # default to the prompt, which is going to be a [y/n]-style confirmation - # because the option is still a boolean flag. That way, instead of [y/n], - # we get [Y/n] or [y/N] depending on the truthy value of the default. - # Refs: https://github.com/pallets/click/pull/3030#discussion_r2289180249 - else: - default = bool(default) - return confirm(self.prompt, default) - - # If show_default is given, provide this to `prompt` as well, - # otherwise we use `prompt`'s default behavior - prompt_kwargs: t.Any = {} - if self.show_default is not None: - prompt_kwargs["show_default"] = self.show_default - - return prompt( - self.prompt, - # Use ``None`` to inform the prompt() function to reiterate until a valid - # value is provided by the user if we have no default. - default=None if default is UNSET else default, - type=self.type, - hide_input=self.hide_input, - show_choices=self.show_choices, - confirmation_prompt=self.confirmation_prompt, - value_proc=lambda x: self.process_value(ctx, x), - **prompt_kwargs, - ) - - def resolve_envvar_value(self, ctx: Context) -> str | None: - """:class:`Option` resolves its environment variable the same way as - :func:`Parameter.resolve_envvar_value`, but it also supports - :attr:`Context.auto_envvar_prefix`. If we could not find an environment from - the :attr:`envvar` property, we fallback on :attr:`Context.auto_envvar_prefix` - to build dynamiccaly the environment variable name using the - :python:`{ctx.auto_envvar_prefix}_{self.name.upper()}` template. - - :meta private: - """ - rv = super().resolve_envvar_value(ctx) - - if rv is not None: - return rv - - if self.allow_from_autoenv and ctx.auto_envvar_prefix is not None and self.name: - envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}" - rv = os.environ.get(envvar) - - if rv: - return rv - - return None - - def value_from_envvar(self, ctx: Context) -> t.Any: - """For :class:`Option`, this method processes the raw environment variable - string the same way as :func:`Parameter.value_from_envvar` does. - - But in the case of non-boolean flags, the value is analyzed to determine if the - flag is activated or not, and returns a boolean of its activation, or the - :attr:`flag_value` if the latter is set. - - This method also takes care of repeated options (i.e. options with - :attr:`multiple` set to ``True``). - - :meta private: - """ - rv = self.resolve_envvar_value(ctx) - - # Absent environment variable or an empty string is interpreted as unset. - if rv is None: - return None - - # Non-boolean flags are more liberal in what they accept. But a flag being a - # flag, its envvar value still needs to be analyzed to determine if the flag is - # activated or not. - if self.is_flag and not self.is_bool_flag: - # If the flag_value is set and match the envvar value, return it - # directly. - if self.flag_value is not UNSET and rv == self.flag_value: - return self.flag_value - # Analyze the envvar value as a boolean to know if the flag is - # activated or not. - return types.BoolParamType.str_to_bool(rv) - - # Split the envvar value if it is allowed to be repeated. - value_depth = (self.nargs != 1) + bool(self.multiple) - if value_depth > 0: - multi_rv = self.type.split_envvar_value(rv) - if self.multiple and self.nargs != 1: - multi_rv = batch(multi_rv, self.nargs) # type: ignore[assignment] - - return multi_rv - - return rv - - def consume_value( - self, ctx: Context, opts: cabc.Mapping[str, Parameter] - ) -> tuple[t.Any, ParameterSource]: - """For :class:`Option`, the value can be collected from an interactive prompt - if the option is a flag that needs a value (and the :attr:`prompt` property is - set). - - Additionally, this method handles flag option that are activated without a - value, in which case the :attr:`flag_value` is returned. - - :meta private: - """ - value, source = super().consume_value(ctx, opts) - - # The parser will emit a sentinel value if the option is allowed to as a flag - # without a value. - if value is FLAG_NEEDS_VALUE: - # If the option allows for a prompt, we start an interaction with the user. - if self.prompt is not None and not ctx.resilient_parsing: - value = self.prompt_for_value(ctx) - source = ParameterSource.PROMPT - # Else the flag takes its flag_value as value. - else: - value = self.flag_value - source = ParameterSource.COMMANDLINE - - # A flag which is activated always returns the flag value, unless the value - # comes from the explicitly sets default. - elif ( - self.is_flag - and value is True - and not self.is_bool_flag - and source < ParameterSource.DEFAULT_MAP - ): - value = self.flag_value - - # Re-interpret a multiple option which has been sent as-is by the parser. - # Here we replace each occurrence of value-less flags (marked by the - # FLAG_NEEDS_VALUE sentinel) with the flag_value. - elif ( - self.multiple - and value is not UNSET - and isinstance(value, cabc.Iterable) - and source < ParameterSource.DEFAULT_MAP - and any(v is FLAG_NEEDS_VALUE for v in value) - ): - value = [self.flag_value if v is FLAG_NEEDS_VALUE else v for v in value] - source = ParameterSource.COMMANDLINE - - # The value wasn't set, or used the param's default, prompt for one to the user - # if prompting is enabled. - elif ( - (value is UNSET or source >= ParameterSource.DEFAULT_MAP) - and self.prompt is not None - and (self.required or self.prompt_required) - and not ctx.resilient_parsing - ): - value = self.prompt_for_value(ctx) - source = ParameterSource.PROMPT - - return value, source - - def process_value(self, ctx: Context, value: t.Any) -> t.Any: - # process_value has to be overridden on Options in order to capture - # `value == UNSET` cases before `type_cast_value()` gets called. - # - # Refs: - # https://github.com/pallets/click/issues/3069 - if self.is_flag and not self.required and self.is_bool_flag and value is UNSET: - value = False - - if self.callback is not None: - value = self.callback(ctx, self, value) - - return value - - # in the normal case, rely on Parameter.process_value - return super().process_value(ctx, value) - - -class Argument(Parameter): - """Arguments are positional parameters to a command. They generally - provide fewer features than options but can have infinite ``nargs`` - and are required by default. - - All parameters are passed onwards to the constructor of :class:`Parameter`. - """ - - param_type_name = "argument" - - def __init__( - self, - param_decls: cabc.Sequence[str], - required: bool | None = None, - **attrs: t.Any, - ) -> None: - # Auto-detect the requirement status of the argument if not explicitly set. - if required is None: - # The argument gets automatically required if it has no explicit default - # value set and is setup to match at least one value. - if attrs.get("default", UNSET) is UNSET: - required = attrs.get("nargs", 1) > 0 - # If the argument has a default value, it is not required. - else: - required = False - - if "multiple" in attrs: - raise TypeError("__init__() got an unexpected keyword argument 'multiple'.") - - super().__init__(param_decls, required=required, **attrs) - - @property - def human_readable_name(self) -> str: - if self.metavar is not None: - return self.metavar - return self.name.upper() - - def make_metavar(self, ctx: Context) -> str: - if self.metavar is not None: - return self.metavar - var = self.type.get_metavar(param=self, ctx=ctx) - if not var: - var = self.name.upper() - # Types like ``Choice`` and ``DateTime`` already surround their metavar - # with square brackets to enumerate the allowed values. Reuse those - # outer brackets as the optional-argument indicator instead of wrapping - # the metavar in a second pair, which would produce ``[[a|b|c]]``. - already_bracketed = var.startswith("[") and var.endswith("]") - if self.deprecated: - var += "!" - if not self.required and not already_bracketed: - var = f"[{var}]" - if self.nargs != 1: - var += "..." - return var - - def _parse_decls( - self, decls: cabc.Sequence[str], expose_value: bool - ) -> tuple[str, list[str], list[str]]: - if not decls: - if not expose_value: - return "", [], [] - raise TypeError("Argument is marked as exposed, but does not have a name.") - if len(decls) == 1: - name = arg = decls[0] - name = name.replace("-", "_").lower() - else: - raise TypeError( - _( - "Arguments take exactly one parameter declaration, got" - " {length}: {decls}." - ).format(length=len(decls), decls=decls) - ) - return name, [arg], [] - - def get_usage_pieces(self, ctx: Context) -> list[str]: - return [self.make_metavar(ctx)] - - def get_error_hint(self, ctx: Context | None) -> str: - if ctx is not None: - return f"'{self.make_metavar(ctx)}'" - return f"'{self.human_readable_name}'" - - def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None: - parser.add_argument(dest=self.name, nargs=self.nargs, obj=self) - - -def __getattr__(name: str) -> object: - import warnings - - if name == "BaseCommand": - warnings.warn( - "'BaseCommand' is deprecated and will be removed in Click 9.0. Use" - " 'Command' instead.", - DeprecationWarning, - stacklevel=2, - ) - return _BaseCommand - - if name == "MultiCommand": - warnings.warn( - "'MultiCommand' is deprecated and will be removed in Click 9.0. Use" - " 'Group' instead.", - DeprecationWarning, - stacklevel=2, - ) - return _MultiCommand - - raise AttributeError(name) diff --git a/bundle/python-cpu/Lib/site-packages/click/decorators.py b/bundle/python-cpu/Lib/site-packages/click/decorators.py deleted file mode 100644 index db6a45ebbaedfdcde339397bbfe936c9440de180..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/decorators.py +++ /dev/null @@ -1,575 +0,0 @@ -from __future__ import annotations - -import inspect -import typing as t -from functools import update_wrapper -from gettext import gettext as _ - -from .core import Argument -from .core import Command -from .core import Context -from .core import Group -from .core import Option -from .core import Parameter -from .globals import get_current_context -from .utils import echo - -if t.TYPE_CHECKING: - import typing_extensions as te - - P = te.ParamSpec("P") - -R = t.TypeVar("R") -T = t.TypeVar("T") -_AnyCallable = t.Callable[..., t.Any] -FC = t.TypeVar("FC", bound="_AnyCallable | Command") - - -def pass_context(f: t.Callable[te.Concatenate[Context, P], R]) -> t.Callable[P, R]: - """Marks a callback as wanting to receive the current context - object as first argument. - """ - - def new_func(*args: P.args, **kwargs: P.kwargs) -> R: - return f(get_current_context(), *args, **kwargs) - - return update_wrapper(new_func, f) - - -def pass_obj(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]: - """Similar to :func:`pass_context`, but only pass the object on the - context onwards (:attr:`Context.obj`). This is useful if that object - represents the state of a nested system. - """ - - def new_func(*args: P.args, **kwargs: P.kwargs) -> R: - return f(get_current_context().obj, *args, **kwargs) - - return update_wrapper(new_func, f) - - -def make_pass_decorator( - object_type: type[T], ensure: bool = False -) -> t.Callable[[t.Callable[te.Concatenate[T, P], R]], t.Callable[P, R]]: - """Given an object type this creates a decorator that will work - similar to :func:`pass_obj` but instead of passing the object of the - current context, it will find the innermost context of type - :func:`object_type`. - - This generates a decorator that works roughly like this:: - - from functools import update_wrapper - - def decorator(f): - @pass_context - def new_func(ctx, *args, **kwargs): - obj = ctx.find_object(object_type) - return ctx.invoke(f, obj, *args, **kwargs) - return update_wrapper(new_func, f) - return decorator - - :param object_type: the type of the object to pass. - :param ensure: if set to `True`, a new object will be created and - remembered on the context if it's not there yet. - """ - - def decorator(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]: - def new_func(*args: P.args, **kwargs: P.kwargs) -> R: - ctx = get_current_context() - - obj: T | None - if ensure: - obj = ctx.ensure_object(object_type) - else: - obj = ctx.find_object(object_type) - - if obj is None: - raise RuntimeError( - "Managed to invoke callback without a context" - f" object of type {object_type.__name__!r}" - " existing." - ) - - return ctx.invoke(f, obj, *args, **kwargs) - - return update_wrapper(new_func, f) - - return decorator - - -def pass_meta_key( - key: str, *, doc_description: str | None = None -) -> t.Callable[[t.Callable[te.Concatenate[T, P], R]], t.Callable[P, R]]: - """Create a decorator that passes a key from - :attr:`click.Context.meta` as the first argument to the decorated - function. - - :param key: Key in ``Context.meta`` to pass. - :param doc_description: Description of the object being passed, - inserted into the decorator's docstring. Defaults to "the 'key' - key from Context.meta". - - .. versionadded:: 8.0 - """ - - def decorator(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]: - def new_func(*args: P.args, **kwargs: P.kwargs) -> R: - ctx = get_current_context() - obj = ctx.meta[key] - return ctx.invoke(f, obj, *args, **kwargs) - - return update_wrapper(new_func, f) - - if doc_description is None: - doc_description = f"the {key!r} key from :attr:`click.Context.meta`" - - decorator.__doc__ = ( - f"Decorator that passes {doc_description} as the first argument" - " to the decorated function." - ) - return decorator - - -CmdType = t.TypeVar("CmdType", bound=Command) - - -# variant: no call, directly as decorator for a function. -@t.overload -def command(name: _AnyCallable) -> Command: ... - - -# variant: with positional name and with positional or keyword cls argument: -# @command(namearg, CommandCls, ...) or @command(namearg, cls=CommandCls, ...) -@t.overload -def command( - name: str | None, - cls: type[CmdType], - **attrs: t.Any, -) -> t.Callable[[_AnyCallable], CmdType]: ... - - -# variant: name omitted, cls _must_ be a keyword argument, @command(cls=CommandCls, ...) -@t.overload -def command( - name: None = None, - *, - cls: type[CmdType], - **attrs: t.Any, -) -> t.Callable[[_AnyCallable], CmdType]: ... - - -# variant: with optional string name, no cls argument provided. -@t.overload -def command( - name: str | None = ..., cls: None = None, **attrs: t.Any -) -> t.Callable[[_AnyCallable], Command]: ... - - -def command( - name: str | _AnyCallable | None = None, - cls: type[CmdType] | None = None, - **attrs: t.Any, -) -> Command | t.Callable[[_AnyCallable], Command | CmdType]: - r"""Creates a new :class:`Command` and uses the decorated function as - callback. This will also automatically attach all decorated - :func:`option`\s and :func:`argument`\s as parameters to the command. - - The name of the command defaults to the name of the function, converted to - lowercase, with underscores ``_`` replaced by dashes ``-``, and the suffixes - ``_command``, ``_cmd``, ``_group``, and ``_grp`` are removed. For example, - ``init_data_command`` becomes ``init-data``. - - All keyword arguments are forwarded to the underlying command class. - For the ``params`` argument, any decorated params are appended to - the end of the list. - - Once decorated the function turns into a :class:`Command` instance - that can be invoked as a command line utility or be attached to a - command :class:`Group`. - - :param name: The name of the command. Defaults to modifying the function's - name as described above. - :param cls: The command class to create. Defaults to :class:`Command`. - - .. versionchanged:: 8.2 - The suffixes ``_command``, ``_cmd``, ``_group``, and ``_grp`` are - removed when generating the name. - - .. versionchanged:: 8.1 - This decorator can be applied without parentheses. - - .. versionchanged:: 8.1 - The ``params`` argument can be used. Decorated params are - appended to the end of the list. - """ - - func: t.Callable[[_AnyCallable], t.Any] | None = None - - if callable(name): - func = name - name = None - assert cls is None, "Use 'command(cls=cls)(callable)' to specify a class." - assert not attrs, "Use 'command(**kwargs)(callable)' to provide arguments." - - if cls is None: - cls = t.cast("type[CmdType]", Command) - - def decorator(f: _AnyCallable) -> CmdType: - if isinstance(f, Command): - raise TypeError("Attempted to convert a callback into a command twice.") - - attr_params = attrs.pop("params", None) - params = attr_params if attr_params is not None else [] - - try: - decorator_params = f.__click_params__ # type: ignore - except AttributeError: - pass - else: - del f.__click_params__ # type: ignore - params.extend(reversed(decorator_params)) - - if attrs.get("help") is None: - attrs["help"] = f.__doc__ - - if t.TYPE_CHECKING: - assert cls is not None - assert not callable(name) - - if name is not None: - cmd_name = name - else: - cmd_name = f.__name__.lower().replace("_", "-") - cmd_left, sep, suffix = cmd_name.rpartition("-") - - if sep and suffix in {"command", "cmd", "group", "grp"}: - cmd_name = cmd_left - - cmd = cls(name=cmd_name, callback=f, params=params, **attrs) - cmd.__doc__ = f.__doc__ - return cmd - - if func is not None: - return decorator(func) - - return decorator - - -GrpType = t.TypeVar("GrpType", bound=Group) - - -# variant: no call, directly as decorator for a function. -@t.overload -def group(name: _AnyCallable) -> Group: ... - - -# variant: with positional name and with positional or keyword cls argument: -# @group(namearg, GroupCls, ...) or @group(namearg, cls=GroupCls, ...) -@t.overload -def group( - name: str | None, - cls: type[GrpType], - **attrs: t.Any, -) -> t.Callable[[_AnyCallable], GrpType]: ... - - -# variant: name omitted, cls _must_ be a keyword argument, @group(cmd=GroupCls, ...) -@t.overload -def group( - name: None = None, - *, - cls: type[GrpType], - **attrs: t.Any, -) -> t.Callable[[_AnyCallable], GrpType]: ... - - -# variant: with optional string name, no cls argument provided. -@t.overload -def group( - name: str | None = ..., cls: None = None, **attrs: t.Any -) -> t.Callable[[_AnyCallable], Group]: ... - - -def group( - name: str | _AnyCallable | None = None, - cls: type[GrpType] | None = None, - **attrs: t.Any, -) -> Group | t.Callable[[_AnyCallable], Group | GrpType]: - """Creates a new :class:`Group` with a function as callback. This - works otherwise the same as :func:`command` just that the `cls` - parameter is set to :class:`Group`. - - .. versionchanged:: 8.1 - This decorator can be applied without parentheses. - """ - if cls is None: - cls = t.cast("type[GrpType]", Group) - - if callable(name): - return command(cls=cls, **attrs)(name) - - return command(name, cls, **attrs) - - -def _param_memo(f: t.Callable[..., t.Any], param: Parameter) -> None: - if isinstance(f, Command): - f.params.append(param) - else: - if not hasattr(f, "__click_params__"): - f.__click_params__ = [] # type: ignore - - f.__click_params__.append(param) # type: ignore - - -def argument( - *param_decls: str, cls: type[Argument] | None = None, **attrs: t.Any -) -> t.Callable[[FC], FC]: - """Attaches an argument to the command. All positional arguments are - passed as parameter declarations to :class:`Argument`; all keyword - arguments are forwarded unchanged (except ``cls``). - This is equivalent to creating an :class:`Argument` instance manually - and attaching it to the :attr:`Command.params` list. - - For the default argument class, refer to :class:`Argument` and - :class:`Parameter` for descriptions of parameters. - - :param cls: the argument class to instantiate. This defaults to - :class:`Argument`. - :param param_decls: Passed as positional arguments to the constructor of - ``cls``. - :param attrs: Passed as keyword arguments to the constructor of ``cls``. - """ - if cls is None: - cls = Argument - - def decorator(f: FC) -> FC: - _param_memo(f, cls(param_decls, **attrs)) - return f - - return decorator - - -def option( - *param_decls: str, cls: type[Option] | None = None, **attrs: t.Any -) -> t.Callable[[FC], FC]: - """Attaches an option to the command. All positional arguments are - passed as parameter declarations to :class:`Option`; all keyword - arguments are forwarded unchanged (except ``cls``). - This is equivalent to creating an :class:`Option` instance manually - and attaching it to the :attr:`Command.params` list. - - For the default option class, refer to :class:`Option` and - :class:`Parameter` for descriptions of parameters. - - :param cls: the option class to instantiate. This defaults to - :class:`Option`. - :param param_decls: Passed as positional arguments to the constructor of - ``cls``. - :param attrs: Passed as keyword arguments to the constructor of ``cls``. - """ - if cls is None: - cls = Option - - def decorator(f: FC) -> FC: - _param_memo(f, cls(param_decls, **attrs)) - return f - - return decorator - - -def confirmation_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: - """Add a ``--yes`` option which shows a prompt before continuing if - not passed. If the prompt is declined, the program will exit. - - :param param_decls: One or more option names. Defaults to the single - value ``"--yes"``. - :param kwargs: Extra arguments are passed to :func:`option`. - """ - - def callback(ctx: Context, param: Parameter, value: bool) -> None: - if not value: - ctx.abort() - - if not param_decls: - param_decls = ("--yes",) - - kwargs.setdefault("is_flag", True) - kwargs.setdefault("callback", callback) - kwargs.setdefault("expose_value", False) - kwargs.setdefault("prompt", _("Do you want to continue?")) - kwargs.setdefault("help", _("Confirm the action without prompting.")) - return option(*param_decls, **kwargs) - - -def password_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: - """Add a ``--password`` option which prompts for a password, hiding - input and asking to enter the value again for confirmation. - - :param param_decls: One or more option names. Defaults to the single - value ``"--password"``. - :param kwargs: Extra arguments are passed to :func:`option`. - """ - if not param_decls: - param_decls = ("--password",) - - kwargs.setdefault("prompt", True) - kwargs.setdefault("confirmation_prompt", True) - kwargs.setdefault("hide_input", True) - return option(*param_decls, **kwargs) - - -def version_option( - version: str | None = None, - *param_decls: str, - package_name: str | None = None, - prog_name: str | None = None, - message: str | None = None, - **kwargs: t.Any, -) -> t.Callable[[FC], FC]: - """Add a ``--version`` option which immediately prints the version - number and exits the program. - - If ``version`` is not provided, Click will try to detect it using - :func:`importlib.metadata.version` to get the version for the - ``package_name``. - - If ``package_name`` is not provided, Click will try to detect it by - inspecting the stack frames. If the detected (or given) name does - not match an installed distribution, Click resolves it as an import - (top-level module) name via - :func:`importlib.metadata.packages_distributions`, so e.g. ``PIL`` - resolves to the ``Pillow`` distribution. - - :param version: The version number to show. If not provided, Click - will try to detect it. - :param param_decls: One or more option names. Defaults to the single - value ``"--version"``. - :param package_name: The package name to detect the version from. If - not provided, Click will try to detect it. - :param prog_name: The name of the CLI to show in the message. If not - provided, it will be detected from the command. - :param message: The message to show. The values ``%(prog)s``, - ``%(package)s``, and ``%(version)s`` are available. Defaults to - ``"%(prog)s, version %(version)s"``. - :param kwargs: Extra arguments are passed to :func:`option`. - :raise RuntimeError: ``version`` could not be detected. - - .. versionchanged:: 8.0 - Add the ``package_name`` parameter, and the ``%(package)s`` - value for messages. - - .. versionchanged:: 8.0 - Use :mod:`importlib.metadata` instead of ``pkg_resources``. The - version is detected based on the package name, not the entry - point name. The Python package name must match the installed - package name, or be passed with ``package_name=``. - - .. versionchanged:: 8.4.2 - When ``package_name`` does not match an installed distribution, - Click now resolves it as an import (top-level module). - """ - if message is None: - message = _("%(prog)s, version %(version)s") - - if version is None and package_name is None: - frame = inspect.currentframe() - f_back = frame.f_back if frame is not None else None - f_globals = f_back.f_globals if f_back is not None else None - # break reference cycle - # https://docs.python.org/3/library/inspect.html#the-interpreter-stack - del frame - - if f_globals is not None: - package_name = f_globals.get("__name__") - - if package_name == "__main__": - package_name = f_globals.get("__package__") - - if package_name: - package_name = package_name.partition(".")[0] - - def callback(ctx: Context, param: Parameter, value: bool) -> None: - if not value or ctx.resilient_parsing: - return - - nonlocal prog_name - nonlocal version - nonlocal package_name - - if prog_name is None: - prog_name = ctx.find_root().info_name - - if version is None and package_name is not None: - import importlib.metadata - - try: - version = importlib.metadata.version(package_name) - except importlib.metadata.PackageNotFoundError: - # The given name didn't match an installed distribution. - # Try resolving it as an import (top-level module) name, - # e.g. ``PIL`` is provided by the ``Pillow`` distribution. - distributions = importlib.metadata.packages_distributions().get( - package_name, [] - ) - if len(distributions) == 1: - package_name = distributions[0] - version = importlib.metadata.version(package_name) - elif len(distributions) > 1: - raise RuntimeError( - f"{package_name!r} maps to multiple installed" - f" distributions ({', '.join(distributions)})." - " Pass 'package_name' to disambiguate." - ) from None - else: - raise RuntimeError( - f"{package_name!r} is not installed. Try passing" - " 'package_name' instead." - ) from None - - if version is None: - raise RuntimeError( - f"Could not determine the version for {package_name!r} automatically." - ) - - echo( - message % {"prog": prog_name, "package": package_name, "version": version}, - color=ctx.color, - ) - ctx.exit() - - if not param_decls: - param_decls = ("--version",) - - kwargs.setdefault("is_flag", True) - kwargs.setdefault("expose_value", False) - kwargs.setdefault("is_eager", True) - kwargs.setdefault("help", _("Show the version and exit.")) - kwargs["callback"] = callback - return option(*param_decls, **kwargs) - - -def help_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: - """Pre-configured ``--help`` option which immediately prints the help page - and exits the program. - - :param param_decls: One or more option names. Defaults to the single - value ``"--help"``. - :param kwargs: Extra arguments are passed to :func:`option`. - """ - - def show_help(ctx: Context, param: Parameter, value: bool) -> None: - """Callback that print the help page on ```` and exits.""" - if value and not ctx.resilient_parsing: - echo(ctx.get_help(), color=ctx.color) - ctx.exit() - - if not param_decls: - param_decls = ("--help",) - - kwargs.setdefault("is_flag", True) - kwargs.setdefault("expose_value", False) - kwargs.setdefault("is_eager", True) - kwargs.setdefault("help", _("Show this message and exit.")) - kwargs.setdefault("callback", show_help) - - return option(*param_decls, **kwargs) diff --git a/bundle/python-cpu/Lib/site-packages/click/exceptions.py b/bundle/python-cpu/Lib/site-packages/click/exceptions.py deleted file mode 100644 index 6272c38a448d57241d9a6bff4191897f4a9bb7d6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/exceptions.py +++ /dev/null @@ -1,378 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import typing as t -from gettext import gettext as _ -from gettext import ngettext - -from ._compat import get_text_stderr -from .globals import resolve_color_default -from .utils import echo -from .utils import format_filename - -if t.TYPE_CHECKING: - from .core import Command - from .core import Context - from .core import Parameter - - -def _join_param_hints(param_hint: cabc.Sequence[str] | str | None) -> str | None: - if param_hint is not None and not isinstance(param_hint, str): - return " / ".join(repr(x) for x in param_hint) - - return param_hint - - -def _format_possibilities(possibilities: list[str]) -> str: - possibility_str = ", ".join(repr(p) for p in sorted(possibilities)) - return ngettext( - "Did you mean {possibility}?", - "(Did you mean one of: {possibilities}?)", - len(possibilities), - ).format(possibility=possibility_str, possibilities=possibility_str) - - -class ClickException(Exception): - """An exception that Click can handle and show to the user.""" - - #: The exit code for this exception. - exit_code: t.ClassVar[int] = 1 - - show_color: t.Final[bool | None] - message: t.Final[str] - - def __init__(self, message: str) -> None: - super().__init__(message) - # The context will be removed by the time we print the message, so cache - # the color settings here to be used later on (in `show`) - self.show_color = resolve_color_default() - self.message = message - - def format_message(self) -> str: - return self.message - - def __str__(self) -> str: - return self.message - - def show(self, file: t.IO[t.Any] | None = None) -> None: - if file is None: - file = get_text_stderr() - - echo( - _("Error: {message}").format(message=self.format_message()), - file=file, - color=self.show_color, - ) - - -class UsageError(ClickException): - """An internal exception that signals a usage error. This typically - aborts any further handling. - - :param message: the error message to display. - :param ctx: optionally the context that caused this error. Click will - fill in the context automatically in some situations. - """ - - exit_code: t.ClassVar[int] = 2 - - ctx: Context | None - cmd: t.Final[Command | None] - - def __init__(self, message: str, ctx: Context | None = None) -> None: - super().__init__(message) - self.ctx = ctx - self.cmd = self.ctx.command if self.ctx else None - - def show(self, file: t.IO[t.Any] | None = None) -> None: - if file is None: - file = get_text_stderr() - color = None - hint = "" - if ( - self.ctx is not None - and self.ctx.command.get_help_option(self.ctx) is not None - ): - help_names = self.ctx.command.get_help_option_names(self.ctx) - # Pick the longest name (like ``--help`` over ``-h``) for - # readability in error messages. - hint = _("Try '{command} {option}' for help.").format( - command=self.ctx.command_path, - option=max(help_names, key=len), - ) - hint = f"{hint}\n" - if self.ctx is not None: - color = self.ctx.color - echo(f"{self.ctx.get_usage()}\n{hint}", file=file, color=color) - echo( - _("Error: {message}").format(message=self.format_message()), - file=file, - color=color, - ) - - -class BadParameter(UsageError): - """An exception that formats out a standardized error message for a - bad parameter. This is useful when thrown from a callback or type as - Click will attach contextual information to it (for instance, which - parameter it is). - - .. versionadded:: 2.0 - - :param param: the parameter object that caused this error. This can - be left out, and Click will attach this info itself - if possible. - :param param_hint: a string that shows up as parameter name. This - can be used as alternative to `param` in cases - where custom validation should happen. If it is - a string it's used as such, if it's a list then - each item is quoted and separated. - """ - - param: Parameter | None - param_hint: cabc.Sequence[str] | str | None - - def __init__( - self, - message: str, - ctx: Context | None = None, - param: Parameter | None = None, - param_hint: cabc.Sequence[str] | str | None = None, - ) -> None: - super().__init__(message, ctx) - self.param = param - self.param_hint = param_hint - - def format_message(self) -> str: - if self.param_hint is not None: - param_hint = self.param_hint - elif self.param is not None: - param_hint = self.param.get_error_hint(self.ctx) - else: - return _("Invalid value: {message}").format(message=self.message) - - return _("Invalid value for {param_hint}: {message}").format( - param_hint=_join_param_hints(param_hint), message=self.message - ) - - -class MissingParameter(BadParameter): - """Raised if click required an option or argument but it was not - provided when invoking the script. - - .. versionadded:: 4.0 - - :param param_type: a string that indicates the type of the parameter. - The default is to inherit the parameter type from - the given `param`. Valid values are ``'parameter'``, - ``'option'`` or ``'argument'``. - """ - - param_type: t.Final[str | None] - - def __init__( - self, - message: str | None = None, - ctx: Context | None = None, - param: Parameter | None = None, - param_hint: cabc.Sequence[str] | str | None = None, - param_type: str | None = None, - ) -> None: - super().__init__(message or "", ctx, param, param_hint) - self.param_type = param_type - - def format_message(self) -> str: - if self.param_hint is not None: - param_hint: cabc.Sequence[str] | str | None = self.param_hint - elif self.param is not None: - param_hint = self.param.get_error_hint(self.ctx) - else: - param_hint = None - - param_hint = _join_param_hints(param_hint) - param_hint = f" {param_hint}" if param_hint else "" - - param_type = self.param_type - if param_type is None and self.param is not None: - param_type = self.param.param_type_name - - msg = self.message - if self.param is not None: - msg_extra = self.param.type.get_missing_message( - param=self.param, ctx=self.ctx - ) - if msg_extra: - if msg: - msg += f". {msg_extra}" - else: - msg = msg_extra - - msg = f" {msg}" if msg else "" - - # Translate param_type for known types. - if param_type == "argument": - missing = _("Missing argument") - elif param_type == "option": - missing = _("Missing option") - elif param_type == "parameter": - missing = _("Missing parameter") - else: - missing = _("Missing {param_type}").format(param_type=param_type) - - return f"{missing}{param_hint}.{msg}" - - def __str__(self) -> str: - if not self.message: - param_name = self.param.name if self.param else None - return _("Missing parameter: {param_name}").format(param_name=param_name) - else: - return self.message - - -class NoSuchOption(UsageError): - """Raised if Click attempted to handle an option that does not exist. - - .. versionadded:: 4.0 - """ - - option_name: t.Final[str] - possibilities: t.Final[list[str] | None] - - def __init__( - self, - option_name: str, - message: str | None = None, - possibilities: cabc.Iterable[str] | None = None, - ctx: Context | None = None, - ) -> None: - if message is None: - message = _("No such option {name!r}.").format(name=option_name) - - super().__init__(message, ctx) - self.option_name = option_name - - if possibilities: - from difflib import get_close_matches - - possibilities_ = get_close_matches(option_name, possibilities) - else: - possibilities_ = None - self.possibilities = possibilities_ - - def format_message(self) -> str: - if not self.possibilities: - return self.message - return f"{self.message} {_format_possibilities(self.possibilities)}" - - -class NoSuchCommand(UsageError): - """Raised if Click attempted to handle a command that does not exist. - - .. versionadded:: 8.4.0 - """ - - command_name: t.Final[str] - possibilities: t.Final[list[str] | None] - - def __init__( - self, - command_name: str, - message: str | None = None, - possibilities: cabc.Iterable[str] | None = None, - ctx: Context | None = None, - ) -> None: - if message is None: - message = _("No such command {name!r}.").format(name=command_name) - - super().__init__(message, ctx) - self.command_name = command_name - - if possibilities: - from difflib import get_close_matches - - possibilities_ = get_close_matches(command_name, possibilities) - else: - possibilities_ = None - self.possibilities = possibilities_ - - def format_message(self) -> str: - if not self.possibilities: - return self.message - return f"{self.message} {_format_possibilities(self.possibilities)}" - - -class BadOptionUsage(UsageError): - """Raised if an option is generally supplied but the use of the option - was incorrect. This is for instance raised if the number of arguments - for an option is not correct. - - .. versionadded:: 4.0 - - :param option_name: the name of the option being used incorrectly. - """ - - option_name: t.Final[str] - - def __init__( - self, option_name: str, message: str, ctx: Context | None = None - ) -> None: - super().__init__(message, ctx) - self.option_name = option_name - - -class BadArgumentUsage(UsageError): - """Raised if an argument is generally supplied but the use of the argument - was incorrect. This is for instance raised if the number of values - for an argument is not correct. - - .. versionadded:: 6.0 - """ - - -class NoArgsIsHelpError(UsageError): - ctx: Context - - def __init__(self, ctx: Context) -> None: - super().__init__(ctx.get_help(), ctx=ctx) - - def show(self, file: t.IO[t.Any] | None = None) -> None: - echo(self.format_message(), file=file, err=True, color=self.ctx.color) - - -class FileError(ClickException): - """Raised if a file cannot be opened.""" - - ui_filename: t.Final[str] - filename: t.Final[str] - - def __init__(self, filename: str, hint: str | None = None) -> None: - if hint is None: - hint = _("unknown error") - - super().__init__(hint) - self.ui_filename = format_filename(filename) - self.filename = filename - - def format_message(self) -> str: - return _("Could not open file {filename!r}: {message}").format( - filename=self.ui_filename, message=self.message - ) - - -class Abort(RuntimeError): - """An internal signalling exception that signals Click to abort.""" - - -class Exit(RuntimeError): - """An exception that indicates that the application should exit with some - status code. - - :param code: the status code to exit with. - """ - - __slots__ = ("exit_code",) - - exit_code: t.Final[int] - - def __init__(self, code: int = 0) -> None: - self.exit_code = code diff --git a/bundle/python-cpu/Lib/site-packages/click/formatting.py b/bundle/python-cpu/Lib/site-packages/click/formatting.py deleted file mode 100644 index c4aa2de571a1eb17bfeb1853e315d28bc968e74c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/formatting.py +++ /dev/null @@ -1,320 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -from contextlib import contextmanager -from gettext import gettext as _ - -from ._compat import term_len -from .parser import _split_opt - -# Can force a width. This is used by the test system -FORCED_WIDTH: int | None = None - - -def measure_table(rows: cabc.Iterable[tuple[str, str]]) -> tuple[int, ...]: - widths: dict[int, int] = {} - - for row in rows: - for idx, col in enumerate(row): - widths[idx] = max(widths.get(idx, 0), term_len(col)) - - return tuple(y for x, y in sorted(widths.items())) - - -def iter_rows( - rows: cabc.Iterable[tuple[str, str]], col_count: int -) -> cabc.Iterator[tuple[str, ...]]: - for row in rows: - yield row + ("",) * (col_count - len(row)) - - -def wrap_text( - text: str, - width: int = 78, - initial_indent: str = "", - subsequent_indent: str = "", - preserve_paragraphs: bool = False, -) -> str: - """A helper function that intelligently wraps text. By default, it - assumes that it operates on a single paragraph of text but if the - `preserve_paragraphs` parameter is provided it will intelligently - handle paragraphs (defined by two empty lines). - - If paragraphs are handled, a paragraph can be prefixed with an empty - line containing the ``\\b`` character (``\\x08``) to indicate that - no rewrapping should happen in that block. - - :param text: the text that should be rewrapped. - :param width: the maximum width for the text. - :param initial_indent: the initial indent that should be placed on the - first line as a string. - :param subsequent_indent: the indent string that should be placed on - each consecutive line. - :param preserve_paragraphs: if this flag is set then the wrapping will - intelligently handle paragraphs. - - .. versionchanged:: 8.4.0 - Width is measured in visible characters. ANSI escape sequences in - ``text``, ``initial_indent``, or ``subsequent_indent`` no longer - count toward the width budget, so styled input wraps based on what - the user sees instead of raw byte length. - """ - from ._textwrap import TextWrapper - - text = text.expandtabs() - wrapper = TextWrapper( - width, - initial_indent=initial_indent, - subsequent_indent=subsequent_indent, - replace_whitespace=False, - ) - if not preserve_paragraphs: - return wrapper.fill(text) - - p: list[tuple[int, bool, str]] = [] - buf: list[str] = [] - indent = None - - def _flush_par() -> None: - if not buf: - return - if buf[0].strip() == "\b": - p.append((indent or 0, True, "\n".join(buf[1:]))) - else: - p.append((indent or 0, False, " ".join(buf))) - del buf[:] - - for line in text.splitlines(): - if not line: - _flush_par() - indent = None - else: - if indent is None: - orig_len = term_len(line) - line = line.lstrip() - indent = orig_len - term_len(line) - buf.append(line) - _flush_par() - - rv = [] - for indent, raw, text in p: - with wrapper.extra_indent(" " * indent): - if raw: - rv.append(wrapper.indent_only(text)) - else: - rv.append(wrapper.fill(text)) - - return "\n\n".join(rv) - - -class HelpFormatter: - """This class helps with formatting text-based help pages. It's - usually just needed for very special internal cases, but it's also - exposed so that developers can write their own fancy outputs. - - At present, it always writes into memory. - - :param indent_increment: the additional increment for each level. - :param width: the width for the text. This defaults to the terminal - width clamped to a maximum of 78. - """ - - indent_increment: int - width: int - current_indent: int - buffer: list[str] - - def __init__( - self, - indent_increment: int = 2, - width: int | None = None, - max_width: int | None = None, - ) -> None: - self.indent_increment = indent_increment - if max_width is None: - max_width = 80 - if width is None: - import shutil - - width = FORCED_WIDTH - if width is None: - width = max(min(shutil.get_terminal_size().columns, max_width) - 2, 50) - self.width = width - self.current_indent = 0 - self.buffer = [] - - def write(self, string: str) -> None: - """Writes a unicode string into the internal buffer.""" - self.buffer.append(string) - - def indent(self) -> None: - """Increases the indentation.""" - self.current_indent += self.indent_increment - - def dedent(self) -> None: - """Decreases the indentation.""" - self.current_indent -= self.indent_increment - - def write_usage(self, prog: str, args: str = "", prefix: str | None = None) -> None: - """Writes a usage line into the buffer. - - :param prog: the program name. - :param args: whitespace separated list of arguments. - :param prefix: The prefix for the first line. Defaults to - ``"Usage: "``. - """ - if prefix is None: - prefix = "{usage} ".format(usage=_("Usage:")) - - usage_prefix = f"{prefix:>{self.current_indent}}{prog} " - text_width = self.width - self.current_indent - - if not args: - # Without args, the prefix's trailing space and the wrap_text - # call that would normally place args on the line are both - # unnecessary. Emit just the prefix line. - self.write(usage_prefix.rstrip(" ")) - self.write("\n") - return - - if text_width >= (term_len(usage_prefix) + 20): - # The arguments will fit to the right of the prefix. - indent = " " * term_len(usage_prefix) - self.write( - wrap_text( - args, - text_width, - initial_indent=usage_prefix, - subsequent_indent=indent, - ) - ) - else: - # The prefix is too long, put the arguments on the next line. - self.write(usage_prefix) - self.write("\n") - indent = " " * (max(self.current_indent, term_len(prefix)) + 4) - self.write( - wrap_text( - args, text_width, initial_indent=indent, subsequent_indent=indent - ) - ) - - self.write("\n") - - def write_heading(self, heading: str) -> None: - """Writes a heading into the buffer.""" - self.write(f"{'':>{self.current_indent}}{heading}:\n") - - def write_paragraph(self) -> None: - """Writes a paragraph into the buffer.""" - if self.buffer: - self.write("\n") - - def write_text(self, text: str) -> None: - """Writes re-indented text into the buffer. This rewraps and - preserves paragraphs. - """ - indent = " " * self.current_indent - self.write( - wrap_text( - text, - self.width, - initial_indent=indent, - subsequent_indent=indent, - preserve_paragraphs=True, - ) - ) - self.write("\n") - - def write_dl( - self, - rows: cabc.Iterable[tuple[str, str]], - col_max: int = 30, - col_spacing: int = 2, - ) -> None: - """Writes a definition list into the buffer. This is how options - and commands are usually formatted. - - :param rows: a list of two item tuples for the terms and values. - :param col_max: the maximum width of the first column. - :param col_spacing: the number of spaces between the first and - second column. - """ - rows = list(rows) - widths = measure_table(rows) - if len(widths) != 2: - raise TypeError("Expected two columns for definition list") - - first_col = min(widths[0], col_max) + col_spacing - - for first, second in iter_rows(rows, len(widths)): - self.write(f"{'':>{self.current_indent}}{first}") - if not second: - self.write("\n") - continue - if term_len(first) <= first_col - col_spacing: - self.write(" " * (first_col - term_len(first))) - else: - self.write("\n") - self.write(" " * (first_col + self.current_indent)) - - text_width = max(self.width - first_col - 2, 10) - wrapped_text = wrap_text(second, text_width, preserve_paragraphs=True) - lines = wrapped_text.splitlines() - - if lines: - self.write(f"{lines[0]}\n") - - for line in lines[1:]: - self.write(f"{'':>{first_col + self.current_indent}}{line}\n") - else: - self.write("\n") - - @contextmanager - def section(self, name: str) -> cabc.Generator[None]: - """Helpful context manager that writes a paragraph, a heading, - and the indents. - - :param name: the section name that is written as heading. - """ - self.write_paragraph() - self.write_heading(name) - self.indent() - try: - yield - finally: - self.dedent() - - @contextmanager - def indentation(self) -> cabc.Generator[None]: - """A context manager that increases the indentation.""" - self.indent() - try: - yield - finally: - self.dedent() - - def getvalue(self) -> str: - """Returns the buffer contents.""" - return "".join(self.buffer) - - -def join_options(options: cabc.Iterable[str]) -> tuple[str, bool]: - """Given a list of option strings this joins them in the most appropriate - way and returns them in the form ``(formatted_string, - any_prefix_is_slash)`` where the second item in the tuple is a flag that - indicates if any of the option prefixes was a slash. - """ - rv = [] - any_prefix_is_slash = False - - for opt in options: - prefix = _split_opt(opt)[0] - - if prefix == "/": - any_prefix_is_slash = True - - rv.append((len(prefix), opt)) - - rv.sort(key=lambda x: x[0]) - return ", ".join(x[1] for x in rv), any_prefix_is_slash diff --git a/bundle/python-cpu/Lib/site-packages/click/globals.py b/bundle/python-cpu/Lib/site-packages/click/globals.py deleted file mode 100644 index a2f91723d21cefdf658be327174e1f8ccdc20fcd..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/globals.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -import typing as t -from threading import local - -if t.TYPE_CHECKING: - from .core import Context - -_local = local() - - -@t.overload -def get_current_context(silent: t.Literal[False] = False) -> Context: ... - - -@t.overload -def get_current_context(silent: bool = ...) -> Context | None: ... - - -def get_current_context(silent: bool = False) -> Context | None: - """Returns the current click context. This can be used as a way to - access the current context object from anywhere. This is a more implicit - alternative to the :func:`pass_context` decorator. This function is - primarily useful for helpers such as :func:`echo` which might be - interested in changing its behavior based on the current context. - - To push the current context, :meth:`Context.scope` can be used. - - .. versionadded:: 5.0 - - :param silent: if set to `True` the return value is `None` if no context - is available. The default behavior is to raise a - :exc:`RuntimeError`. - """ - try: - return t.cast("Context", _local.stack[-1]) - except (AttributeError, IndexError) as e: - if not silent: - raise RuntimeError("There is no active click context.") from e - - return None - - -def push_context(ctx: Context) -> None: - """Pushes a new context to the current stack.""" - _local.__dict__.setdefault("stack", []).append(ctx) - - -def pop_context() -> None: - """Removes the top level from the stack.""" - _local.stack.pop() - - -def resolve_color_default(color: bool | None = None) -> bool | None: - """Internal helper to get the default value of the color flag. If a - value is passed it's returned unchanged, otherwise it's looked up from - the current context. - """ - if color is not None: - return color - - ctx = get_current_context(silent=True) - - if ctx is not None: - return ctx.color - - return None diff --git a/bundle/python-cpu/Lib/site-packages/click/parser.py b/bundle/python-cpu/Lib/site-packages/click/parser.py deleted file mode 100644 index 4fcbf7caa83a474cee2d3ea25da56b0399dd893a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/parser.py +++ /dev/null @@ -1,533 +0,0 @@ -""" -This module started out as largely a copy paste from the stdlib's -optparse module with the features removed that we do not need from -optparse because we implement them in Click on a higher level (for -instance type handling, help formatting and a lot more). - -The plan is to remove more and more from here over time. - -The reason this is a different module and not optparse from the stdlib -is that there are differences in 2.x and 3.x about the error messages -generated and optparse in the stdlib uses gettext for no good reason -and might cause us issues. - -Click uses parts of optparse written by Gregory P. Ward and maintained -by the Python Software Foundation. This is limited to code in parser.py. - -Copyright 2001-2006 Gregory P. Ward. All rights reserved. -Copyright 2002-2006 Python Software Foundation. All rights reserved. -""" - -# This code uses parts of optparse written by Gregory P. Ward and -# maintained by the Python Software Foundation. -# Copyright 2001-2006 Gregory P. Ward -# Copyright 2002-2006 Python Software Foundation -from __future__ import annotations - -import collections.abc as cabc -import typing as t -from collections import deque -from gettext import gettext as _ -from gettext import ngettext - -from ._utils import FLAG_NEEDS_VALUE -from ._utils import UNSET -from .exceptions import BadArgumentUsage -from .exceptions import BadOptionUsage -from .exceptions import NoSuchOption -from .exceptions import UsageError - -if t.TYPE_CHECKING: - from ._utils import T_FLAG_NEEDS_VALUE - from ._utils import T_UNSET - from .core import Argument as CoreArgument - from .core import Context - from .core import Option as CoreOption - from .core import Parameter as CoreParameter - -V = t.TypeVar("V") - - -def _unpack_args( - args: cabc.Sequence[str], nargs_spec: cabc.Sequence[int] -) -> tuple[cabc.Sequence[str | cabc.Sequence[str | T_UNSET] | T_UNSET], list[str]]: - """Given an iterable of arguments and an iterable of nargs specifications, - it returns a tuple with all the unpacked arguments at the first index - and all remaining arguments as the second. - - The nargs specification is the number of arguments that should be consumed - or `-1` to indicate that this position should eat up all the remainders. - - Missing items are filled with ``UNSET``. - """ - args = deque(args) - nargs_spec = deque(nargs_spec) - rv: list[str | tuple[str | T_UNSET, ...] | T_UNSET] = [] - spos: int | None = None - - def _fetch(c: deque[str]) -> str | T_UNSET: - try: - if spos is None: - return c.popleft() - else: - return c.pop() - except IndexError: - return UNSET - - while nargs_spec: - if spos is None: - nargs = nargs_spec.popleft() - else: - nargs = nargs_spec.pop() - - if nargs == 1: - rv.append(_fetch(args)) - elif nargs > 1: - x: list[str | T_UNSET] = [_fetch(args) for _ in range(nargs)] - - # If we're reversed, we're pulling in the arguments in reverse, - # so we need to turn them around. - if spos is not None: - x.reverse() - - rv.append(tuple(x)) - elif nargs < 0: - if spos is not None: - raise TypeError("Cannot have two nargs < 0") - - spos = len(rv) - rv.append(UNSET) - - # spos is the position of the wildcard (star). If it's not `None`, - # we fill it with the remainder. - if spos is not None: - rv[spos] = tuple(args) - args = [] - rv[spos + 1 :] = reversed(rv[spos + 1 :]) - - return tuple(rv), list(args) - - -def _split_opt(opt: str) -> tuple[str, str]: - first = opt[:1] - if first.isalnum(): - return "", opt - if opt[1:2] == first: - return opt[:2], opt[2:] - return first, opt[1:] - - -def _normalize_opt(opt: str, ctx: Context | None) -> str: - if ctx is None or ctx.token_normalize_func is None: - return opt - prefix, opt = _split_opt(opt) - return f"{prefix}{ctx.token_normalize_func(opt)}" - - -class _Option: - def __init__( - self, - obj: CoreOption, - opts: cabc.Sequence[str], - dest: str | None, - action: str | None = None, - nargs: int = 1, - const: t.Any | None = None, - ): - self._short_opts = [] - self._long_opts = [] - self.prefixes: set[str] = set() - - for opt in opts: - prefix, value = _split_opt(opt) - if not prefix: - raise ValueError( - _("Invalid start character for option ({option})").format( - option=opt - ) - ) - self.prefixes.add(prefix[0]) - if len(prefix) == 1 and len(value) == 1: - self._short_opts.append(opt) - else: - self._long_opts.append(opt) - self.prefixes.add(prefix) - - if action is None: - action = "store" - - self.dest = dest - self.action = action - self.nargs = nargs - self.const = const - self.obj = obj - - @property - def takes_value(self) -> bool: - return self.action in ("store", "append") - - def process(self, value: t.Any, state: _ParsingState) -> None: - if self.action == "store": - state.opts[self.dest] = value # type: ignore - elif self.action == "store_const": - state.opts[self.dest] = self.const # type: ignore - elif self.action == "append": - state.opts.setdefault(self.dest, []).append(value) # type: ignore - elif self.action == "append_const": - state.opts.setdefault(self.dest, []).append(self.const) # type: ignore - elif self.action == "count": - state.opts[self.dest] = state.opts.get(self.dest, 0) + 1 # type: ignore - else: - raise ValueError(f"unknown action '{self.action}'") - state.order.append(self.obj) - - -class _Argument: - def __init__(self, obj: CoreArgument, dest: str | None, nargs: int = 1): - self.dest = dest - self.nargs = nargs - self.obj = obj - - def process( - self, - value: str | cabc.Sequence[str | T_UNSET] | T_UNSET, - state: _ParsingState, - ) -> None: - if self.nargs > 1: - assert isinstance(value, cabc.Sequence) - holes = sum(x is UNSET for x in value) - if holes == len(value): - value = UNSET - elif holes != 0: - raise BadArgumentUsage( - _("Argument {name!r} takes {nargs} values.").format( - name=self.dest, nargs=self.nargs - ) - ) - - # We failed to collect any argument value so we consider the argument as unset. - if value == (): - value = UNSET - - state.opts[self.dest] = value # type: ignore - state.order.append(self.obj) - - -class _ParsingState: - def __init__(self, rargs: list[str]) -> None: - self.opts: dict[str, t.Any] = {} - self.largs: list[str] = [] - self.rargs = rargs - self.order: list[CoreParameter] = [] - - -class _OptionParser: - """The option parser is an internal class that is ultimately used to - parse options and arguments. It's modelled after optparse and brings - a similar but vastly simplified API. It should generally not be used - directly as the high level Click classes wrap it for you. - - It's not nearly as extensible as optparse or argparse as it does not - implement features that are implemented on a higher level (such as - types or defaults). - - :param ctx: optionally the :class:`~click.Context` where this parser - should go with. - - .. deprecated:: 8.2 - Will be removed in Click 9.0. - """ - - def __init__(self, ctx: Context | None = None) -> None: - #: The :class:`~click.Context` for this parser. This might be - #: `None` for some advanced use cases. - self.ctx = ctx - #: This controls how the parser deals with interspersed arguments. - #: If this is set to `False`, the parser will stop on the first - #: non-option. Click uses this to implement nested subcommands - #: safely. - self.allow_interspersed_args: bool = True - #: This tells the parser how to deal with unknown options. By - #: default it will error out (which is sensible), but there is a - #: second mode where it will ignore it and continue processing - #: after shifting all the unknown options into the resulting args. - self.ignore_unknown_options: bool = False - - if ctx is not None: - self.allow_interspersed_args = ctx.allow_interspersed_args - self.ignore_unknown_options = ctx.ignore_unknown_options - - self._short_opt: dict[str, _Option] = {} - self._long_opt: dict[str, _Option] = {} - self._opt_prefixes = {"-", "--"} - self._args: list[_Argument] = [] - - def add_option( - self, - obj: CoreOption, - opts: cabc.Sequence[str], - dest: str | None, - action: str | None = None, - nargs: int = 1, - const: t.Any | None = None, - ) -> None: - """Adds a new option named `dest` to the parser. The destination - is not inferred (unlike with optparse) and needs to be explicitly - provided. Action can be any of ``store``, ``store_const``, - ``append``, ``append_const`` or ``count``. - - The `obj` can be used to identify the option in the order list - that is returned from the parser. - """ - opts = [_normalize_opt(opt, self.ctx) for opt in opts] - option = _Option(obj, opts, dest, action=action, nargs=nargs, const=const) - self._opt_prefixes.update(option.prefixes) - for opt in option._short_opts: - self._short_opt[opt] = option - for opt in option._long_opts: - self._long_opt[opt] = option - - def add_argument(self, obj: CoreArgument, dest: str | None, nargs: int = 1) -> None: - """Adds a positional argument named `dest` to the parser. - - The `obj` can be used to identify the option in the order list - that is returned from the parser. - """ - self._args.append(_Argument(obj, dest=dest, nargs=nargs)) - - def parse_args( - self, args: list[str] - ) -> tuple[dict[str, t.Any], list[str], list[CoreParameter]]: - """Parses positional arguments and returns ``(values, args, order)`` - for the parsed options and arguments as well as the leftover - arguments if there are any. The order is a list of objects as they - appear on the command line. If arguments appear multiple times they - will be memorized multiple times as well. - """ - state = _ParsingState(args) - try: - self._process_args_for_options(state) - self._process_args_for_args(state) - except UsageError: - if self.ctx is None or not self.ctx.resilient_parsing: - raise - return state.opts, state.largs, state.order - - def _process_args_for_args(self, state: _ParsingState) -> None: - pargs, args = _unpack_args( - state.largs + state.rargs, [x.nargs for x in self._args] - ) - - for idx, arg in enumerate(self._args): - arg.process(pargs[idx], state) - - state.largs = args - state.rargs = [] - - def _process_args_for_options(self, state: _ParsingState) -> None: - while state.rargs: - arg = state.rargs.pop(0) - arglen = len(arg) - # Double dashes always handled explicitly regardless of what - # prefixes are valid. - if arg == "--": - return - elif arg[:1] in self._opt_prefixes and arglen > 1: - self._process_opts(arg, state) - elif self.allow_interspersed_args: - state.largs.append(arg) - else: - state.rargs.insert(0, arg) - return - - # Say this is the original argument list: - # [arg0, arg1, ..., arg(i-1), arg(i), arg(i+1), ..., arg(N-1)] - # ^ - # (we are about to process arg(i)). - # - # Then rargs is [arg(i), ..., arg(N-1)] and largs is a *subset* of - # [arg0, ..., arg(i-1)] (any options and their arguments will have - # been removed from largs). - # - # The while loop will usually consume 1 or more arguments per pass. - # If it consumes 1 (eg. arg is an option that takes no arguments), - # then after _process_arg() is done the situation is: - # - # largs = subset of [arg0, ..., arg(i)] - # rargs = [arg(i+1), ..., arg(N-1)] - # - # If allow_interspersed_args is false, largs will always be - # *empty* -- still a subset of [arg0, ..., arg(i-1)], but - # not a very interesting subset! - - def _match_long_opt( - self, opt: str, explicit_value: str | None, state: _ParsingState - ) -> None: - if opt not in self._long_opt: - raise NoSuchOption(opt, possibilities=self._long_opt, ctx=self.ctx) - - option = self._long_opt[opt] - if option.takes_value: - # At this point it's safe to modify rargs by injecting the - # explicit value, because no exception is raised in this - # branch. This means that the inserted value will be fully - # consumed. - if explicit_value is not None: - state.rargs.insert(0, explicit_value) - - value = self._get_value_from_state(opt, option, state) - - elif explicit_value is not None: - raise BadOptionUsage( - opt, _("Option {name!r} does not take a value.").format(name=opt) - ) - - else: - value = UNSET - - option.process(value, state) - - def _match_short_opt(self, arg: str, state: _ParsingState) -> None: - stop = False - i = 1 - prefix = arg[0] - unknown_options = [] - - for ch in arg[1:]: - opt = _normalize_opt(f"{prefix}{ch}", self.ctx) - option = self._short_opt.get(opt) - i += 1 - - if not option: - if self.ignore_unknown_options: - unknown_options.append(ch) - continue - raise NoSuchOption(opt, ctx=self.ctx) - if option.takes_value: - # Any characters left in arg? Pretend they're the - # next arg, and stop consuming characters of arg. - if i < len(arg): - state.rargs.insert(0, arg[i:]) - stop = True - - value = self._get_value_from_state(opt, option, state) - - else: - value = UNSET - - option.process(value, state) - - if stop: - break - - # If we got any unknown options we recombine the string of the - # remaining options and re-attach the prefix, then report that - # to the state as new large. This way there is basic combinatorics - # that can be achieved while still ignoring unknown arguments. - if self.ignore_unknown_options and unknown_options: - state.largs.append(f"{prefix}{''.join(unknown_options)}") - - def _get_value_from_state( - self, option_name: str, option: _Option, state: _ParsingState - ) -> str | cabc.Sequence[str] | T_UNSET | T_FLAG_NEEDS_VALUE: - nargs = option.nargs - - value: str | cabc.Sequence[str] | T_UNSET | T_FLAG_NEEDS_VALUE - - if len(state.rargs) < nargs: - if option.obj._flag_needs_value: - # Option allows omitting the value. - value = FLAG_NEEDS_VALUE - else: - raise BadOptionUsage( - option_name, - ngettext( - "Option {name!r} requires an argument.", - "Option {name!r} requires {nargs} arguments.", - nargs, - ).format(name=option_name, nargs=nargs), - ) - elif nargs == 1: - next_rarg = state.rargs[0] - - if ( - option.obj._flag_needs_value - and isinstance(next_rarg, str) - and next_rarg[:1] in self._opt_prefixes - and len(next_rarg) > 1 - ): - # The next arg looks like the start of an option, don't - # use it as the value if omitting the value is allowed. - value = FLAG_NEEDS_VALUE - else: - value = state.rargs.pop(0) - else: - value = tuple(state.rargs[:nargs]) - del state.rargs[:nargs] - - return value - - def _process_opts(self, arg: str, state: _ParsingState) -> None: - explicit_value = None - # Long option handling happens in two parts. The first part is - # supporting explicitly attached values. In any case, we will try - # to long match the option first. - if "=" in arg: - long_opt, explicit_value = arg.split("=", 1) - else: - long_opt = arg - norm_long_opt = _normalize_opt(long_opt, self.ctx) - - # At this point we will match the (assumed) long option through - # the long option matching code. Note that this allows options - # like "-foo" to be matched as long options. - try: - self._match_long_opt(norm_long_opt, explicit_value, state) - except NoSuchOption: - # At this point the long option matching failed, and we need - # to try with short options. However there is a special rule - # which says, that if we have a two character options prefix - # (applies to "--foo" for instance), we do not dispatch to the - # short option code and will instead raise the no option - # error. - if arg[:2] not in self._opt_prefixes: - self._match_short_opt(arg, state) - return - - if not self.ignore_unknown_options: - raise - - state.largs.append(arg) - - -def __getattr__(name: str) -> object: - import warnings - - if name in { - "OptionParser", - "Argument", - "Option", - "split_opt", - "normalize_opt", - "ParsingState", - }: - warnings.warn( - f"'parser.{name}' is deprecated and will be removed in Click 9.0." - " The old parser is available in 'optparse'.", - DeprecationWarning, - stacklevel=2, - ) - return globals()[f"_{name}"] - - if name == "split_arg_string": - from .shell_completion import split_arg_string - - warnings.warn( - "Importing 'parser.split_arg_string' is deprecated, it will only be" - " available in 'shell_completion' in Click 9.0.", - DeprecationWarning, - stacklevel=2, - ) - return split_arg_string - - raise AttributeError(name) diff --git a/bundle/python-cpu/Lib/site-packages/click/py.typed b/bundle/python-cpu/Lib/site-packages/click/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/click/shell_completion.py b/bundle/python-cpu/Lib/site-packages/click/shell_completion.py deleted file mode 100644 index 468ee7720d934396d0a309067d800ea819af7da2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/shell_completion.py +++ /dev/null @@ -1,705 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import os -import re -import typing as t -from gettext import gettext as _ - -from .core import Argument -from .core import Command -from .core import Context -from .core import Group -from .core import Option -from .core import Parameter -from .core import ParameterSource -from .utils import echo - - -def shell_complete( - cli: Command, - ctx_args: cabc.MutableMapping[str, t.Any], - prog_name: str, - complete_var: str, - instruction: str, -) -> t.Literal[0, 1]: - """Perform shell completion for the given CLI program. - - :param cli: Command being called. - :param ctx_args: Extra arguments to pass to - ``cli.make_context``. - :param prog_name: Name of the executable in the shell. - :param complete_var: Name of the environment variable that holds - the completion instruction. - :param instruction: Value of ``complete_var`` with the completion - instruction and shell, in the form ``instruction_shell``. - :return: Status code to exit with. - """ - shell, _, instruction = instruction.partition("_") - comp_cls = get_completion_class(shell) - - if comp_cls is None: - return 1 - - comp = comp_cls(cli, ctx_args, prog_name, complete_var) - - # Write bytes, otherwise Windows text stdout translates LF to CRLF and breaks. - if instruction == "source": - echo(comp.source().encode(), nl=False) - return 0 - - if instruction == "complete": - echo(comp.complete().encode()) - return 0 - - return 1 - - -if t.TYPE_CHECKING: - from typing_extensions import TypeVar - - # `Any` is used as default for backwards compatibility (instead of e.g. `str`) - _ValueT_co = TypeVar("_ValueT_co", covariant=True, default=t.Any) -else: - _ValueT_co = t.TypeVar("_ValueT_co", covariant=True) - - -class CompletionItem(t.Generic[_ValueT_co]): - """Represents a completion value and metadata about the value. The - default metadata is ``type`` to indicate special shell handling, - and ``help`` if a shell supports showing a help string next to the - value. - - Arbitrary parameters can be passed when creating the object, and - accessed using ``item.attr``. If an attribute wasn't passed, - accessing it returns ``None``. - - :param value: The completion suggestion. - :param type: Tells the shell script to provide special completion - support for the type. Click uses ``"dir"`` and ``"file"``. - :param help: String shown next to the value if supported. - :param kwargs: Arbitrary metadata. The built-in implementations - don't use this, but custom type completions paired with custom - shell support could use it. - """ - - __slots__ = ("value", "type", "help", "_info") - - def __init__( - self, - value: _ValueT_co, - type: str = "plain", - help: str | None = None, - **kwargs: t.Any, - ) -> None: - self.value: _ValueT_co = value - self.type: str = type - self.help: str | None = help - self._info = kwargs - - def __getattr__(self, name: str) -> t.Any: - return self._info.get(name) - - -# Only Bash >= 4.4 has the nosort option. -_SOURCE_BASH = """\ -%(complete_func)s() { - local IFS=$'\\n' - local response - - response=$(env COMP_WORDS="${COMP_WORDS[*]}" COMP_CWORD=$COMP_CWORD \ -%(complete_var)s=bash_complete $1) - - for completion in $response; do - IFS=',' read type value <<< "$completion" - - if [[ $type == 'dir' ]]; then - COMPREPLY=() - compopt -o dirnames - elif [[ $type == 'file' ]]; then - COMPREPLY=() - compopt -o default - elif [[ $type == 'plain' ]]; then - COMPREPLY+=($value) - fi - done - - return 0 -} - -%(complete_func)s_setup() { - complete -o nosort -F %(complete_func)s %(prog_name)s -} - -%(complete_func)s_setup; -""" - -# See ZshComplete.format_completion below, and issue #2703, before -# changing this script. -# -# (TL;DR: _describe is picky about the format, but this Zsh script snippet -# is already widely deployed. So freeze this script, and use clever-ish -# handling of colons in ZshComplet.format_completion.) -_SOURCE_ZSH = """\ -#compdef %(prog_name)s - -%(complete_func)s() { - local -a completions - local -a completions_with_descriptions - local -a response - (( ! $+commands[%(prog_name)s] )) && return 1 - - response=("${(@f)$(env COMP_WORDS="${words[*]}" COMP_CWORD=$((CURRENT-1)) \ -%(complete_var)s=zsh_complete %(prog_name)s)}") - - for type key descr in ${response}; do - if [[ "$type" == "plain" ]]; then - if [[ "$descr" == "_" ]]; then - completions+=("$key") - else - completions_with_descriptions+=("$key":"$descr") - fi - elif [[ "$type" == "dir" ]]; then - _path_files -/ - elif [[ "$type" == "file" ]]; then - _path_files -f - fi - done - - if [ -n "$completions_with_descriptions" ]; then - _describe -V unsorted completions_with_descriptions -U - fi - - if [ -n "$completions" ]; then - compadd -U -V unsorted -a completions - fi -} - -if [[ $zsh_eval_context[-1] == loadautofunc ]]; then - # autoload from fpath, call function directly - %(complete_func)s "$@" -else - # eval/source/. command, register function for later - compdef %(complete_func)s %(prog_name)s -fi -""" - -_SOURCE_FISH = """\ -function %(complete_func)s; - set -l response (env %(complete_var)s=fish_complete COMP_WORDS=(commandline -cp) \ -COMP_CWORD=(commandline -t) %(prog_name)s); - - for completion in $response; - set -l metadata (string split "," $completion); - - if test $metadata[1] = "dir"; - __fish_complete_directories $metadata[2]; - else if test $metadata[1] = "file"; - __fish_complete_path $metadata[2]; - else if test $metadata[1] = "plain"; - echo $metadata[2]; - end; - end; -end; - -complete --no-files --command %(prog_name)s --arguments \ -"(%(complete_func)s)"; -""" - - -class _SourceVarsDict(t.TypedDict): - complete_func: str - complete_var: str - prog_name: str - - -class ShellComplete: - """Base class for providing shell completion support. A subclass for - a given shell will override attributes and methods to implement the - completion instructions (``source`` and ``complete``). - - :param cli: Command being called. - :param prog_name: Name of the executable in the shell. - :param complete_var: Name of the environment variable that holds - the completion instruction. - - .. versionadded:: 8.0 - """ - - name: t.ClassVar[str] - """Name to register the shell as with :func:`add_completion_class`. - This is used in completion instructions (``{name}_source`` and - ``{name}_complete``). - """ - - source_template: t.ClassVar[str] - """Completion script template formatted by :meth:`source`. This must - be provided by subclasses. - """ - - cli: Command - ctx_args: cabc.MutableMapping[str, t.Any] - prog_name: str - complete_var: str - - def __init__( - self, - cli: Command, - ctx_args: cabc.MutableMapping[str, t.Any], - prog_name: str, - complete_var: str, - ) -> None: - self.cli = cli - self.ctx_args = ctx_args - self.prog_name = prog_name - self.complete_var = complete_var - - @property - def func_name(self) -> str: - """The name of the shell function defined by the completion - script. - """ - safe_name = re.sub(r"\W*", "", self.prog_name.replace("-", "_"), flags=re.ASCII) - return f"_{safe_name}_completion" - - def source_vars(self) -> _SourceVarsDict: - """Vars for formatting :attr:`source_template`. - - By default this provides ``complete_func``, ``complete_var``, - and ``prog_name``. - """ - return { - "complete_func": self.func_name, - "complete_var": self.complete_var, - "prog_name": self.prog_name, - } - - def source(self) -> str: - """Produce the shell script that defines the completion - function. By default this ``%``-style formats - :attr:`source_template` with the dict returned by - :meth:`source_vars`. - """ - return self.source_template % self.source_vars() - - def get_completion_args(self) -> tuple[list[str], str]: - """Use the env vars defined by the shell script to return a - tuple of ``args, incomplete``. This must be implemented by - subclasses. - """ - raise NotImplementedError - - def get_completions( - self, args: list[str], incomplete: str - ) -> list[CompletionItem[str]]: - """Determine the context and last complete command or parameter - from the complete args. Call that object's ``shell_complete`` - method to get the completions for the incomplete value. - - :param args: List of complete args before the incomplete value. - :param incomplete: Value being completed. May be empty. - """ - ctx = _resolve_context(self.cli, self.ctx_args, self.prog_name, args) - obj, incomplete = _resolve_incomplete(ctx, args, incomplete) - return obj.shell_complete(ctx, incomplete) - - def format_completion(self, item: CompletionItem[str]) -> str: - """Format a completion item into the form recognized by the - shell script. This must be implemented by subclasses. - - :param item: Completion item to format. - """ - raise NotImplementedError - - def complete(self) -> str: - """Produce the completion data to send back to the shell. - - By default this calls :meth:`get_completion_args`, gets the - completions, then calls :meth:`format_completion` for each - completion. - """ - args, incomplete = self.get_completion_args() - completions = self.get_completions(args, incomplete) - out = [self.format_completion(item) for item in completions] - return "\n".join(out) - - -class BashComplete(ShellComplete): - """Shell completion for Bash.""" - - name: t.ClassVar[str] = "bash" - source_template: t.ClassVar[str] = _SOURCE_BASH - - @staticmethod - def _check_version() -> None: - import shutil - import subprocess - - bash_exe = shutil.which("bash") - - if bash_exe is None: - match = None - else: - output = subprocess.run( - [bash_exe, "--norc", "-c", 'echo "${BASH_VERSION}"'], - stdout=subprocess.PIPE, - ) - match = re.search(r"^(\d+)\.(\d+)\.\d+", output.stdout.decode()) - - if match is not None: - major, minor = match.groups() - - if major < "4" or major == "4" and minor < "4": - echo( - _( - "Shell completion is not supported for Bash" - " versions older than 4.4." - ), - err=True, - ) - else: - echo( - _("Couldn't detect Bash version, shell completion is not supported."), - err=True, - ) - - def source(self) -> str: - self._check_version() - return super().source() - - def get_completion_args(self) -> tuple[list[str], str]: - cwords = split_arg_string(os.environ["COMP_WORDS"]) - cword = int(os.environ["COMP_CWORD"]) - args = cwords[1:cword] - - try: - incomplete = cwords[cword] - except IndexError: - incomplete = "" - - return args, incomplete - - def format_completion(self, item: CompletionItem[t.Any]) -> str: - return f"{item.type},{item.value}" - - -class ZshComplete(ShellComplete): - """Shell completion for Zsh.""" - - name: t.ClassVar[str] = "zsh" - source_template: t.ClassVar[str] = _SOURCE_ZSH - - def get_completion_args(self) -> tuple[list[str], str]: - cwords = split_arg_string(os.environ["COMP_WORDS"]) - cword = int(os.environ["COMP_CWORD"]) - args = cwords[1:cword] - - try: - incomplete = cwords[cword] - except IndexError: - incomplete = "" - - return args, incomplete - - def format_completion(self, item: CompletionItem[str]) -> str: - help_ = item.help or "_" - # The zsh completion script uses `_describe` on items with help - # texts (which splits the item help from the item value at the - # first unescaped colon) and `compadd` on items without help - # text (which uses the item value as-is and does not support - # colon escaping). So escape colons in the item value if and - # only if the item help is not the sentinel "_" value, as used - # by the completion script. - # - # (The zsh completion script is potentially widely deployed, and - # thus harder to fix than this method.) - # - # See issue #1812 and issue #2703 for further context. - value = item.value.replace(":", r"\:") if help_ != "_" else item.value - return f"{item.type}\n{value}\n{help_}" - - -class FishComplete(ShellComplete): - """Shell completion for Fish.""" - - name: t.ClassVar[str] = "fish" - source_template: t.ClassVar[str] = _SOURCE_FISH - - def get_completion_args(self) -> tuple[list[str], str]: - cwords = split_arg_string(os.environ["COMP_WORDS"]) - incomplete = os.environ["COMP_CWORD"] - if incomplete: - incomplete = split_arg_string(incomplete)[0] - args = cwords[1:] - - # Fish stores the partial word in both COMP_WORDS and - # COMP_CWORD, remove it from complete args. - if incomplete and args and args[-1] == incomplete: - args.pop() - - return args, incomplete - - def format_completion(self, item: CompletionItem[str]) -> str: - """ - .. versionchanged:: 8.4.2 - Escape newlines and replace tabs with spaces in the help text to - fix completion errors with multi-line help strings. - """ - # According to https://fishshell.com/docs/current/cmds/complete.html - # Command substitutions found in ARGUMENTS should return a newline- - # separated list of arguments, and each argument may optionally have a tab - # character followed by the argument description. - if item.help: - help_ = item.help.replace("\n", "\\n").replace("\t", " ") - return f"{item.type},{item.value}\t{help_}" - - return f"{item.type},{item.value}" - - -_available_shells: t.Final[dict[str, type[ShellComplete]]] = { - "bash": BashComplete, - "fish": FishComplete, - "zsh": ZshComplete, -} - -_ShellCompleteT = t.TypeVar("_ShellCompleteT", bound="ShellComplete") - - -def add_completion_class( - cls: type[_ShellCompleteT], name: str | None = None -) -> type[_ShellCompleteT]: - """Register a :class:`ShellComplete` subclass under the given name. - The name will be provided by the completion instruction environment - variable during completion. - - :param cls: The completion class that will handle completion for the - shell. - :param name: Name to register the class under. Defaults to the - class's ``name`` attribute. - """ - if name is None: - name = cls.name - - _available_shells[name] = cls - - return cls - - -@t.overload -def get_completion_class(shell: t.Literal["bash"]) -> type[BashComplete]: ... -@t.overload -def get_completion_class(shell: t.Literal["fish"]) -> type[FishComplete]: ... -@t.overload -def get_completion_class(shell: t.Literal["zsh"]) -> type[ZshComplete]: ... -@t.overload -def get_completion_class(shell: str) -> type[ShellComplete] | None: ... -def get_completion_class(shell: str) -> type[ShellComplete] | None: - """Look up a registered :class:`ShellComplete` subclass by the name - provided by the completion instruction environment variable. If the - name isn't registered, returns ``None``. - - :param shell: Name the class is registered under. - """ - return _available_shells.get(shell) - - -def split_arg_string(string: str) -> list[str]: - """Split an argument string as with :func:`shlex.split`, but don't - fail if the string is incomplete. Ignores a missing closing quote or - incomplete escape sequence and uses the partial token as-is. - - .. code-block:: python - - split_arg_string("example 'my file") - ["example", "my file"] - - split_arg_string("example my\\") - ["example", "my"] - - :param string: String to split. - - .. versionchanged:: 8.2 - Moved to ``shell_completion`` from ``parser``. - """ - import shlex - - lex = shlex.shlex(string, posix=True) - lex.whitespace_split = True - lex.commenters = "" - out = [] - - try: - for token in lex: - out.append(token) - except ValueError: - # Raised when end-of-string is reached in an invalid state. Use - # the partial token as-is. The quote or escape character is in - # lex.state, not lex.token. - out.append(lex.token) - - return out - - -def _is_incomplete_argument(ctx: Context, param: Parameter) -> bool: - """Determine if the given parameter is an argument that can still - accept values. - - :param ctx: Invocation context for the command represented by the - parsed complete args. - :param param: Argument object being checked. - """ - if not isinstance(param, Argument): - return False - - value = ctx.params.get(param.name) - return ( - param.nargs == -1 - or ctx.get_parameter_source(param.name) is not ParameterSource.COMMANDLINE - or ( - param.nargs > 1 - and isinstance(value, (tuple, list)) - and len(value) < param.nargs - ) - ) - - -def _start_of_option(ctx: Context, value: str) -> bool: - """Check if the value looks like the start of an option.""" - if not value: - return False - - c = value[0] - return c in ctx._opt_prefixes - - -def _is_incomplete_option(ctx: Context, args: list[str], param: Parameter) -> bool: - """Determine if the given parameter is an option that needs a value. - - :param args: List of complete args before the incomplete value. - :param param: Option object being checked. - """ - if not isinstance(param, Option): - return False - - if param.is_flag or param.count: - return False - - last_option = None - - for index, arg in enumerate(reversed(args)): - if index + 1 > param.nargs: - break - - if _start_of_option(ctx, arg): - last_option = arg - break - - return last_option is not None and last_option in param.opts - - -def _resolve_context( - cli: Command, - ctx_args: cabc.MutableMapping[str, t.Any], - prog_name: str, - args: list[str], -) -> Context: - """Produce the context hierarchy starting with the command and - traversing the complete arguments. This only follows the commands, - it doesn't trigger input prompts or callbacks. - - :param cli: Command being called. - :param prog_name: Name of the executable in the shell. - :param args: List of complete args before the incomplete value. - """ - ctx_args["resilient_parsing"] = True - with cli.make_context(prog_name, args.copy(), **ctx_args) as ctx: - args = ctx._protected_args + ctx.args - - while args: - command = ctx.command - - if isinstance(command, Group): - if not command.chain: - name, cmd, args = command.resolve_command(ctx, args) - - if cmd is None: - return ctx - - with cmd.make_context( - name, args, parent=ctx, resilient_parsing=True - ) as sub_ctx: - ctx = sub_ctx - args = ctx._protected_args + ctx.args - else: - sub_ctx = ctx - - while args: - name, cmd, args = command.resolve_command(ctx, args) - - if cmd is None: - return ctx - - with cmd.make_context( - name, - args, - parent=ctx, - allow_extra_args=True, - allow_interspersed_args=False, - resilient_parsing=True, - ) as sub_sub_ctx: - sub_ctx = sub_sub_ctx - args = sub_ctx.args - - ctx = sub_ctx - args = [*sub_ctx._protected_args, *sub_ctx.args] - else: - break - - return ctx - - -def _resolve_incomplete( - ctx: Context, args: list[str], incomplete: str -) -> tuple[Command | Parameter, str]: - """Find the Click object that will handle the completion of the - incomplete value. Return the object and the incomplete value. - - :param ctx: Invocation context for the command represented by - the parsed complete args. - :param args: List of complete args before the incomplete value. - :param incomplete: Value being completed. May be empty. - """ - # Different shells treat an "=" between a long option name and - # value differently. Might keep the value joined, return the "=" - # as a separate item, or return the split name and value. Always - # split and discard the "=" to make completion easier. - if incomplete == "=": - incomplete = "" - elif "=" in incomplete and _start_of_option(ctx, incomplete): - name, _, incomplete = incomplete.partition("=") - args.append(name) - - # The "--" marker tells Click to stop treating values as options - # even if they start with the option character. If it hasn't been - # given and the incomplete arg looks like an option, the current - # command will provide option name completions. - if "--" not in args and _start_of_option(ctx, incomplete): - return ctx.command, incomplete - - params = ctx.command.get_params(ctx) - - # If the last complete arg is an option name with an incomplete - # value, the option will provide value completions. - for param in params: - if _is_incomplete_option(ctx, args, param): - return param, incomplete - - # It's not an option name or value. The first argument without a - # parsed value will provide value completions. - for param in params: - if _is_incomplete_argument(ctx, param): - return param, incomplete - - # There were no unparsed arguments, the command may be a group that - # will provide command name completions. - return ctx.command, incomplete diff --git a/bundle/python-cpu/Lib/site-packages/click/termui.py b/bundle/python-cpu/Lib/site-packages/click/termui.py deleted file mode 100644 index 9bc88db14dd66c59ccc99e386b385573d474dfd3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/termui.py +++ /dev/null @@ -1,945 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import inspect -import io -import itertools -import re -import sys -import typing as t -from contextlib import AbstractContextManager -from contextlib import redirect_stdout -from gettext import gettext as _ - -from ._compat import isatty -from ._compat import strip_ansi -from ._compat import WIN -from .exceptions import Abort -from .exceptions import UsageError -from .globals import resolve_color_default -from .types import Choice -from .types import convert_type -from .types import ParamType -from .utils import echo -from .utils import LazyFile - -if t.TYPE_CHECKING: - from ._termui_impl import ProgressBar - -V = t.TypeVar("V") - -# The prompt functions to use. The doc tools currently override these -# functions to customize how they work. -visible_prompt_func: t.Callable[[str], str] = input - -_ansi_colors = { - "black": 30, - "red": 31, - "green": 32, - "yellow": 33, - "blue": 34, - "magenta": 35, - "cyan": 36, - "white": 37, - "reset": 39, - "bright_black": 90, - "bright_red": 91, - "bright_green": 92, - "bright_yellow": 93, - "bright_blue": 94, - "bright_magenta": 95, - "bright_cyan": 96, - "bright_white": 97, -} -_ansi_reset_all = "\033[0m" - - -_HIDDEN_INPUT_MASK = "'***'" - - -def _mask_hidden_input(message: str, value: str) -> str: - """Replace occurrences of ``value`` in ``message`` with a fixed mask. - - Both ``repr(value)`` (the form built-in :class:`ParamType` errors use - via ``{value!r}``) and the raw value are masked. The raw-value pass - uses word-boundary lookarounds so a substring like ``"1"`` does not - match inside ``"10"``, and ``"ent"`` does not match inside - ``"Authentication"``. The empty string is skipped to avoid matching - at every boundary. - """ - message = message.replace(repr(value), _HIDDEN_INPUT_MASK) - if value: - message = re.sub( - rf"(? str: - import getpass - - return getpass.getpass(prompt) - - -def _readline_prompt(func: t.Callable[[str], str], text: str, err: bool) -> str: - """Call a prompt function, passing the full prompt on non-Windows so - readline can handle line editing and cursor positioning correctly. - - On Windows the prompt is written separately via :func:`echo` for - colorama support, with only the last character passed to *func*. - """ - if WIN: - # Write the prompt separately so that we get nice coloring - # through colorama on Windows. - echo(text[:-1], nl=False, err=err) - # Echo the last character to stdout to work around an issue - # where readline causes backspace to clear the whole line. - return func(text[-1:]) - if err: - with redirect_stdout(sys.stderr): - return func(text) - return func(text) - - -def _build_prompt( - text: str, - suffix: str, - show_default: bool | str = False, - default: t.Any | None = None, - show_choices: bool = True, - type: ParamType[t.Any] | None = None, -) -> str: - prompt = text - if type is not None and show_choices and isinstance(type, Choice): - prompt += f" ({', '.join(map(str, type.choices))})" - if isinstance(show_default, str): - default = f"({show_default})" - if default is not None and show_default: - prompt = f"{prompt} [{_format_default(default)}]" - return f"{prompt}{suffix}" - - -def _format_default(default: t.Any) -> t.Any: - if isinstance(default, (io.IOBase, LazyFile)) and hasattr(default, "name"): - return default.name - - return default - - -def prompt( - text: str, - default: t.Any | None = None, - hide_input: bool = False, - confirmation_prompt: bool | str = False, - type: ParamType[t.Any] | t.Any | None = None, - value_proc: t.Callable[[str], t.Any] | None = None, - prompt_suffix: str = ": ", - show_default: bool | str = True, - err: bool = False, - show_choices: bool = True, -) -> t.Any: - """Prompts a user for input. This is a convenience function that can - be used to prompt a user for input later. - - If the user aborts the input by sending an interrupt signal, this - function will catch it and raise a :exc:`Abort` exception. - - :param text: the text to show for the prompt. - :param default: the default value to use if no input happens. If this - is not given it will prompt until it's aborted. - :param hide_input: if this is set to true then the input value will - be hidden. - :param confirmation_prompt: Prompt a second time to confirm the - value. Can be set to a string instead of ``True`` to customize - the message. - :param type: the type to use to check the value against. - :param value_proc: if this parameter is provided it's a function that - is invoked instead of the type conversion to - convert a value. - :param prompt_suffix: a suffix that should be added to the prompt. - :param show_default: shows or hides the default value in the prompt. - If this value is a string, it shows that string - in parentheses instead of the actual value. - :param err: if set to true the file defaults to ``stderr`` instead of - ``stdout``, the same as with echo. - :param show_choices: Show or hide choices if the passed type is a Choice. - For example if type is a Choice of either day or week, - show_choices is true and text is "Group by" then the - prompt will be "Group by (day, week): ". - - .. versionchanged:: 8.3.3 - ``show_default`` can be a string to show a custom value instead - of the actual default, matching the help text behavior. - - .. versionchanged:: 8.3.1 - A space is no longer appended to the prompt. - - .. versionadded:: 8.0 - ``confirmation_prompt`` can be a custom string. - - .. versionadded:: 7.0 - Added the ``show_choices`` parameter. - - .. versionadded:: 6.0 - Added unicode support for cmd.exe on Windows. - - .. versionadded:: 4.0 - Added the `err` parameter. - - """ - - def prompt_func(text: str) -> str: - f = hidden_prompt_func if hide_input else visible_prompt_func - try: - return _readline_prompt(f, text, err) - except (KeyboardInterrupt, EOFError): - # getpass doesn't print a newline if the user aborts input with ^C. - # Allegedly this behavior is inherited from getpass(3). - # A doc bug has been filed at https://bugs.python.org/issue24711 - if hide_input: - echo(None, err=err) - raise Abort() from None - - if value_proc is None: - value_proc = convert_type(type, default) - - prompt = _build_prompt( - text, prompt_suffix, show_default, default, show_choices, type - ) - - if confirmation_prompt: - if confirmation_prompt is True: - confirmation_prompt = _("Repeat for confirmation") - - confirmation_prompt = _build_prompt(confirmation_prompt, prompt_suffix) - - while True: - while True: - value = prompt_func(prompt) - if value: - break - elif default is not None: - value = default - break - try: - result = value_proc(value) - except UsageError as e: - message = _mask_hidden_input(e.message, value) if hide_input else e.message - echo(_("Error: {message}").format(message=message), err=err) - continue - if not confirmation_prompt: - return result - while True: - value2 = prompt_func(confirmation_prompt) - is_empty = not value and not value2 - if value2 or is_empty: - break - if value == value2: - return result - echo(_("Error: The two entered values do not match."), err=err) - - -def confirm( - text: str, - default: bool | None = False, - abort: bool = False, - prompt_suffix: str = ": ", - show_default: bool = True, - err: bool = False, -) -> bool: - """Prompts for confirmation (yes/no question). - - If the user aborts the input by sending a interrupt signal this - function will catch it and raise a :exc:`Abort` exception. - - :param text: the question to ask. - :param default: The default value to use when no input is given. If - ``None``, repeat until input is given. - :param abort: if this is set to `True` a negative answer aborts the - exception by raising :exc:`Abort`. - :param prompt_suffix: a suffix that should be added to the prompt. - :param show_default: shows or hides the default value in the prompt. - :param err: if set to true the file defaults to ``stderr`` instead of - ``stdout``, the same as with echo. - - .. versionchanged:: 8.3.1 - A space is no longer appended to the prompt. - - .. versionchanged:: 8.0 - Repeat until input is given if ``default`` is ``None``. - - .. versionadded:: 4.0 - Added the ``err`` parameter. - """ - prompt = _build_prompt( - text, - prompt_suffix, - show_default, - "y/n" if default is None else ("Y/n" if default else "y/N"), - ) - - while True: - try: - value = _readline_prompt(visible_prompt_func, prompt, err).lower().strip() - except (KeyboardInterrupt, EOFError): - raise Abort() from None - if value in ("y", "yes"): - rv = True - elif value in ("n", "no"): - rv = False - elif default is not None and value == "": - rv = default - else: - echo(_("Error: invalid input"), err=err) - continue - break - if abort and not rv: - raise Abort() - return rv - - -def get_pager_file( - color: bool | None = None, -) -> t.ContextManager[t.TextIO]: - """Context manager. - - Yields a writable file-like object which can be used as an output pager. - - .. versionadded:: 8.4.0 - - :param color: controls if the pager supports ANSI colors or not. The - default is autodetection. - """ - from ._termui_impl import get_pager_file - - color = resolve_color_default(color) - - return get_pager_file(color=color) - - -def echo_via_pager( - text_or_generator: cabc.Iterable[str] | t.Callable[[], cabc.Iterable[str]] | str, - color: bool | None = None, -) -> None: - """This function takes a text and shows it via an environment specific - pager on stdout. - - .. versionchanged:: 3.0 - Added the `color` flag. - - :param text_or_generator: the text to page, or alternatively, a - generator emitting the text to page. - :param color: controls if the pager supports ANSI colors or not. The - default is autodetection. - """ - - if inspect.isgeneratorfunction(text_or_generator): - i = t.cast("t.Callable[[], cabc.Iterable[str]]", text_or_generator)() - elif isinstance(text_or_generator, str): - i = [text_or_generator] - else: - i = iter(t.cast("cabc.Iterable[str]", text_or_generator)) - - # convert every element of i to a text type if necessary - text_generator = (el if isinstance(el, str) else str(el) for el in i) - - with get_pager_file(color=color) as pager: - for text in itertools.chain(text_generator, "\n"): - pager.write(text) - # Flush after each write so a slow generator streams to the pager - # incrementally rather than staying invisible until the pipe buffer - # fills (~8 KB). - pager.flush() - - -@t.overload -def progressbar( - *, - length: int, - label: str | None = None, - hidden: bool = False, - show_eta: bool = True, - show_percent: bool | None = None, - show_pos: bool = False, - fill_char: str = "#", - empty_char: str = "-", - bar_template: str = "%(label)s [%(bar)s] %(info)s", - info_sep: str = " ", - width: int = 36, - file: t.TextIO | None = None, - color: bool | None = None, - update_min_steps: int = 1, -) -> ProgressBar[int]: ... - - -@t.overload -def progressbar( - iterable: cabc.Iterable[V] | None = None, - length: int | None = None, - label: str | None = None, - hidden: bool = False, - show_eta: bool = True, - show_percent: bool | None = None, - show_pos: bool = False, - item_show_func: t.Callable[[V | None], str | None] | None = None, - fill_char: str = "#", - empty_char: str = "-", - bar_template: str = "%(label)s [%(bar)s] %(info)s", - info_sep: str = " ", - width: int = 36, - file: t.TextIO | None = None, - color: bool | None = None, - update_min_steps: int = 1, -) -> ProgressBar[V]: ... - - -def progressbar( - iterable: cabc.Iterable[V] | None = None, - length: int | None = None, - label: str | None = None, - hidden: bool = False, - show_eta: bool = True, - show_percent: bool | None = None, - show_pos: bool = False, - item_show_func: t.Callable[[V | None], str | None] | None = None, - fill_char: str = "#", - empty_char: str = "-", - bar_template: str = "%(label)s [%(bar)s] %(info)s", - info_sep: str = " ", - width: int = 36, - file: t.TextIO | None = None, - color: bool | None = None, - update_min_steps: int = 1, -) -> ProgressBar[V]: - """This function creates an iterable context manager that can be used - to iterate over something while showing a progress bar. It will - either iterate over the `iterable` or `length` items (that are counted - up). While iteration happens, this function will print a rendered - progress bar to the given `file` (defaults to stdout) and will attempt - to calculate remaining time and more. By default, this progress bar - will not be rendered if the file is not a terminal. - - The context manager creates the progress bar. When the context - manager is entered the progress bar is already created. With every - iteration over the progress bar, the iterable passed to the bar is - advanced and the bar is updated. When the context manager exits, - a newline is printed and the progress bar is finalized on screen. - - Note: The progress bar is currently designed for use cases where the - total progress can be expected to take at least several seconds. - Because of this, the ProgressBar class object won't display - progress that is considered too fast, and progress where the time - between steps is less than a second. - - No printing must happen or the progress bar will be unintentionally - destroyed. - - Example usage:: - - with progressbar(items) as bar: - for item in bar: - do_something_with(item) - - Alternatively, if no iterable is specified, one can manually update the - progress bar through the `update()` method instead of directly - iterating over the progress bar. The update method accepts the number - of steps to increment the bar with:: - - with progressbar(length=chunks.total_bytes) as bar: - for chunk in chunks: - process_chunk(chunk) - bar.update(chunks.bytes) - - The ``update()`` method also takes an optional value specifying the - ``current_item`` at the new position. This is useful when used - together with ``item_show_func`` to customize the output for each - manual step:: - - with click.progressbar( - length=total_size, - label='Unzipping archive', - item_show_func=lambda a: a.filename - ) as bar: - for archive in zip_file: - archive.extract() - bar.update(archive.size, archive) - - :param iterable: an iterable to iterate over. If not provided the length - is required. - :param length: the number of items to iterate over. By default the - progressbar will attempt to ask the iterator about its - length, which might or might not work. If an iterable is - also provided this parameter can be used to override the - length. If an iterable is not provided the progress bar - will iterate over a range of that length. - :param label: the label to show next to the progress bar. - :param hidden: hide the progressbar. Defaults to ``False``. When no tty is - detected, it will only print the progressbar label. Setting this to - ``False`` also disables that. - :param show_eta: enables or disables the estimated time display. This is - automatically disabled if the length cannot be - determined. - :param show_percent: enables or disables the percentage display. The - default is `True` if the iterable has a length or - `False` if not. - :param show_pos: enables or disables the absolute position display. The - default is `False`. - :param item_show_func: A function called with the current item which - can return a string to show next to the progress bar. If the - function returns ``None`` nothing is shown. The current item can - be ``None``, such as when entering and exiting the bar. - :param fill_char: the character to use to show the filled part of the - progress bar. - :param empty_char: the character to use to show the non-filled part of - the progress bar. - :param bar_template: the format string to use as template for the bar. - The parameters in it are ``label`` for the label, - ``bar`` for the progress bar and ``info`` for the - info section. - :param info_sep: the separator between multiple info items (eta etc.) - :param width: the width of the progress bar in characters, 0 means full - terminal width - :param file: The file to write to. If this is not a terminal then - only the label is printed. - :param color: controls if the terminal supports ANSI colors or not. The - default is autodetection. This is only needed if ANSI - codes are included anywhere in the progress bar output - which is not the case by default. - :param update_min_steps: Render only when this many updates have - completed. This allows tuning for very fast iterators. - - .. versionadded:: 8.2 - The ``hidden`` argument. - - .. versionchanged:: 8.0 - Output is shown even if execution time is less than 0.5 seconds. - - .. versionchanged:: 8.0 - ``item_show_func`` shows the current item, not the previous one. - - .. versionchanged:: 8.0 - Labels are echoed if the output is not a TTY. Reverts a change - in 7.0 that removed all output. - - .. versionadded:: 8.0 - The ``update_min_steps`` parameter. - - .. versionadded:: 4.0 - The ``color`` parameter and ``update`` method. - - .. versionadded:: 2.0 - """ - from ._termui_impl import ProgressBar - - color = resolve_color_default(color) - return ProgressBar( - iterable=iterable, - length=length, - hidden=hidden, - show_eta=show_eta, - show_percent=show_percent, - show_pos=show_pos, - item_show_func=item_show_func, - fill_char=fill_char, - empty_char=empty_char, - bar_template=bar_template, - info_sep=info_sep, - file=file, - label=label, - width=width, - color=color, - update_min_steps=update_min_steps, - ) - - -def clear() -> None: - """Clears the terminal screen. This will have the effect of clearing - the whole visible space of the terminal and moving the cursor to the - top left. This does not do anything if not connected to a terminal. - - .. versionadded:: 2.0 - """ - if not isatty(sys.stdout): - return - - # ANSI escape \033[2J clears the screen, \033[1;1H moves the cursor - echo("\033[2J\033[1;1H", nl=False) - - -def _interpret_color(color: int | tuple[int, int, int] | str, offset: int = 0) -> str: - if isinstance(color, int): - return f"{38 + offset};5;{color:d}" - - if isinstance(color, (tuple, list)): - r, g, b = color - return f"{38 + offset};2;{r:d};{g:d};{b:d}" - - return str(_ansi_colors[color] + offset) - - -def style( - text: t.Any, - fg: int | tuple[int, int, int] | str | None = None, - bg: int | tuple[int, int, int] | str | None = None, - bold: bool | None = None, - dim: bool | None = None, - underline: bool | None = None, - overline: bool | None = None, - italic: bool | None = None, - blink: bool | None = None, - reverse: bool | None = None, - strikethrough: bool | None = None, - reset: bool = True, -) -> str: - """Styles a text with ANSI styles and returns the new string. By - default the styling is self contained which means that at the end - of the string a reset code is issued. This can be prevented by - passing ``reset=False``. - - Examples:: - - click.echo(click.style('Hello World!', fg='green')) - click.echo(click.style('ATTENTION!', blink=True)) - click.echo(click.style('Some things', reverse=True, fg='cyan')) - click.echo(click.style('More colors', fg=(255, 12, 128), bg=117)) - - Supported color names: - - * ``black`` (might be a gray) - * ``red`` - * ``green`` - * ``yellow`` (might be an orange) - * ``blue`` - * ``magenta`` - * ``cyan`` - * ``white`` (might be light gray) - * ``bright_black`` - * ``bright_red`` - * ``bright_green`` - * ``bright_yellow`` - * ``bright_blue`` - * ``bright_magenta`` - * ``bright_cyan`` - * ``bright_white`` - * ``reset`` (reset the color code only) - - If the terminal supports it, color may also be specified as: - - - An integer in the interval [0, 255]. The terminal must support - 8-bit/256-color mode. - - An RGB tuple of three integers in [0, 255]. The terminal must - support 24-bit/true-color mode. - - See https://en.wikipedia.org/wiki/ANSI_color and - https://gist.github.com/XVilka/8346728 for more information. - - :param text: the string to style with ansi codes. - :param fg: if provided this will become the foreground color. - :param bg: if provided this will become the background color. - :param bold: if provided this will enable or disable bold mode. - :param dim: if provided this will enable or disable dim mode. This is - badly supported. - :param underline: if provided this will enable or disable underline. - :param overline: if provided this will enable or disable overline. - :param italic: if provided this will enable or disable italic. - :param blink: if provided this will enable or disable blinking. - :param reverse: if provided this will enable or disable inverse - rendering (foreground becomes background and the - other way round). - :param strikethrough: if provided this will enable or disable - striking through text. - :param reset: by default a reset-all code is added at the end of the - string which means that styles do not carry over. This - can be disabled to compose styles. - - .. versionchanged:: 8.0 - A non-string ``message`` is converted to a string. - - .. versionchanged:: 8.0 - Added support for 256 and RGB color codes. - - .. versionchanged:: 8.0 - Added the ``strikethrough``, ``italic``, and ``overline`` - parameters. - - .. versionchanged:: 7.0 - Added support for bright colors. - - .. versionadded:: 2.0 - """ - if not isinstance(text, str): - text = str(text) - - bits = [] - - if fg: - try: - bits.append(f"\033[{_interpret_color(fg)}m") - except KeyError: - raise TypeError(_("Unknown color {colour!r}").format(colour=fg)) from None - - if bg: - try: - bits.append(f"\033[{_interpret_color(bg, 10)}m") - except KeyError: - raise TypeError(_("Unknown color {colour!r}").format(colour=bg)) from None - - if bold is not None: - bits.append(f"\033[{1 if bold else 22}m") - if dim is not None: - bits.append(f"\033[{2 if dim else 22}m") - if underline is not None: - bits.append(f"\033[{4 if underline else 24}m") - if overline is not None: - bits.append(f"\033[{53 if overline else 55}m") - if italic is not None: - bits.append(f"\033[{3 if italic else 23}m") - if blink is not None: - bits.append(f"\033[{5 if blink else 25}m") - if reverse is not None: - bits.append(f"\033[{7 if reverse else 27}m") - if strikethrough is not None: - bits.append(f"\033[{9 if strikethrough else 29}m") - bits.append(text) - if reset: - bits.append(_ansi_reset_all) - return "".join(bits) - - -def unstyle(text: str) -> str: - """Removes ANSI styling information from a string. Usually it's not - necessary to use this function as Click's echo function will - automatically remove styling if necessary. - - .. versionadded:: 2.0 - - :param text: the text to remove style information from. - """ - return strip_ansi(text) - - -def secho( - message: t.Any | None = None, - file: t.IO[t.AnyStr] | None = None, - nl: bool = True, - err: bool = False, - color: bool | None = None, - **styles: t.Any, -) -> None: - """This function combines :func:`echo` and :func:`style` into one - call. As such the following two calls are the same:: - - click.secho('Hello World!', fg='green') - click.echo(click.style('Hello World!', fg='green')) - - All keyword arguments are forwarded to the underlying functions - depending on which one they go with. - - Non-string types will be converted to :class:`str`. However, - :class:`bytes` are passed directly to :meth:`echo` without applying - style. If you want to style bytes that represent text, call - :meth:`bytes.decode` first. - - .. versionchanged:: 8.0 - A non-string ``message`` is converted to a string. Bytes are - passed through without style applied. - - .. versionadded:: 2.0 - """ - if message is not None and not isinstance(message, (bytes, bytearray)): - message = style(message, **styles) - - return echo(message, file=file, nl=nl, err=err, color=color) - - -@t.overload -def edit( - text: bytes | bytearray, - editor: str | None = None, - env: cabc.Mapping[str, str] | None = None, - require_save: bool = False, - extension: str = ".txt", -) -> bytes | None: ... - - -@t.overload -def edit( - text: str, - editor: str | None = None, - env: cabc.Mapping[str, str] | None = None, - require_save: bool = True, - extension: str = ".txt", -) -> str | None: ... - - -@t.overload -def edit( - text: None = None, - editor: str | None = None, - env: cabc.Mapping[str, str] | None = None, - require_save: bool = True, - extension: str = ".txt", - filename: str | cabc.Iterable[str] | None = None, -) -> None: ... - - -def edit( - text: str | bytes | bytearray | None = None, - editor: str | None = None, - env: cabc.Mapping[str, str] | None = None, - require_save: bool = True, - extension: str = ".txt", - filename: str | cabc.Iterable[str] | None = None, -) -> str | bytes | bytearray | None: - r"""Edits the given text in the defined editor. If an editor is given - (should be the full path to the executable but the regular operating - system search path is used for finding the executable) it overrides - the detected editor. Optionally, some environment variables can be - used. If the editor is closed without changes, `None` is returned. In - case a file is edited directly the return value is always `None` and - `require_save` and `extension` are ignored. - - If the editor cannot be opened a :exc:`UsageError` is raised. - - Note for Windows: to simplify cross-platform usage, the newlines are - automatically converted from POSIX to Windows and vice versa. As such, - the message here will have ``\n`` as newline markers. - - :param text: the text to edit. - :param editor: optionally the editor to use. Defaults to automatic - detection. - :param env: environment variables to forward to the editor. - :param require_save: if this is true, then not saving in the editor - will make the return value become `None`. - :param extension: the extension to tell the editor about. This defaults - to `.txt` but changing this might change syntax - highlighting. - :param filename: if provided it will edit this file instead of the - provided text contents. It will not use a temporary - file as an indirection in that case. If the editor supports - editing multiple files at once, a sequence of files may be - passed as well. Invoke `click.file` once per file instead - if multiple files cannot be managed at once or editing the - files serially is desired. - - .. versionchanged:: 8.2.0 - ``filename`` now accepts any ``Iterable[str]`` in addition to a ``str`` - if the ``editor`` supports editing multiple files at once. - - """ - from ._termui_impl import Editor - - ed = Editor(editor=editor, env=env, require_save=require_save, extension=extension) - - if filename is None: - return ed.edit(text) - - if isinstance(filename, str): - filename = (filename,) - - ed.edit_files(filenames=filename) - return None - - -def launch(url: str, wait: bool = False, locate: bool = False) -> int: - """This function launches the given URL (or filename) in the default - viewer application for this file type. If this is an executable, it - might launch the executable in a new session. The return value is - the exit code of the launched application. Usually, ``0`` indicates - success. - - Examples:: - - click.launch('https://click.palletsprojects.com/') - click.launch('/my/downloaded/file', locate=True) - - .. versionadded:: 2.0 - - :param url: URL or filename of the thing to launch. - :param wait: Wait for the program to exit before returning. This - only works if the launched program blocks. In particular, - ``xdg-open`` on Linux does not block. - :param locate: if this is set to `True` then instead of launching the - application associated with the URL it will attempt to - launch a file manager with the file located. This - might have weird effects if the URL does not point to - the filesystem. - """ - from ._termui_impl import open_url - - return open_url(url, wait=wait, locate=locate) - - -# If this is provided, getchar() calls into this instead. This is used -# for unittesting purposes. -_getchar: t.Callable[[bool], str] | None = None - - -def getchar(echo: bool = False) -> str: - """Fetches a single character from the terminal and returns it. This - will always return a unicode character and under certain rare - circumstances this might return more than one character. The - situations which more than one character is returned is when for - whatever reason multiple characters end up in the terminal buffer or - standard input was not actually a terminal. - - Note that this will always read from the terminal, even if something - is piped into the standard input. - - Note for Windows: in rare cases when typing non-ASCII characters, this - function might wait for a second character and then return both at once. - This is because certain Unicode characters look like special-key markers. - - .. versionadded:: 2.0 - - :param echo: if set to `True`, the character read will also show up on - the terminal. The default is to not show it. - """ - global _getchar - - if _getchar is None: - from ._termui_impl import getchar as f - - _getchar = f - - return _getchar(echo) - - -def raw_terminal() -> AbstractContextManager[int]: - from ._termui_impl import raw_terminal as f - - return f() - - -def pause(info: str | None = None, err: bool = False) -> None: - """This command stops execution and waits for the user to press any - key to continue. This is similar to the Windows batch "pause" - command. If the program is not run through a terminal, this command - will instead do nothing. - - .. versionadded:: 2.0 - - .. versionadded:: 4.0 - Added the `err` parameter. - - :param info: The message to print before pausing. Defaults to - ``"Press any key to continue..."``. - :param err: if set to message goes to ``stderr`` instead of - ``stdout``, the same as with echo. - """ - if not isatty(sys.stdin) or not isatty(sys.stdout): - return - - if info is None: - info = _("Press any key to continue...") - - try: - if info: - echo(info, nl=False, err=err) - try: - getchar() - except (KeyboardInterrupt, EOFError): - pass - finally: - if info: - echo(err=err) diff --git a/bundle/python-cpu/Lib/site-packages/click/testing.py b/bundle/python-cpu/Lib/site-packages/click/testing.py deleted file mode 100644 index 19fae4a620ea8ae28410cb8b29d965c1ac8cc524..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/testing.py +++ /dev/null @@ -1,772 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import contextlib -import io -import os -import pdb -import shlex -import sys -import tempfile -import typing as t -from types import TracebackType - -from . import _compat -from . import formatting -from . import termui -from . import utils -from ._compat import _find_binary_reader - -if t.TYPE_CHECKING: - from _typeshed import ReadableBuffer - - from .core import Command - -if sys.platform == "win32": - CaptureMode: t.TypeAlias = t.Literal["sys"] # pyright: ignore[reportRedeclaration] -else: - CaptureMode: t.TypeAlias = t.Literal["sys", "fd"] # pyright: ignore[reportRedeclaration] -ExceptionInfo: t.TypeAlias = tuple[type[BaseException], BaseException, TracebackType] - - -class EchoingStdin: - _input: t.BinaryIO - _output: t.BinaryIO - _paused: bool - - def __init__(self, input: t.BinaryIO, output: t.BinaryIO) -> None: - self._input = input - self._output = output - self._paused = False - - def __getattr__(self, x: str) -> t.Any: - return getattr(self._input, x) - - def _echo(self, rv: bytes) -> bytes: - if not self._paused: - self._output.write(rv) - - return rv - - def read(self, n: int = -1) -> bytes: - return self._echo(self._input.read(n)) - - def read1(self, n: int = -1) -> bytes: - return self._echo(self._input.read1(n)) # type: ignore - - def readline(self, n: int = -1) -> bytes: - return self._echo(self._input.readline(n)) - - def readlines(self) -> list[bytes]: - return [self._echo(x) for x in self._input.readlines()] - - def __iter__(self) -> cabc.Iterator[bytes]: - return iter(self._echo(x) for x in self._input) - - def __repr__(self) -> str: - return repr(self._input) - - -@contextlib.contextmanager -def _pause_echo(stream: EchoingStdin | None) -> cabc.Generator[None]: - if stream is None: - yield - else: - stream._paused = True - yield - stream._paused = False - - -class _FDCapture: - """Redirect a file descriptor to a temporary file for capture. - - Saves the current target of *targetfd* via :func:`os.dup`, then - redirects it to a temporary file via :func:`os.dup2`. On - :meth:`stop`, restores the original ``fd`` and returns the captured - bytes. Inspired by Pytest's ``FDCapture``. - - .. versionadded:: 8.4.0 - """ - - _targetfd: int - saved_fd: int - _tmpfile: t.BinaryIO | None - - def __init__(self, targetfd: int) -> None: - self._targetfd = targetfd - self.saved_fd = -1 - self._tmpfile = None - - def start(self) -> None: - self.saved_fd = os.dup(self._targetfd) - self._tmpfile = tempfile.TemporaryFile(buffering=0) - os.dup2(self._tmpfile.fileno(), self._targetfd) - - def stop(self) -> bytes: - assert self._tmpfile is not None, "_FDCapture.start() was not called" - os.dup2(self.saved_fd, self._targetfd) - os.close(self.saved_fd) - self.saved_fd = -1 - self._tmpfile.seek(0) - data = self._tmpfile.read() - self._tmpfile.close() - self._tmpfile = None - return data - - -class BytesIOCopy(io.BytesIO): - """Patch ``io.BytesIO`` to let the written stream be copied to another. - - .. versionadded:: 8.2 - """ - - copy_to: io.BytesIO - - def __init__(self, copy_to: io.BytesIO) -> None: - super().__init__() - self.copy_to = copy_to - - def flush(self) -> None: - super().flush() - self.copy_to.flush() - - def write(self, b: ReadableBuffer) -> int: - self.copy_to.write(b) - return super().write(b) - - -class StreamMixer: - """Mixes `` and `` streams. - - The result is available in the ``output`` attribute. - - .. versionadded:: 8.2 - """ - - output: io.BytesIO - stdout: BytesIOCopy - stderr: BytesIOCopy - - def __init__(self) -> None: - self.output = io.BytesIO() - self.stdout = BytesIOCopy(copy_to=self.output) - self.stderr = BytesIOCopy(copy_to=self.output) - - -class _NamedTextIOWrapper(io.TextIOWrapper): - """A :class:`~io.TextIOWrapper` with custom ``name`` and ``mode`` - that does not close its underlying buffer. - - When ``CliRunner`` runs in ``fd`` mode, ``_original_fd`` is patched to - point at the saved (pre-redirection) ``fd``, so C-level consumers that call - :meth:`fileno` (like ``faulthandler`` or ``subprocess``) keep working. In - the default ``sys`` mode ``_original_fd`` stays at ``-1`` and - :meth:`fileno` raises :exc:`io.UnsupportedOperation`, matching the - pre-``8.3.3`` behavior. - """ - - _name: str - _mode: str - _original_fd: int - - def __init__( - self, - buffer: t.BinaryIO, - name: str, - mode: str, - **kwargs: t.Any, - ) -> None: - super().__init__(buffer, **kwargs) - self._name = name - self._mode = mode - self._original_fd = -1 - - def close(self) -> None: - """The buffer this object contains belongs to some other object, - so prevent the default ``__del__`` implementation from closing - that buffer. - - .. versionadded:: 8.3.2 - """ - - def fileno(self) -> int: - """Return the file descriptor of the saved original stream when - ``CliRunner`` runs in ``fd`` mode. Otherwise delegate to - :class:`~io.TextIOWrapper`, which raises - :exc:`io.UnsupportedOperation` for a ``BytesIO``-backed buffer. - """ - if self._original_fd >= 0: - return self._original_fd - return super().fileno() - - @property - def name(self) -> str: - return self._name - - @property - def mode(self) -> str: - return self._mode - - -def make_input_stream( - input: str | bytes | t.IO[t.Any] | None, charset: str -) -> t.BinaryIO: - # Is already an input stream. - if hasattr(input, "read"): - rv = _find_binary_reader(t.cast("t.IO[t.Any]", input)) - - if rv is not None: - return rv - - raise TypeError("Could not find binary reader for input stream.") - - if input is None: - input = b"" - elif isinstance(input, str): - input = input.encode(charset) - - return io.BytesIO(input) - - -class Result: - """Holds the captured result of an invoked CLI script. - - :param runner: The runner that created the result - :param stdout_bytes: The standard output as bytes. - :param stderr_bytes: The standard error as bytes. - :param output_bytes: A mix of ``stdout_bytes`` and ``stderr_bytes``, as the - user would see it in its terminal. - :param return_value: The value returned from the invoked command. - :param exit_code: The exit code as integer. - :param exception: The exception that happened if one did. - :param exc_info: Exception information (exception type, exception instance, - traceback type). - - .. versionchanged:: 8.2 - ``stderr_bytes`` no longer optional, ``output_bytes`` introduced and - ``mix_stderr`` has been removed. - - .. versionadded:: 8.0 - Added ``return_value``. - """ - - runner: CliRunner - stdout_bytes: bytes - stderr_bytes: bytes - output_bytes: bytes - return_value: t.Any - exit_code: int - exception: BaseException | None - exc_info: ExceptionInfo | None - - def __init__( - self, - runner: CliRunner, - stdout_bytes: bytes, - stderr_bytes: bytes, - output_bytes: bytes, - return_value: t.Any, - exit_code: int, - exception: BaseException | None, - exc_info: ExceptionInfo | None = None, - ) -> None: - self.runner = runner - self.stdout_bytes = stdout_bytes - self.stderr_bytes = stderr_bytes - self.output_bytes = output_bytes - self.return_value = return_value - self.exit_code = exit_code - self.exception = exception - self.exc_info = exc_info - - @property - def output(self) -> str: - """The terminal output as unicode string, as the user would see it. - - .. versionchanged:: 8.2 - No longer a proxy for ``self.stdout``. Now has its own independent stream - that is mixing `` and ``, in the order they were written. - """ - return self.output_bytes.decode(self.runner.charset, "replace").replace( - "\r\n", "\n" - ) - - @property - def stdout(self) -> str: - """The standard output as unicode string.""" - return self.stdout_bytes.decode(self.runner.charset, "replace").replace( - "\r\n", "\n" - ) - - @property - def stderr(self) -> str: - """The standard error as unicode string. - - .. versionchanged:: 8.2 - No longer raise an exception, always returns the `` string. - """ - return self.stderr_bytes.decode(self.runner.charset, "replace").replace( - "\r\n", "\n" - ) - - def __repr__(self) -> str: - exc_str = repr(self.exception) if self.exception else "okay" - return f"<{type(self).__name__} {exc_str}>" - - -class CliRunner: - """The CLI runner provides functionality to invoke a Click command line - script for unittesting purposes in a isolated environment. This only - works in single-threaded systems without any concurrency as it changes the - global interpreter state. - - :param charset: the character set for the input and output data. - :param env: a dictionary with environment variables for overriding. - :param echo_stdin: if this is set to `True`, then reading from `` writes - to ``. This is useful for showing examples in - some circumstances. Note that regular prompts - will automatically echo the input. - :param catch_exceptions: Whether to catch any exceptions other than - ``SystemExit`` when running :meth:`~CliRunner.invoke`. - :param capture: Selects the output capture strategy. ``sys`` (default) - captures Python-level writes only and leaves - :meth:`sys.stdout.fileno` raising :exc:`io.UnsupportedOperation`, so - user code that calls :func:`os.dup2` on ``sys.stdout.fileno()`` cannot - clobber the host runner's stdout. ``fd`` redirects file descriptors - ``1`` and ``2`` via :func:`os.dup2` to a temporary file, also catching - output from stale stream references, C extensions, and subprocesses. - ``fd`` is not supported on Windows. - - .. versionchanged:: 8.4.0 - Added the ``capture`` parameter. The default ``sys`` mode no longer - exposes the original fd through :meth:`fileno`, reverting the change - introduced in ``8.3.3`` that broke Pytest's ``fd``-level capture - teardown. Use ``capture="fd"`` to restore that behavior with proper - isolation. :issue:`3384` - - .. versionchanged:: 8.2 - Added the ``catch_exceptions`` parameter. - - .. versionchanged:: 8.2 - ``mix_stderr`` parameter has been removed. - """ - - charset: str - env: cabc.Mapping[str, str | None] - echo_stdin: bool - catch_exceptions: bool - capture: CaptureMode - - def __init__( - self, - charset: str = "utf-8", - env: cabc.Mapping[str, str | None] | None = None, - echo_stdin: bool = False, - catch_exceptions: bool = True, - capture: CaptureMode = "sys", - ) -> None: - if capture not in {"sys", "fd"}: - raise ValueError( - f"capture={capture!r} is not valid. Choose from 'sys' or 'fd'." - ) - if capture == "fd" and sys.platform == "win32": - raise ValueError( - f"capture={capture!r} is not supported on Windows. Use 'sys'." - ) - self.charset = charset - self.env = env or {} - self.echo_stdin = echo_stdin - self.catch_exceptions = catch_exceptions - self.capture = capture - - def get_default_prog_name(self, cli: Command) -> str: - """Given a command object it will return the default program name - for it. The default is the `name` attribute or ``"root"`` if not - set. - """ - return cli.name or "root" - - def make_env( - self, overrides: cabc.Mapping[str, str | None] | None = None - ) -> cabc.Mapping[str, str | None]: - """Returns the environment overrides for invoking a script.""" - rv = dict(self.env) - if overrides: - rv.update(overrides) - return rv - - @contextlib.contextmanager - def isolation( - self, - input: str | bytes | t.IO[t.Any] | None = None, - env: cabc.Mapping[str, str | None] | None = None, - color: bool = False, - ) -> cabc.Generator[tuple[io.BytesIO, io.BytesIO, io.BytesIO]]: - """A context manager that sets up the isolation for invoking of a - command line tool. This sets up `` with the given input data - and `os.environ` with the overrides from the given dictionary. - This also rebinds some internals in Click to be mocked (like the - prompt functionality). - - This is automatically done in the :meth:`invoke` method. - - :param input: the input stream to put into `sys.stdin`. - :param env: the environment overrides as dictionary. - :param color: whether the output should contain color codes. The - application can still override this explicitly. - - .. versionadded:: 8.2 - An additional output stream is returned, which is a mix of - `` and `` streams. - - .. versionchanged:: 8.2 - Always returns the `` stream. - - .. versionchanged:: 8.0 - `` is opened with ``errors="backslashreplace"`` - instead of the default ``"strict"``. - - .. versionchanged:: 4.0 - Added the ``color`` parameter. - """ - bytes_input = make_input_stream(input, self.charset) - echo_input = None - - old_stdin = sys.stdin - old_stdout = sys.stdout - old_stderr = sys.stderr - old_forced_width = formatting.FORCED_WIDTH - formatting.FORCED_WIDTH = 80 - - env = self.make_env(env) - - stream_mixer = StreamMixer() - - if self.echo_stdin: - bytes_input = echo_input = t.cast( - t.BinaryIO, EchoingStdin(bytes_input, stream_mixer.stdout) - ) - - sys.stdin = text_input = _NamedTextIOWrapper( - bytes_input, encoding=self.charset, name="", mode="r" - ) - - if self.echo_stdin: - # Force unbuffered reads, otherwise TextIOWrapper reads a - # large chunk which is echoed early. - text_input._CHUNK_SIZE = 1 # type: ignore - - sys.stdout = _NamedTextIOWrapper( - stream_mixer.stdout, - encoding=self.charset, - name="", - mode="w", - ) - - sys.stderr = _NamedTextIOWrapper( - stream_mixer.stderr, - encoding=self.charset, - name="", - mode="w", - errors="backslashreplace", - ) - - @_pause_echo(echo_input) # type: ignore - def visible_input(prompt: str | None = None) -> str: - sys.stdout.write(prompt or "") - try: - val = next(text_input).rstrip("\r\n") - except StopIteration as e: - raise EOFError() from e - sys.stdout.write(f"{val}\n") - sys.stdout.flush() - return val - - @_pause_echo(echo_input) # type: ignore - def hidden_input(prompt: str | None = None) -> str: - sys.stdout.write(f"{prompt or ''}\n") - sys.stdout.flush() - try: - return next(text_input).rstrip("\r\n") - except StopIteration as e: - raise EOFError() from e - - @_pause_echo(echo_input) # type: ignore - def _getchar(echo: bool) -> str: - char = sys.stdin.read(1) - - if echo: - sys.stdout.write(char) - - sys.stdout.flush() - return char - - default_color = color - - def should_strip_ansi( - stream: t.IO[t.Any] | None = None, color: bool | None = None - ) -> bool: - if color is None: - return not default_color - return not color - - old_visible_prompt_func = termui.visible_prompt_func - old_hidden_prompt_func = termui.hidden_prompt_func - old__getchar_func = termui._getchar - old_should_strip_ansi = utils.should_strip_ansi # type: ignore - old__compat_should_strip_ansi = _compat.should_strip_ansi - old_pdb_init = pdb.Pdb.__init__ - termui.visible_prompt_func = visible_input - termui.hidden_prompt_func = hidden_input - termui._getchar = _getchar - utils.should_strip_ansi = should_strip_ansi # type: ignore - _compat.should_strip_ansi = should_strip_ansi - - def _patched_pdb_init( - self: pdb.Pdb, - completekey: str = "tab", - stdin: t.IO[str] | None = None, - stdout: t.IO[str] | None = None, - **kwargs: t.Any, - ) -> None: - """Default ``pdb.Pdb`` to real terminal streams during - ``CliRunner`` isolation. - - Without this patch, ``pdb.Pdb.__init__`` inherits from - ``cmd.Cmd`` which falls back to ``sys.stdin``/``sys.stdout`` - when no explicit streams are provided. During isolation - those are ``BytesIO``-backed wrappers, so the debugger - reads from an empty buffer and writes to captured output, - making interactive debugging impossible. - - By defaulting to ``sys.__stdin__``/``sys.__stdout__`` (the - original terminal streams Python preserves regardless of - redirection), debuggers can interact with the user while - ``click.echo`` output is still captured normally. - - This covers ``pdb.set_trace()``, ``breakpoint()``, - ``pdb.post_mortem()``, and debuggers that subclass - ``pdb.Pdb`` (ipdb, pdbpp). Explicit ``stdin``/``stdout`` - arguments are honored and not overridden. Debuggers that - do not subclass ``pdb.Pdb`` (pudb, debugpy) are not - covered. - """ - if stdin is None: - stdin = sys.__stdin__ - if stdout is None: - stdout = sys.__stdout__ - old_pdb_init( - self, completekey=completekey, stdin=stdin, stdout=stdout, **kwargs - ) - - pdb.Pdb.__init__ = _patched_pdb_init # type: ignore[assignment] - - old_env = {} - try: - for key, value in env.items(): - old_env[key] = os.environ.get(key) - if value is None: - try: - del os.environ[key] - except Exception: - pass - else: - os.environ[key] = value - yield (stream_mixer.stdout, stream_mixer.stderr, stream_mixer.output) - finally: - for key, value in old_env.items(): - if value is None: - try: - del os.environ[key] - except Exception: - pass - else: - os.environ[key] = value - sys.stdout = old_stdout - sys.stderr = old_stderr - sys.stdin = old_stdin - termui.visible_prompt_func = old_visible_prompt_func - termui.hidden_prompt_func = old_hidden_prompt_func - termui._getchar = old__getchar_func - utils.should_strip_ansi = old_should_strip_ansi # type: ignore - _compat.should_strip_ansi = old__compat_should_strip_ansi - formatting.FORCED_WIDTH = old_forced_width - pdb.Pdb.__init__ = old_pdb_init # type: ignore[method-assign] - - def invoke( - self, - cli: Command, - args: str | cabc.Sequence[str] | None = None, - input: str | bytes | t.IO[t.Any] | None = None, - env: cabc.Mapping[str, str | None] | None = None, - catch_exceptions: bool | None = None, - color: bool = False, - **extra: t.Any, - ) -> Result: - """Invokes a command in an isolated environment. The arguments are - forwarded directly to the command line script, the `extra` keyword - arguments are passed to the :meth:`~clickpkg.Command.main` function of - the command. - - This returns a :class:`Result` object. - - :param cli: the command to invoke - :param args: the arguments to invoke. It may be given as an iterable - or a string. When given as string it will be interpreted - as a Unix shell command. More details at - :func:`shlex.split`. - :param input: the input data for `sys.stdin`. - :param env: the environment overrides. - :param catch_exceptions: Whether to catch any other exceptions than - ``SystemExit``. If :data:`None`, the value - from :class:`CliRunner` is used. - :param extra: the keyword arguments to pass to :meth:`main`. - :param color: whether the output should contain color codes. The - application can still override this explicitly. - - .. versionadded:: 8.2 - The result object has the ``output_bytes`` attribute with - the mix of ``stdout_bytes`` and ``stderr_bytes``, as the user would - see it in its terminal. - - .. versionchanged:: 8.2 - The result object always returns the ``stderr_bytes`` stream. - - .. versionchanged:: 8.0 - The result object has the ``return_value`` attribute with - the value returned from the invoked command. - - .. versionchanged:: 4.0 - Added the ``color`` parameter. - - .. versionchanged:: 3.0 - Added the ``catch_exceptions`` parameter. - - .. versionchanged:: 3.0 - The result object has the ``exc_info`` attribute with the - traceback if available. - """ - exc_info = None - if catch_exceptions is None: - catch_exceptions = self.catch_exceptions - - # Set up fd capture before isolation replaces sys.stdout and sys.stderr. - cap_out: _FDCapture | None = None - cap_err: _FDCapture | None = None - - if self.capture == "fd": - cap_out = _FDCapture(1) - cap_err = _FDCapture(2) - try: - cap_out.start() - cap_err.start() - except OSError: - cap_out = cap_err = None - - with self.isolation(input=input, env=env, color=color) as outstreams: - # Point the captured streams' fileno() at the saved (original) - # fd so that C-level consumers like faulthandler keep working - # while fd 1/2 are redirected to the capture tmpfile. - if cap_out is not None and cap_err is not None: - sys.stdout._original_fd = cap_out.saved_fd # type: ignore[union-attr] - sys.stderr._original_fd = cap_err.saved_fd # type: ignore[union-attr] - - return_value = None - exception: BaseException | None = None - exit_code = 0 - - if isinstance(args, str): - args = shlex.split(args) - - try: - prog_name = extra.pop("prog_name") - except KeyError: - prog_name = self.get_default_prog_name(cli) - - try: - return_value = cli.main(args=args or (), prog_name=prog_name, **extra) - except SystemExit as e: - exc_info = sys.exc_info() - e_code = t.cast("int | t.Any | None", e.code) - - if e_code is None: - e_code = 0 - - if e_code != 0: - exception = e - - if not isinstance(e_code, int): - sys.stdout.write(str(e_code)) - sys.stdout.write("\n") - e_code = 1 - - exit_code = e_code - - except Exception as e: - if not catch_exceptions: - raise - exception = e - exit_code = 1 - exc_info = sys.exc_info() - finally: - sys.stdout.flush() - sys.stderr.flush() - - # Stop fd capture and merge the captured bytes into - # the stdout/stderr BytesIO streams. BytesIOCopy mirrors - # those writes into outstreams[2] automatically. - if cap_out is not None and cap_err is not None: - fd_out = cap_out.stop() - fd_err = cap_err.stop() - if fd_out: - outstreams[0].write(fd_out) - if fd_err: - outstreams[1].write(fd_err) - - stdout = outstreams[0].getvalue() - stderr = outstreams[1].getvalue() - output = outstreams[2].getvalue() - - return Result( - runner=self, - stdout_bytes=stdout, - stderr_bytes=stderr, - output_bytes=output, - return_value=return_value, - exit_code=exit_code, - exception=exception, - exc_info=exc_info, # type: ignore - ) - - @contextlib.contextmanager - def isolated_filesystem( - self, temp_dir: str | os.PathLike[str] | None = None - ) -> cabc.Generator[str]: - """A context manager that creates a temporary directory and - changes the current working directory to it. This isolates tests - that affect the contents of the CWD to prevent them from - interfering with each other. - - :param temp_dir: Create the temporary directory under this - directory. If given, the created directory is not removed - when exiting. - - .. versionchanged:: 8.0 - Added the ``temp_dir`` parameter. - """ - cwd = os.getcwd() - dt = tempfile.mkdtemp(dir=temp_dir) - os.chdir(dt) - - try: - yield dt - finally: - os.chdir(cwd) - - if temp_dir is None: - import shutil - - try: - shutil.rmtree(dt) - except OSError: - pass diff --git a/bundle/python-cpu/Lib/site-packages/click/types.py b/bundle/python-cpu/Lib/site-packages/click/types.py deleted file mode 100644 index 1e9872e410a3371f091e03ddadb90de3120211dc..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/types.py +++ /dev/null @@ -1,1374 +0,0 @@ -from __future__ import annotations - -import abc -import collections.abc as cabc -import enum -import os -import stat -import sys -import typing as t -import uuid -from datetime import datetime -from gettext import gettext as _ -from gettext import ngettext - -from ._compat import _get_argv_encoding -from ._compat import open_stream -from .exceptions import BadParameter -from .utils import format_filename -from .utils import LazyFile -from .utils import safecall - -if t.TYPE_CHECKING: - import typing_extensions as te - - from .core import Context - from .core import Parameter - from .shell_completion import CompletionItem - -_ValueT = t.TypeVar("_ValueT") -_ValueT_contra = t.TypeVar("_ValueT_contra", contravariant=True) -_ValueT_co = t.TypeVar("_ValueT_co", covariant=True) - -_FloatValueT = t.TypeVar("_FloatValueT", bound=float) -_FloatValueT_co = t.TypeVar("_FloatValueT_co", bound=float, covariant=True) - - -class ParamTypeInfoDict(t.TypedDict): - param_type: str - name: str - - -class ParamType(t.Generic[_ValueT_co], abc.ABC): - """Represents the type of a parameter. Validates and converts values - from the command line or Python into the correct type. - - To implement a custom type, subclass and implement at least the - following: - - - The :attr:`name` class attribute must be set. - - Calling an instance of the type with ``None`` must return - ``None``. This is already implemented by default. - - :meth:`convert` must convert string values to the correct type. - - :meth:`convert` must accept values that are already the correct - type. - - It must be able to convert a value if the ``ctx`` and ``param`` - arguments are ``None``. This can occur when converting prompt - input. - - .. versionchanged:: 8.4.0 - Now a generic abstract base class. Parameterize with the - converted value type (``ParamType[int]`` for an integer-returning - type) so that :meth:`convert` and downstream consumers carry the - narrowed return type. - """ - - is_composite: t.ClassVar[bool] = False - arity: int = 1 # read-only - - #: the descriptive name of this type - name: str - - #: if a list of this type is expected and the value is pulled from a - #: string environment variable, this is what splits it up. `None` - #: means any whitespace. For all parameters the general rule is that - #: whitespace splits them up. The exception are paths and files which - #: are split by ``os.path.pathsep`` by default (":" on Unix and ";" on - #: Windows). - envvar_list_splitter: t.ClassVar[str | None] = None - - def to_info_dict(self) -> ParamTypeInfoDict: - """Gather information that could be useful for a tool generating - user-facing documentation. - - Use :meth:`click.Context.to_info_dict` to traverse the entire - CLI structure. - - .. versionadded:: 8.0 - """ - # The class name without the "ParamType" suffix. - param_type = type(self).__name__.partition("ParamType")[0] - param_type = param_type.partition("ParameterType")[0] - - # Custom subclasses might not remember to set a name. - if hasattr(self, "name"): - name = self.name - else: - name = param_type - - return {"param_type": param_type, "name": name} - - def __call__( - self, - value: t.Any, - param: Parameter | None = None, - ctx: Context | None = None, - ) -> _ValueT_co | None: - if value is not None: - return self.convert(value, param, ctx) - return None - - def get_metavar(self, param: Parameter, ctx: Context) -> str | None: - """Returns the metavar default for this param if it provides one.""" - - def get_missing_message(self, param: Parameter, ctx: Context | None) -> str | None: - """Optionally might return extra information about a missing - parameter. - - .. versionadded:: 2.0 - """ - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> _ValueT_co: - """Convert the value to the correct type. This is not called if - the value is ``None`` (the missing value). - - This must accept string values from the command line, as well as - values that are already the correct type. It may also convert - other compatible types. - - The ``param`` and ``ctx`` arguments may be ``None`` in certain - situations, such as when converting prompt input. - - If the value cannot be converted, call :meth:`fail` with a - descriptive message. - - :param value: The value to convert. - :param param: The parameter that is using this type to convert - its value. May be ``None``. - :param ctx: The current context that arrived at this value. May - be ``None``. - """ - # The default returns the value as-is so subclasses that only customize - # metadata are not forced to redeclare ``convert``. - return t.cast("_ValueT_co", value) - - def split_envvar_value(self, rv: str) -> cabc.Sequence[str]: - """Given a value from an environment variable this splits it up - into small chunks depending on the defined envvar list splitter. - - If the splitter is set to `None`, which means that whitespace splits, - then leading and trailing whitespace is ignored. Otherwise, leading - and trailing splitters usually lead to empty items being included. - """ - return (rv or "").split(self.envvar_list_splitter) - - def fail( - self, - message: str, - param: Parameter | None = None, - ctx: Context | None = None, - ) -> t.NoReturn: - """Helper method to fail with an invalid value message.""" - raise BadParameter(message, ctx=ctx, param=param) - - def shell_complete( - self, ctx: Context, param: Parameter, incomplete: str - ) -> list[CompletionItem]: - """Return a list of - :class:`~click.shell_completion.CompletionItem` objects for the - incomplete value. Most types do not provide completions, but - some do, and this allows custom types to provide custom - completions as well. - - :param ctx: Invocation context for this command. - :param param: The parameter that is requesting completion. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - return [] - - -class CompositeParamType(ParamType[_ValueT_co]): - is_composite: t.ClassVar[bool] = True - - @property - @abc.abstractmethod - def arity(self) -> int: ... # type: ignore[override] - - -if t.TYPE_CHECKING: - # on Python 3.10 this will raise a TypeError - - class FuncParamTypeInfoDict( - ParamTypeInfoDict, - t.Generic[_ValueT_contra, _ValueT_co], - ): - func: t.Callable[[_ValueT_contra], _ValueT_co] -else: - - class FuncParamTypeInfoDict(ParamTypeInfoDict): - func: t.Callable[[t.Any], t.Any] - - -class FuncParamType(ParamType[_ValueT_co], t.Generic[_ValueT_contra, _ValueT_co]): - name: str - func: t.Callable[[_ValueT_contra], _ValueT_co] - - def __init__(self, func: t.Callable[[_ValueT_contra], _ValueT_co]) -> None: - self.name = func.__name__ - self.func = func - - def to_info_dict(self) -> FuncParamTypeInfoDict[_ValueT_contra, _ValueT_co]: - return {"func": self.func, **super().to_info_dict()} - - def convert( - self, value: _ValueT_contra, param: Parameter | None, ctx: Context | None - ) -> _ValueT_co: - try: - return self.func(value) - except ValueError as exc: - message = str(exc) - - if not message: - try: - message = str(value) - except UnicodeError: - message = t.cast("bytes", value).decode("utf-8", "replace") - - self.fail(message, param, ctx) - - -class UnprocessedParamType(ParamType[t.Any]): - name = "text" - - def convert( - self, value: _ValueT, param: Parameter | None, ctx: Context | None - ) -> _ValueT: - return value - - def __repr__(self) -> str: - return "UNPROCESSED" - - -class StringParamType(ParamType[str]): - name = "text" - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> str: - if isinstance(value, bytes): - enc = _get_argv_encoding() - try: - return value.decode(enc) - except UnicodeError: - fs_enc = sys.getfilesystemencoding() - if fs_enc != enc: - try: - return value.decode(fs_enc) - except UnicodeError: - return value.decode("utf-8", "replace") - else: - return value.decode("utf-8", "replace") - return str(value) - - def __repr__(self) -> str: - return "STRING" - - -if t.TYPE_CHECKING: - # on Python 3.10 this will raise a TypeError - - class ChoiceInfoDict(ParamTypeInfoDict, t.Generic[_ValueT_co]): - choices: tuple[_ValueT_co, ...] - case_sensitive: bool -else: - - class ChoiceInfoDict(ParamTypeInfoDict): - choices: tuple[t.Any, ...] - case_sensitive: bool - - -class Choice(ParamType[_ValueT_co], t.Generic[_ValueT_co]): - """The choice type allows a value to be checked against a fixed set - of supported values. - - You may pass any iterable value which will be converted to a tuple - and thus will only be iterated once. - - The resulting value will always be one of the originally passed choices. - See :meth:`normalize_choice` for more info on the mapping of strings - to choices. See :ref:`choice-opts` for an example. - - :param case_sensitive: Set to false to make choices case - insensitive. Defaults to true. - - .. versionchanged:: 8.4.0 - Now generic in the choice value type. Parameterize with the type of - the choice values (``Choice[HashType]`` for an enum, ``Choice[str]`` - for plain strings) to enable type-checked consumers. - - .. versionchanged:: 8.2.0 - Non-``str`` ``choices`` are now supported. It can additionally be any - iterable. Before you were not recommended to pass anything but a list or - tuple. - - .. versionadded:: 8.2.0 - Choice normalization can be overridden via :meth:`normalize_choice`. - """ - - name: str = "choice" - - choices: tuple[_ValueT_co, ...] - case_sensitive: bool - - def __init__( - self, choices: cabc.Iterable[_ValueT_co], case_sensitive: bool = True - ) -> None: - self.choices = tuple(choices) - self.case_sensitive = case_sensitive - - def to_info_dict(self) -> ChoiceInfoDict[_ValueT_co]: - return { - "choices": self.choices, - "case_sensitive": self.case_sensitive, - **super().to_info_dict(), - } - - def _normalized_mapping( - self, ctx: Context | None = None - ) -> cabc.Mapping[_ValueT_co, str]: - """ - Returns mapping where keys are the original choices and the values are - the normalized values that are accepted via the command line. - - This is a simple wrapper around :meth:`normalize_choice`, use that - instead which is supported. - """ - return { - choice: self.normalize_choice( - choice=choice, - ctx=ctx, - ) - for choice in self.choices - } - - def normalize_choice(self, choice: object, ctx: Context | None) -> str: - """ - Normalize a choice value, used to map a passed string to a choice. - Each choice must have a unique normalized value. - - By default uses :meth:`Context.token_normalize_func` and if not case - sensitive, convert it to a casefolded value. - - .. versionadded:: 8.2.0 - """ - normed_value = choice.name if isinstance(choice, enum.Enum) else str(choice) - - if ctx is not None and ctx.token_normalize_func is not None: - normed_value = ctx.token_normalize_func(normed_value) - - if not self.case_sensitive: - normed_value = normed_value.casefold() - - return normed_value - - def get_metavar(self, param: Parameter, ctx: Context) -> str | None: - if param.param_type_name == "option" and not param.show_choices: # type: ignore[attr-defined] - choice_metavars = [ - convert_type(type(choice)).name.upper() for choice in self.choices - ] - choices_str = "|".join([*dict.fromkeys(choice_metavars)]) - else: - choices_str = "|".join( - [str(i) for i in self._normalized_mapping(ctx=ctx).values()] - ) - - # Use curly braces to indicate a required argument. - if param.required and param.param_type_name == "argument": - return f"{{{choices_str}}}" - - # Use square braces to indicate an option or optional argument. - return f"[{choices_str}]" - - def get_missing_message(self, param: Parameter, ctx: Context | None) -> str: - """ - Message shown when no choice is passed. - - .. versionchanged:: 8.2.0 Added ``ctx`` argument. - """ - return _("Choose from:\n\t{choices}").format( - choices=",\n\t".join(self._normalized_mapping(ctx=ctx).values()) - ) - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> _ValueT_co: - """ - For a given value from the parser, normalize it and find its - matching normalized value in the list of choices. Then return the - matched "original" choice. - """ - normed_value = self.normalize_choice(choice=value, ctx=ctx) - normalized_mapping = self._normalized_mapping(ctx=ctx) - - try: - return next( - original - for original, normalized in normalized_mapping.items() - if normalized == normed_value - ) - except StopIteration: - self.fail( - self.get_invalid_choice_message(value=value, ctx=ctx), - param=param, - ctx=ctx, - ) - - def get_invalid_choice_message(self, value: t.Any, ctx: Context | None) -> str: - """Get the error message when the given choice is invalid. - - :param value: The invalid value. - - .. versionadded:: 8.2 - """ - choices_str = ", ".join(map(repr, self._normalized_mapping(ctx=ctx).values())) - return ngettext( - "{value!r} is not {choice}.", - "{value!r} is not one of {choices}.", - len(self.choices), - ).format(value=value, choice=choices_str, choices=choices_str) - - def __repr__(self) -> str: - return _("Choice({choices})").format(choices=list(self.choices)) - - def shell_complete( - self, ctx: Context, param: Parameter, incomplete: str - ) -> list[CompletionItem]: - """Complete choices that start with the incomplete value. - - :param ctx: Invocation context for this command. - :param param: The parameter that is requesting completion. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - from click.shell_completion import CompletionItem - - str_choices = [self.normalize_choice(choice, ctx) for choice in self.choices] - if self.case_sensitive: - matched = (c for c in str_choices if c.startswith(incomplete)) - else: - incomplete = incomplete.lower() - matched = (c for c in str_choices if c.lower().startswith(incomplete)) - - return [CompletionItem(c) for c in matched] - - -class DateTimeInfoDict(ParamTypeInfoDict): - formats: cabc.Sequence[str] - - -class DateTime(ParamType[datetime]): - """The DateTime type converts date strings into `datetime` objects. - - The format strings which are checked are configurable, but default to some - common (non-timezone aware) ISO 8601 formats. - - When specifying *DateTime* formats, you should only pass a list or a tuple. - Other iterables, like generators, may lead to surprising results. - - The format strings are processed using ``datetime.strptime``, and this - consequently defines the format strings which are allowed. - - Parsing is tried using each format, in order, and the first format which - parses successfully is used. - - :param formats: A list or tuple of date format strings, in the order in - which they should be tried. Defaults to - ``'%Y-%m-%d'``, ``'%Y-%m-%dT%H:%M:%S'``, - ``'%Y-%m-%d %H:%M:%S'``. - """ - - name = "datetime" - - formats: cabc.Sequence[str] - - def __init__(self, formats: cabc.Sequence[str] | None = None): - self.formats = formats or [ - "%Y-%m-%d", - "%Y-%m-%dT%H:%M:%S", - "%Y-%m-%d %H:%M:%S", - ] - - def to_info_dict(self) -> DateTimeInfoDict: - return {"formats": self.formats, **super().to_info_dict()} - - def get_metavar(self, param: Parameter, ctx: Context) -> str: - return f"[{'|'.join(self.formats)}]" - - def _try_to_convert_date(self, value: t.Any, format: str) -> datetime | None: - try: - return datetime.strptime(value, format) - except ValueError: - return None - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> datetime: - if isinstance(value, datetime): - return value - - for format in self.formats: - converted = self._try_to_convert_date(value, format) - - if converted is not None: - return converted - - formats_str = ", ".join(map(repr, self.formats)) - self.fail( - ngettext( - "{value!r} does not match the format {format}.", - "{value!r} does not match the formats {formats}.", - len(self.formats), - ).format(value=value, format=formats_str, formats=formats_str), - param, - ctx, - ) - - def __repr__(self) -> str: - return "DateTime" - - -class _NumberParamTypeBase( - ParamType[_ValueT_co], t.Generic[_ValueT_contra, _ValueT_co] -): - _number_class: t.Callable[[_ValueT_contra], _ValueT_co] - - def convert( - self, value: _ValueT_contra, param: Parameter | None, ctx: Context | None - ) -> _ValueT_co: - try: - return self._number_class(value) - except ValueError: - self.fail( - _("{value!r} is not a valid {number_type}.").format( - value=value, number_type=self.name - ), - param, - ctx, - ) - - -if t.TYPE_CHECKING: - # on Python 3.10 this will raise a TypeError - - class NumberRangeInfoDict(ParamTypeInfoDict, t.Generic[_FloatValueT_co]): - min: _FloatValueT_co | None - max: _FloatValueT_co | None - min_open: bool - max_open: bool - clamp: bool -else: - - class NumberRangeInfoDict(ParamTypeInfoDict): - min: t.Any | None - max: t.Any | None - min_open: bool - max_open: bool - clamp: bool - - -class _NumberRangeBase( - _NumberParamTypeBase[_ValueT_contra, _FloatValueT_co], - t.Generic[_ValueT_contra, _FloatValueT_co], -): - min: _FloatValueT_co | None - max: _FloatValueT_co | None - min_open: bool - max_open: bool - clamp: bool - - def __init__( - self, - min: _FloatValueT_co | None = None, - max: _FloatValueT_co | None = None, - min_open: bool = False, - max_open: bool = False, - clamp: bool = False, - ) -> None: - self.min = min - self.max = max - self.min_open = min_open - self.max_open = max_open - self.clamp = clamp - - def to_info_dict(self) -> NumberRangeInfoDict[_FloatValueT_co]: - return { - "min": self.min, - "max": self.max, - "min_open": self.min_open, - "max_open": self.max_open, - "clamp": self.clamp, - **super().to_info_dict(), - } - - def convert( - self, value: _ValueT_contra, param: Parameter | None, ctx: Context | None - ) -> _FloatValueT_co: - import operator - - rv = super().convert(value, param, ctx) - min = self.min - max = self.max - lt_min: bool = min is not None and ( - operator.le if self.min_open else operator.lt - )(rv, min) - gt_max: bool = max is not None and ( - operator.ge if self.max_open else operator.gt - )(rv, max) - - if self.clamp: - if min is not None and lt_min: - return self._clamp(min, 1, self.min_open) - - if max is not None and gt_max: - return self._clamp(max, -1, self.max_open) - - if lt_min or gt_max: - self.fail( - _("{value} is not in the range {range}.").format( - value=rv, range=self._describe_range() - ), - param, - ctx, - ) - - return rv - - @abc.abstractmethod - def _clamp( - # Covariant type variables cannot be used in input positions, so we use a - # separate method-scoped type variable instead. - self: _NumberRangeBase[t.Any, _FloatValueT], - bound: _FloatValueT, - dir: t.Literal[1, -1], - open: bool, - ) -> _FloatValueT: - """Find the valid value to clamp to bound in the given - direction. - - :param bound: The boundary value. - :param dir: 1 or -1 indicating the direction to move. - :param open: If true, the range does not include the bound. - """ - ... - - def _describe_range(self) -> str: - """Describe the range for use in help text.""" - if self.min is None: - op = "<" if self.max_open else "<=" - return f"x{op}{self.max}" - - if self.max is None: - op = ">" if self.min_open else ">=" - return f"x{op}{self.min}" - - lop = "<" if self.min_open else "<=" - rop = "<" if self.max_open else "<=" - return f"{self.min}{lop}x{rop}{self.max}" - - def __repr__(self) -> str: - clamp = " clamped" if self.clamp else "" - return f"<{type(self).__name__} {self._describe_range()}{clamp}>" - - -class IntParamType(_NumberParamTypeBase[t.SupportsInt | t.SupportsIndex, int]): - name = "integer" - _number_class = int - - def __repr__(self) -> str: - return "INT" - - -class IntRange(_NumberRangeBase[int, int], IntParamType): - """Restrict an :data:`click.INT` value to a range of accepted - values. See :ref:`ranges`. - - If ``min`` or ``max`` are not passed, any value is accepted in that - direction. If ``min_open`` or ``max_open`` are enabled, the - corresponding boundary is not included in the range. - - If ``clamp`` is enabled, a value outside the range is clamped to the - boundary instead of failing. - - .. versionchanged:: 8.0 - Added the ``min_open`` and ``max_open`` parameters. - """ - - name = "integer range" - - def _clamp(self, bound: int, dir: t.Literal[1, -1], open: bool) -> int: - if not open: - return bound - - return bound + dir - - -class FloatParamType(_NumberParamTypeBase[t.SupportsFloat | t.SupportsIndex, float]): - name = "float" - _number_class = float - - def __repr__(self) -> str: - return "FLOAT" - - -class FloatRange(_NumberRangeBase[float, float], FloatParamType): - """Restrict a :data:`click.FLOAT` value to a range of accepted - values. See :ref:`ranges`. - - If ``min`` or ``max`` are not passed, any value is accepted in that - direction. If ``min_open`` or ``max_open`` are enabled, the - corresponding boundary is not included in the range. - - If ``clamp`` is enabled, a value outside the range is clamped to the - boundary instead of failing. This is not supported if either - boundary is marked ``open``. - - .. versionchanged:: 8.0 - Added the ``min_open`` and ``max_open`` parameters. - """ - - name = "float range" - - def __init__( - self, - min: float | None = None, - max: float | None = None, - min_open: bool = False, - max_open: bool = False, - clamp: bool = False, - ) -> None: - super().__init__( - min=min, max=max, min_open=min_open, max_open=max_open, clamp=clamp - ) - - if (min_open or max_open) and clamp: - raise TypeError("Clamping is not supported for open bounds.") - - def _clamp(self, bound: float, dir: t.Literal[1, -1], open: bool) -> float: - if not open: - return bound - - # Could use math.nextafter here, but clamping an - # open float range doesn't seem to be particularly useful. It's - # left up to the user to write a callback to do it if needed. - raise RuntimeError("Clamping is not supported for open bounds.") - - -class BoolParamType(ParamType[bool]): - name = "boolean" - - bool_states: dict[str, bool] = { - "1": True, - "0": False, - "yes": True, - "no": False, - "true": True, - "false": False, - "on": True, - "off": False, - "t": True, - "f": False, - "y": True, - "n": False, - # Absence of value is considered False. - "": False, - } - """A mapping of string values to boolean states. - - Mapping is inspired by :py:attr:`configparser.ConfigParser.BOOLEAN_STATES` - and extends it. - - .. caution:: - String values are lower-cased, as the ``str_to_bool`` comparison function - below is case-insensitive. - - .. warning:: - The mapping is not exhaustive, and does not cover all possible boolean strings - representations. It will remains as it is to avoid endless bikeshedding. - - Future work my be considered to make this mapping user-configurable from public - API. - """ - - @staticmethod - def str_to_bool(value: str | bool) -> bool | None: - """Convert a string to a boolean value. - - If the value is already a boolean, it is returned as-is. If the value is a - string, it is stripped of whitespaces and lower-cased, then checked against - the known boolean states pre-defined in the `BoolParamType.bool_states` mapping - above. - - Returns `None` if the value does not match any known boolean state. - """ - if isinstance(value, bool): - return value - return BoolParamType.bool_states.get(value.strip().lower()) - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> bool: - normalized = self.str_to_bool(value) - if normalized is None: - self.fail( - _( - "{value!r} is not a valid boolean. Recognized values: {states}" - ).format(value=value, states=", ".join(sorted(self.bool_states))), - param, - ctx, - ) - return normalized - - def __repr__(self) -> str: - return "BOOL" - - -class UUIDParameterType(ParamType[uuid.UUID]): - name = "uuid" - - def convert( - self, value: uuid.UUID | str, param: Parameter | None, ctx: Context | None - ) -> uuid.UUID: - if isinstance(value, uuid.UUID): - return value - - value = value.strip() - - try: - return uuid.UUID(value) - except ValueError: - self.fail( - _("{value!r} is not a valid UUID.").format(value=value), param, ctx - ) - - def __repr__(self) -> str: - return "UUID" - - -class FileInfoDict(ParamTypeInfoDict): - mode: str - encoding: str | None - - -class File(ParamType[t.IO[t.Any]]): - """Declares a parameter to be a file for reading or writing. The file - is automatically closed once the context tears down (after the command - finished working). - - Files can be opened for reading or writing. The special value ``-`` - indicates stdin or stdout depending on the mode. - - By default, the file is opened for reading text data, but it can also be - opened in binary mode or for writing. The encoding parameter can be used - to force a specific encoding. - - The `lazy` flag controls if the file should be opened immediately or upon - first IO. The default is to be non-lazy for standard input and output - streams as well as files opened for reading, `lazy` otherwise. When opening a - file lazily for reading, it is still opened temporarily for validation, but - will not be held open until first IO. lazy is mainly useful when opening - for writing to avoid creating the file until it is needed. - - Files can also be opened atomically in which case all writes go into a - separate file in the same folder and upon completion the file will - be moved over to the original location. This is useful if a file - regularly read by other users is modified. - - See :ref:`file-args` for more information. - - .. versionchanged:: 2.0 - Added the ``atomic`` parameter. - """ - - name = "filename" - envvar_list_splitter: t.ClassVar[str] = os.path.pathsep - - mode: str - encoding: str | None - errors: str | None - lazy: bool | None - atomic: bool - - def __init__( - self, - mode: str = "r", - encoding: str | None = None, - errors: str | None = "strict", - lazy: bool | None = None, - atomic: bool = False, - ) -> None: - self.mode = mode - self.encoding = encoding - self.errors = errors - self.lazy = lazy - self.atomic = atomic - - def to_info_dict(self) -> FileInfoDict: - return { - "mode": self.mode, - "encoding": self.encoding, - **super().to_info_dict(), - } - - def resolve_lazy_flag(self, value: str | os.PathLike[str]) -> bool: - if self.lazy is not None: - return self.lazy - if os.fspath(value) == "-": - return False - elif "w" in self.mode: - return True - return False - - def convert( - self, - value: str | os.PathLike[str] | t.IO[t.Any], - param: Parameter | None, - ctx: Context | None, - ) -> t.IO[t.Any]: - if _is_file_like(value): - return value - - try: - lazy = self.resolve_lazy_flag(value) - - if lazy: - lf = LazyFile( - value, self.mode, self.encoding, self.errors, atomic=self.atomic - ) - - if ctx is not None: - ctx.call_on_close(lf.close_intelligently) - - return t.cast("t.IO[t.Any]", lf) - - f, should_close = open_stream( - value, self.mode, self.encoding, self.errors, atomic=self.atomic - ) - - # If a context is provided, we automatically close the file - # at the end of the context execution (or flush out). If a - # context does not exist, it's the caller's responsibility to - # properly close the file. This for instance happens when the - # type is used with prompts. - if ctx is not None: - if should_close: - ctx.call_on_close(safecall(f.close)) - else: - ctx.call_on_close(safecall(f.flush)) - - return f - except OSError as e: - self.fail( - f"'{format_filename(value)}': {e.strerror}", - param, - ctx, - ) - - def shell_complete( - self, ctx: Context, param: Parameter, incomplete: str - ) -> list[CompletionItem]: - """Return a special completion marker that tells the completion - system to use the shell to provide file path completions. - - :param ctx: Invocation context for this command. - :param param: The parameter that is requesting completion. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - from click.shell_completion import CompletionItem - - return [CompletionItem(incomplete, type="file")] - - -def _is_file_like(value: t.Any) -> te.TypeIs[t.IO[t.Any]]: - return hasattr(value, "read") or hasattr(value, "write") - - -class PathInfoDict(ParamTypeInfoDict): - exists: bool - file_okay: bool - dir_okay: bool - writable: bool - readable: bool - allow_dash: bool - - -class Path(ParamType[str | bytes | os.PathLike[str]]): - """The ``Path`` type is similar to the :class:`File` type, but - returns the filename instead of an open file. Various checks can be - enabled to validate the type of file and permissions. - - :param exists: The file or directory needs to exist for the value to - be valid. If this is not set to ``True``, and the file does not - exist, then all further checks are silently skipped. - :param file_okay: Allow a file as a value. - :param dir_okay: Allow a directory as a value. - :param readable: if true, a readable check is performed. - :param writable: if true, a writable check is performed. - :param executable: if true, an executable check is performed. - :param resolve_path: Make the value absolute and resolve any - symlinks. A ``~`` is not expanded, as this is supposed to be - done by the shell only. - :param allow_dash: Allow a single dash as a value, which indicates - a standard stream (but does not open it). Use - :func:`~click.open_file` to handle opening this value. - :param path_type: Convert the incoming path value to this type. If - ``None``, keep Python's default, which is ``str``. Useful to - convert to :class:`pathlib.Path`. - - .. versionchanged:: 8.1 - Added the ``executable`` parameter. - - .. versionchanged:: 8.0 - Allow passing ``path_type=pathlib.Path``. - - .. versionchanged:: 6.0 - Added the ``allow_dash`` parameter. - """ - - envvar_list_splitter: t.ClassVar[str] = os.path.pathsep - - exists: bool - file_okay: bool - dir_okay: bool - readable: bool - writable: bool - executable: bool - resolve_path: bool - allow_dash: bool - name: str - - def __init__( - self, - exists: bool = False, - file_okay: bool = True, - dir_okay: bool = True, - writable: bool = False, - readable: bool = True, - resolve_path: bool = False, - allow_dash: bool = False, - path_type: type | None = None, - executable: bool = False, - ) -> None: - self.exists = exists - self.file_okay = file_okay - self.dir_okay = dir_okay - self.readable = readable - self.writable = writable - self.executable = executable - self.resolve_path = resolve_path - self.allow_dash = allow_dash - self.type: type | None = path_type - - if self.file_okay and not self.dir_okay: - self.name = _("file") - elif self.dir_okay and not self.file_okay: - self.name = _("directory") - else: - self.name = _("path") - - def to_info_dict(self) -> PathInfoDict: - return { - "exists": self.exists, - "file_okay": self.file_okay, - "dir_okay": self.dir_okay, - "writable": self.writable, - "readable": self.readable, - "allow_dash": self.allow_dash, - **super().to_info_dict(), - } - - def coerce_path_result( - self, value: str | os.PathLike[str] - ) -> str | bytes | os.PathLike[str]: - if self.type is not None and not isinstance(value, self.type): - if self.type is str: - return os.fsdecode(value) - elif self.type is bytes: - return os.fsencode(value) - else: - return t.cast("os.PathLike[str]", self.type(value)) - - return value - - def convert( - self, - value: str | os.PathLike[str], - param: Parameter | None, - ctx: Context | None, - ) -> str | bytes | os.PathLike[str]: - rv = value - - is_dash = self.file_okay and self.allow_dash and rv in (b"-", "-") - - if not is_dash: - if self.resolve_path: - rv = os.path.realpath(rv) - - try: - st = os.stat(rv) - except OSError: - if not self.exists: - return self.coerce_path_result(rv) - self.fail( - _("{name} {filename!r} does not exist.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - - if not self.file_okay and stat.S_ISREG(st.st_mode): - self.fail( - _("{name} {filename!r} is a file.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - if not self.dir_okay and stat.S_ISDIR(st.st_mode): - self.fail( - _("{name} {filename!r} is a directory.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - - if self.readable and not os.access(rv, os.R_OK): - self.fail( - _("{name} {filename!r} is not readable.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - - if self.writable and not os.access(rv, os.W_OK): - self.fail( - _("{name} {filename!r} is not writable.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - - if self.executable and not os.access(value, os.X_OK): - self.fail( - _("{name} {filename!r} is not executable.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - - return self.coerce_path_result(rv) - - def shell_complete( - self, ctx: Context, param: Parameter, incomplete: str - ) -> list[CompletionItem]: - """Return a special completion marker that tells the completion - system to use the shell to provide path completions for only - directories or any paths. - - :param ctx: Invocation context for this command. - :param param: The parameter that is requesting completion. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - from click.shell_completion import CompletionItem - - type = "dir" if self.dir_okay and not self.file_okay else "file" - return [CompletionItem(incomplete, type=type)] - - -class TupleInfoDict(ParamTypeInfoDict): - types: cabc.Sequence[ParamTypeInfoDict] - - -class Tuple(CompositeParamType[tuple[t.Any, ...]]): - """The default behavior of Click is to apply a type on a value directly. - This works well in most cases, except for when `nargs` is set to a fixed - count and different types should be used for different items. In this - case the :class:`Tuple` type can be used. This type can only be used - if `nargs` is set to a fixed number. - - For more information see :ref:`tuple-type`. - - This can be selected by using a Python tuple literal as a type. - - :param types: a list of types that should be used for the tuple items. - """ - - def __init__(self, types: cabc.Sequence[type[t.Any] | ParamType[t.Any]]) -> None: - self.types: cabc.Sequence[ParamType[t.Any]] = [convert_type(ty) for ty in types] - - def to_info_dict(self) -> TupleInfoDict: - return { - "types": [ty.to_info_dict() for ty in self.types], - **super().to_info_dict(), - } - - @property - def name(self) -> str: # type: ignore[override] - return f"<{' '.join(ty.name for ty in self.types)}>" - - @property - def arity(self) -> int: # type: ignore[override] - return len(self.types) - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> tuple[t.Any, ...]: - len_type = len(self.types) - len_value = len(value) - - if len_value != len_type: - self.fail( - ngettext( - "{len_type} values are required, but {len_value} was given.", - "{len_type} values are required, but {len_value} were given.", - len_value, - ).format(len_type=len_type, len_value=len_value), - param=param, - ctx=ctx, - ) - - return tuple( - ty(x, param, ctx) for ty, x in zip(self.types, value, strict=False) - ) - - -def _guess_type( - ty: type[t.Any] | ParamType[t.Any] | None, - default: t.Any | None, -) -> type[t.Any] | tuple[type[t.Any], ...] | ParamType[t.Any] | None: - """Infer a type from *ty* or *default*. - - Returns *ty* unchanged when it is not ``None``. Otherwise inspects - *default* to produce a ``type``, a ``tuple`` of types (for tuple - defaults), or ``None``. - """ - if ty is not None: - return ty - - if default is None: - return None - - if not isinstance(default, (tuple, list)): - return type(default) - - # If the default is empty, return None so convert_type falls - # through to STRING. - if not default: - return None - - item = default[0] - - # A sequence of iterables needs to detect the inner types. - # Can't call convert_type recursively because that would - # incorrectly unwind the tuple to a single type. - if isinstance(item, (tuple, list)): - return tuple(map(type, item)) - - return type(item) - - -@t.overload -def convert_type(ty: None, default: None = None) -> StringParamType: ... -@t.overload -def convert_type( - ty: type | ParamType[t.Any], default: t.Any | None = None -) -> ParamType[t.Any]: ... -@t.overload -def convert_type( - ty: t.Any | None, default: t.Any | None = None -) -> ParamType[t.Any]: ... -def convert_type( - ty: t.Any | None = None, default: t.Any | None = None -) -> ParamType[t.Any]: - """Find the most appropriate :class:`ParamType` for the given Python - type. If the type isn't provided, it can be inferred from a default - value. - """ - guessed = _guess_type(ty, default) - is_guessed = guessed is not ty - - if isinstance(guessed, tuple): - return Tuple(guessed) - - if isinstance(guessed, ParamType): - return guessed - - if guessed is str or guessed is None: - return STRING - - if guessed is int: - return INT - - if guessed is float: - return FLOAT - - if guessed is bool: - return BOOL - - if is_guessed: - return STRING - - if __debug__: - try: - if issubclass(guessed, ParamType): - raise AssertionError( - f"Attempted to use an uninstantiated parameter type ({guessed})." - ) - except TypeError: - # guessed is an instance (correct), so issubclass fails. - pass - - return FuncParamType(guessed) - - -#: A dummy parameter type that just does nothing. From a user's -#: perspective this appears to just be the same as `STRING` but -#: internally no string conversion takes place if the input was bytes. -#: This is usually useful when working with file paths as they can -#: appear in bytes and unicode. -#: -#: For path related uses the :class:`Path` type is a better choice but -#: there are situations where an unprocessed type is useful which is why -#: it is provided. -#: -#: .. versionadded:: 4.0 -UNPROCESSED: t.Final[UnprocessedParamType] = UnprocessedParamType() - -#: A unicode string parameter type which is the implicit default. This -#: can also be selected by using ``str`` as type. -STRING: t.Final[StringParamType] = StringParamType() - -#: An integer parameter. This can also be selected by using ``int`` as -#: type. -INT: t.Final[IntParamType] = IntParamType() - -#: A floating point value parameter. This can also be selected by using -#: ``float`` as type. -FLOAT: t.Final[FloatParamType] = FloatParamType() - -#: A boolean parameter. This is the default for boolean flags. This can -#: also be selected by using ``bool`` as a type. -BOOL: t.Final[BoolParamType] = BoolParamType() - -#: A UUID parameter. -UUID: t.Final[UUIDParameterType] = UUIDParameterType() - - -class OptionHelpExtra(t.TypedDict, total=False): - envvars: tuple[str, ...] - default: str - range: str - required: str diff --git a/bundle/python-cpu/Lib/site-packages/click/utils.py b/bundle/python-cpu/Lib/site-packages/click/utils.py deleted file mode 100644 index c0cb22d683b4ea2ff3b035efdf1e6ef392b08367..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/click/utils.py +++ /dev/null @@ -1,653 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import os -import re -import sys -import typing as t -from functools import update_wrapper -from gettext import gettext as _ -from types import ModuleType -from types import TracebackType - -from ._compat import _default_text_stderr -from ._compat import _default_text_stdout -from ._compat import _find_binary_writer -from ._compat import auto_wrap_for_ansi -from ._compat import binary_streams -from ._compat import open_stream -from ._compat import should_strip_ansi -from ._compat import strip_ansi -from ._compat import text_streams -from ._compat import WIN -from .globals import resolve_color_default - -if t.TYPE_CHECKING: - import typing_extensions as te - - P = te.ParamSpec("P") - -R = t.TypeVar("R") - - -def _posixify(name: str) -> str: - return "-".join(name.split()).lower() - - -def safecall(func: t.Callable[P, R]) -> t.Callable[P, R | None]: - """Wraps a function so that it swallows exceptions.""" - - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R | None: - try: - return func(*args, **kwargs) - except Exception: - pass - return None - - return update_wrapper(wrapper, func) - - -def make_str(value: t.Any) -> str: - """Converts a value into a valid string.""" - if isinstance(value, bytes): - try: - return value.decode(sys.getfilesystemencoding()) - except UnicodeError: - return value.decode("utf-8", "replace") - return str(value) - - -def make_default_short_help(help: str, max_length: int = 45) -> str: - """Returns a condensed version of help string. - - :meta private: - """ - # Consider only the first paragraph. - paragraph_end = help.find("\n\n") - - if paragraph_end != -1: - help = help[:paragraph_end] - - # Collapse newlines, tabs, and spaces. - words = help.split() - - if not words: - return "" - - # The first paragraph started with a "no rewrap" marker, ignore it. - if words[0] == "\b": - words = words[1:] - - total_length = 0 - last_index = len(words) - 1 - - for i, word in enumerate(words): - total_length += len(word) + (i > 0) - - if total_length > max_length: # too long, truncate - break - - if word[-1] == ".": # sentence end, truncate without "..." - return " ".join(words[: i + 1]) - - if total_length == max_length and i != last_index: - break # not at sentence end, truncate with "..." - else: - return " ".join(words) # no truncation needed - - # Account for the length of the suffix. - total_length += len("...") - - # remove words until the length is short enough - while i > 0: - total_length -= len(words[i]) + (i > 0) - - if total_length <= max_length: - break - - i -= 1 - - return " ".join(words[:i]) + "..." - - -class LazyFile: - """A lazy file works like a regular file but it does not fully open - the file but it does perform some basic checks early to see if the - filename parameter does make sense. This is useful for safely opening - files for writing. - """ - - name: str - mode: str - encoding: str | None - errors: str | None - atomic: bool - _f: t.IO[t.Any] | None - should_close: bool - - def __init__( - self, - filename: str | os.PathLike[str], - mode: str = "r", - encoding: str | None = None, - errors: str | None = "strict", - atomic: bool = False, - ) -> None: - self.name = os.fspath(filename) - self.mode = mode - self.encoding = encoding - self.errors = errors - self.atomic = atomic - - if self.name == "-": - self._f, self.should_close = open_stream(filename, mode, encoding, errors) - else: - if "r" in mode: - # Open and close the file in case we're opening it for - # reading so that we can catch at least some errors in - # some cases early. - open(filename, mode).close() - self._f = None - self.should_close = True - - def __getattr__(self, name: str) -> t.Any: - return getattr(self.open(), name) - - def __repr__(self) -> str: - if self._f is not None: - return repr(self._f) - return f"" - - def open(self) -> t.IO[t.Any]: - """Opens the file if it's not yet open. This call might fail with - a :exc:`FileError`. Not handling this error will produce an error - that Click shows. - """ - if self._f is not None: - return self._f - try: - rv, self.should_close = open_stream( - self.name, self.mode, self.encoding, self.errors, atomic=self.atomic - ) - except OSError as e: - from .exceptions import FileError - - raise FileError(self.name, hint=e.strerror) from e - self._f = rv - return rv - - def close(self) -> None: - """Closes the underlying file, no matter what.""" - if self._f is not None: - self._f.close() - - def close_intelligently(self) -> None: - """This function only closes the file if it was opened by the lazy - file wrapper. For instance this will never close stdin. - """ - if self.should_close: - self.close() - - def __enter__(self) -> LazyFile: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> None: - self.close_intelligently() - - def __iter__(self) -> cabc.Iterator[t.AnyStr]: - self.open() - return iter(self._f) # type: ignore - - -class KeepOpenFile: - """Proxy a file object but keep it open across a ``with`` block. - - Wraps a borrowed file (such as ``sys.stdin`` or ``sys.stdout``) so that - leaving a ``with`` block does not close it, as used by :func:`open_file` - for the ``-`` filename. The caller stays responsible for the file: an - explicit :meth:`close` still passes through to the wrapped object. - - Dunder methods are proxied explicitly: implicit special-method lookups - bypass :meth:`__getattr__`, because Python resolves them on the type rather - than the instance. - """ - - _file: t.IO[t.Any] - - def __init__(self, file: t.IO[t.Any]) -> None: - self._file = file - - def __getattr__(self, name: str) -> t.Any: - return getattr(self._file, name) - - def __enter__(self) -> KeepOpenFile: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> None: - pass - - def __repr__(self) -> str: - return repr(self._file) - - def __iter__(self) -> cabc.Iterator[t.AnyStr]: - return iter(self._file) - - -def echo( - message: object = None, - file: t.IO[t.Any] | None = None, - nl: bool = True, - err: bool = False, - color: bool | None = None, -) -> None: - """Print a message and newline to stdout or a file. This should be - used instead of :func:`print` because it provides better support - for different data, files, and environments. - - Compared to :func:`print`, this does the following: - - - Ensures that the output encoding is not misconfigured on Linux. - - Supports Unicode in the Windows console. - - Supports writing to binary outputs, and supports writing bytes - to text outputs. - - Supports colors and styles on Windows. - - Removes ANSI color and style codes if the output does not look - like an interactive terminal. - - Always flushes the output. - - :param message: The string or bytes to output. Other objects are - converted to strings. - :param file: The file to write to. Defaults to ``stdout``. - :param err: Write to ``stderr`` instead of ``stdout``. - :param nl: Print a newline after the message. Enabled by default. - :param color: Force showing or hiding colors and other styles. By - default Click will remove color if the output does not look like - an interactive terminal. - - .. versionchanged:: 6.0 - Support Unicode output on the Windows console. Click does not - modify ``sys.stdout``, so ``sys.stdout.write()`` and ``print()`` - will still not support Unicode. - - .. versionchanged:: 4.0 - Added the ``color`` parameter. - - .. versionadded:: 3.0 - Added the ``err`` parameter. - - .. versionchanged:: 2.0 - Support colors on Windows if colorama is installed. - """ - if file is None: - if err: - file = _default_text_stderr() - else: - file = _default_text_stdout() - - # There are no standard streams attached to write to. For example, - # pythonw on Windows. - if file is None: - return - - match message: - case str() | bytes() | bytearray(): - out = message - case None: - out = "" - case _: - out = str(message) - - if nl: - if isinstance(out, str): - out += "\n" - else: - out += b"\n" - - if not out: - file.flush() - return - - # If there is a message and the value looks like bytes, we manually - # need to find the binary stream and write the message in there. - # This is done separately so that most stream types will work as you - # would expect. Eg: you can write to StringIO for other cases. - if isinstance(out, (bytes, bytearray)): - binary_file = _find_binary_writer(file) - if binary_file is not None: - file.flush() - binary_file.write(out) - binary_file.flush() - return - - # ANSI style code support. For no message or bytes, nothing happens. - # When outputting to a file instead of a terminal, strip codes. - else: - color = resolve_color_default(color) - - if should_strip_ansi(file, color): - out = strip_ansi(out) - elif WIN: - if auto_wrap_for_ansi is not None: - file = auto_wrap_for_ansi(file, color) # type: ignore - elif not color: - out = strip_ansi(out) - - file.write(out) # type: ignore - file.flush() - - -def get_binary_stream(name: t.Literal["stdin", "stdout", "stderr"]) -> t.BinaryIO: - """Returns a system stream for byte processing. - - :param name: the name of the stream to open. Valid names are ``'stdin'``, - ``'stdout'`` and ``'stderr'`` - """ - opener = binary_streams.get(name) - if opener is None: - raise TypeError(_("Unknown standard stream '{name}'").format(name=name)) - return opener() - - -def get_text_stream( - name: t.Literal["stdin", "stdout", "stderr"], - encoding: str | None = None, - errors: str | None = "strict", -) -> t.TextIO: - """Returns a system stream for text processing. This usually returns - a wrapped stream around a binary stream returned from - :func:`get_binary_stream` but it also can take shortcuts for already - correctly configured streams. - - :param name: the name of the stream to open. Valid names are ``'stdin'``, - ``'stdout'`` and ``'stderr'`` - :param encoding: overrides the detected default encoding. - :param errors: overrides the default error mode. - """ - opener = text_streams.get(name) - if opener is None: - raise TypeError(_("Unknown standard stream '{name}'").format(name=name)) - return opener(encoding, errors) - - -def open_file( - filename: str | os.PathLike[str], - mode: str = "r", - encoding: str | None = None, - errors: str | None = "strict", - lazy: bool = False, - atomic: bool = False, -) -> t.IO[t.Any]: - """Open a file, with extra behavior to handle ``'-'`` to indicate - a standard stream, lazy open on write, and atomic write. Similar to - the behavior of the :class:`~click.File` param type. - - If ``'-'`` is given to open ``stdout`` or ``stdin``, the stream is - wrapped so that using it in a context manager will not close it. - This makes it possible to use the function without accidentally - closing a standard stream: - - .. code-block:: python - - with open_file(filename) as f: - ... - - :param filename: The name or Path of the file to open, or ``'-'`` for - ``stdin``/``stdout``. - :param mode: The mode in which to open the file. - :param encoding: The encoding to decode or encode a file opened in - text mode. - :param errors: The error handling mode. - :param lazy: Wait to open the file until it is accessed. For read - mode, the file is temporarily opened to raise access errors - early, then closed until it is read again. - :param atomic: Write to a temporary file and replace the given file - on close. - - .. versionadded:: 3.0 - """ - if lazy: - return t.cast( - "t.IO[t.Any]", LazyFile(filename, mode, encoding, errors, atomic=atomic) - ) - - f, should_close = open_stream(filename, mode, encoding, errors, atomic=atomic) - - if not should_close: - f = t.cast("t.IO[t.Any]", KeepOpenFile(f)) - - return f - - -def format_filename( - filename: str | bytes | os.PathLike[str] | os.PathLike[bytes], - shorten: bool = False, -) -> str: - """Format a filename as a string for display. Ensures the filename can be - displayed by replacing any invalid bytes or surrogate escapes in the name - with the replacement character ``�``. - - Invalid bytes or surrogate escapes will raise an error when written to a - stream with ``errors="strict"``. This will typically happen with ``stdout`` - when the locale is something like ``en_GB.UTF-8``. - - Many scenarios *are* safe to write surrogates though, due to PEP 538 and - PEP 540, including: - - - Writing to ``stderr``, which uses ``errors="backslashreplace"``. - - The system has ``LANG=C.UTF-8``, ``C``, or ``POSIX``. Python opens - stdout and stderr with ``errors="surrogateescape"``. - - None of ``LANG/LC_*`` are set. Python assumes ``LANG=C.UTF-8``. - - Python is started in UTF-8 mode with ``PYTHONUTF8=1`` or ``-X utf8``. - Python opens stdout and stderr with ``errors="surrogateescape"``. - - :param filename: formats a filename for UI display. This will also convert - the filename into unicode without failing. - :param shorten: this optionally shortens the filename to strip of the - path that leads up to it. - """ - if shorten: - filename = os.path.basename(filename) - else: - filename = os.fspath(filename) - - if isinstance(filename, bytes): - filename = filename.decode(sys.getfilesystemencoding(), "replace") - else: - filename = filename.encode("utf-8", "surrogateescape").decode( - "utf-8", "replace" - ) - - return filename - - -def get_app_dir(app_name: str, roaming: bool = True, force_posix: bool = False) -> str: - r"""Returns the config folder for the application. The default behavior - is to return whatever is most appropriate for the operating system. - - To give you an idea, for an app called ``"Foo Bar"``, something like - the following folders could be returned: - - Mac OS X: - ``~/Library/Application Support/Foo Bar`` - Mac OS X (POSIX): - ``~/.foo-bar`` - Unix: - ``~/.config/foo-bar`` - Unix (POSIX): - ``~/.foo-bar`` - Windows (roaming): - ``C:\Users\\AppData\Roaming\Foo Bar`` - Windows (not roaming): - ``C:\Users\\AppData\Local\Foo Bar`` - - .. versionadded:: 2.0 - - :param app_name: the application name. This should be properly capitalized - and can contain whitespace. - :param roaming: controls if the folder should be roaming or not on Windows. - Has no effect otherwise. - :param force_posix: if this is set to `True` then on any POSIX system the - folder will be stored in the home folder with a leading - dot instead of the XDG config home or darwin's - application support folder. - """ - if WIN: - key = "APPDATA" if roaming else "LOCALAPPDATA" - folder = os.environ.get(key) - if folder is None: - folder = os.path.expanduser("~") - return os.path.join(folder, app_name) - if force_posix: - return os.path.join(os.path.expanduser(f"~/.{_posixify(app_name)}")) - if sys.platform == "darwin": - return os.path.join( - os.path.expanduser("~/Library/Application Support"), app_name - ) - return os.path.join( - os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), - _posixify(app_name), - ) - - -class PacifyFlushWrapper: - """This wrapper is used to catch and suppress BrokenPipeErrors resulting - from ``.flush()`` being called on broken pipe during the shutdown/final-GC - of the Python interpreter. Notably ``.flush()`` is always called on - ``sys.stdout`` and ``sys.stderr``. So as to have minimal impact on any - other cleanup code, and the case where the underlying file is not a broken - pipe, all calls and attributes are proxied. - """ - - wrapped: t.IO[t.Any] - - def __init__(self, wrapped: t.IO[t.Any]) -> None: - self.wrapped = wrapped - - def flush(self) -> None: - try: - self.wrapped.flush() - except OSError as e: - import errno - - if e.errno != errno.EPIPE: - raise - - def __getattr__(self, attr: str) -> t.Any: - return getattr(self.wrapped, attr) - - -def _detect_program_name( - path: str | None = None, _main: ModuleType | None = None -) -> str: - """Determine the command used to run the program, for use in help - text. If a file or entry point was executed, the file name is - returned. If ``python -m`` was used to execute a module or package, - ``python -m name`` is returned. - - This doesn't try to be too precise, the goal is to give a concise - name for help text. Files are only shown as their name without the - path. ``python`` is only shown for modules, and the full path to - ``sys.executable`` is not shown. - - :param path: The Python file being executed. Python puts this in - ``sys.argv[0]``, which is used by default. - :param _main: The ``__main__`` module. This should only be passed - during internal testing. - - .. versionadded:: 8.0 - Based on command args detection in the Werkzeug reloader. - - :meta private: - """ - if _main is None: - _main = sys.modules["__main__"] - - if not path: - path = sys.argv[0] - - # The value of __package__ indicates how Python was called. It may - # not exist if a setuptools script is installed as an egg. It may be - # set incorrectly for entry points created with pip on Windows. - # It is set to "" inside a Shiv or PEX zipapp. - if getattr(_main, "__package__", None) in {None, ""} or ( - os.name == "nt" - and _main.__package__ == "" - and not os.path.exists(path) - and os.path.exists(f"{path}.exe") - ): - # Executed a file, like "python app.py". - return os.path.basename(path) - - # Executed a module, like "python -m example". - # Rewritten by Python from "-m script" to "/path/to/script.py". - # Need to look at main module to determine how it was executed. - py_module = t.cast(str, _main.__package__) - name = os.path.splitext(os.path.basename(path))[0] - - # A submodule like "example.cli". - if name != "__main__": - py_module = f"{py_module}.{name}" - - return f"python -m {py_module.lstrip('.')}" - - -def _expand_args( - args: cabc.Iterable[str], - *, - user: bool = True, - env: bool = True, - glob_recursive: bool = True, -) -> list[str]: - """Simulate Unix shell expansion with Python functions. - - See :func:`glob.glob`, :func:`os.path.expanduser`, and - :func:`os.path.expandvars`. - - This is intended for use on Windows, where the shell does not do any - expansion. It may not exactly match what a Unix shell would do. - - :param args: List of command line arguments to expand. - :param user: Expand user home directory. - :param env: Expand environment variables. - :param glob_recursive: ``**`` matches directories recursively. - - .. versionchanged:: 8.1 - Invalid glob patterns are treated as empty expansions rather - than raising an error. - - .. versionadded:: 8.0 - - :meta private: - """ - from glob import glob - - out = [] - - for arg in args: - if user: - arg = os.path.expanduser(arg) - - if env: - arg = os.path.expandvars(arg) - - try: - matches = glob(arg, recursive=glob_recursive) - except re.error: - matches = [] - - if not matches: - out.append(arg) - else: - out.extend(matches) - - return out diff --git a/bundle/python-cpu/Lib/site-packages/distutils-precedence.pth b/bundle/python-cpu/Lib/site-packages/distutils-precedence.pth deleted file mode 100644 index c659194195f07bd6f19b5522515551309af14a3d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/distutils-precedence.pth +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2638ce9e2500e572a5e0de7faed6661eb569d1b696fcba07b0dd223da5f5d224 -size 151 diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/METADATA deleted file mode 100644 index e1a9a27f56e8ab864801962bbd8c89b71c1818b4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/METADATA +++ /dev/null @@ -1,159 +0,0 @@ -Metadata-Version: 2.4 -Name: exceptiongroup -Version: 1.3.1 -Summary: Backport of PEP 654 (exception groups) -Author-email: Alex Grönholm -Requires-Python: >=3.7 -Description-Content-Type: text/x-rst -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Typing :: Typed -License-File: LICENSE -Requires-Dist: typing-extensions >= 4.6.0; python_version < '3.13' -Requires-Dist: pytest >= 6 ; extra == "test" -Project-URL: Changelog, https://github.com/agronholm/exceptiongroup/blob/main/CHANGES.rst -Project-URL: Issue Tracker, https://github.com/agronholm/exceptiongroup/issues -Project-URL: Source code, https://github.com/agronholm/exceptiongroup -Provides-Extra: test - -.. image:: https://github.com/agronholm/exceptiongroup/actions/workflows/test.yml/badge.svg - :target: https://github.com/agronholm/exceptiongroup/actions/workflows/test.yml - :alt: Build Status -.. image:: https://coveralls.io/repos/github/agronholm/exceptiongroup/badge.svg?branch=main - :target: https://coveralls.io/github/agronholm/exceptiongroup?branch=main - :alt: Code Coverage - -This is a backport of the ``BaseExceptionGroup`` and ``ExceptionGroup`` classes from -Python 3.11. - -It contains the following: - -* The ``exceptiongroup.BaseExceptionGroup`` and ``exceptiongroup.ExceptionGroup`` - classes -* A utility function (``exceptiongroup.catch()``) for catching exceptions possibly - nested in an exception group -* Patches to the ``TracebackException`` class that properly formats exception groups - (installed on import) -* An exception hook that handles formatting of exception groups through - ``TracebackException`` (installed on import) -* Special versions of some of the functions from the ``traceback`` module, modified to - correctly handle exception groups even when monkey patching is disabled, or blocked by - another custom exception hook: - - * ``traceback.format_exception()`` - * ``traceback.format_exception_only()`` - * ``traceback.print_exception()`` - * ``traceback.print_exc()`` -* A backported version of ``contextlib.suppress()`` from Python 3.12.1 which also - handles suppressing exceptions inside exception groups - -If this package is imported on Python 3.11 or later, the built-in implementations of the -exception group classes are used instead, ``TracebackException`` is not monkey patched -and the exception hook won't be installed. - -See the `standard library documentation`_ for more information on exception groups. - -.. _standard library documentation: https://docs.python.org/3/library/exceptions.html - -Catching exceptions -=================== - -Due to the lack of the ``except*`` syntax introduced by `PEP 654`_ in earlier Python -versions, you need to use ``exceptiongroup.catch()`` to catch exceptions that are -potentially nested inside an exception group. This function returns a context manager -that calls the given handler for any exceptions matching the sole argument. - -The argument to ``catch()`` must be a dict (or any ``Mapping``) where each key is either -an exception class or an iterable of exception classes. Each value must be a callable -that takes a single positional argument. The handler will be called at most once, with -an exception group as an argument which will contain all the exceptions that are any -of the given types, or their subclasses. The exception group may contain nested groups -containing more matching exceptions. - -Thus, the following Python 3.11+ code: - -.. code-block:: python - - try: - ... - except* (ValueError, KeyError) as excgroup: - for exc in excgroup.exceptions: - print('Caught exception:', type(exc)) - except* RuntimeError: - print('Caught runtime error') - -would be written with this backport like this: - -.. code-block:: python - - from exceptiongroup import BaseExceptionGroup, catch - - def value_key_err_handler(excgroup: BaseExceptionGroup) -> None: - for exc in excgroup.exceptions: - print('Caught exception:', type(exc)) - - def runtime_err_handler(exc: BaseExceptionGroup) -> None: - print('Caught runtime error') - - with catch({ - (ValueError, KeyError): value_key_err_handler, - RuntimeError: runtime_err_handler - }): - ... - -**NOTE**: Just like with ``except*``, you cannot handle ``BaseExceptionGroup`` or -``ExceptionGroup`` with ``catch()``. - -Suppressing exceptions -====================== - -This library contains a backport of the ``contextlib.suppress()`` context manager from -Python 3.12.1. It allows you to selectively ignore certain exceptions, even when they're -inside exception groups: - -.. code-block:: python - - from exceptiongroup import suppress - - with suppress(RuntimeError): - raise ExceptionGroup("", [RuntimeError("boo")]) - -Notes on monkey patching -======================== - -To make exception groups render properly when an unhandled exception group is being -printed out, this package does two things when it is imported on any Python version -earlier than 3.11: - -#. The ``traceback.TracebackException`` class is monkey patched to store extra - information about exception groups (in ``__init__()``) and properly format them (in - ``format()``) -#. An exception hook is installed at ``sys.excepthook``, provided that no other hook is - already present. This hook causes the exception to be formatted using - ``traceback.TracebackException`` rather than the built-in rendered. - -If ``sys.exceptionhook`` is found to be set to something else than the default when -``exceptiongroup`` is imported, no monkeypatching is done at all. - -To prevent the exception hook and patches from being installed, set the environment -variable ``EXCEPTIONGROUP_NO_PATCH`` to ``1``. - -Formatting exception groups ---------------------------- - -Normally, the monkey patching applied by this library on import will cause exception -groups to be printed properly in tracebacks. But in cases when the monkey patching is -blocked by a third party exception hook, or monkey patching is explicitly disabled, -you can still manually format exceptions using the special versions of the ``traceback`` -functions, like ``format_exception()``, listed at the top of this page. They work just -like their counterparts in the ``traceback`` module, except that they use a separately -patched subclass of ``TracebackException`` to perform the rendering. - -Particularly in cases where a library installs its own exception hook, it is recommended -to use these special versions to do the actual formatting of exceptions/tracebacks. - -.. _PEP 654: https://www.python.org/dev/peps/pep-0654/ - diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/RECORD deleted file mode 100644 index df9547454e2f807a20fc17b91b89ca2a81853261..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/RECORD +++ /dev/null @@ -1,13 +0,0 @@ -exceptiongroup-1.3.1.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -exceptiongroup-1.3.1.dist-info/METADATA,sha256=gZhKUjovelIq0SvqeEqLuF7ewIBeu9D7TjUBaaNt2AI,6725 -exceptiongroup-1.3.1.dist-info/RECORD,, -exceptiongroup-1.3.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -exceptiongroup-1.3.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 -exceptiongroup-1.3.1.dist-info/licenses/LICENSE,sha256=blBw12UDHgrUA6HL-Qrm0ZoCKPgC4yC3rP9GCqcu1Hw,3704 -exceptiongroup/__init__.py,sha256=7DHS0hDk-RIs3IQc3SbZVB0-1MhiSCJ9XgvEyEloL7M,1049 -exceptiongroup/_catch.py,sha256=CaJez3E-Jkr-7B7RT3fzusdLWnuyeekooSFn7KyWt9s,4680 -exceptiongroup/_exceptions.py,sha256=wPwPsZ64SXEptuwb4XrTIa1Mc78uqF5vmCrXTdllLn4,11463 -exceptiongroup/_formatting.py,sha256=OYTuT_T6TzM8G2v3DVt8LRBwMNyNK0tNl0fKMls3chM,21063 -exceptiongroup/_suppress.py,sha256=LX11PRNpchwfNWwEMY92nYN1F_5qFenQcS8EjIONXKE,1772 -exceptiongroup/_version.py,sha256=-4u7pjQ4caDQqa-1Qgms81j5hpkXjmjUYRCVEaLmb88,704 -exceptiongroup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/WHEEL deleted file mode 100644 index d8b9936dad9ab2513fa6979f411560d3b6b57e37..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: flit 3.12.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/licenses/LICENSE b/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/licenses/LICENSE deleted file mode 100644 index 50d4fa5e68439ce837f6eef437b299c0dd7c8594..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/exceptiongroup-1.3.1.dist-info/licenses/LICENSE +++ /dev/null @@ -1,73 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2022 Alex Grönholm - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -This project contains code copied from the Python standard library. -The following is the required license notice for those parts. - -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF hereby -grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -analyze, test, perform and/or display publicly, prepare derivative works, -distribute, and otherwise use Python alone or in any derivative version, -provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Python Software Foundation; -All Rights Reserved" are retained in Python alone or in any derivative version -prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup/__init__.py b/bundle/python-cpu/Lib/site-packages/exceptiongroup/__init__.py deleted file mode 100644 index d8e36b2e65d11e7f3b2c540c1b292a39a6cc219d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/exceptiongroup/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -__all__ = [ - "BaseExceptionGroup", - "ExceptionGroup", - "catch", - "format_exception", - "format_exception_only", - "print_exception", - "print_exc", - "suppress", -] - -import os -import sys - -from ._catch import catch -from ._version import version as __version__ # noqa: F401 - -if sys.version_info < (3, 11): - from ._exceptions import BaseExceptionGroup, ExceptionGroup - from ._formatting import ( - format_exception, - format_exception_only, - print_exc, - print_exception, - ) - - if os.getenv("EXCEPTIONGROUP_NO_PATCH") != "1": - from . import _formatting # noqa: F401 - - BaseExceptionGroup.__module__ = __name__ - ExceptionGroup.__module__ = __name__ -else: - from traceback import ( - format_exception, - format_exception_only, - print_exc, - print_exception, - ) - - BaseExceptionGroup = BaseExceptionGroup - ExceptionGroup = ExceptionGroup - -if sys.version_info < (3, 12, 1): - from ._suppress import suppress -else: - from contextlib import suppress diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup/_catch.py b/bundle/python-cpu/Lib/site-packages/exceptiongroup/_catch.py deleted file mode 100644 index 0246568bd05013ed797e0514181aa43bdc59c63e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/exceptiongroup/_catch.py +++ /dev/null @@ -1,138 +0,0 @@ -from __future__ import annotations - -import inspect -import sys -from collections.abc import Callable, Iterable, Mapping -from contextlib import AbstractContextManager -from types import TracebackType -from typing import TYPE_CHECKING, Any - -if sys.version_info < (3, 11): - from ._exceptions import BaseExceptionGroup - -if TYPE_CHECKING: - _Handler = Callable[[BaseExceptionGroup[Any]], Any] - - -class _Catcher: - def __init__(self, handler_map: Mapping[tuple[type[BaseException], ...], _Handler]): - self._handler_map = handler_map - - def __enter__(self) -> None: - pass - - def __exit__( - self, - etype: type[BaseException] | None, - exc: BaseException | None, - tb: TracebackType | None, - ) -> bool: - if exc is not None: - unhandled = self.handle_exception(exc) - if unhandled is exc: - return False - elif unhandled is None: - return True - else: - if isinstance(exc, BaseExceptionGroup): - try: - raise unhandled from exc.__cause__ - except BaseExceptionGroup: - # Change __context__ to __cause__ because Python 3.11 does this - # too - unhandled.__context__ = exc.__cause__ - raise - - raise unhandled from exc - - return False - - def handle_exception(self, exc: BaseException) -> BaseException | None: - excgroup: BaseExceptionGroup | None - if isinstance(exc, BaseExceptionGroup): - excgroup = exc - else: - excgroup = BaseExceptionGroup("", [exc]) - - new_exceptions: list[BaseException] = [] - for exc_types, handler in self._handler_map.items(): - matched, excgroup = excgroup.split(exc_types) - if matched: - try: - try: - raise matched - except BaseExceptionGroup: - result = handler(matched) - except BaseExceptionGroup as new_exc: - if new_exc is matched: - new_exceptions.append(new_exc) - else: - new_exceptions.extend(new_exc.exceptions) - except BaseException as new_exc: - new_exceptions.append(new_exc) - else: - if inspect.iscoroutine(result): - raise TypeError( - f"Error trying to handle {matched!r} with {handler!r}. " - "Exception handler must be a sync function." - ) from exc - - if not excgroup: - break - - if new_exceptions: - if len(new_exceptions) == 1: - return new_exceptions[0] - - return BaseExceptionGroup("", new_exceptions) - elif ( - excgroup and len(excgroup.exceptions) == 1 and excgroup.exceptions[0] is exc - ): - return exc - else: - return excgroup - - -def catch( - __handlers: Mapping[type[BaseException] | Iterable[type[BaseException]], _Handler], -) -> AbstractContextManager[None]: - if not isinstance(__handlers, Mapping): - raise TypeError("the argument must be a mapping") - - handler_map: dict[ - tuple[type[BaseException], ...], Callable[[BaseExceptionGroup]] - ] = {} - for type_or_iterable, handler in __handlers.items(): - iterable: tuple[type[BaseException]] - if isinstance(type_or_iterable, type) and issubclass( - type_or_iterable, BaseException - ): - iterable = (type_or_iterable,) - elif isinstance(type_or_iterable, Iterable): - iterable = tuple(type_or_iterable) - else: - raise TypeError( - "each key must be either an exception classes or an iterable thereof" - ) - - if not callable(handler): - raise TypeError("handlers must be callable") - - for exc_type in iterable: - if not isinstance(exc_type, type) or not issubclass( - exc_type, BaseException - ): - raise TypeError( - "each key must be either an exception classes or an iterable " - "thereof" - ) - - if issubclass(exc_type, BaseExceptionGroup): - raise TypeError( - "catching ExceptionGroup with catch() is not allowed. " - "Use except instead." - ) - - handler_map[iterable] = handler - - return _Catcher(handler_map) diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup/_exceptions.py b/bundle/python-cpu/Lib/site-packages/exceptiongroup/_exceptions.py deleted file mode 100644 index f42c1ad3628d927776f82dbd180cd499f26cd34d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/exceptiongroup/_exceptions.py +++ /dev/null @@ -1,336 +0,0 @@ -from __future__ import annotations - -import sys -from collections.abc import Callable, Sequence -from functools import partial -from inspect import getmro, isclass -from typing import TYPE_CHECKING, Generic, Type, TypeVar, cast, overload - -if sys.version_info < (3, 13): - from typing_extensions import TypeVar - -_BaseExceptionT_co = TypeVar( - "_BaseExceptionT_co", bound=BaseException, covariant=True, default=BaseException -) -_BaseExceptionT = TypeVar("_BaseExceptionT", bound=BaseException) -_ExceptionT_co = TypeVar( - "_ExceptionT_co", bound=Exception, covariant=True, default=Exception -) -_ExceptionT = TypeVar("_ExceptionT", bound=Exception) -# using typing.Self would require a typing_extensions dependency on py<3.11 -_ExceptionGroupSelf = TypeVar("_ExceptionGroupSelf", bound="ExceptionGroup") -_BaseExceptionGroupSelf = TypeVar("_BaseExceptionGroupSelf", bound="BaseExceptionGroup") - - -def check_direct_subclass( - exc: BaseException, parents: tuple[type[BaseException]] -) -> bool: - for cls in getmro(exc.__class__)[:-1]: - if cls in parents: - return True - - return False - - -def get_condition_filter( - condition: type[_BaseExceptionT] - | tuple[type[_BaseExceptionT], ...] - | Callable[[_BaseExceptionT_co], bool], -) -> Callable[[_BaseExceptionT_co], bool]: - if isclass(condition) and issubclass( - cast(Type[BaseException], condition), BaseException - ): - return partial(check_direct_subclass, parents=(condition,)) - elif isinstance(condition, tuple): - if all(isclass(x) and issubclass(x, BaseException) for x in condition): - return partial(check_direct_subclass, parents=condition) - elif callable(condition): - return cast("Callable[[BaseException], bool]", condition) - - raise TypeError("expected a function, exception type or tuple of exception types") - - -def _derive_and_copy_attributes(self, excs): - eg = self.derive(excs) - eg.__cause__ = self.__cause__ - eg.__context__ = self.__context__ - eg.__traceback__ = self.__traceback__ - if hasattr(self, "__notes__"): - # Create a new list so that add_note() only affects one exceptiongroup - eg.__notes__ = list(self.__notes__) - return eg - - -class BaseExceptionGroup(BaseException, Generic[_BaseExceptionT_co]): - """A combination of multiple unrelated exceptions.""" - - def __new__( - cls: type[_BaseExceptionGroupSelf], - __message: str, - __exceptions: Sequence[_BaseExceptionT_co], - ) -> _BaseExceptionGroupSelf: - if not isinstance(__message, str): - raise TypeError(f"argument 1 must be str, not {type(__message)}") - if not isinstance(__exceptions, Sequence): - raise TypeError("second argument (exceptions) must be a sequence") - if not __exceptions: - raise ValueError( - "second argument (exceptions) must be a non-empty sequence" - ) - - for i, exc in enumerate(__exceptions): - if not isinstance(exc, BaseException): - raise ValueError( - f"Item {i} of second argument (exceptions) is not an exception" - ) - - if cls is BaseExceptionGroup: - if all(isinstance(exc, Exception) for exc in __exceptions): - cls = ExceptionGroup - - if issubclass(cls, Exception): - for exc in __exceptions: - if not isinstance(exc, Exception): - if cls is ExceptionGroup: - raise TypeError( - "Cannot nest BaseExceptions in an ExceptionGroup" - ) - else: - raise TypeError( - f"Cannot nest BaseExceptions in {cls.__name__!r}" - ) - - instance = super().__new__(cls, __message, __exceptions) - instance._exceptions = tuple(__exceptions) - return instance - - def __init__( - self, - __message: str, - __exceptions: Sequence[_BaseExceptionT_co], - *args: object, - ) -> None: - BaseException.__init__(self, __message, __exceptions, *args) - - def add_note(self, note: str) -> None: - if not isinstance(note, str): - raise TypeError( - f"Expected a string, got note={note!r} (type {type(note).__name__})" - ) - - if not hasattr(self, "__notes__"): - self.__notes__: list[str] = [] - - self.__notes__.append(note) - - @property - def message(self) -> str: - return self.args[0] - - @property - def exceptions( - self, - ) -> tuple[_BaseExceptionT_co | BaseExceptionGroup[_BaseExceptionT_co], ...]: - return tuple(self._exceptions) - - @overload - def subgroup( - self, __condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...] - ) -> ExceptionGroup[_ExceptionT] | None: ... - - @overload - def subgroup( - self, __condition: type[_BaseExceptionT] | tuple[type[_BaseExceptionT], ...] - ) -> BaseExceptionGroup[_BaseExceptionT] | None: ... - - @overload - def subgroup( - self, - __condition: Callable[[_BaseExceptionT_co | _BaseExceptionGroupSelf], bool], - ) -> BaseExceptionGroup[_BaseExceptionT_co] | None: ... - - def subgroup( - self, - __condition: type[_BaseExceptionT] - | tuple[type[_BaseExceptionT], ...] - | Callable[[_BaseExceptionT_co | _BaseExceptionGroupSelf], bool], - ) -> BaseExceptionGroup[_BaseExceptionT] | None: - condition = get_condition_filter(__condition) - modified = False - if condition(self): - return self - - exceptions: list[BaseException] = [] - for exc in self.exceptions: - if isinstance(exc, BaseExceptionGroup): - subgroup = exc.subgroup(__condition) - if subgroup is not None: - exceptions.append(subgroup) - - if subgroup is not exc: - modified = True - elif condition(exc): - exceptions.append(exc) - else: - modified = True - - if not modified: - return self - elif exceptions: - group = _derive_and_copy_attributes(self, exceptions) - return group - else: - return None - - @overload - def split( - self, __condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...] - ) -> tuple[ - ExceptionGroup[_ExceptionT] | None, - BaseExceptionGroup[_BaseExceptionT_co] | None, - ]: ... - - @overload - def split( - self, __condition: type[_BaseExceptionT] | tuple[type[_BaseExceptionT], ...] - ) -> tuple[ - BaseExceptionGroup[_BaseExceptionT] | None, - BaseExceptionGroup[_BaseExceptionT_co] | None, - ]: ... - - @overload - def split( - self, - __condition: Callable[[_BaseExceptionT_co | _BaseExceptionGroupSelf], bool], - ) -> tuple[ - BaseExceptionGroup[_BaseExceptionT_co] | None, - BaseExceptionGroup[_BaseExceptionT_co] | None, - ]: ... - - def split( - self, - __condition: type[_BaseExceptionT] - | tuple[type[_BaseExceptionT], ...] - | Callable[[_BaseExceptionT_co], bool], - ) -> ( - tuple[ - ExceptionGroup[_ExceptionT] | None, - BaseExceptionGroup[_BaseExceptionT_co] | None, - ] - | tuple[ - BaseExceptionGroup[_BaseExceptionT] | None, - BaseExceptionGroup[_BaseExceptionT_co] | None, - ] - | tuple[ - BaseExceptionGroup[_BaseExceptionT_co] | None, - BaseExceptionGroup[_BaseExceptionT_co] | None, - ] - ): - condition = get_condition_filter(__condition) - if condition(self): - return self, None - - matching_exceptions: list[BaseException] = [] - nonmatching_exceptions: list[BaseException] = [] - for exc in self.exceptions: - if isinstance(exc, BaseExceptionGroup): - matching, nonmatching = exc.split(condition) - if matching is not None: - matching_exceptions.append(matching) - - if nonmatching is not None: - nonmatching_exceptions.append(nonmatching) - elif condition(exc): - matching_exceptions.append(exc) - else: - nonmatching_exceptions.append(exc) - - matching_group: _BaseExceptionGroupSelf | None = None - if matching_exceptions: - matching_group = _derive_and_copy_attributes(self, matching_exceptions) - - nonmatching_group: _BaseExceptionGroupSelf | None = None - if nonmatching_exceptions: - nonmatching_group = _derive_and_copy_attributes( - self, nonmatching_exceptions - ) - - return matching_group, nonmatching_group - - @overload - def derive(self, __excs: Sequence[_ExceptionT]) -> ExceptionGroup[_ExceptionT]: ... - - @overload - def derive( - self, __excs: Sequence[_BaseExceptionT] - ) -> BaseExceptionGroup[_BaseExceptionT]: ... - - def derive( - self, __excs: Sequence[_BaseExceptionT] - ) -> BaseExceptionGroup[_BaseExceptionT]: - return BaseExceptionGroup(self.message, __excs) - - def __str__(self) -> str: - suffix = "" if len(self._exceptions) == 1 else "s" - return f"{self.message} ({len(self._exceptions)} sub-exception{suffix})" - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.args[0]!r}, {self.args[1]!r})" - - -class ExceptionGroup(BaseExceptionGroup[_ExceptionT_co], Exception): - def __new__( - cls: type[_ExceptionGroupSelf], - __message: str, - __exceptions: Sequence[_ExceptionT_co], - ) -> _ExceptionGroupSelf: - return super().__new__(cls, __message, __exceptions) - - if TYPE_CHECKING: - - @property - def exceptions( - self, - ) -> tuple[_ExceptionT_co | ExceptionGroup[_ExceptionT_co], ...]: ... - - @overload # type: ignore[override] - def subgroup( - self, __condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...] - ) -> ExceptionGroup[_ExceptionT] | None: ... - - @overload - def subgroup( - self, __condition: Callable[[_ExceptionT_co | _ExceptionGroupSelf], bool] - ) -> ExceptionGroup[_ExceptionT_co] | None: ... - - def subgroup( - self, - __condition: type[_ExceptionT] - | tuple[type[_ExceptionT], ...] - | Callable[[_ExceptionT_co], bool], - ) -> ExceptionGroup[_ExceptionT] | None: - return super().subgroup(__condition) - - @overload - def split( - self, __condition: type[_ExceptionT] | tuple[type[_ExceptionT], ...] - ) -> tuple[ - ExceptionGroup[_ExceptionT] | None, ExceptionGroup[_ExceptionT_co] | None - ]: ... - - @overload - def split( - self, __condition: Callable[[_ExceptionT_co | _ExceptionGroupSelf], bool] - ) -> tuple[ - ExceptionGroup[_ExceptionT_co] | None, ExceptionGroup[_ExceptionT_co] | None - ]: ... - - def split( - self: _ExceptionGroupSelf, - __condition: type[_ExceptionT] - | tuple[type[_ExceptionT], ...] - | Callable[[_ExceptionT_co], bool], - ) -> tuple[ - ExceptionGroup[_ExceptionT_co] | None, ExceptionGroup[_ExceptionT_co] | None - ]: - return super().split(__condition) diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup/_formatting.py b/bundle/python-cpu/Lib/site-packages/exceptiongroup/_formatting.py deleted file mode 100644 index 490e2e0cafcd5ed4f6c53da5dbd5b517e10a7baa..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/exceptiongroup/_formatting.py +++ /dev/null @@ -1,602 +0,0 @@ -# traceback_exception_init() adapted from trio -# -# _ExceptionPrintContext and traceback_exception_format() copied from the standard -# library -from __future__ import annotations - -import collections.abc -import sys -import textwrap -import traceback -from functools import singledispatch -from types import TracebackType -from typing import Any, List, Optional - -from ._exceptions import BaseExceptionGroup - -max_group_width = 15 -max_group_depth = 10 -_cause_message = ( - "\nThe above exception was the direct cause of the following exception:\n\n" -) - -_context_message = ( - "\nDuring handling of the above exception, another exception occurred:\n\n" -) - - -def _format_final_exc_line(etype, value): - valuestr = _safe_string(value, "exception") - if value is None or not valuestr: - line = f"{etype}\n" - else: - line = f"{etype}: {valuestr}\n" - - return line - - -def _safe_string(value, what, func=str): - try: - return func(value) - except BaseException: - return f"<{what} {func.__name__}() failed>" - - -class _ExceptionPrintContext: - def __init__(self): - self.seen = set() - self.exception_group_depth = 0 - self.need_close = False - - def indent(self): - return " " * (2 * self.exception_group_depth) - - def emit(self, text_gen, margin_char=None): - if margin_char is None: - margin_char = "|" - indent_str = self.indent() - if self.exception_group_depth: - indent_str += margin_char + " " - - if isinstance(text_gen, str): - yield textwrap.indent(text_gen, indent_str, lambda line: True) - else: - for text in text_gen: - yield textwrap.indent(text, indent_str, lambda line: True) - - -def exceptiongroup_excepthook( - etype: type[BaseException], value: BaseException, tb: TracebackType | None -) -> None: - sys.stderr.write("".join(traceback.format_exception(etype, value, tb))) - - -class PatchedTracebackException(traceback.TracebackException): - def __init__( - self, - exc_type: type[BaseException], - exc_value: BaseException, - exc_traceback: TracebackType | None, - *, - limit: int | None = None, - lookup_lines: bool = True, - capture_locals: bool = False, - compact: bool = False, - _seen: set[int] | None = None, - ) -> None: - kwargs: dict[str, Any] = {} - if sys.version_info >= (3, 10): - kwargs["compact"] = compact - - is_recursive_call = _seen is not None - if _seen is None: - _seen = set() - _seen.add(id(exc_value)) - - self.stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_traceback), - limit=limit, - lookup_lines=lookup_lines, - capture_locals=capture_locals, - ) - self.exc_type = exc_type - # Capture now to permit freeing resources: only complication is in the - # unofficial API _format_final_exc_line - self._str = _safe_string(exc_value, "exception") - try: - self.__notes__ = getattr(exc_value, "__notes__", None) - except KeyError: - # Workaround for https://github.com/python/cpython/issues/98778 on Python - # <= 3.9, and some 3.10 and 3.11 patch versions. - HTTPError = getattr(sys.modules.get("urllib.error", None), "HTTPError", ()) - if sys.version_info[:2] <= (3, 11) and isinstance(exc_value, HTTPError): - self.__notes__ = None - else: - raise - - if exc_type and issubclass(exc_type, SyntaxError): - # Handle SyntaxError's specially - self.filename = exc_value.filename - lno = exc_value.lineno - self.lineno = str(lno) if lno is not None else None - self.text = exc_value.text - self.offset = exc_value.offset - self.msg = exc_value.msg - if sys.version_info >= (3, 10): - end_lno = exc_value.end_lineno - self.end_lineno = str(end_lno) if end_lno is not None else None - self.end_offset = exc_value.end_offset - elif ( - exc_type - and issubclass(exc_type, (NameError, AttributeError)) - and getattr(exc_value, "name", None) is not None - ): - suggestion = _compute_suggestion_error(exc_value, exc_traceback) - if suggestion: - self._str += f". Did you mean: '{suggestion}'?" - - if lookup_lines: - # Force all lines in the stack to be loaded - for frame in self.stack: - frame.line - - self.__suppress_context__ = ( - exc_value.__suppress_context__ if exc_value is not None else False - ) - - # Convert __cause__ and __context__ to `TracebackExceptions`s, use a - # queue to avoid recursion (only the top-level call gets _seen == None) - if not is_recursive_call: - queue = [(self, exc_value)] - while queue: - te, e = queue.pop() - - if e and e.__cause__ is not None and id(e.__cause__) not in _seen: - cause = PatchedTracebackException( - type(e.__cause__), - e.__cause__, - e.__cause__.__traceback__, - limit=limit, - lookup_lines=lookup_lines, - capture_locals=capture_locals, - _seen=_seen, - ) - else: - cause = None - - if compact: - need_context = ( - cause is None and e is not None and not e.__suppress_context__ - ) - else: - need_context = True - if ( - e - and e.__context__ is not None - and need_context - and id(e.__context__) not in _seen - ): - context = PatchedTracebackException( - type(e.__context__), - e.__context__, - e.__context__.__traceback__, - limit=limit, - lookup_lines=lookup_lines, - capture_locals=capture_locals, - _seen=_seen, - ) - else: - context = None - - # Capture each of the exceptions in the ExceptionGroup along with each - # of their causes and contexts - if e and isinstance(e, BaseExceptionGroup): - exceptions = [] - for exc in e.exceptions: - texc = PatchedTracebackException( - type(exc), - exc, - exc.__traceback__, - lookup_lines=lookup_lines, - capture_locals=capture_locals, - _seen=_seen, - ) - exceptions.append(texc) - else: - exceptions = None - - te.__cause__ = cause - te.__context__ = context - te.exceptions = exceptions - if cause: - queue.append((te.__cause__, e.__cause__)) - if context: - queue.append((te.__context__, e.__context__)) - if exceptions: - queue.extend(zip(te.exceptions, e.exceptions)) - - def format(self, *, chain=True, _ctx=None, **kwargs): - if _ctx is None: - _ctx = _ExceptionPrintContext() - - output = [] - exc = self - if chain: - while exc: - if exc.__cause__ is not None: - chained_msg = _cause_message - chained_exc = exc.__cause__ - elif exc.__context__ is not None and not exc.__suppress_context__: - chained_msg = _context_message - chained_exc = exc.__context__ - else: - chained_msg = None - chained_exc = None - - output.append((chained_msg, exc)) - exc = chained_exc - else: - output.append((None, exc)) - - for msg, exc in reversed(output): - if msg is not None: - yield from _ctx.emit(msg) - if getattr(exc, "exceptions", None) is None: - if exc.stack: - yield from _ctx.emit("Traceback (most recent call last):\n") - yield from _ctx.emit(exc.stack.format()) - yield from _ctx.emit(exc.format_exception_only()) - elif _ctx.exception_group_depth > max_group_depth: - # exception group, but depth exceeds limit - yield from _ctx.emit(f"... (max_group_depth is {max_group_depth})\n") - else: - # format exception group - is_toplevel = _ctx.exception_group_depth == 0 - if is_toplevel: - _ctx.exception_group_depth += 1 - - if exc.stack: - yield from _ctx.emit( - "Exception Group Traceback (most recent call last):\n", - margin_char="+" if is_toplevel else None, - ) - yield from _ctx.emit(exc.stack.format()) - - yield from _ctx.emit(exc.format_exception_only()) - num_excs = len(exc.exceptions) - if num_excs <= max_group_width: - n = num_excs - else: - n = max_group_width + 1 - _ctx.need_close = False - for i in range(n): - last_exc = i == n - 1 - if last_exc: - # The closing frame may be added by a recursive call - _ctx.need_close = True - - if max_group_width is not None: - truncated = i >= max_group_width - else: - truncated = False - title = f"{i + 1}" if not truncated else "..." - yield ( - _ctx.indent() - + ("+-" if i == 0 else " ") - + f"+---------------- {title} ----------------\n" - ) - _ctx.exception_group_depth += 1 - if not truncated: - yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx) - else: - remaining = num_excs - max_group_width - plural = "s" if remaining > 1 else "" - yield from _ctx.emit( - f"and {remaining} more exception{plural}\n" - ) - - if last_exc and _ctx.need_close: - yield _ctx.indent() + "+------------------------------------\n" - _ctx.need_close = False - _ctx.exception_group_depth -= 1 - - if is_toplevel: - assert _ctx.exception_group_depth == 1 - _ctx.exception_group_depth = 0 - - def format_exception_only(self, **kwargs): - """Format the exception part of the traceback. - The return value is a generator of strings, each ending in a newline. - Normally, the generator emits a single string; however, for - SyntaxError exceptions, it emits several lines that (when - printed) display detailed information about where the syntax - error occurred. - The message indicating which exception occurred is always the last - string in the output. - """ - if self.exc_type is None: - yield traceback._format_final_exc_line(None, self._str) - return - - stype = self.exc_type.__qualname__ - smod = self.exc_type.__module__ - if smod not in ("__main__", "builtins"): - if not isinstance(smod, str): - smod = "" - stype = smod + "." + stype - - if not issubclass(self.exc_type, SyntaxError): - yield _format_final_exc_line(stype, self._str) - elif traceback_exception_format_syntax_error is not None: - yield from traceback_exception_format_syntax_error(self, stype) - else: - yield from traceback_exception_original_format_exception_only(self) - - notes = getattr(self, "__notes__", None) - if isinstance(notes, collections.abc.Sequence): - for note in notes: - note = _safe_string(note, "note") - yield from [line + "\n" for line in note.split("\n")] - elif notes is not None: - yield _safe_string(notes, "__notes__", func=repr) - - -traceback_exception_original_format = traceback.TracebackException.format -traceback_exception_original_format_exception_only = ( - traceback.TracebackException.format_exception_only -) -traceback_exception_format_syntax_error = getattr( - traceback.TracebackException, "_format_syntax_error", None -) -if sys.excepthook is sys.__excepthook__: - traceback.TracebackException.__init__ = ( # type: ignore[assignment] - PatchedTracebackException.__init__ - ) - traceback.TracebackException.format = ( # type: ignore[assignment] - PatchedTracebackException.format - ) - traceback.TracebackException.format_exception_only = ( # type: ignore[assignment] - PatchedTracebackException.format_exception_only - ) - sys.excepthook = exceptiongroup_excepthook - -# Ubuntu's system Python has a sitecustomize.py file that imports -# apport_python_hook and replaces sys.excepthook. -# -# The custom hook captures the error for crash reporting, and then calls -# sys.__excepthook__ to actually print the error. -# -# We don't mind it capturing the error for crash reporting, but we want to -# take over printing the error. So we monkeypatch the apport_python_hook -# module so that instead of calling sys.__excepthook__, it calls our custom -# hook. -# -# More details: https://github.com/python-trio/trio/issues/1065 -if getattr(sys.excepthook, "__name__", None) in ( - "apport_excepthook", - # on ubuntu 22.10 the hook was renamed to partial_apport_excepthook - "partial_apport_excepthook", -): - # patch traceback like above - traceback.TracebackException.__init__ = ( # type: ignore[assignment] - PatchedTracebackException.__init__ - ) - traceback.TracebackException.format = ( # type: ignore[assignment] - PatchedTracebackException.format - ) - traceback.TracebackException.format_exception_only = ( # type: ignore[assignment] - PatchedTracebackException.format_exception_only - ) - - from types import ModuleType - - import apport_python_hook - - # monkeypatch the sys module that apport has imported - fake_sys = ModuleType("exceptiongroup_fake_sys") - fake_sys.__dict__.update(sys.__dict__) - fake_sys.__excepthook__ = exceptiongroup_excepthook - apport_python_hook.sys = fake_sys - - -@singledispatch -def format_exception_only(__exc: BaseException, **kwargs: Any) -> List[str]: - return list( - PatchedTracebackException( - type(__exc), __exc, None, compact=True - ).format_exception_only() - ) - - -@format_exception_only.register -def _(__exc: type, value: BaseException, **kwargs: Any) -> List[str]: - return format_exception_only(value) - - -@singledispatch -def format_exception( - __exc: BaseException, limit: Optional[int] = None, chain: bool = True, **kwargs: Any -) -> List[str]: - return list( - PatchedTracebackException( - type(__exc), __exc, __exc.__traceback__, limit=limit, compact=True - ).format(chain=chain) - ) - - -@format_exception.register -def _( - __exc: type, - value: BaseException, - tb: TracebackType, - limit: Optional[int] = None, - chain: bool = True, - **kwargs: Any, -) -> List[str]: - return format_exception(value, limit, chain) - - -@singledispatch -def print_exception( - __exc: BaseException, - limit: Optional[int] = None, - file: Any = None, - chain: bool = True, - **kwargs: Any, -) -> None: - if file is None: - file = sys.stderr - - for line in PatchedTracebackException( - type(__exc), __exc, __exc.__traceback__, limit=limit - ).format(chain=chain): - print(line, file=file, end="") - - -@print_exception.register -def _( - __exc: type, - value: BaseException, - tb: TracebackType, - limit: Optional[int] = None, - file: Any = None, - chain: bool = True, -) -> None: - print_exception(value, limit, file, chain) - - -def print_exc( - limit: Optional[int] = None, - file: Any | None = None, - chain: bool = True, -) -> None: - value = sys.exc_info()[1] - print_exception(value, limit, file, chain) - - -# Python levenshtein edit distance code for NameError/AttributeError -# suggestions, backported from 3.12 - -_MAX_CANDIDATE_ITEMS = 750 -_MAX_STRING_SIZE = 40 -_MOVE_COST = 2 -_CASE_COST = 1 -_SENTINEL = object() - - -def _substitution_cost(ch_a, ch_b): - if ch_a == ch_b: - return 0 - if ch_a.lower() == ch_b.lower(): - return _CASE_COST - return _MOVE_COST - - -def _compute_suggestion_error(exc_value, tb): - wrong_name = getattr(exc_value, "name", None) - if wrong_name is None or not isinstance(wrong_name, str): - return None - if isinstance(exc_value, AttributeError): - obj = getattr(exc_value, "obj", _SENTINEL) - if obj is _SENTINEL: - return None - obj = exc_value.obj - try: - d = dir(obj) - except Exception: - return None - else: - assert isinstance(exc_value, NameError) - # find most recent frame - if tb is None: - return None - while tb.tb_next is not None: - tb = tb.tb_next - frame = tb.tb_frame - - d = list(frame.f_locals) + list(frame.f_globals) + list(frame.f_builtins) - if len(d) > _MAX_CANDIDATE_ITEMS: - return None - wrong_name_len = len(wrong_name) - if wrong_name_len > _MAX_STRING_SIZE: - return None - best_distance = wrong_name_len - suggestion = None - for possible_name in d: - if possible_name == wrong_name: - # A missing attribute is "found". Don't suggest it (see GH-88821). - continue - # No more than 1/3 of the involved characters should need changed. - max_distance = (len(possible_name) + wrong_name_len + 3) * _MOVE_COST // 6 - # Don't take matches we've already beaten. - max_distance = min(max_distance, best_distance - 1) - current_distance = _levenshtein_distance( - wrong_name, possible_name, max_distance - ) - if current_distance > max_distance: - continue - if not suggestion or current_distance < best_distance: - suggestion = possible_name - best_distance = current_distance - return suggestion - - -def _levenshtein_distance(a, b, max_cost): - # A Python implementation of Python/suggestions.c:levenshtein_distance. - - # Both strings are the same - if a == b: - return 0 - - # Trim away common affixes - pre = 0 - while a[pre:] and b[pre:] and a[pre] == b[pre]: - pre += 1 - a = a[pre:] - b = b[pre:] - post = 0 - while a[: post or None] and b[: post or None] and a[post - 1] == b[post - 1]: - post -= 1 - a = a[: post or None] - b = b[: post or None] - if not a or not b: - return _MOVE_COST * (len(a) + len(b)) - if len(a) > _MAX_STRING_SIZE or len(b) > _MAX_STRING_SIZE: - return max_cost + 1 - - # Prefer shorter buffer - if len(b) < len(a): - a, b = b, a - - # Quick fail when a match is impossible - if (len(b) - len(a)) * _MOVE_COST > max_cost: - return max_cost + 1 - - # Instead of producing the whole traditional len(a)-by-len(b) - # matrix, we can update just one row in place. - # Initialize the buffer row - row = list(range(_MOVE_COST, _MOVE_COST * (len(a) + 1), _MOVE_COST)) - - result = 0 - for bindex in range(len(b)): - bchar = b[bindex] - distance = result = bindex * _MOVE_COST - minimum = sys.maxsize - for index in range(len(a)): - # 1) Previous distance in this row is cost(b[:b_index], a[:index]) - substitute = distance + _substitution_cost(bchar, a[index]) - # 2) cost(b[:b_index], a[:index+1]) from previous row - distance = row[index] - # 3) existing result is cost(b[:b_index+1], a[index]) - - insert_delete = min(result, distance) + _MOVE_COST - result = min(insert_delete, substitute) - - # cost(b[:b_index+1], a[:index+1]) - row[index] = result - if result < minimum: - minimum = result - if minimum > max_cost: - # Everything in this row is too big, so bail early. - return max_cost + 1 - return result diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup/_suppress.py b/bundle/python-cpu/Lib/site-packages/exceptiongroup/_suppress.py deleted file mode 100644 index 11467eeda9b317cbf5d378beea30e31a51d35d1c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/exceptiongroup/_suppress.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations - -import sys -from contextlib import AbstractContextManager -from types import TracebackType -from typing import TYPE_CHECKING, Optional, Type, cast - -if sys.version_info < (3, 11): - from ._exceptions import BaseExceptionGroup - -if TYPE_CHECKING: - # requires python 3.9 - BaseClass = AbstractContextManager[None] -else: - BaseClass = AbstractContextManager - - -class suppress(BaseClass): - """Backport of :class:`contextlib.suppress` from Python 3.12.1.""" - - def __init__(self, *exceptions: type[BaseException]): - self._exceptions = exceptions - - def __enter__(self) -> None: - pass - - def __exit__( - self, - exctype: Optional[Type[BaseException]], - excinst: Optional[BaseException], - exctb: Optional[TracebackType], - ) -> bool: - # Unlike isinstance and issubclass, CPython exception handling - # currently only looks at the concrete type hierarchy (ignoring - # the instance and subclass checking hooks). While Guido considers - # that a bug rather than a feature, it's a fairly hard one to fix - # due to various internal implementation details. suppress provides - # the simpler issubclass based semantics, rather than trying to - # exactly reproduce the limitations of the CPython interpreter. - # - # See http://bugs.python.org/issue12029 for more details - if exctype is None: - return False - - if issubclass(exctype, self._exceptions): - return True - - if issubclass(exctype, BaseExceptionGroup): - match, rest = cast(BaseExceptionGroup, excinst).split(self._exceptions) - if rest is None: - return True - - raise rest - - return False diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup/_version.py b/bundle/python-cpu/Lib/site-packages/exceptiongroup/_version.py deleted file mode 100644 index ebbbcb239f0fb8240eada674229f81e66f1c7022..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/exceptiongroup/_version.py +++ /dev/null @@ -1,34 +0,0 @@ -# file generated by setuptools-scm -# don't change, don't track in version control - -__all__ = [ - "__version__", - "__version_tuple__", - "version", - "version_tuple", - "__commit_id__", - "commit_id", -] - -TYPE_CHECKING = False -if TYPE_CHECKING: - from typing import Tuple - from typing import Union - - VERSION_TUPLE = Tuple[Union[int, str], ...] - COMMIT_ID = Union[str, None] -else: - VERSION_TUPLE = object - COMMIT_ID = object - -version: str -__version__: str -__version_tuple__: VERSION_TUPLE -version_tuple: VERSION_TUPLE -commit_id: COMMIT_ID -__commit_id__: COMMIT_ID - -__version__ = version = '1.3.1' -__version_tuple__ = version_tuple = (1, 3, 1) - -__commit_id__ = commit_id = None diff --git a/bundle/python-cpu/Lib/site-packages/exceptiongroup/py.typed b/bundle/python-cpu/Lib/site-packages/exceptiongroup/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/faiss/__init__.py b/bundle/python-cpu/Lib/site-packages/faiss/__init__.py deleted file mode 100644 index a0bbc9c0882ff0d21d568005c5856b308c44bcc3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/__init__.py +++ /dev/null @@ -1,614 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -# start delvewheel patch -def _delvewheel_patch_1_13_0(): - import os - if os.path.isdir(libs_dir := os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'faiss_cpu.libs'))): - os.add_dll_directory(libs_dir) - - -_delvewheel_patch_1_13_0() -del _delvewheel_patch_1_13_0 -# end delvewheel patch - -# @nolint - -# not linting this file because it imports * from swigfaiss, which -# causes a ton of useless warnings. - -import numpy as np -import logging -import sys -import inspect - - -def _preload_gpu_libs(): - """Pre-load CUDA / RAPIDS shared libs from pip wheels with RTLD_GLOBAL. - - These libs ship in nvidia-*-cuNN / libcuvs-cuNN wheels, off ld.so's search - path, so we dlopen them before the SWIG extension loads libfaiss.so. Gated - on the `faiss._gpu_build` marker (CMake writes it only for GPU builds); the - `_cuvs_build` marker selects the CUDA 13 cuVS variant (else CUDA 12) and - adds the cuVS stack. Missing wheels raise a fix-it. - """ - try: - from . import _gpu_build # noqa: F401 - except ImportError: - return # faiss-cpu install: marker absent, nothing to preload - - import ctypes - import os - - def _load(path): - """dlopen one .so with global visibility, or raise a fix-it.""" - try: - ctypes.CDLL(path, mode=ctypes.RTLD_GLOBAL) - except OSError as e: - raise RuntimeError( - f"faiss-gpu: failed to load {os.path.basename(path)} from " - f"{path} — corrupt or incomplete nvidia CUDA wheel?" - ) from e - - # faiss-gpu-cuvs wheels carry the `_cuvs_build` marker and are built against - # CUDA 13 (+ the cuVS stack); the plain faiss-gpu wheel is CUDA 12. The two - # use different pip wheel layouts (handled separately below). Load order - # matters: libcudart first (others resolve symbols against it), and load - # RTLD_GLOBAL so the SWIG extension's later dlopen of libfaiss.so sees them. - try: - from . import _cuvs_build # noqa: F401 - - is_cuvs = True - except ImportError: - is_cuvs = False - - if is_cuvs: - # CUDA 13 nvidia wheels (nvidia-cuda-runtime / -cublas / -curand / - # -nvjitlink — all unsuffixed) ship every lib in one PEP 420 namespace - # dir, nvidia/cu13/lib/, with no importable per-component module. This - # differs from CUDA 12's per-component nvidia//lib/ layout. - try: - import nvidia.cu13 as _cu13 - - _libdir = os.path.join(list(_cu13.__path__)[0], "lib") - except ImportError as e: - raise RuntimeError( - "faiss-gpu-cuvs installed but the CUDA 13 runtime wheels are " - "missing — pip install 'nvidia-cuda-runtime>=13.2,<14' " - "'nvidia-cublas>=13.2,<14' 'nvidia-curand>=10.3.7,<11' " - "'nvidia-nvjitlink>=13.2,<14'" - ) from e - # nvjitlink before cublas: cublas 13 links it. - for _soname in ( - "libcudart.so.13", - "libnvJitLink.so.13", - "libcublas.so.13", - "libcublasLt.so.13", - "libcurand.so.10", - ): - _load(os.path.join(_libdir, _soname)) - else: - # CUDA 12 per-component layout: each nvidia-*-cu12 wheel exposes an - # importable module whose lib/ dir holds the .so. - def _nvidia_lib_dir(import_name, pip_spec): - """Return the lib/ dir of an nvidia-*-cu12 wheel, or raise a - fix-it.""" - try: - mod = __import__( - "nvidia." + import_name, fromlist=[import_name] - ) - except ImportError as e: - pip_name = pip_spec.split(">")[0].split("<")[0].split("=")[0] - raise RuntimeError( - f"faiss-gpu installed but {pip_name} is missing — " - f"pip install '{pip_spec}'" - ) from e - # __path__[0] not __file__: PEP 420 namespace pkgs have - # __file__ = None. - return os.path.join(mod.__path__[0], "lib") - - _cudart = _nvidia_lib_dir( - "cuda_runtime", "nvidia-cuda-runtime-cu12>=12.6,<13" - ) - _cublas = _nvidia_lib_dir("cublas", "nvidia-cublas-cu12>=12.6,<13") - _curand = _nvidia_lib_dir("curand", "nvidia-curand-cu12>=10.3.7,<11") - _load(os.path.join(_cudart, "libcudart.so.12")) - _load(os.path.join(_cublas, "libcublas.so.12")) - _load(os.path.join(_cublas, "libcublasLt.so.12")) - _load(os.path.join(_curand, "libcurand.so.10")) - return # faiss-gpu (non-cuVS) build: base CUDA preload is sufficient. - - # faiss-gpu-cuvs wheels also need the cuVS stack. Delegate to RAPIDS' - # load_library() (loads each .so RTLD_GLOBAL + its CUDA deps); order - # rmm -> raft -> cuvs makes every symbol global before the SWIG extension - # loads. - try: - import libcuvs - import libraft - import librmm - except ImportError as e: - raise RuntimeError( - "faiss-gpu-cuvs installed but the cuVS runtime wheels are " - "missing — " - "pip install 'libcuvs-cu13>=26.06,<27' " - "--extra-index-url https://pypi.nvidia.com" - ) from e - - for _mod in (librmm, libraft, libcuvs): - _mod.load_library() - - -_preload_gpu_libs() -del _preload_gpu_libs - -# We import * so that the symbol foo can be accessed as faiss.foo. -from .loader import * - -# additional wrappers -from faiss import class_wrappers -from faiss.gpu_wrappers import * -from faiss.array_conversions import * -from faiss.extra_wrappers import ( - kmin, - kmax, - pairwise_distances, - rand, - randint, - lrand, - randn, - rand_smooth_vectors, - eval_intersection, - normalize_L2, - ResultHeap, - knn, - Kmeans, - SuperKmeans, - checksum, - matrix_bucket_sort_inplace, - bucket_sort, - merge_knn_results, - MapInt64ToInt64, - knn_hamming, - pack_bitstrings, - unpack_bitstrings, -) - - -__version__ = "%d.%d.%d" % ( - FAISS_VERSION_MAJOR, - FAISS_VERSION_MINOR, - FAISS_VERSION_PATCH, -) - -logger = logging.getLogger(__name__) - -class_wrappers.handle_Clustering(Clustering) -class_wrappers.handle_SuperKMeans(SuperKMeans) -class_wrappers.handle_Clustering1D(Clustering1D) -class_wrappers.handle_MatrixStats(MatrixStats) -class_wrappers.handle_IOWriter(IOWriter) -class_wrappers.handle_IOReader(IOReader) -class_wrappers.handle_AutoTuneCriterion(AutoTuneCriterion) -class_wrappers.handle_ParameterSpace(ParameterSpace) -class_wrappers.handle_NSG(IndexNSG) -class_wrappers.handle_MapLong2Long(MapLong2Long) -class_wrappers.handle_IDSelectorSubset(IDSelectorBatch, class_owns=True) -class_wrappers.handle_IDSelectorSubset(IDSelectorArray, class_owns=False) -class_wrappers.handle_IDSelectorSubset( - IDSelectorBitmap, class_owns=False, force_int64=False -) -class_wrappers.handle_CodeSet(CodeSet) - -class_wrappers.handle_Tensor2D(Tensor2D) -class_wrappers.handle_Tensor2D(Int32Tensor2D) -class_wrappers.handle_Embedding(Embedding) -class_wrappers.handle_Linear(Linear) -class_wrappers.handle_QINCo(QINCo) -class_wrappers.handle_QINCoStep(QINCoStep) -shard_ivf_index_centroids = class_wrappers.handle_shard_ivf_index_centroids( - shard_ivf_index_centroids -) - - -this_module = sys.modules[__name__] - -# handle sub-classes -for symbol in dir(this_module): - obj = getattr(this_module, symbol) - # print symbol, isinstance(obj, (type, types.ClassType)) - if inspect.isclass(obj): - the_class = obj - if issubclass(the_class, Index): - class_wrappers.handle_Index(the_class) - - if issubclass(the_class, IndexBinary): - class_wrappers.handle_IndexBinary(the_class) - - if issubclass(the_class, VectorTransform): - class_wrappers.handle_VectorTransform(the_class) - - if issubclass(the_class, Quantizer): - class_wrappers.handle_Quantizer(the_class) - - if issubclass(the_class, IndexRowwiseMinMax) or issubclass( - the_class, IndexRowwiseMinMaxFP16 - ): - class_wrappers.handle_IndexRowwiseMinMax(the_class) - - if issubclass(the_class, SearchParameters): - class_wrappers.handle_SearchParameters(the_class) - - if issubclass(the_class, CodePacker): - class_wrappers.handle_CodePacker(the_class) - -############################################################################## -# For some classes (IndexIVF, IDSelector), the object holds a reference to -# a C++ object (eg. the quantizer object of IndexIVF). We don't transfer the -# ownership to the C++ object (ie. set own_quantizer=true), but instead we add -# a reference in the Python class wrapper instead. This is done via an -# additional referenced_objects field. -# -# Since the semantics of ownership in the C++ classes are sometimes irregular, -# these references are added manually using the functions below. -############################################################################## - - -def add_ref_in_constructor(the_class, parameter_no): - # adds a reference to parameter parameter_no in self - # so that that parameter does not get deallocated before self - original_init = the_class.__init__ - - def replacement_init(self, *args): - original_init(self, *args) - self.referenced_objects = [args[parameter_no]] - - def replacement_init_multiple(self, *args): - original_init(self, *args) - pset = parameter_no[len(args)] - self.referenced_objects = [args[no] for no in pset] - - if type(parameter_no) == dict: - # a list of parameters to keep, depending on the number of arguments - the_class.__init__ = replacement_init_multiple - else: - the_class.__init__ = replacement_init - - -def add_to_referenced_objects(self, ref): - if not hasattr(self, "referenced_objects"): - self.referenced_objects = [ref] - else: - self.referenced_objects.append(ref) - - -def add_ref_in_method(the_class, method_name, parameter_no): - original_method = getattr(the_class, method_name) - - def replacement_method(self, *args): - ref = args[parameter_no] - add_to_referenced_objects(self, ref) - return original_method(self, *args) - - setattr(the_class, method_name, replacement_method) - - -def add_ref_in_method_explicit_own(the_class, method_name): - # for methods of format set_XXX(object, own) - original_method = getattr(the_class, method_name) - - def replacement_method(self, ref, own=False): - if not own: - if not hasattr(self, "referenced_objects"): - self.referenced_objects = [ref] - else: - self.referenced_objects.append(ref) - else: - # transfer ownership to C++ class - ref.this.disown() - return original_method(self, ref, own) - - setattr(the_class, method_name, replacement_method) - - -def add_ref_in_function(function_name, parameter_no): - # assumes the function returns an object - original_function = getattr(this_module, function_name) - - def replacement_function(*args): - result = original_function(*args) - ref = args[parameter_no] - result.referenced_objects = [ref] - return result - - setattr(this_module, function_name, replacement_function) - - -if "GPU" in get_compile_options(): - add_ref_in_constructor(GpuIndexIVFFlat, 1) - add_ref_in_constructor(GpuIndexBinaryFlat, 1) - add_ref_in_constructor(GpuIndexFlat, 1) - add_ref_in_constructor(GpuIndexIVFPQ, 1) - add_ref_in_constructor(GpuIndexIVFScalarQuantizer, 1) - -add_ref_in_constructor(IndexIVFFlat, 0) -add_ref_in_constructor(IndexIVFFlatDedup, 0) -add_ref_in_constructor(IndexIVFFlatPanorama, 0) -add_ref_in_constructor(IndexPreTransform, {2: [0, 1], 1: [0]}) -add_ref_in_method(IndexPreTransform, "prepend_transform", 0) -add_ref_in_constructor(IndexIVFPQ, 0) -add_ref_in_constructor(IndexIVFPQR, 0) -add_ref_in_constructor(IndexIVFPQFastScan, 0) -add_ref_in_constructor(IndexIVFResidualQuantizer, 0) -add_ref_in_constructor(IndexIVFLocalSearchQuantizer, 0) -add_ref_in_constructor(IndexIVFResidualQuantizerFastScan, 0) -add_ref_in_constructor(IndexIVFLocalSearchQuantizerFastScan, 0) -add_ref_in_constructor(IndexIVFSpectralHash, 0) -add_ref_in_method_explicit_own(IndexIVFSpectralHash, "replace_vt") - -add_ref_in_constructor(Index2Layer, 0) -add_ref_in_constructor(Level1Quantizer, 0) -add_ref_in_constructor(IndexIVFScalarQuantizer, 0) -add_ref_in_constructor(IndexRowwiseMinMax, 0) -add_ref_in_constructor(IndexRowwiseMinMaxFP16, 0) -add_ref_in_constructor(IndexIDMap, 0) -add_ref_in_constructor(IndexIDMap2, 0) -add_ref_in_constructor(IndexHNSW, 0) -add_ref_in_method(IndexShards, "add_shard", 0) -add_ref_in_method(IndexBinaryShards, "add_shard", 0) -add_ref_in_constructor(IndexRefineFlat, {2: [0], 1: [0]}) -add_ref_in_constructor(IndexRefinePanorama, {2: [0, 1]}) -add_ref_in_constructor(IndexRefine, {2: [0, 1]}) - -add_ref_in_constructor(IndexBinaryIVF, 0) -add_ref_in_constructor(IndexBinaryFromFloat, 0) -add_ref_in_constructor(IndexBinaryIDMap, 0) -add_ref_in_constructor(IndexBinaryIDMap2, 0) - -add_ref_in_method(IndexReplicas, "addIndex", 0) -add_ref_in_method(IndexBinaryReplicas, "addIndex", 0) - -add_ref_in_constructor(BufferedIOWriter, 0) -add_ref_in_constructor(BufferedIOReader, 0) - -add_ref_in_constructor(IDSelectorNot, 0) -add_ref_in_constructor(IDSelectorAnd, slice(2)) -add_ref_in_constructor(IDSelectorOr, slice(2)) -add_ref_in_constructor(IDSelectorXOr, slice(2)) -add_ref_in_constructor(IDSelectorTranslated, slice(2)) - -add_ref_in_constructor(IDSelectorXOr, slice(2)) -add_ref_in_constructor(IndexIVFIndependentQuantizer, slice(3)) - -add_ref_in_constructor(IndexIVFRaBitQ, 0) -add_ref_in_constructor(IndexIVFRaBitQFastScan, 0) -add_ref_in_constructor(IndexIVFEDEN, 0) - -if "SVS" in get_compile_options(): - add_ref_in_constructor(IndexSVSVamana, 0) - add_ref_in_constructor(IndexSVSVamanaLVQ, 0) - add_ref_in_constructor(IndexSVSVamanaLeanVec, 0) - add_ref_in_constructor(IndexSVSFlat, 0) - -# seems really marginal... -# remove_ref_from_method(IndexReplicas, 'removeIndex', 0) - - -###################################################### -# search_with_parameters interface -###################################################### - -search_with_parameters_c = search_with_parameters - - -def search_with_parameters(index, x, k, params=None, output_stats=False): - x = np.ascontiguousarray(x, dtype="float32") - n, d = x.shape - assert d == index.d - if not params: - # if not provided use the ones set in the IVF object - params = IVFSearchParameters() - index_ivf = extract_index_ivf(index) - params.nprobe = index_ivf.nprobe - params.max_codes = index_ivf.max_codes - nb_dis = np.empty(1, "uint64") - ms_per_stage = np.empty(3, "float64") - distances = np.empty((n, k), dtype=np.float32) - labels = np.empty((n, k), dtype=np.int64) - search_with_parameters_c( - index, - n, - swig_ptr(x), - k, - swig_ptr(distances), - swig_ptr(labels), - params, - swig_ptr(nb_dis), - swig_ptr(ms_per_stage), - ) - if not output_stats: - return distances, labels - else: - stats = { - "ndis": nb_dis[0], - "pre_transform_ms": ms_per_stage[0], - "coarse_quantizer_ms": ms_per_stage[1], - "invlist_scan_ms": ms_per_stage[2], - } - return distances, labels, stats - - -range_search_with_parameters_c = range_search_with_parameters - - -def range_search_with_parameters( - index, x, radius, params=None, output_stats=False -): - x = np.ascontiguousarray(x, dtype="float32") - n, d = x.shape - assert d == index.d - if not params: - # if not provided use the ones set in the IVF object - params = IVFSearchParameters() - index_ivf = extract_index_ivf(index) - params.nprobe = index_ivf.nprobe - params.max_codes = index_ivf.max_codes - nb_dis = np.empty(1, "uint64") - ms_per_stage = np.empty(3, "float64") - res = RangeSearchResult(n) - range_search_with_parameters_c( - index, - n, - swig_ptr(x), - radius, - res, - params, - swig_ptr(nb_dis), - swig_ptr(ms_per_stage), - ) - lims = rev_swig_ptr(res.lims, n + 1).copy() - nd = int(lims[-1]) - Dout = rev_swig_ptr(res.distances, nd).copy() - Iout = rev_swig_ptr(res.labels, nd).copy() - if not output_stats: - return lims, Dout, Iout - else: - stats = { - "ndis": nb_dis[0], - "pre_transform_ms": ms_per_stage[0], - "coarse_quantizer_ms": ms_per_stage[1], - "invlist_scan_ms": ms_per_stage[2], - } - return lims, Dout, Iout, stats - - -super_kmeans_assign_iteration_c = super_kmeans_assign_iteration - - -def super_kmeans_assign_iteration( - X_tilde, Y_tilde, tau, assignments, d_prime, ad_coeff, cp, -): - """Run one SuperKMeans iter-1+ pruned assignment pass on caller-managed state. - - All arrays must be C-contiguous. `X_tilde`, `Y_tilde`, `tau`, and `ad_coeff` - must be float32; `assignments` must be int32. These mirror the C++ pointer - contract, and passing another dtype (e.g. int64 assignments, numpy's default - integer type) would otherwise reinterpret the buffer and corrupt results. - - Shapes: `X_tilde` is (n, d), `Y_tilde` is (k, d), `tau` and `assignments` - have length n, and `ad_coeff` has length d + 1. - - Returns (total_pairs, pruned_at_gemm) for a d_prime controller. - Mutates `tau` and `assignments` in place. - """ - for name, arr, dtype in ( - ("X_tilde", X_tilde, "float32"), - ("Y_tilde", Y_tilde, "float32"), - ("tau", tau, "float32"), - ("ad_coeff", ad_coeff, "float32"), - ("assignments", assignments, "int32"), - ): - if arr.dtype != dtype: - raise TypeError( - f"super_kmeans_assign_iteration: {name} must be {dtype}, " - f"got {arr.dtype}" - ) - if not arr.flags["C_CONTIGUOUS"]: - raise ValueError( - f"super_kmeans_assign_iteration: {name} must be C-contiguous" - ) - if X_tilde.ndim != 2: - raise ValueError( - f"super_kmeans_assign_iteration: X_tilde must be 2D (n, d), " - f"got shape {X_tilde.shape}" - ) - n, d = X_tilde.shape - if Y_tilde.ndim != 2 or Y_tilde.shape[1] != d: - raise ValueError( - f"super_kmeans_assign_iteration: Y_tilde must have shape (k, {d}), " - f"got {Y_tilde.shape}" - ) - k = Y_tilde.shape[0] - if tau.shape != (n,): - raise ValueError( - f"super_kmeans_assign_iteration: tau must have shape ({n},), " - f"got {tau.shape}" - ) - if assignments.shape != (n,): - raise ValueError( - f"super_kmeans_assign_iteration: assignments must have shape ({n},), " - f"got {assignments.shape}" - ) - if ad_coeff.shape != (d + 1,): - raise ValueError( - f"super_kmeans_assign_iteration: ad_coeff must have shape ({d + 1},), " - f"got {ad_coeff.shape}" - ) - total = np.zeros(1, dtype=np.int64) - pruned = np.zeros(1, dtype=np.int64) - super_kmeans_assign_iteration_c( - swig_ptr(X_tilde), n, d, - swig_ptr(Y_tilde), k, - swig_ptr(tau), swig_ptr(assignments), - d_prime, swig_ptr(ad_coeff), cp, - swig_ptr(total), swig_ptr(pruned), - ) - return int(total[0]), int(pruned[0]) - - -# IndexProxy was renamed to IndexReplicas, remap the old name for any old code -# people may have -IndexProxy = IndexReplicas -ConcatenatedInvertedLists = HStackInvertedLists -IndexResidual = IndexResidualQuantizer - -IVFSearchParameters = SearchParametersIVF - -########################################### -# serialization of indexes to byte arrays -########################################### - - -def serialize_index(index, io_flags=0): - """convert an index to a numpy uint8 array""" - writer = VectorIOWriter() - write_index(index, writer, io_flags) - return vector_to_array(writer.data) - - -def deserialize_index(data, io_flags=0): - reader = VectorIOReader() - copy_array_to_vector(data, reader.data) - return read_index(reader, io_flags) - - -def serialize_index_binary(index): - """convert an index to a numpy uint8 array""" - writer = VectorIOWriter() - write_index_binary(index, writer) - return vector_to_array(writer.data) - - -def deserialize_index_binary(data): - reader = VectorIOReader() - copy_array_to_vector(data, reader.data) - return read_index_binary(reader) - - -class TimeoutGuard: - def __init__(self, timeout_in_seconds: float): - self.timeout = timeout_in_seconds - - def __enter__(self): - TimeoutCallback.reset(self.timeout) - - def __exit__(self, exc_type, exc_value, traceback): - PythonInterruptCallback.reset() - - -try: - post_init_hook() -except NameError: - pass diff --git a/bundle/python-cpu/Lib/site-packages/faiss/__init__.pyi b/bundle/python-cpu/Lib/site-packages/faiss/__init__.pyi deleted file mode 100644 index 00f9e3705022a69ace27dcb583eaf13d639e7609..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/__init__.pyi +++ /dev/null @@ -1,5122 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# Faiss Python API type stubs -# Provides type information for IDE autocompletion and static type checkers -# (mypy, pyright, Pyre, etc.) - -from __future__ import annotations - -from typing import Any, Callable, Literal, overload - -import numpy as np -import numpy.typing as npt - -try: - import torch -except ImportError: - pass - -idx_t = int -MetricType = int -EDENScaleType = Literal[1, 2] -METRIC_INNER_PRODUCT: int -METRIC_L2: int -METRIC_L1: int -METRIC_Linf: int -METRIC_Lp: int -METRIC_Canberra: int -METRIC_BrayCurtis: int -METRIC_JensenShannon: int -METRIC_Jaccard: int -METRIC_NaNEuclidean: int -METRIC_GOWER: int - -ClusteringInitMethod = int -ClusteringInitMethod_RANDOM: int -ClusteringInitMethod_KMEANS_PLUS_PLUS: int -ClusteringInitMethod_AFK_MC2: int - -# Storage kind for SVS (Intel Scalable Vector Search) indexes -SVSStorageKind = int -SVS_FP32: int -SVS_FP16: int -SVS_SQ8: int -SVS_LVQ4x0: int -SVS_LVQ4x4: int -SVS_LVQ4x8: int -SVS_LVQ8x0: int -SVS_LeanVec4x4: int -SVS_LeanVec4x8: int -SVS_LeanVec8x8: int -SVS_count: int - -# I/O flag constants for reading/writing indexes -IO_FLAG_SKIP_STORAGE: int # skip the storage for graph-based indexes -IO_FLAG_READ_ONLY: int # read-only mode -IO_FLAG_ONDISK_SAME_DIR: int # strip directory component from ondisk filename -IO_FLAG_SKIP_IVF_DATA: int # don't load IVF data to RAM, only list sizes -IO_FLAG_SKIP_PRECOMPUTE_TABLE: ( - int # don't initialize precomputed table after loading -) -IO_FLAG_PQ_SKIP_SDC_TABLE: ( - int # don't compute the sdc table for PQ-based indices -) -IO_FLAG_MMAP: int # try to memmap data (useful to load as OnDiskInvertedLists) -IO_FLAG_MMAP_IFC: int # mmap for IndexFlatCodes-derived indices and HNSW - -# EDEN scale type enum -EDENScaleType_UNBIASED: Literal[1] -EDENScaleType_BIASED: Literal[2] - -# Numeric type enum -Float32: int -Float16: int -UInt8: int -Int8: int - -def get_numeric_type_size(numeric_type: int) -> int: ... -def normalize_L2(x: npt.NDArray[np.float32]) -> None: ... -def real_to_binary(d: int, x_in: Any, x_out: Any) -> None: ... -def bucket_sort( - tab: npt.NDArray[np.int64], nbucket: int | None = None, nt: int = 0 -) -> tuple[npt.NDArray[np.int64], npt.NDArray[np.int64]]: ... -def matrix_bucket_sort_inplace( - tab: npt.NDArray[np.int32 | np.int64], - nbucket: int | None = None, - nt: int = 0, -) -> npt.NDArray[np.int64]: ... -def eval_intersection( - I1: npt.NDArray[np.int64], I2: npt.NDArray[np.int64] -) -> int: ... -def checksum(a: npt.NDArray[np.uint8]) -> int | npt.NDArray[np.uint64]: ... -def rand_smooth_vectors( - n: int, d: int, seed: int = 1234 -) -> npt.NDArray[np.float32]: ... -def merge_knn_results( - Dall: npt.NDArray[np.float32], - Iall: npt.NDArray[np.int64], - keep_max: bool = False, -) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... -@overload -def knn( - xq: torch.Tensor, - xb: torch.Tensor, - k: int, - metric: int = METRIC_L2, - metric_arg: float = 0.0, -) -> tuple[torch.Tensor, torch.Tensor]: ... -@overload -def knn( - xq: npt.NDArray[np.float32], - xb: npt.NDArray[np.float32], - k: int, - metric: int = METRIC_L2, - metric_arg: float = 0.0, -) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... -def knn_hamming( - xq: npt.NDArray[np.uint8], - xb: npt.NDArray[np.uint8], - k: int, - variant: str = "hc", -) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.int64]]: ... -def pack_bitstrings( - a: npt.NDArray[np.int32], nbit: int | npt.NDArray[np.int32] -) -> npt.NDArray[np.uint8]: ... -def unpack_bitstrings( - b: npt.NDArray[np.uint8], - M_or_nbits: int | npt.NDArray[np.int32], - nbit: int | None = None, -) -> npt.NDArray[np.int32]: ... -def popcount32(x: int) -> int: ... -def popcount64(x: int) -> int: ... - -# Version information -FAISS_VERSION_MAJOR: int -FAISS_VERSION_MINOR: int -FAISS_VERSION_PATCH: int - -# Vector types (std::vector templates) -class Float32Vector: - def __init__(self) -> None: ... - def push_back(self, x: float) -> None: ... - def clear(self) -> None: ... - def data(self) -> Any: ... # float* - def size(self) -> int: ... - def at(self, n: int) -> float: ... - def __getitem__(self, n: int) -> float: ... - def __setitem__(self, n: int, val: float) -> None: ... - def resize(self, n: int) -> None: ... - def swap(self, other: Float32Vector) -> None: ... - -class Float64Vector: - def __init__(self) -> None: ... - def push_back(self, x: float) -> None: ... - def clear(self) -> None: ... - def data(self) -> Any: ... - def size(self) -> int: ... - def at(self, n: int) -> float: ... - def __getitem__(self, n: int) -> float: ... - def __setitem__(self, n: int, val: float) -> None: ... - def resize(self, n: int) -> None: ... - def swap(self, other: Float64Vector) -> None: ... - -class Int8Vector: - def __init__(self) -> None: ... - def push_back(self, x: int) -> None: ... - def clear(self) -> None: ... - def data(self) -> Any: ... - def size(self) -> int: ... - def at(self, n: int) -> int: ... - def __getitem__(self, n: int) -> int: ... - def __setitem__(self, n: int, val: int) -> None: ... - def resize(self, n: int) -> None: ... - -class Int16Vector: - def __init__(self) -> None: ... - def push_back(self, x: int) -> None: ... - def clear(self) -> None: ... - def data(self) -> Any: ... - def size(self) -> int: ... - def at(self, n: int) -> int: ... - def __getitem__(self, n: int) -> int: ... - def __setitem__(self, n: int, val: int) -> None: ... - def resize(self, n: int) -> None: ... - def swap(self, other: Int16Vector) -> None: ... - -class Int32Vector: - def __init__(self) -> None: ... - def push_back(self, x: int) -> None: ... - def clear(self) -> None: ... - def data(self) -> Any: ... - def size(self) -> int: ... - def at(self, n: int) -> int: ... - def __getitem__(self, n: int) -> int: ... - def __setitem__(self, n: int, val: int) -> None: ... - def resize(self, n: int) -> None: ... - def swap(self, other: Int32Vector) -> None: ... - -class Int64Vector: - def __init__(self) -> None: ... - def push_back(self, x: int) -> None: ... - def clear(self) -> None: ... - def data(self) -> Any: ... - def size(self) -> int: ... - def at(self, n: int) -> int: ... - def __getitem__(self, n: int) -> int: ... - def __setitem__(self, n: int, val: int) -> None: ... - def resize(self, n: int) -> None: ... - def swap(self, other: Int64Vector) -> None: ... - -class UInt8Vector: - def __init__(self) -> None: ... - def push_back(self, x: int) -> None: ... - def clear(self) -> None: ... - def data(self) -> Any: ... - def size(self) -> int: ... - def at(self, n: int) -> int: ... - def __getitem__(self, n: int) -> int: ... - def __setitem__(self, n: int, val: int) -> None: ... - def resize(self, n: int) -> None: ... - def swap(self, other: UInt8Vector) -> None: ... - -class UInt16Vector: - def __init__(self) -> None: ... - def push_back(self, x: int) -> None: ... - def clear(self) -> None: ... - def data(self) -> Any: ... - def size(self) -> int: ... - def at(self, n: int) -> int: ... - def __getitem__(self, n: int) -> int: ... - def __setitem__(self, n: int, val: int) -> None: ... - def resize(self, n: int) -> None: ... - def swap(self, other: UInt16Vector) -> None: ... - -class UInt32Vector: - def __init__(self) -> None: ... - def push_back(self, x: int) -> None: ... - def clear(self) -> None: ... - def data(self) -> Any: ... - def size(self) -> int: ... - def at(self, n: int) -> int: ... - def __getitem__(self, n: int) -> int: ... - def __setitem__(self, n: int, val: int) -> None: ... - def resize(self, n: int) -> None: ... - def swap(self, other: UInt32Vector) -> None: ... - -class UInt64Vector: - def __init__(self) -> None: ... - def push_back(self, x: int) -> None: ... - def clear(self) -> None: ... - def data(self) -> Any: ... - def size(self) -> int: ... - def at(self, n: int) -> int: ... - def __getitem__(self, n: int) -> int: ... - def __setitem__(self, n: int, val: int) -> None: ... - def resize(self, n: int) -> None: ... - def swap(self, other: UInt64Vector) -> None: ... - -# Forward declarations -class IDSelector: ... -class DistanceComputer: ... - -# Range search result structure -class RangeSearchResult: - nq: int # number of queries - lims: Any # size_t* - size (nq + 1) array - labels: Any # idx_t* - result for query i is labels[lims[i]:lims[i+1]] - distances: Any # float* - corresponding distances (not sorted) - buffer_size: int # size of the result buffers used - - def __init__(self, nq: int, alloc_lims: bool = True) -> None: ... - def do_allocation(self) -> None: ... - -# IDSelector implementations -class IDSelectorTranslated(IDSelector): - def __init__( - self, - id_map: npt.NDArray[np.int64] | list[int], - sel: IDSelector, - ) -> None: ... - def is_member(self, id: int) -> bool: ... - -# Search parameters -class SearchParameters: - sel: IDSelector | None - def __init__(self) -> None: ... - -class SearchParametersPreTransform(SearchParameters): - index_params: SearchParameters | None - def __init__(self) -> None: ... - -class SearchParametersSVSVamana(SearchParameters): - search_window_size: int - search_buffer_capacity: int - def __init__(self) -> None: ... - -class SearchParametersSVSIVF(SearchParameters): - n_probes: int - k_reorder: float - def __init__(self) -> None: ... - -# Base Index class -class Index: - d: int # vector dimension - ntotal: int # total number of indexed vectors - verbose: bool # verbosity level - is_trained: bool # whether the index is trained - metric_type: MetricType # metric type for search - metric_arg: float # metric argument - - def __init__(self, d: int = 0, metric: MetricType = METRIC_L2) -> None: ... - # Python wrapper interface (what users actually see) - @overload - def train(self, x: torch.Tensor) -> None: ... - @overload - def train(self, x: npt.NDArray[np.float32]) -> None: ... - @overload - def add(self, x: torch.Tensor) -> None: ... - @overload - def add(self, x: npt.NDArray[np.float32]) -> None: ... - @overload - def add_with_ids(self, x: torch.Tensor, ids: torch.Tensor) -> None: ... - @overload - def add_with_ids( - self, x: npt.NDArray[np.float32], ids: npt.NDArray[np.int64] - ) -> None: ... - @overload - def search( - self, - x: torch.Tensor, - k: int, - *, - params: SearchParameters | None = None, - D: torch.Tensor | None = None, - I: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: ... - @overload - def search( - self, - x: npt.NDArray[np.float32], - k: int, - *, - params: SearchParameters | None = None, - D: npt.NDArray[np.float32] | None = None, - I: npt.NDArray[np.int64] | None = None, - ) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... - @overload - def range_search( - self, - x: torch.Tensor, - thresh: float, - *, - params: SearchParameters | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ... - @overload - def range_search( - self, - x: npt.NDArray[np.float32], - thresh: float, - *, - params: SearchParameters | None = None, - ) -> tuple[ - npt.NDArray[np.int64], npt.NDArray[np.float32], npt.NDArray[np.int64] - ]: ... - @overload - def assign(self, x: torch.Tensor, k: int = 1) -> torch.Tensor: ... - @overload - def assign( - self, x: npt.NDArray[np.float32], k: int = 1 - ) -> npt.NDArray[np.int64]: ... - def reset(self) -> None: ... - def remove_ids(self, sel: IDSelector) -> int: ... - @overload - def reconstruct( - self, key: int, x: torch.Tensor | None = None - ) -> torch.Tensor: ... - @overload - def reconstruct( - self, key: int, x: npt.NDArray[np.float32] | None = None - ) -> npt.NDArray[np.float32]: ... - @overload - def reconstruct_batch( - self, keys: torch.Tensor, x: torch.Tensor | None = None - ) -> torch.Tensor: ... - @overload - def reconstruct_batch( - self, - keys: npt.NDArray[np.int64], - x: npt.NDArray[np.float32] | None = None, - ) -> npt.NDArray[np.float32]: ... - # Without an `x` buffer the return type is not statically knowable: importing - # faiss.contrib.torch_utils replaces this method with one that allocates a - # torch.Tensor, while the default numpy implementation allocates an ndarray. - @overload - def reconstruct_n( - self, n0: int = ..., ni: int = ..., x: None = ... - ) -> Any: ... - @overload - def reconstruct_n( - self, n0: int = ..., ni: int = ..., x: torch.Tensor = ... - ) -> torch.Tensor: ... - @overload - def reconstruct_n( - self, - n0: int = ..., - ni: int = ..., - x: npt.NDArray[np.float32] = ..., - ) -> npt.NDArray[np.float32]: ... - @overload - def search_and_reconstruct( - self, - x: torch.Tensor, - k: int, - *, - params: SearchParameters | None = None, - D: torch.Tensor | None = None, - I: torch.Tensor | None = None, - R: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ... - @overload - def search_and_reconstruct( - self, - x: npt.NDArray[np.float32], - k: int, - *, - params: SearchParameters | None = None, - D: npt.NDArray[np.float32] | None = None, - I: npt.NDArray[np.int64] | None = None, - R: npt.NDArray[np.float32] | None = None, - ) -> tuple[ - npt.NDArray[np.float32], npt.NDArray[np.int64], npt.NDArray[np.float32] - ]: ... - def compute_residual( - self, - x: npt.NDArray[np.float32], - residual: npt.NDArray[np.float32], - key: int, - ) -> None: ... - def compute_residual_n( - self, - n: int, - xs: npt.NDArray[np.float32], - residuals: npt.NDArray[np.float32], - keys: npt.NDArray[np.int64], - ) -> None: ... - def get_distance_computer(self) -> DistanceComputer: ... - - # Standalone codec interface with tensor overloads - def sa_code_size(self) -> int: ... - @overload - def sa_encode( - self, x: torch.Tensor, codes: torch.Tensor | None = None - ) -> torch.Tensor: ... - @overload - def sa_encode( - self, - x: npt.NDArray[np.float32], - codes: npt.NDArray[np.uint8] | None = None, - ) -> npt.NDArray[np.uint8]: ... - @overload - def sa_decode( - self, codes: torch.Tensor, x: torch.Tensor | None = None - ) -> torch.Tensor: ... - @overload - def sa_decode( - self, - codes: npt.NDArray[np.uint8], - x: npt.NDArray[np.float32] | None = None, - ) -> npt.NDArray[np.float32]: ... - @overload - def add_sa_codes( - self, codes: torch.Tensor, ids: torch.Tensor | None = None - ) -> None: ... - @overload - def add_sa_codes( - self, - codes: npt.NDArray[np.uint8], - ids: npt.NDArray[np.int64] | None = None, - ) -> None: ... - def merge_from(self, other_index: Index, add_id: int = 0) -> None: ... - def check_compatible_for_merge(self, other_index: Index) -> None: ... - @overload - def search_and_return_codes( - self, - x: torch.Tensor, - k: int, - include_listnos: bool = False, - params: SearchParameters | None = None, - D: torch.Tensor | None = None, - I: torch.Tensor | None = None, - codes: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ... - @overload - def search_and_return_codes( - self, - x: npt.NDArray[np.float32], - k: int, - include_listnos: bool = False, - params: SearchParameters | None = None, - D: npt.NDArray[np.float32] | None = None, - I: npt.NDArray[np.int64] | None = None, - codes: npt.NDArray[np.uint8] | None = None, - ) -> tuple[ - npt.NDArray[np.float32], npt.NDArray[np.int64], npt.NDArray[np.uint8] - ]: ... - @overload - def update_vectors(self, keys: torch.Tensor, x: torch.Tensor) -> None: ... - @overload - def update_vectors( - self, keys: npt.NDArray[np.int64], x: npt.NDArray[np.float32] - ) -> None: ... - @overload - def permute_entries(self, perm: torch.Tensor) -> None: ... - @overload - def permute_entries(self, perm: npt.NDArray[np.int64]) -> None: ... - -# Vector transform classes -class VectorTransform: - d_in: int # input dimension - d_out: int # output dimension - is_trained: bool - - def __init__(self, d_in: int = 0, d_out: int = 0) -> None: ... - @overload - def train(self, x: torch.Tensor) -> None: ... - @overload - def train(self, x: npt.NDArray[np.float32]) -> None: ... - @overload - def apply(self, x: torch.Tensor) -> torch.Tensor: ... - @overload - def apply(self, x: npt.NDArray[np.float32]) -> npt.NDArray[np.float32]: ... - @overload - def apply_py(self, x: torch.Tensor) -> torch.Tensor: ... - @overload - def apply_py( - self, x: npt.NDArray[np.float32] - ) -> npt.NDArray[np.float32]: ... - @overload - def reverse_transform( - self, xt: torch.Tensor, x: torch.Tensor | None = None - ) -> torch.Tensor: ... - @overload - def reverse_transform( - self, - xt: npt.NDArray[np.float32], - x: npt.NDArray[np.float32] | None = None, - ) -> npt.NDArray[np.float32]: ... - def check_identical(self, other: VectorTransform) -> None: ... - -class LinearTransform(VectorTransform): - have_bias: bool - is_orthonormal: bool - A: Float32Vector # transformation matrix - b: Float32Vector # bias vector - verbose: bool - - def __init__( - self, d_in: int = 0, d_out: int = 0, have_bias: bool = False - ) -> None: ... - def set_is_orthonormal(self) -> None: ... - def check_identical(self, other: VectorTransform) -> None: ... - -class RandomRotationMatrix(LinearTransform): - def __init__(self, d_in: int, d_out: int) -> None: ... - def init(self, seed: int) -> None: ... - -class HadamardRotation(VectorTransform): - """Three rounds of random sign-flip + Fast Walsh-Hadamard Transform. - Produces a pseudo-random rotation in O(d log d) time. - d_out is the smallest power of 2 >= d_in (zero-padded as needed). - """ - - seed: int - - def __init__(self, d: int, seed: int = 12345) -> None: ... - def init(self, seed: int) -> None: ... - -class PCAMatrix(LinearTransform): - eigen_power: float - epsilon: float - random_rotation: bool - max_points_per_d: int - balanced_bins: int - mean: Float32Vector - eigenvalues: Float32Vector - PCAMat: Float32Vector - - def __init__( - self, - d_in: int = 0, - d_out: int = 0, - eigen_power: float = 0, - random_rotation: bool = False, - ) -> None: ... - def copy_from(self, other: PCAMatrix) -> None: ... - def prepare_Ab(self) -> None: ... - # Explicit overloads for apply method to ensure torch.Tensor support - @overload - def apply(self, x: torch.Tensor) -> torch.Tensor: ... - @overload - def apply(self, x: npt.NDArray[np.float32]) -> npt.NDArray[np.float32]: ... - -class ITQMatrix(LinearTransform): - max_iter: int - seed: int - init_rotation: Float64Vector - - def __init__(self, d: int = 0) -> None: ... - -class ITQTransform(VectorTransform): - mean: Float32Vector - do_pca: bool - itq: ITQMatrix - max_train_per_dim: int - pca_then_itq: LinearTransform - - def __init__( - self, d_in: int = 0, d_out: int = 0, do_pca: bool = False - ) -> None: ... - def check_identical(self, other: VectorTransform) -> None: ... - -class OPQMatrix(LinearTransform): - M: int # number of subquantizers - niter: int - niter_pq: int - niter_pq_0: int - max_train_points: int - verbose: bool - pq: ProductQuantizer | None - - def __init__(self, d: int = 0, M: int = 1, d2: int = -1) -> None: ... - -class RemapDimensionsTransform(VectorTransform): - map: Int32Vector - - def __init__( - self, - d_in: int, - d_out: int, - map: npt.NDArray[np.int32] | None = None, - uniform: bool = True, - ) -> None: ... - def check_identical(self, other: VectorTransform) -> None: ... - -class NormalizationTransform(VectorTransform): - norm: float - - def __init__(self, d: int, norm: float = 2.0) -> None: ... - def check_identical(self, other: VectorTransform) -> None: ... - -class CenteringTransform(VectorTransform): - mean: Float32Vector - - def __init__(self, d: int = 0) -> None: ... - def check_identical(self, other: VectorTransform) -> None: ... - -# Specific Index implementations -class IndexFlatCodes(Index): - code_size: int - codes: Any # MaybeOwnedVector - - def __init__( - self, code_size: int = 0, d: int = 0, metric: MetricType = METRIC_L2 - ) -> None: ... - def sa_code_size(self) -> int: ... - def permute_entries(self, perm: npt.NDArray[np.int64]) -> None: ... - -class IndexFlat(IndexFlatCodes): - def __init__(self, d: int, metric: MetricType = METRIC_L2) -> None: ... - @overload - def search( - self, - x: torch.Tensor, - k: int, - *, - params: SearchParameters | None = None, - D: torch.Tensor | None = None, - I: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: ... - @overload - def search( - self, - x: npt.NDArray[np.float32], - k: int, - *, - params: SearchParameters | None = None, - D: npt.NDArray[np.float32] | None = None, - I: npt.NDArray[np.int64] | None = None, - ) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... - @overload - def range_search( - self, - x: torch.Tensor, - thresh: float, - *, - params: SearchParameters | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ... - @overload - def range_search( - self, - x: npt.NDArray[np.float32], - thresh: float, - *, - params: SearchParameters | None = None, - ) -> tuple[ - npt.NDArray[np.int64], npt.NDArray[np.float32], npt.NDArray[np.int64] - ]: ... - @overload - def reconstruct( - self, key: int, recons: torch.Tensor | None = None - ) -> torch.Tensor: ... - @overload - def reconstruct( - self, key: int, recons: npt.NDArray[np.float32] | None = None - ) -> npt.NDArray[np.float32]: ... - def get_xb(self) -> npt.NDArray[np.float32]: ... - -class IndexFlatIP(IndexFlat): - def __init__(self, d: int) -> None: ... - -class IndexFlatL2(IndexFlat): - cached_l2norms: Float32Vector - - def __init__(self, d: int) -> None: ... - def sync_l2norms(self) -> None: ... - def clear_l2norms(self) -> None: ... - -class IndexFlat1D(IndexFlatL2): - continuous_update: bool - perm: Int64Vector - - def __init__(self, continuous_update: bool = True) -> None: ... - def update_permutation(self) -> None: ... - def reset(self) -> None: ... - -class IndexFlatPanorama(IndexFlat): - """Panorama implementation of IndexFlat following https://arxiv.org/abs/2510.00566""" - - batch_size: int - n_levels: int - - def __init__( - self, - d: int, - metric: MetricType, - n_levels: int, - batch_size: int, - ) -> None: ... - def permute_entries(self, perm: npt.NDArray[np.int64]) -> None: ... - -class IndexFlatL2Panorama(IndexFlatPanorama): - def __init__( - self, d: int, n_levels: int, batch_size: int = 512 - ) -> None: ... - -class IndexFlatIPPanorama(IndexFlatPanorama): - def __init__( - self, d: int, n_levels: int, batch_size: int = 512 - ) -> None: ... - -class IndexPreTransform(Index): - chain: list[VectorTransform] # std::vector chain - index: Index - own_fields: bool - - @overload - def __init__(self, index: Index) -> None: ... - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, ltrans: VectorTransform, index: Index) -> None: ... - def prepend_transform(self, ltrans: VectorTransform) -> None: ... - @overload - def train(self, x: torch.Tensor) -> None: ... - @overload - def train(self, x: npt.NDArray[np.float32]) -> None: ... - @overload - def add(self, x: torch.Tensor) -> None: ... - @overload - def add(self, x: npt.NDArray[np.float32]) -> None: ... - @overload - def add_with_ids(self, x: torch.Tensor, ids: torch.Tensor) -> None: ... - @overload - def add_with_ids( - self, x: npt.NDArray[np.float32], ids: npt.NDArray[np.int64] - ) -> None: ... - def reset(self) -> None: ... - def remove_ids(self, sel: IDSelector) -> int: ... - @overload - def search( - self, - x: torch.Tensor, - k: int, - *, - params: SearchParameters | None = None, - D: torch.Tensor | None = None, - I: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: ... - @overload - def search( - self, - x: npt.NDArray[np.float32], - k: int, - *, - params: SearchParameters | None = None, - D: npt.NDArray[np.float32] | None = None, - I: npt.NDArray[np.int64] | None = None, - ) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... - @overload - def range_search( - self, - x: torch.Tensor, - thresh: float, - *, - params: SearchParameters | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ... - @overload - def range_search( - self, - x: npt.NDArray[np.float32], - thresh: float, - *, - params: SearchParameters | None = None, - ) -> tuple[ - npt.NDArray[np.int64], npt.NDArray[np.float32], npt.NDArray[np.int64] - ]: ... - @overload - def reconstruct( - self, key: int, x: torch.Tensor | None = None - ) -> torch.Tensor: ... - @overload - def reconstruct( - self, key: int, x: npt.NDArray[np.float32] | None = None - ) -> npt.NDArray[np.float32]: ... - # Without an `x` buffer the return type is not statically knowable: importing - # faiss.contrib.torch_utils replaces this method with one that allocates a - # torch.Tensor, while the default numpy implementation allocates an ndarray. - @overload - def reconstruct_n( - self, n0: int = ..., ni: int = ..., x: None = ... - ) -> Any: ... - @overload - def reconstruct_n( - self, n0: int = ..., ni: int = ..., x: torch.Tensor = ... - ) -> torch.Tensor: ... - @overload - def reconstruct_n( - self, - n0: int = ..., - ni: int = ..., - x: npt.NDArray[np.float32] = ..., - ) -> npt.NDArray[np.float32]: ... - @overload - def search_and_reconstruct( - self, - x: torch.Tensor, - k: int, - *, - params: SearchParameters | None = None, - D: torch.Tensor | None = None, - I: torch.Tensor | None = None, - R: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ... - @overload - def search_and_reconstruct( - self, - x: npt.NDArray[np.float32], - k: int, - *, - params: SearchParameters | None = None, - D: npt.NDArray[np.float32] | None = None, - I: npt.NDArray[np.int64] | None = None, - R: npt.NDArray[np.float32] | None = None, - ) -> tuple[ - npt.NDArray[np.float32], npt.NDArray[np.int64], npt.NDArray[np.float32] - ]: ... - def apply_chain( - self, x: npt.NDArray[np.float32] - ) -> npt.NDArray[np.float32]: ... - def reverse_chain( - self, xt: npt.NDArray[np.float32], x: npt.NDArray[np.float32] - ) -> None: ... - def get_distance_computer(self) -> DistanceComputer: ... - def sa_code_size(self) -> int: ... - @overload - def sa_encode( - self, x: torch.Tensor, codes: torch.Tensor | None = None - ) -> torch.Tensor: ... - @overload - def sa_encode( - self, - x: npt.NDArray[np.float32], - codes: npt.NDArray[np.uint8] | None = None, - ) -> npt.NDArray[np.uint8]: ... - @overload - def sa_decode( - self, codes: torch.Tensor, x: torch.Tensor | None = None - ) -> torch.Tensor: ... - @overload - def sa_decode( - self, - codes: npt.NDArray[np.uint8], - x: npt.NDArray[np.float32] | None = None, - ) -> npt.NDArray[np.float32]: ... - def merge_from(self, other_index: Index, add_id: int = 0) -> None: ... - def check_compatible_for_merge(self, other_index: Index) -> None: ... - -# Quantizer classes -class Quantizer: - d: int - code_size: int - is_trained: bool - - def __init__(self, d: int = 0, code_size: int = 0) -> None: ... - @overload - def train(self, x: torch.Tensor) -> None: ... - @overload - def train(self, x: npt.NDArray[np.float32]) -> None: ... - @overload - def compute_codes(self, x: torch.Tensor) -> torch.Tensor: ... - @overload - def compute_codes( - self, x: npt.NDArray[np.float32] - ) -> npt.NDArray[np.uint8]: ... - @overload - def decode(self, codes: torch.Tensor) -> torch.Tensor: ... - @overload - def decode( - self, codes: npt.NDArray[np.uint8] - ) -> npt.NDArray[np.float32]: ... - -class ProductQuantizer(Quantizer): - # Core attributes from C++ struct - M: int # number of subquantizers - nbits: int # bits per subquantizer - - # Derived values - dsub: int # dimensionality of each subvector - ksub: int # number of centroids for each subquantizer - - verbose: bool # verbose during training? - - # Training configuration - train_type: int # enum train_type_t - Train_default: int - Train_hot_start: int - Train_shared: int - Train_hypercube: int - Train_hypercube_pca: int - - cp: ClusteringParameters - assign_index: Index | None # optional index for assignment - - # Centroid storage - centroids: Float32Vector # M * ksub * dsub layout - transposed_centroids: Float32Vector # dsub * M * ksub layout - centroids_sq_lengths: Float32Vector # M * ksub layout - - # Symmetric Distance Table - sdc_table: Float32Vector - - def __init__(self, d: int = 0, M: int = 0, nbits: int = 0) -> None: ... - - # Basic encoding/decoding - def compute_code( - self, x: npt.NDArray[np.float32], code: npt.NDArray[np.uint8] - ) -> None: ... - def compute_codes_with_assign_index( - self, x: npt.NDArray[np.float32], codes: npt.NDArray[np.uint8], n: int - ) -> None: ... - def decode( - self, code: npt.NDArray[np.uint8], x: npt.NDArray[np.float32] - ) -> None: ... - - # Distance computation methods - def compute_distance_table( - self, x: npt.NDArray[np.float32], dis_table: npt.NDArray[np.float32] - ) -> None: ... - def compute_inner_prod_table( - self, x: npt.NDArray[np.float32], dis_table: npt.NDArray[np.float32] - ) -> None: ... - def compute_distance_tables( - self, - nx: int, - x: npt.NDArray[np.float32], - dis_tables: npt.NDArray[np.float32], - ) -> None: ... - def compute_inner_prod_tables( - self, - nx: int, - x: npt.NDArray[np.float32], - dis_tables: npt.NDArray[np.float32], - ) -> None: ... - - # Advanced encoding methods - def compute_code_from_distance_table( - self, tab: npt.NDArray[np.float32], code: npt.NDArray[np.uint8] - ) -> None: ... - - # Search methods - def search( - self, - x: npt.NDArray[np.float32], - nx: int, - codes: npt.NDArray[np.uint8], - ncodes: int, - res: Any, # float_maxheap_array_t* - init_finalize_heap: bool = True, - ) -> None: ... - def search_ip( - self, - x: npt.NDArray[np.float32], - nx: int, - codes: npt.NDArray[np.uint8], - ncodes: int, - res: Any, # float_minheap_array_t* - init_finalize_heap: bool = True, - ) -> None: ... - def search_sdc( - self, - qcodes: npt.NDArray[np.uint8], - nq: int, - bcodes: npt.NDArray[np.uint8], - ncodes: int, - res: Any, # float_maxheap_array_t* - init_finalize_heap: bool = True, - ) -> None: ... - - # Centroid management - def set_derived_values(self) -> None: ... - def set_params( - self, centroids: npt.NDArray[np.float32], m: int - ) -> None: ... - def get_centroids(self, m: int, i: int) -> npt.NDArray[np.float32]: ... - def sync_transposed_centroids(self) -> None: ... - def clear_transposed_centroids(self) -> None: ... - def compute_sdc_table(self) -> None: ... - -class AdditiveQuantizer(Quantizer): - # Core attributes from C++ struct - M: int # number of codebooks - nbits: Int32Vector # bits for each step (variable length) - codebooks: Float32Vector # codebooks - - # Derived values - codebook_offsets: Int64Vector # codebook offsets - tot_bits: int # total number of bits - norm_bits: int # bits allocated for norms - total_codebook_size: int # size of codebook in vectors - only_8bit: bool # are all nbits = 8 (use faster decoder) - - verbose: bool # verbose during training - is_trained: bool # is trained or not - - # Auxiliary data for special search types - norm_tabs: Float32Vector # norms of codebook entries for 4-bit fastscan - qnorm: IndexFlat1D # store and search norms - centroid_norms: Float32Vector # norms of all codebook entries - codebook_cross_products: ( - Float32Vector # dot products with previous codebooks - ) - max_mem_distances: int # memory limit for beam search - - # Search type configuration - search_type: int # Search_type_t enum value - norm_min: float # min for quantization of norms - norm_max: float # max for quantization of norms - - # Search type constants - ST_decompress: int - ST_LUT_nonorm: int - ST_norm_from_LUT: int - ST_norm_float: int - ST_norm_qint8: int - ST_norm_qint4: int - ST_norm_cqint8: int - ST_norm_cqint4: int - ST_norm_lsq2x4: int - ST_norm_rq2x4: int - - @overload - def __init__( - self, d: int, nbits: list[int], search_type: int = 0 - ) -> None: ... - @overload - def __init__(self) -> None: ... - def set_derived_values(self) -> None: ... - @overload - def train_norm(self, n: int, norms: torch.Tensor) -> None: ... - @overload - def train_norm(self, n: int, norms: npt.NDArray[np.float32]) -> None: ... - @overload - def compute_codes_add_centroids( - self, - x: torch.Tensor, - codes: torch.Tensor, - n: int, - centroids: torch.Tensor | None = None, - ) -> None: ... - @overload - def compute_codes_add_centroids( - self, - x: npt.NDArray[np.float32], - codes: npt.NDArray[np.uint8], - n: int, - centroids: npt.NDArray[np.float32] | None = None, - ) -> None: ... - def pack_codes( - self, - n: int, - codes: npt.NDArray[np.int32], - packed_codes: npt.NDArray[np.uint8], - ld_codes: int = -1, - norms: npt.NDArray[np.float32] | None = None, - centroids: npt.NDArray[np.float32] | None = None, - ) -> None: ... - @overload - def decode_unpacked( - self, - codes: npt.NDArray[np.int32], - x: npt.NDArray[np.float32], - n: int, - ld_codes: int = -1, - ) -> None: ... - def decode_64bit(self, n: int, x: npt.NDArray[np.float32]) -> None: ... - @overload - def compute_LUT( - self, - n: int, - xq: npt.NDArray[np.float32], - LUT: npt.NDArray[np.float32], - alpha: float = 1.0, - ld_lut: int = -1, - ) -> None: ... - def knn_centroids_inner_product( - self, - n: int, - xq: npt.NDArray[np.float32], - k: int, - distances: npt.NDArray[np.float32], - labels: npt.NDArray[np.int64], - ) -> None: ... - def compute_centroid_norms( - self, norms: npt.NDArray[np.float32] - ) -> None: ... - def knn_centroids_L2( - self, - n: int, - xq: npt.NDArray[np.float32], - k: int, - distances: npt.NDArray[np.float32], - labels: npt.NDArray[np.int64], - centroid_norms: npt.NDArray[np.float32], - ) -> None: ... - def encode_norm(self, norm: float) -> int: ... - def encode_qcint(self, x: float) -> int: ... - def decode_qcint(self, c: int) -> float: ... - -class ResidualQuantizer(AdditiveQuantizer): - # Training configuration - train_type: int # train_type_t enum value - - # Training type constants - Train_default: int - Train_progressive_dim: int - Train_refine_codebook: int - Train_top_beam: int - Skip_codebook_tables: int - - niter_codebook_refine: int # iterations for codebook refinement - max_beam_size: int # beam size for training and encoding - use_beam_LUT: int # use LUT for beam search - approx_topk_mode: int # ApproxTopK_mode_t enum value - - # Clustering parameters - cp: ProgressiveDimClusteringParameters - assign_index_factory: ProgressiveDimIndexFactory - - @overload - def __init__( - self, d: int, nbits: list[int], search_type: int = 0 - ) -> None: ... - @overload - def __init__( - self, d: int, M: int, nbits: int, search_type: int = 0 - ) -> None: ... - @overload - def __init__(self) -> None: ... - def initialize_from( - self, other: ResidualQuantizer, skip_M: int = 0 - ) -> None: ... - @overload - def retrain_AQ_codebook(self, n: int, x: torch.Tensor) -> float: ... - @overload - def retrain_AQ_codebook( - self, n: int, x: npt.NDArray[np.float32] - ) -> float: ... - def refine_beam( - self, - n: int, - beam_size: int, - residuals: npt.NDArray[np.float32], - new_beam_size: int, - new_codes: npt.NDArray[np.int32], - new_residuals: npt.NDArray[np.float32] | None = None, - new_distances: npt.NDArray[np.float32] | None = None, - ) -> None: ... - def refine_beam_LUT( - self, - n: int, - query_norms: npt.NDArray[np.float32], - query_cp: npt.NDArray[np.float32], - new_beam_size: int, - new_codes: npt.NDArray[np.int32], - new_distances: npt.NDArray[np.float32] | None = None, - ) -> None: ... - def memory_per_point(self, beam_size: int = -1) -> int: ... - def train_type_to_str(self, train_type: int) -> str: ... - -class IcmEncoder: - """ICM (Iterated Conditional Modes) encoder for LSQ""" - - def __init__(self, lsq: LocalSearchQuantizer) -> None: ... - -class IcmEncoderFactory: - """Factory class for ICM (Iterated Conditional Modes) encoders in LSQ""" - - def __init__(self) -> None: ... - def get(self, lsq: LocalSearchQuantizer) -> IcmEncoder: ... - -class LocalSearchQuantizer(AdditiveQuantizer): - K: int # number of codes per codebook - - # Training parameters - train_iters: int # iterations in training - encode_ils_iters: int # iterations of local search in encoding - train_ils_iters: int # iterations of local search in training - icm_iters: int # iterations in ICM - - # Algorithm parameters - p: float # temperature factor - lambd: float # regularization factor - - # Processing parameters - chunk_size: int # vectors to encode at a time - random_seed: int # seed for random generator - nperts: int # number of perturbations in each code - - # Encoder configuration - icm_encoder_factory: IcmEncoderFactory | None # lsq::IcmEncoderFactory* - update_codebooks_with_double: bool - - @overload - def __init__( - self, d: int, M: int, nbits: int, search_type: int = 0 - ) -> None: ... - @overload - def __init__(self) -> None: ... - @overload - def update_codebooks( - self, x: torch.Tensor, codes: npt.NDArray[np.int32], n: int - ) -> None: ... - @overload - def update_codebooks( - self, x: npt.NDArray[np.float32], codes: npt.NDArray[np.int32], n: int - ) -> None: ... - def icm_encode( - self, - codes: npt.NDArray[np.int32], - x: npt.NDArray[np.float32], - n: int, - ils_iters: int, - gen: Any, # std::mt19937& - ) -> None: ... - def icm_encode_impl( - self, - codes: npt.NDArray[np.int32], - x: npt.NDArray[np.float32], - unaries: npt.NDArray[np.float32], - gen: Any, # std::mt19937& - n: int, - ils_iters: int, - verbose: bool, - ) -> None: ... - def icm_encode_step( - self, - codes: npt.NDArray[np.int32], - unaries: npt.NDArray[np.float32], - binaries: npt.NDArray[np.float32], - n: int, - n_iters: int, - ) -> None: ... - def perturb_codes( - self, - codes: npt.NDArray[np.int32], - n: int, - gen: Any, # std::mt19937& - ) -> None: ... - def perturb_codebooks( - self, - T: float, - stddev: list[float], - gen: Any, # std::mt19937& - ) -> None: ... - def compute_binary_terms( - self, binaries: npt.NDArray[np.float32] - ) -> None: ... - @overload - def compute_unary_terms( - self, x: torch.Tensor, unaries: npt.NDArray[np.float32], n: int - ) -> None: ... - @overload - def compute_unary_terms( - self, - x: npt.NDArray[np.float32], - unaries: npt.NDArray[np.float32], - n: int, - ) -> None: ... - @overload - def evaluate( - self, - codes: npt.NDArray[np.int32], - x: torch.Tensor, - n: int, - objs: npt.NDArray[np.float32] | None = None, - ) -> float: ... - @overload - def evaluate( - self, - codes: npt.NDArray[np.int32], - x: npt.NDArray[np.float32], - n: int, - objs: npt.NDArray[np.float32] | None = None, - ) -> float: ... - -class RaBitQuantizer(Quantizer): - # Core attributes - centroid: Any # float* - pointer to centroid (not serialized) - metric_type: MetricType # metric type for the quantizer - - def __init__(self, d: int = 0, metric: MetricType = METRIC_L2) -> None: ... - @overload - def compute_codes_core( - self, - x: torch.Tensor, - codes: torch.Tensor, - n: int, - centroid_in: torch.Tensor, - ) -> None: ... - @overload - def compute_codes_core( - self, - x: npt.NDArray[np.float32], - codes: npt.NDArray[np.uint8], - n: int, - centroid_in: npt.NDArray[np.float32], - ) -> None: ... - @overload - def decode_core( - self, - codes: torch.Tensor, - x: torch.Tensor, - n: int, - centroid_in: torch.Tensor, - ) -> None: ... - @overload - def decode_core( - self, - codes: npt.NDArray[np.uint8], - x: npt.NDArray[np.float32], - n: int, - centroid_in: npt.NDArray[np.float32], - ) -> None: ... - def get_distance_computer( - self, - qb: int, - centroid_in: npt.NDArray[np.float32] | None = None, - ) -> Any: ... # FlatCodesDistanceComputer* - -class ProductAdditiveQuantizer(AdditiveQuantizer): - nsplits: int # number of sub-vectors we split a vector into - quantizers: list[AdditiveQuantizer] # sub-additive quantizers - - @overload - def __init__( - self, - d: int, - aqs: list[AdditiveQuantizer], - search_type: int = 0, - ) -> None: ... - @overload - def __init__(self) -> None: ... - def init( - self, - d: int, - aqs: list[AdditiveQuantizer], - search_type: int, - ) -> None: ... - def subquantizer(self, m: int) -> AdditiveQuantizer: ... - @overload - def compute_unpacked_codes( - self, - x: torch.Tensor, - codes: npt.NDArray[np.int32], - n: int, - centroids: torch.Tensor | None = None, - ) -> None: ... - @overload - def compute_unpacked_codes( - self, - x: npt.NDArray[np.float32], - codes: npt.NDArray[np.int32], - n: int, - centroids: npt.NDArray[np.float32] | None = None, - ) -> None: ... - -class ProductLocalSearchQuantizer(ProductAdditiveQuantizer): - @overload - def __init__( - self, - d: int, - nsplits: int, - Msub: int, - nbits: int, - search_type: int = 0, - ) -> None: ... - @overload - def __init__(self) -> None: ... - -class ProductResidualQuantizer(ProductAdditiveQuantizer): - @overload - def __init__( - self, - d: int, - nsplits: int, - Msub: int, - nbits: int, - search_type: int = 0, - ) -> None: ... - @overload - def __init__(self) -> None: ... - -class ScalarQuantizer(Quantizer): - qtype: int - rangestat: int - rangestat_arg: float - d: int - code_size: int - trained: Float32Vector - - def __init__(self, d: int = 0, qtype: int = 0) -> None: ... - - # ScalarQuantizer quantization type constants (as class attributes) - QT_8bit: int - QT_4bit: int - QT_8bit_uniform: int - QT_4bit_uniform: int - QT_fp16: int - QT_8bit_direct: int - QT_6bit: int - QT_bf16: int - QT_8bit_direct_signed: int - QT_0bit: int - QT_1bit_tqmse: int - QT_2bit_tqmse: int - QT_3bit_tqmse: int - QT_4bit_tqmse: int - QT_8bit_tqmse: int - QT_2bit_tq: int - QT_3bit_tq: int - QT_4bit_tq: int - QT_5bit_tq: int - QT_1bit_eden: int - QT_2bit_eden: int - QT_3bit_eden: int - QT_4bit_eden: int - QT_5bit_eden: int - QT_6bit_eden: int - QT_7bit_eden: int - QT_8bit_eden: int - - # RangeStat constants (as class attributes) - RS_minmax: int - RS_meanstd: int - RS_quantiles: int - RS_optim: int - -# LSH index -class IndexLSH(IndexFlatCodes): - nbits: int - bytes_per_vec: int - rrot: RandomRotationMatrix - bytes: UInt8Vector - train_thresholds: bool - sign_bit: float - - def __init__( - self, - d: int, - nbits: int, - rotate_data: bool = True, - train_thresholds: bool = True, - ) -> None: ... - -# PQ index -class IndexPQ(IndexFlatCodes): - pq: ProductQuantizer - codes: UInt8Vector - - def __init__( - self, d: int, M: int, nbits: int, metric: MetricType = METRIC_L2 - ) -> None: ... - -# Scalar Quantizer index -class IndexScalarQuantizer(IndexFlatCodes): - sq: ScalarQuantizer - codes: UInt8Vector - - def __init__( - self, d: int, qtype: int, metric: MetricType = METRIC_L2 - ) -> None: ... - -# IVF classes -class InvertedLists: - nlist: int - code_size: int - - def __init__(self, nlist: int, code_size: int) -> None: ... - def list_size(self, list_no: int) -> int: ... - def get_codes(self, list_no: int) -> npt.NDArray[np.uint8]: ... - def get_ids(self, list_no: int) -> npt.NDArray[np.int64]: ... - def add_entries( - self, - list_no: int, - n_entry: int, - ids: npt.NDArray[np.int64], - codes: npt.NDArray[np.uint8], - ) -> int: ... - def update_entries( - self, - list_no: int, - offset: int, - n_entry: int, - ids: npt.NDArray[np.int64], - codes: npt.NDArray[np.uint8], - ) -> None: ... - def resize(self, list_no: int, new_size: int) -> None: ... - def merge_from(self, other: InvertedLists, add_id: int = 0) -> None: ... - def imbalance_factor(self) -> float: ... - -class ArrayInvertedLists(InvertedLists): - def __init__(self, nlist: int, code_size: int) -> None: ... - -# InvertedListsIOHook base class for I/O operations -class InvertedListsIOHook: - key: str # fourcc key for identification - classname: str # typeid.name for the associated class - - def __init__(self, key: str, classname: str) -> None: ... - def write(self, ils: InvertedLists, f: IOWriter) -> None: ... - def read(self, f: IOReader, io_flags: int) -> InvertedLists: ... - def read_ArrayInvertedLists( - self, - f: IOReader, - io_flags: int, - nlist: int, - code_size: int, - sizes: list[int], - ) -> InvertedLists: ... - - # Static methods for managing callbacks - @staticmethod - def add_callback(cb: InvertedListsIOHook) -> None: ... - @staticmethod - def print_callbacks() -> None: ... - @staticmethod - def lookup(h: int) -> InvertedListsIOHook: ... - @staticmethod - def lookup_classname(classname: str) -> InvertedListsIOHook: ... - -# OnDiskOneList structure -class OnDiskOneList: - size: int # size of inverted list (entries) - capacity: int # allocated size (entries) - offset: int # offset in buffer (bytes) - - def __init__(self) -> None: ... - -# OnDiskInvertedLists Slot structure -class OnDiskSlot: - offset: int # offset in bytes - capacity: int # capacity in bytes - - def __init__(self, offset: int = 0, capacity: int = 0) -> None: ... - -# OnDiskInvertedLists class for memory-mapped inverted lists -class OnDiskInvertedLists(InvertedLists): - # Core attributes - lists: list[OnDiskOneList] # vector of OnDiskOneList - slots: Any # std::list - available slots sorted by size - filename: str # filename of the mmapped file - totsize: int # total size of mmapped region - ptr: Any # uint8_t* - mmap base pointer - read_only: bool # are inverted lists mapped read-only - - # Slot management (private in C++ but accessible in Python) - locks: Any # LockLevels* - thread synchronization - pf: Any # OngoingPrefetch* - prefetch management - prefetch_nthread: int # number of prefetch threads - - @overload - def __init__(self, nlist: int, code_size: int, filename: str) -> None: ... - @overload - def __init__(self) -> None: ... # empty constructor for I/O functions - - # Override base InvertedLists methods - def list_size(self, list_no: int) -> int: ... - def get_codes(self, list_no: int) -> npt.NDArray[np.uint8]: ... - def get_ids(self, list_no: int) -> npt.NDArray[np.int64]: ... - def add_entries( - self, - list_no: int, - n_entry: int, - ids: npt.NDArray[np.int64], - codes: npt.NDArray[np.uint8], - ) -> int: ... - def update_entries( - self, - list_no: int, - offset: int, - n_entry: int, - ids: npt.NDArray[np.int64], - codes: npt.NDArray[np.uint8], - ) -> None: ... - def resize(self, list_no: int, new_size: int) -> None: ... - - # OnDiskInvertedLists specific methods - def merge_from_multiple( - self, - ils: Any, # const InvertedLists** (C pointer array) - n_il: int, # number of InvertedLists - shift_ids: bool = False, - verbose: bool = False, - ) -> int: ... - def merge_from_1(self, il: InvertedLists, verbose: bool = False) -> int: ... - def crop_invlists(self, l0: int, l1: int) -> None: ... - def prefetch_lists( - self, list_nos: npt.NDArray[np.int64], nlist: int - ) -> None: ... - - # Memory management methods - def do_mmap(self) -> None: ... - def update_totsize(self, new_totsize: int) -> None: ... - def resize_locked(self, list_no: int, new_size: int) -> None: ... - def allocate_slot(self, capacity: int) -> int: ... - def free_slot(self, offset: int, capacity: int) -> None: ... - def set_all_lists_sizes(self, sizes: npt.NDArray[np.int64]) -> None: ... - -# OnDiskInvertedListsIOHook for I/O operations -class OnDiskInvertedListsIOHook(InvertedListsIOHook): - def __init__(self) -> None: ... - def write(self, ils: InvertedLists, f: IOWriter) -> None: ... - def read(self, f: IOReader, io_flags: int) -> InvertedLists: ... - def read_ArrayInvertedLists( - self, - f: IOReader, - io_flags: int, - nlist: int, - code_size: int, - sizes: list[int], - ) -> InvertedLists: ... - -class IndexIVF(Index): - # Core attributes from C++ struct - invlists: InvertedLists | None - own_invlists: bool - code_size: int - parallel_mode: int - PARALLEL_MODE_NO_HEAP_INIT: int = 1024 - direct_map: Any # DirectMap type - by_residual: bool - - # Additional Python wrapper attributes (legacy compatibility) - quantizer: Index - nprobe: int - nlist: int - quantizer_trains_alone: ( - str # char in C++ -> single character string in Python - ) - own_fields: bool - cp: ClusteringParameters - clustering_index: Index - max_codes: int - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - metric: MetricType = METRIC_L2, - ) -> None: ... - - # Core IndexIVF methods from C++ - def add_core( - self, - n: int, - x: npt.NDArray[np.float32], - xids: npt.NDArray[np.int64] | None = None, - precomputed_idx: npt.NDArray[np.int64] | None = None, - inverted_list_context: Any | None = None, - ) -> None: ... - def search_preassigned( - self, - n: int, - x: npt.NDArray[np.float32], - k: int, - assign: npt.NDArray[np.int64], - centroid_dis: npt.NDArray[np.float32], - distances: npt.NDArray[np.float32], - labels: npt.NDArray[np.int64], - store_pairs: bool, - params: IVFSearchParameters | None = None, - stats: IndexIVFStats | None = None, - ) -> None: ... - def range_search_preassigned( - self, - nx: int, - x: npt.NDArray[np.float32], - radius: float, - keys: npt.NDArray[np.int64], - coarse_dis: npt.NDArray[np.float32], - result: RangeSearchResult, - store_pairs: bool = False, - params: Any | None = None, # IVFSearchParameters - stats: Any | None = None, # IndexIVFStats - ) -> None: ... - def set_beam_factor(self, beam_factor: float) -> None: ... - def encode_vectors( - self, - n: int, - x: npt.NDArray[np.float32], - list_nos: npt.NDArray[np.int64], - codes: npt.NDArray[np.uint8], - include_listno: bool = False, - ) -> None: ... - def decode_vectors( - self, - n: int, - codes: npt.NDArray[np.uint8], - list_nos: npt.NDArray[np.int64], - x: npt.NDArray[np.float32], - ) -> None: ... - def train_encoder( - self, - n: int, - x: npt.NDArray[np.float32], - assign: npt.NDArray[np.int64] | None = None, - ) -> None: ... - def train_encoder_num_vectors(self) -> int: ... - def get_InvertedListScanner( - self, - store_pairs: bool = False, - sel: IDSelector | None = None, - params: Any | None = None, # IVFSearchParameters - ) -> Any: ... # InvertedListScanner* - def update_vectors( - self, - nv: int, - idx: npt.NDArray[np.int64], - v: npt.NDArray[np.float32], - ) -> None: ... - def reconstruct_from_offset( - self, list_no: int, offset: int, recons: npt.NDArray[np.float32] - ) -> None: ... - def get_CodePacker(self) -> Any: ... # CodePacker* - def copy_subset_to( - self, - other: IndexIVF, - subset_type: int, # InvertedLists::subset_type_t - a1: int, - a2: int, - ) -> None: ... - def get_list_size(self, list_no: int) -> int: ... - def check_ids_sorted(self) -> bool: ... - def make_direct_map(self, new_maintain_direct_map: bool = True) -> None: ... - def set_direct_map_type(self, type: int) -> None: ... # DirectMap::Type - def replace_invlists( - self, invlists: InvertedLists, own: bool = False - ) -> None: ... - @overload - def search_preassigned( - self, - x: torch.Tensor, - k: int, - Iq: torch.Tensor, - Dq: torch.Tensor | None, - *, - params: SearchParameters | None = None, - D: torch.Tensor | None = None, - I: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: ... - @overload - def search_preassigned( - self, - x: npt.NDArray[np.float32], - k: int, - Iq: npt.NDArray[np.int64], - Dq: npt.NDArray[np.float32] | None, - *, - params: SearchParameters | None = None, - D: npt.NDArray[np.float32] | None = None, - I: npt.NDArray[np.int64] | None = None, - ) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... - -class IndexIVFFlat(IndexIVF): - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - metric: MetricType = METRIC_L2, - ) -> None: ... - -class IndexIVFFlatPanorama(IndexIVFFlat): - """Panorama adaptation of IndexIVFFlat following https://arxiv.org/abs/2510.00566""" - - n_levels: int - batch_size: int - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - n_levels: int, - metric: MetricType = METRIC_L2, - own_invlists: bool = True, - batch_size: int = ..., - ) -> None: ... - -class IndexIVFPQ(IndexIVF): - pq: ProductQuantizer - code_size: int - by_residual: bool - use_precomputed_table: int - polysemous_ht: int - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - ) -> None: ... - def precompute_table(self) -> None: ... - -class IndexIVFScalarQuantizer(IndexIVF): - sq: ScalarQuantizer - code_size: int - by_residual: bool - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - qtype: int, - metric: MetricType = METRIC_L2, - encode_residual: bool = True, - ) -> None: ... - -# IVF FastScan base class -class IndexIVFFastScan(IndexIVF): - M: int - nbits: int - ksub: int - M2: int - bbs: int - qbs: int - implem: int - skip: int - orig_invlists: InvertedLists | None - - def __init__( - self, - quantizer: Index | None = None, - d: int = 0, - nlist: int = 0, - code_size: int = 0, - metric: MetricType = METRIC_L2, - own_invlists: bool = True, - ) -> None: ... - def init_fastscan( - self, - fine_quantizer: Quantizer, - M: int, - nbits: int, - nlist: int, - metric: MetricType, - bbs: int = 32, - own_invlists: bool = True, - ) -> None: ... - def init_code_packer(self) -> None: ... - def get_CodePacker(self) -> Any: ... # CodePacker* - @overload - def permute_entries(self, perm: torch.Tensor) -> None: ... - @overload - def permute_entries(self, perm: npt.NDArray[np.int64]) -> None: ... - def reconstruct_orig_invlists(self) -> None: ... - -class IndexIVFPQFastScan(IndexIVFFastScan): - pq: ProductQuantizer - use_precomputed_table: int - precomputed_table: AlignedTableFloat32 - - def __init__( - self, - quantizer: Index | None = None, - d: int = 0, - nlist: int = 0, - M: int = 0, - nbits: int = 0, - metric: MetricType = METRIC_L2, - bbs: int = 32, - own_invlists: bool = True, - ) -> None: ... - def train_encoder_num_vectors(self) -> int: ... - def precompute_table(self) -> None: ... - def lookup_table_is_3d(self) -> bool: ... - -# HNSW classes -class HNSWStats: - n1: int - n2: int - n3: int - ndis: int - nreorder: int - -class HNSW: - max_level: int - entry_point: int - efConstruction: int - efSearch: int - hnsw_stats: HNSWStats - assign_probas: Float32Vector - cum_nneighbor_per_level: Int32Vector - levels: Int32Vector - graph: Int32Vector - - def __init__(self, M: int = 32) -> None: ... - def reset(self) -> None: ... - -class IndexHNSW(Index): - hnsw: HNSW - storage: Index - own_fields: bool - reconstruct_from_neighbors: Callable[ - [int, npt.NDArray[np.float32], npt.NDArray[np.float32]], None - ] - - def __init__( - self, d: int, M: int, metric: MetricType = METRIC_L2 - ) -> None: ... - -class IndexHNSWFlat(IndexHNSW): - def __init__( - self, d: int, M: int, metric: MetricType = METRIC_L2 - ) -> None: ... - -class IndexHNSWFlatPanorama(IndexHNSWFlat): - """Panorama implementation of IndexHNSWFlat. - Uses progressive distance refinement to prune candidates early. - """ - - num_panorama_levels: int - - def __init__( - self, - d: int, - M: int, - num_panorama_levels: int, - metric: MetricType = METRIC_L2, - ) -> None: ... - -class IndexHNSWPQ(IndexHNSW): - pq: ProductQuantizer - - def __init__( - self, - d: int, - pq: ProductQuantizer, - M: int, - metric: MetricType = METRIC_L2, - ) -> None: ... - -class IndexHNSWSQ(IndexHNSW): - sq: ScalarQuantizer - - def __init__( - self, - d: int, - sq: ScalarQuantizer, - M: int, - metric: MetricType = METRIC_L2, - ) -> None: ... - -class IndexHNSW2Level(IndexHNSW): - q1: Any # Level1Quantizer - pq: ProductQuantizer # from Index2Layer - - def __init__( - self, - d: int, - q1: Quantizer, - nlist: int, - M: int, - cu: int, - metric: MetricType = METRIC_L2, - ) -> None: ... - -# Binary Index classes -class IndexBinary: - d: int - code_size: int - ntotal: int - verbose: bool - is_trained: bool - - def __init__(self, d: int) -> None: ... - @overload - def add(self, x: torch.Tensor) -> None: ... - @overload - def add(self, x: npt.NDArray[np.uint8]) -> None: ... - @overload - def add_with_ids(self, x: torch.Tensor, ids: torch.Tensor) -> None: ... - @overload - def add_with_ids( - self, x: npt.NDArray[np.uint8], ids: npt.NDArray[np.int64] - ) -> None: ... - @overload - def train(self, x: torch.Tensor) -> None: ... - @overload - def train(self, x: npt.NDArray[np.uint8]) -> None: ... - @overload - def search( - self, - x: torch.Tensor, - k: int, - params: SearchParameters | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: ... - @overload - def search( - self, - x: npt.NDArray[np.uint8], - k: int, - params: SearchParameters | None = None, - ) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.int64]]: ... - @overload - def range_search( - self, - x: torch.Tensor, - thresh: int, - params: SearchParameters | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ... - @overload - def range_search( - self, - x: npt.NDArray[np.uint8], - thresh: int, - params: SearchParameters | None = None, - ) -> tuple[ - npt.NDArray[np.int64], npt.NDArray[np.int32], npt.NDArray[np.int64] - ]: ... - @overload - def reconstruct(self, key: int) -> torch.Tensor: ... - @overload - def reconstruct(self, key: int) -> npt.NDArray[np.uint8]: ... - # Without an `x` buffer the return type is not statically knowable: importing - # faiss.contrib.torch_utils replaces this method with one that allocates a - # torch.Tensor, while the default numpy implementation allocates an ndarray. - @overload - def reconstruct_n( - self, n0: int = ..., ni: int = ..., x: None = ... - ) -> Any: ... - @overload - def reconstruct_n( - self, n0: int = ..., ni: int = ..., x: torch.Tensor = ... - ) -> torch.Tensor: ... - @overload - def reconstruct_n( - self, n0: int = ..., ni: int = ..., x: npt.NDArray[np.uint8] = ... - ) -> npt.NDArray[np.uint8]: ... - def reset(self) -> None: ... - @overload - def remove_ids(self, x: torch.Tensor) -> int: ... - @overload - def remove_ids(self, x: npt.NDArray[np.int64]) -> int: ... - @overload - def remove_ids(self, sel: IDSelector) -> int: ... - @overload - def assign(self, x: torch.Tensor, k: int = 1) -> torch.Tensor: ... - @overload - def assign( - self, x: npt.NDArray[np.uint8], k: int = 1 - ) -> npt.NDArray[np.int64]: ... - @overload - def search_preassigned( - self, - x: torch.Tensor, - k: int, - Iq: torch.Tensor, - Dq: torch.Tensor | None, - ) -> tuple[torch.Tensor, torch.Tensor]: ... - @overload - def search_preassigned( - self, - x: npt.NDArray[np.uint8], - k: int, - Iq: npt.NDArray[np.int64], - Dq: npt.NDArray[np.int32] | torch.Tensor | None, - ) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.int64]]: ... - @overload - def range_search_preassigned( - self, - x: torch.Tensor, - thresh: int, - Iq: torch.Tensor, - Dq: torch.Tensor | None, - params: SearchParameters | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ... - @overload - def range_search_preassigned( - self, - x: npt.NDArray[np.uint8], - thresh: int, - Iq: npt.NDArray[np.int64], - Dq: npt.NDArray[np.int32] | None, - params: SearchParameters | None = None, - ) -> tuple[ - npt.NDArray[np.int64], npt.NDArray[np.int32], npt.NDArray[np.int64] - ]: ... - -class IndexBinaryFlat(IndexBinary): - def __init__(self, d: int) -> None: ... - -class IndexBinaryIVF(IndexBinary): - nlist: int - nprobe: int - quantizer: IndexBinary - invlists: InvertedLists - own_fields: bool - cp: Any # ClusteringParameters - clustering_index: Index | None # to override index used during clustering - max_codes: int - use_heap: bool - per_invlist_search: bool - direct_map: Any # DirectMap type - - def __init__(self, quantizer: IndexBinary, d: int, nlist: int) -> None: ... - @overload - def search_preassigned( - self, - x: torch.Tensor, - k: int, - Iq: torch.Tensor, - Dq: torch.Tensor | None, - *, - params: SearchParameters | None = None, - D: torch.Tensor | None = None, - I: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: ... - @overload - def search_preassigned( - self, - x: npt.NDArray[np.uint8], - k: int, - Iq: npt.NDArray[np.int64], - Dq: npt.NDArray[np.int32] | None, - *, - params: SearchParameters | None = None, - D: npt.NDArray[np.int32] | None = None, - I: npt.NDArray[np.int64] | None = None, - ) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.int64]]: ... - @overload - def range_search_preassigned( - self, - x: torch.Tensor, - thresh: int, - Iq: torch.Tensor, - Dq: torch.Tensor | None = None, - *, - params: SearchParameters | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ... - @overload - def range_search_preassigned( - self, - x: npt.NDArray[np.uint8], - thresh: int, - Iq: npt.NDArray[np.int64], - Dq: npt.NDArray[np.int32] | None = None, - *, - params: SearchParameters | None = None, - ) -> tuple[ - npt.NDArray[np.int64], npt.NDArray[np.int32], npt.NDArray[np.int64] - ]: ... - def reconstruct_from_offset( - self, - list_no: int, - offset: int, - key: npt.NDArray[np.uint8] | None = None, - ) -> npt.NDArray[np.uint8]: ... - def make_direct_map(self, new_maintain_direct_map: bool = True) -> None: ... - def set_direct_map_type(self, type: int) -> None: ... - def replace_invlists( - self, invlists: InvertedLists, own: bool = False - ) -> None: ... - def get_list_size(self, list_no: int) -> int: ... - def merge_from(self, other: IndexBinaryIVF, add_id: int = 0) -> None: ... - def check_compatible_for_merge(self, other: IndexBinaryIVF) -> None: ... - -class IndexBinaryFromFloat(IndexBinary): - index: Index - - def __init__(self, index: Index) -> None: ... - -class IndexBinaryHNSW(IndexBinary): - hnsw: Any - storage: IndexBinary - own_fields: bool - - def __init__(self, d: int, M: int) -> None: ... - -class IndexBinaryHash(IndexBinary): - nflip: int - - def __init__(self, d: int, nflip: int) -> None: ... - -class IndexBinaryMultiHash(IndexBinaryHash): - maps: UInt8VectorVector - - def __init__(self, d: int, nhash: int, nflip: int) -> None: ... - -# Index refinement classes -class IndexRefineSearchParameters(SearchParameters): - k_factor: float - base_index_params: SearchParameters | None - -class IndexRefine(Index): - base_index: Index - refine_index: Index - own_fields: bool - own_refine_index: bool - k_factor: float - - def __init__( - self, base_index: Index | None = None, refine_index: Index | None = None - ) -> None: ... - def reset(self) -> None: ... - def sa_code_size(self) -> int: ... - -class IndexRefineFlat(IndexRefine): - def __init__( - self, - base_index: Index | None = None, - xb: npt.NDArray[np.float32] | None = None, - ) -> None: ... - -class IndexRefinePanorama(IndexRefine): - """Version where the search calls search_subset, allowing for Panorama refinement.""" - - def __init__( - self, - base_index: Index | None = None, - refine_index: Index | None = None, - ) -> None: ... - -# FastScan base class -class IndexFastScan(Index): - M: int - nbits: int - ksub: int - M2: int - bbs: int - qbs: int - implem: int - skip: int - quantizer: Quantizer - codes: AlignedTableUint8 - - def __init__(self) -> None: ... - -# FastScan PQ index -class IndexPQFastScan(IndexFastScan): - pq: ProductQuantizer - - def __init__( - self, - d: int = 0, - M: int = 0, - nbits: int = 0, - metric: MetricType = METRIC_L2, - bbs: int = 32, - ) -> None: ... - -# FastScan additive quantizer base classes -class IndexAdditiveQuantizerFastScan(IndexFastScan): - aq: AdditiveQuantizer - rescale_norm: bool - norm_scale: int - max_train_points: int - - def __init__(self) -> None: ... - -class IndexResidualQuantizerFastScan(IndexAdditiveQuantizerFastScan): - rq: ResidualQuantizer - - @overload - def __init__( - self, - d: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - bbs: int = 32, - ) -> None: ... - @overload - def __init__(self) -> None: ... - -class IndexLocalSearchQuantizerFastScan(IndexAdditiveQuantizerFastScan): - lsq: LocalSearchQuantizer - - @overload - def __init__( - self, - d: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - bbs: int = 32, - ) -> None: ... - @overload - def __init__(self) -> None: ... - -# Index factory and related functions -def downcast_index(index: Index) -> Index: ... -def downcast_IndexBinary(index: IndexBinary) -> IndexBinary: ... -def downcast_VectorTransform(vt: VectorTransform) -> VectorTransform: ... -def downcast_InvertedLists(il: InvertedLists) -> InvertedLists: ... - -# IO Writer/Reader classes - for serialization -class IOWriter: - def __init__(self) -> None: ... - -class IOReader: - def __init__(self) -> None: ... - -class VectorIOWriter(IOWriter): - data: UInt8Vector - def __init__(self) -> None: ... - -class VectorIOReader(IOReader): - data: UInt8Vector - def __init__(self) -> None: ... - -class BufferedIOWriter(IOWriter): - def __init__(self, writer: IOWriter) -> None: ... - -class BufferedIOReader(IOReader): - def __init__(self, reader: IOReader) -> None: ... - -class FileIOWriter(IOWriter): - def __init__(self, fname: str) -> None: ... - -class FileIOReader(IOReader): - def __init__(self, fname: str) -> None: ... - -# Index serialization functions (THE MAIN ONES FROM __init__.py) -def serialize_index( - index: Index, io_flags: int = 0 -) -> npt.NDArray[np.uint8]: ... -def deserialize_index( - data: npt.NDArray[np.uint8], io_flags: int = 0 -) -> Index: ... -def serialize_index_binary(index: IndexBinary) -> npt.NDArray[np.uint8]: ... -def deserialize_index_binary(data: npt.NDArray[np.uint8]) -> IndexBinary: ... - -# Index file I/O functions with overloads -@overload -def write_index(index: Index, fname: str) -> None: ... -@overload -def write_index(index: Index, writer: IOWriter, io_flags: int = 0) -> None: ... -@overload -def read_index(fname: str, io_flags: int = 0) -> Index: ... -@overload -def read_index(reader: IOReader, io_flags: int = 0) -> Index: ... -@overload -def write_index_binary(index: IndexBinary, fname: str) -> None: ... -@overload -def write_index_binary(index: IndexBinary, writer: IOWriter) -> None: ... -@overload -def read_index_binary(fname: str, io_flags: int = 0) -> IndexBinary: ... -@overload -def read_index_binary(reader: IOReader) -> IndexBinary: ... -def write_VectorTransform(vt: VectorTransform, fname: str) -> None: ... -def read_VectorTransform(fname: str) -> VectorTransform: ... - -# InvertedLists I/O functions -def write_InvertedLists(invlists: InvertedLists, writer: IOWriter) -> None: ... -def read_InvertedLists( - reader: IOReader, io_flags: int = 0 -) -> InvertedLists: ... - -# Deserialization safety limits -def get_deserialization_loop_limit() -> int: ... -def set_deserialization_loop_limit(value: int) -> None: ... -def get_deserialization_vector_byte_limit() -> int: ... -def set_deserialization_vector_byte_limit(value: int) -> None: ... -def get_deserialization_lattice_r2_limit() -> int: ... -def set_deserialization_lattice_r2_limit(value: int) -> None: ... - -# Search with parameters functions -@overload -def search_with_parameters( - index: Index, - x: torch.Tensor, - k: int, - params: SearchParameters | None = None, - output_stats: bool = False, -) -> ( - tuple[torch.Tensor, torch.Tensor] - | tuple[torch.Tensor, torch.Tensor, dict[str, Any]] -): ... -@overload -def search_with_parameters( - index: Index, - x: npt.NDArray[np.float32], - k: int, - params: SearchParameters | None = None, - output_stats: bool = False, -) -> ( - tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]] - | tuple[npt.NDArray[np.float32], npt.NDArray[np.int64], dict[str, Any]] -): ... -@overload -def range_search_with_parameters( - index: Index, - x: torch.Tensor, - radius: float, - params: SearchParameters | None = None, - output_stats: bool = False, -) -> ( - tuple[torch.Tensor, torch.Tensor, torch.Tensor] - | tuple[torch.Tensor, torch.Tensor, torch.Tensor, dict[str, Any]] -): ... -@overload -def range_search_with_parameters( - index: Index, - x: npt.NDArray[np.float32], - radius: float, - params: SearchParameters | None = None, - output_stats: bool = False, -) -> ( - tuple[npt.NDArray[np.int64], npt.NDArray[np.float32], npt.NDArray[np.int64]] - | tuple[ - npt.NDArray[np.int64], - npt.NDArray[np.float32], - npt.NDArray[np.int64], - dict[str, Any], - ] -): ... - -# IVF Search Parameters -class IVFSearchParameters(SearchParameters): - nprobe: int - max_codes: int - max_lists_num: int - ensure_topk_full: bool - max_empty_result_buckets: int - quantizer_params: SearchParameters | None - inverted_list_context: Any - sel: IDSelector | None - def __init__(self) -> None: ... - -class IVFSQTurboQSearchParameters(IVFSearchParameters): - qb: int - int_qjl: bool - - def __init__(self) -> None: ... - -# IVF Statistics tracking -class IndexIVFStats: - nq: int # number of queries run - nlist: int # number of inverted lists scanned - ndis: int # number of distances computed - nheap_updates: int # number of times the heap was updated - quantization_time: float # time spent quantizing vectors (in ms) - search_time: float # time spent searching lists (in ms) - - def __init__(self) -> None: ... - def reset(self) -> None: ... - def add(self, other: IndexIVFStats) -> None: ... - -# Legacy aliases and remapped classes -IndexProxy = IndexReplicas -ConcatenatedInvertedLists = HStackInvertedLists -IndexResidual = IndexResidualQuantizer -SearchParametersIVF = IVFSearchParameters - -# TimeoutGuard class for managing timeouts -class TimeoutGuard: - timeout: float - def __init__(self, timeout_in_seconds: float) -> None: ... - def __enter__(self) -> None: ... - def __exit__( - self, exc_type: Any, exc_value: Any, traceback: Any - ) -> None: ... - -# Callback classes for timeout handling -class TimeoutCallback: - @staticmethod - def reset(timeout: float) -> None: ... - -class PythonInterruptCallback: - @staticmethod - def reset() -> None: ... - -# Array conversion utilities -def vector_to_array(v: Any) -> npt.NDArray[Any]: ... -def vector_float_to_array(v: Float32Vector) -> npt.NDArray[np.float32]: ... -@overload -def copy_array_to_vector(a: torch.Tensor, v: Any) -> None: ... -@overload -def copy_array_to_vector(a: npt.NDArray[Any], v: Any) -> None: ... -@overload -def copy_array_to_AlignedTable(a: torch.Tensor, v: Any) -> None: ... -@overload -def copy_array_to_AlignedTable(a: npt.NDArray[Any], v: Any) -> None: ... - -# Pointer conversion utilities -def swig_ptr(a: npt.NDArray[Any]) -> Any: ... -def rev_swig_ptr(ptr: Any, n: int) -> npt.NDArray[Any]: ... -def cast_integer_to_float_ptr(x: int) -> Any: ... -def cast_integer_to_idx_t_ptr(x: int) -> Any: ... -def cast_integer_to_int_ptr(x: int) -> Any: ... -def cast_integer_to_void_ptr(x: int) -> Any: ... - -# Additional vector types not covered yet -class UInt8VectorVector: - def __init__(self) -> None: ... - def size(self) -> int: ... - def at(self, n: int) -> UInt8Vector: ... - def push_back(self, v: UInt8Vector) -> None: ... - def resize(self, n: int) -> None: ... - -class ParameterRangeVector: - def __init__(self) -> None: ... - def size(self) -> int: ... - def at(self, n: int) -> ParameterRange: ... - def push_back(self, v: ParameterRange) -> None: ... - def resize(self, n: int) -> None: ... - -class OperatingPointVector: - def __init__(self) -> None: ... - def size(self) -> int: ... - def at(self, n: int) -> OperatingPoint: ... - def push_back(self, v: OperatingPoint) -> None: ... - def resize(self, n: int) -> None: ... - -# Missing index classes from loader imports -class IndexShards(Index): - own_fields: bool - threaded: bool - successive_ids: bool - - def __init__(self, d: int, threaded: bool = False) -> None: ... - def add_shard(self, index: Index) -> None: ... - def remove_shard(self, index: Index) -> None: ... - -class IndexReplicas(Index): - own_fields: bool - - @overload - def __init__(self, threaded: bool = True) -> None: ... - @overload - def __init__(self, d: int, threaded: bool = True) -> None: ... - def addIndex(self, index: Index) -> None: ... - def removeIndex(self, index: Index) -> None: ... - -class IndexBinaryShards(IndexBinary): - own_fields: bool - threaded: bool - successive_ids: bool - - def __init__(self, d: int, threaded: bool = False) -> None: ... - def add_shard(self, index: IndexBinary) -> None: ... - def remove_shard(self, index: IndexBinary) -> None: ... - -class IndexBinaryReplicas(IndexBinary): - own_fields: bool - - def __init__(self, d: int) -> None: ... - def addIndex(self, index: IndexBinary) -> None: ... - def removeIndex(self, index: IndexBinary) -> None: ... - -class HStackInvertedLists(InvertedLists): - def __init__(self, nil: int, invlists: list[InvertedLists]) -> None: ... - -# Additional classes mentioned in __init__.py -# Clustering parameters base class -class ClusteringParameters: - niter: int - nredo: int - verbose: bool - spherical: bool - int_centroids: bool - update_index: bool - frozen_centroids: bool - min_points_per_centroid: int - max_points_per_centroid: int - seed: int - decode_block_size: int - check_input_data_for_NaNs: bool - use_faster_subsampling: bool - init_method: ClusteringInitMethod - afkmc2_chain_length: int # chain length for AFK-MC² initialization - early_stop_threshold: float # early stop threshold [0, 1] - - def __init__(self) -> None: ... - -class ClusteringIterationStats: - obj: float - time: float - time_search: float - imbalance_factor: float - nsplit: int - - def __init__(self) -> None: ... - -# K-means clustering class -class Clustering(ClusteringParameters): - d: int - k: int - centroids: Float32Vector - iteration_stats: list[ClusteringIterationStats] - - @overload - def __init__(self, d: int, k: int) -> None: ... - @overload - def __init__(self, d: int, k: int, cp: ClusteringParameters) -> None: ... - @overload - def train( - self, - x: torch.Tensor, - index: Index, - x_weights: torch.Tensor | None = None, - ) -> None: ... - @overload - def train( - self, - x: npt.NDArray[np.float32], - index: Index, - x_weights: npt.NDArray[np.float32] | None = None, - ) -> None: ... - @overload - def train_encoded( - self, - x: torch.Tensor, - codec: Index, - index: Index, - weights: torch.Tensor | None = None, - ) -> None: ... - @overload - def train_encoded( - self, - x: npt.NDArray[np.uint8], - codec: Index, - index: Index, - weights: npt.NDArray[np.float32] | None = None, - ) -> None: ... - def post_process_centroids(self) -> None: ... - -class Clustering1D(Clustering): - @overload - def __init__(self, k: int) -> None: ... - @overload - def __init__(self, k: int, cp: ClusteringParameters) -> None: ... - @overload - def train_exact(self, x: torch.Tensor) -> None: ... - @overload - def train_exact(self, x: npt.NDArray[np.float32]) -> None: ... - -class ProgressiveDimClusteringParameters(ClusteringParameters): - progressive_dim_steps: int # number of incremental steps - apply_pca: bool # apply PCA on input - - def __init__(self) -> None: ... - -class ProgressiveDimIndexFactory: - """Generates an index suitable for clustering when called""" - - def __call__(self, dim: int) -> Index: ... - # Note: ownership transferred to caller - -class ProgressiveDimClustering(ProgressiveDimClusteringParameters): - """K-means clustering with progressive dimensions used - - The clustering first happens in dim 1, then with exponentially increasing - dimension until d (I steps). This is typically applied after a PCA - transformation (optional). Reference: - - "Improved Residual Vector Quantization for High-dimensional Approximate - Nearest Neighbor Search" - - Shicong Liu, Hongtao Lu, Junru Shao, AAAI'15 - - https://arxiv.org/abs/1509.05195 - """ - - d: int # dimension of the vectors - k: int # nb of centroids - centroids: Float32Vector # centroids (k * d) - iteration_stats: list[ - ClusteringIterationStats - ] # stats at every iteration of clustering - - @overload - def __init__(self, d: int, k: int) -> None: ... - @overload - def __init__( - self, d: int, k: int, cp: ProgressiveDimClusteringParameters - ) -> None: ... - @overload - def train( - self, x: torch.Tensor, factory: ProgressiveDimIndexFactory - ) -> None: ... - @overload - def train( - self, x: npt.NDArray[np.float32], factory: ProgressiveDimIndexFactory - ) -> None: ... - -# Standalone k-means clustering function -@overload -def kmeans_clustering( - d: int, - n: int, - k: int, - x: torch.Tensor, - centroids: torch.Tensor | None = None, -) -> float: ... -@overload -def kmeans_clustering( - d: int, - n: int, - k: int, - x: npt.NDArray[np.float32], - centroids: npt.NDArray[np.float32] | None = None, -) -> float: ... - -class MatrixStats: - comments: str - n: int - d: int - n_collision: int - hash_value: int - - @overload - def __init__(self, x: torch.Tensor) -> None: ... - @overload - def __init__(self, x: npt.NDArray[np.float32]) -> None: ... - def reset(self) -> None: ... - -# AutoTune related classes (from AutoTune.h) -class AutoTuneCriterion: - nq: int - nnn: int - gt_nnn: int - gt_D: Float32Vector - gt_I: Int64Vector - - def __init__(self, nq: int, nnn: int) -> None: ... - @overload - def set_groundtruth( - self, - gt_nnn: int, - gt_D_in: torch.Tensor, - gt_I_in: torch.Tensor, - ) -> None: ... - @overload - def set_groundtruth( - self, - gt_nnn: int, - gt_D_in: npt.NDArray[np.float32], - gt_I_in: npt.NDArray[np.int64], - ) -> None: ... - @overload - def evaluate(self, D: torch.Tensor, I: torch.Tensor) -> float: ... - @overload - def evaluate( - self, D: npt.NDArray[np.float32], I: npt.NDArray[np.int64] - ) -> float: ... - -class OneRecallAtRCriterion(AutoTuneCriterion): - R: int - - def __init__(self, nq: int, R: int) -> None: ... - -class IntersectionCriterion(AutoTuneCriterion): - R: int - - def __init__(self, nq: int, R: int) -> None: ... - -# Neural Network classes from NeuralNet.h -class Tensor2D: - shape: tuple[int, int] - v: Float32Vector - - @overload - def __init__( - self, n0: int, n1: int, data: npt.NDArray[np.float32] | None = None - ) -> None: ... - @overload - def __init__(self, array: npt.NDArray[np.float32]) -> None: ... - def numel(self) -> int: ... - def data(self) -> npt.NDArray[np.float32]: ... - def numpy(self) -> npt.NDArray[np.float32]: ... - def column(self, j: int) -> Tensor2D: ... - def __iadd__(self, other: Tensor2D) -> Tensor2D: ... - -class Int32Tensor2D: - shape: tuple[int, int] - v: Int32Vector - - @overload - def __init__( - self, n0: int, n1: int, data: npt.NDArray[np.int32] | None = None - ) -> None: ... - @overload - def __init__(self, array: npt.NDArray[np.int32]) -> None: ... - def numel(self) -> int: ... - def data(self) -> npt.NDArray[np.int32]: ... - def numpy(self) -> npt.NDArray[np.int32]: ... - def column(self, j: int) -> Int32Tensor2D: ... - def __iadd__(self, other: Int32Tensor2D) -> Int32Tensor2D: ... - -# Neural network layer classes from NeuralNet.h -class Linear: - in_features: int - out_features: int - weight: Float32Vector - bias: Float32Vector - - @overload - def __init__( - self, in_features: int, out_features: int, bias: bool = True - ) -> None: ... - @overload - def __init__(self, torch_linear: Any) -> None: ... # torch.nn.Linear - def __call__(self, x: Tensor2D) -> Tensor2D: ... - def from_torch(self, linear: Any) -> None: ... # torch.nn.Linear - def from_array( - self, - array: npt.NDArray[np.float32], - bias: npt.NDArray[np.float32] | None = None, - ) -> None: ... - -class Embedding: - num_embeddings: int - embedding_dim: int - weight: Float32Vector - - @overload - def __init__(self, num_embeddings: int, embedding_dim: int) -> None: ... - @overload - def __init__(self, torch_embedding: Any) -> None: ... # torch.nn.Embedding - def __call__(self, indices: Int32Tensor2D) -> Tensor2D: ... - def data(self) -> npt.NDArray[np.float32]: ... - def from_torch(self, emb: Any) -> None: ... # torch.nn.Embedding - def from_array(self, array: npt.NDArray[np.float32]) -> None: ... - -class FFN: - linear1: Linear - linear2: Linear - - def __init__(self, d: int, h: int) -> None: ... - def __call__(self, x: Tensor2D) -> Tensor2D: ... - -# QINCo neural net codec classes from NeuralNet.h -class QINCoStep: - d: int - K: int - L: int - h: int - codebook: Embedding - MLPconcat: Linear - residual_blocks: list[FFN] - - @overload - def __init__(self, d: int, K: int, L: int, h: int) -> None: ... - @overload - def __init__(self, torch_qinco_step: Any) -> None: ... # torch QINCoStep - def get_residual_block(self, i: int) -> FFN: ... - def encode( - self, xhat: Tensor2D, x: Tensor2D, residuals: Tensor2D | None = None - ) -> Int32Tensor2D: ... - def decode(self, xhat: Tensor2D, codes: Int32Tensor2D) -> Tensor2D: ... - def from_torch(self, step: Any) -> None: ... # torch QINCoStep - -class NeuralNetCodec: - d: int - M: int - - def __init__(self, d: int, M: int) -> None: ... - def decode(self, codes: Int32Tensor2D) -> Tensor2D: ... - def encode(self, x: Tensor2D) -> Int32Tensor2D: ... - -class QINCo(NeuralNetCodec): - K: int - L: int - h: int - codebook0: Embedding - steps: list[QINCoStep] - - @overload - def __init__(self, d: int, K: int, L: int, M: int, h: int) -> None: ... - @overload - def __init__(self, torch_qinco: Any) -> None: ... # torch QINCo - def get_step(self, i: int) -> QINCoStep: ... - def decode(self, codes: Int32Tensor2D) -> Tensor2D: ... - def encode(self, x: Tensor2D) -> Int32Tensor2D: ... - def from_torch(self, qinco: Any) -> None: ... # torch QINCo - -# Code set utility from utils.h -class CodeSet: - d: int - - def __init__(self, d: int) -> None: ... - @overload - def insert( - self, codes: torch.Tensor, inserted: torch.Tensor | None = None - ) -> torch.Tensor: ... - @overload - def insert( - self, - codes: npt.NDArray[np.uint8], - inserted: npt.NDArray[np.bool_] | None = None, - ) -> npt.NDArray[np.bool_]: ... - -# ID selector implementations (from impl/IDSelector.h) with correct C++ signatures -class IDSelectorRange(IDSelector): - imin: int - imax: int - assume_sorted: bool - - def __init__( - self, imin: int, imax: int, assume_sorted: bool = False - ) -> None: ... - def is_member(self, id: int) -> bool: ... - def find_sorted_ids_bounds( - self, - list_size: int, - ids: npt.NDArray[np.int64], - ) -> tuple[int, int]: ... - -class IDSelectorArray(IDSelector): - n: int - # Note: C++ uses const idx_t* ids (raw pointer), Python wrapper handles this - @overload - def __init__(self, n: int, ids: npt.NDArray[np.int64]) -> None: ... - @overload - def __init__( - self, ids: npt.NDArray[np.int64] - ) -> None: ... # Python wrapper convenience - def is_member(self, id: int) -> bool: ... - -class IDSelectorBatch(IDSelector): - # C++ has std::unordered_set set and bloom filter internals - - @overload - def __init__(self, n: int, indices: npt.NDArray[np.int64]) -> None: ... - @overload - def __init__( - self, indices: npt.NDArray[np.int64] - ) -> None: ... # Python wrapper convenience - def is_member(self, id: int) -> bool: ... - -class IDSelectorBitmap(IDSelector): - n: int - # Note: C++ uses const uint8_t* bitmap (raw pointer), Python wrapper handles this - def __init__(self, n: int, bitmap: npt.NDArray[np.uint8]) -> None: ... - def is_member(self, id: int) -> bool: ... - -class IDSelectorNot(IDSelector): - sel: IDSelector - - def __init__(self, sel: IDSelector) -> None: ... - def is_member(self, id: int) -> bool: ... - -class IDSelectorAll(IDSelector): - def __init__(self) -> None: ... - def is_member(self, id: int) -> bool: ... - -class IDSelectorAnd(IDSelector): - lhs: IDSelector - rhs: IDSelector - - def __init__(self, lhs: IDSelector, rhs: IDSelector) -> None: ... - def is_member(self, id: int) -> bool: ... - -class IDSelectorOr(IDSelector): - lhs: IDSelector - rhs: IDSelector - - def __init__(self, lhs: IDSelector, rhs: IDSelector) -> None: ... - def is_member(self, id: int) -> bool: ... - -class IDSelectorXOr(IDSelector): - lhs: IDSelector - rhs: IDSelector - - def __init__(self, lhs: IDSelector, rhs: IDSelector) -> None: ... - def is_member(self, id: int) -> bool: ... - -# CodePacker classes (from impl/CodePacker.h) with correct C++ API -class CodePacker: - code_size: int - nvec: int - block_size: int - - # Abstract base class - no direct constructor - def pack_1( - self, - flat_code: npt.NDArray[np.uint8], - offset: int, - block: npt.NDArray[np.uint8], - ) -> None: ... - def unpack_1( - self, - block: npt.NDArray[np.uint8], - offset: int, - flat_code: npt.NDArray[np.uint8], - ) -> None: ... - def pack_all( - self, - flat_codes: npt.NDArray[np.uint8], - block: npt.NDArray[np.uint8], - ) -> None: ... - def unpack_all( - self, - block: npt.NDArray[np.uint8], - flat_codes: npt.NDArray[np.uint8], - ) -> None: ... - -class CodePackerFlat(CodePacker): - def __init__(self, code_size: int) -> None: ... - -# Utility functions -def get_mem_usage_kb() -> int: ... -def get_compile_options() -> str: ... -def check_openmp() -> bool: ... -def shard_ivf_index_centroids( - index: IndexIVF, - shard_count: int = 20, - filename_template: str = "shard.%d.index", - sharding_function: ShardingFunction | None = None, - generate_ids: bool = False, -) -> None: ... -def shard_binary_ivf_index_centroids( - index: IndexBinaryIVF, - shard_count: int = 20, - filename_template: str = "shard.%d.index", - sharding_function: ShardingFunction | None = None, - generate_ids: bool = False, -) -> None: ... - -# IVF extraction utility functions (from IVFlib.h) -def extract_index_ivf(index: Index) -> IndexIVF: ... -def try_extract_index_ivf(index: Index) -> IndexIVF | None: ... - -# IVFlib utility functions and classes (from IVFlib.h) -def check_compatible_for_merge(index1: Index, index2: Index) -> None: ... -def merge_into(index0: Index, index1: Index, shift_ids: bool) -> None: ... -def search_centroid( - index: Index, - x: npt.NDArray[np.float32], - n: int, - centroid_ids: npt.NDArray[np.int64], -) -> None: ... -def search_and_return_centroids( - index: Index, - n: int, - xin: npt.NDArray[np.float32], - k: int, - distances: npt.NDArray[np.float32], - labels: npt.NDArray[np.int64], - query_centroid_ids: npt.NDArray[np.int64], - result_centroid_ids: npt.NDArray[np.int64], -) -> None: ... -def get_invlist_range(index: Index, i0: int, i1: int) -> ArrayInvertedLists: ... -def set_invlist_range( - index: Index, i0: int, i1: int, src: ArrayInvertedLists -) -> None: ... -def search_with_parameters( - index: Index, - n: int, - x: npt.NDArray[np.float32], - k: int, - distances: npt.NDArray[np.float32], - labels: npt.NDArray[np.int64], - params: IVFSearchParameters, - nb_dis: npt.NDArray[np.int64] | None = None, - ms_per_stage: npt.NDArray[np.float64] | None = None, -) -> None: ... -def range_search_with_parameters( - index: Index, - n: int, - x: npt.NDArray[np.float32], - radius: float, - result: RangeSearchResult, - params: IVFSearchParameters, - nb_dis: npt.NDArray[np.int64] | None = None, - ms_per_stage: npt.NDArray[np.float64] | None = None, -) -> None: ... -def ivf_residual_from_quantizer( - rq: ResidualQuantizer, nlevel: int -) -> IndexIVFResidualQuantizer: ... -def ivf_residual_add_from_flat_codes( - ivfrq: IndexIVFResidualQuantizer, - ncode: int, - codes: npt.NDArray[np.uint8], - code_size: int = -1, -) -> None: ... - -class SlidingIndexWindow: - index: Index - ils: ArrayInvertedLists - n_slice: int - nlist: int - sizes: list[list[int]] - - def __init__(self, index: Index) -> None: ... - def step(self, sub_index: Index | None, remove_oldest: bool) -> None: ... - -class ShardingFunction: - def __call__(self, i: int, shard_count: int) -> int: ... - -class DefaultShardingFunction(ShardingFunction): - def __call__(self, i: int, shard_count: int) -> int: ... - -# Version information -__version__: str - -# Logger -logger: Any - -# Additional missing index types from SWIG includes - -# IDMap index classes - for mapping external IDs to internal indices -class IndexIDMap(Index): - index: Index - own_fields: bool - id_map: Int64Vector - - def __init__(self, index: Index) -> None: ... - # Override methods that don't use add() directly - @overload - def add_with_ids(self, x: torch.Tensor, ids: torch.Tensor) -> None: ... - @overload - def add_with_ids( - self, x: npt.NDArray[np.float32], ids: npt.NDArray[np.int64] - ) -> None: ... - @overload - def train(self, x: torch.Tensor) -> None: ... - @overload - def train(self, x: npt.NDArray[np.float32]) -> None: ... - def reset(self) -> None: ... - def remove_ids(self, sel: IDSelector) -> int: ... - @overload - def search( - self, - x: torch.Tensor, - k: int, - *, - params: SearchParameters | None = None, - D: torch.Tensor | None = None, - I: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: ... - @overload - def search( - self, - x: npt.NDArray[np.float32], - k: int, - *, - params: SearchParameters | None = None, - D: npt.NDArray[np.float32] | None = None, - I: npt.NDArray[np.int64] | None = None, - ) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... - @overload - def range_search( - self, - x: torch.Tensor, - thresh: float, - *, - params: SearchParameters | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ... - @overload - def range_search( - self, - x: npt.NDArray[np.float32], - thresh: float, - *, - params: SearchParameters | None = None, - ) -> tuple[ - npt.NDArray[np.int64], npt.NDArray[np.float32], npt.NDArray[np.int64] - ]: ... - def sa_code_size(self) -> int: ... - @overload - def add_sa_codes(self, codes: torch.Tensor, ids: torch.Tensor) -> None: ... - @overload - def add_sa_codes( - self, codes: npt.NDArray[np.uint8], ids: npt.NDArray[np.int64] - ) -> None: ... - def merge_from(self, other_index: Index, add_id: int = 0) -> None: ... - def check_compatible_for_merge(self, other_index: Index) -> None: ... - -class IndexBinaryIDMap(IndexBinary): - index: IndexBinary - own_fields: bool - id_map: Int64Vector - - def __init__(self, index: IndexBinary) -> None: ... - # Override methods that don't use add() directly - @overload - def add_with_ids(self, x: torch.Tensor, ids: torch.Tensor) -> None: ... - @overload - def add_with_ids( - self, x: npt.NDArray[np.uint8], ids: npt.NDArray[np.int64] - ) -> None: ... - @overload - def train(self, x: torch.Tensor) -> None: ... - @overload - def train(self, x: npt.NDArray[np.uint8]) -> None: ... - def reset(self) -> None: ... - def remove_ids(self, sel: IDSelector) -> int: ... - @overload - def search( - self, - x: torch.Tensor, - k: int, - *, - params: SearchParameters | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: ... - @overload - def search( - self, - x: npt.NDArray[np.uint8], - k: int, - *, - params: SearchParameters | None = None, - ) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.int64]]: ... - @overload - def range_search( - self, - x: torch.Tensor, - thresh: int, - *, - params: SearchParameters | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ... - @overload - def range_search( - self, - x: npt.NDArray[np.uint8], - thresh: int, - *, - params: SearchParameters | None = None, - ) -> tuple[ - npt.NDArray[np.int64], npt.NDArray[np.int32], npt.NDArray[np.int64] - ]: ... - @overload - def reconstruct(self, key: int) -> torch.Tensor: ... - @overload - def reconstruct(self, key: int) -> npt.NDArray[np.uint8]: ... - -class IndexIDMap2(IndexIDMap): - rev_map: Any # std::unordered_map - - def __init__(self, index: Index) -> None: ... - @overload - def add_with_ids(self, x: torch.Tensor, ids: torch.Tensor) -> None: ... - @overload - def add_with_ids( - self, x: npt.NDArray[np.float32], ids: npt.NDArray[np.int64] - ) -> None: ... - def check_consistency(self) -> None: ... - def construct_rev_map(self) -> None: ... - def merge_from(self, other_index: Index, add_id: int = 0) -> None: ... - def remove_ids(self, sel: IDSelector) -> int: ... - @overload - def reconstruct( - self, key: int, x: torch.Tensor | None = None - ) -> torch.Tensor: ... - @overload - def reconstruct( - self, key: int, x: npt.NDArray[np.float32] | None = None - ) -> npt.NDArray[np.float32]: ... - -class IndexBinaryIDMap2(IndexBinaryIDMap): - rev_map: Any # std::unordered_map - - def __init__(self, index: IndexBinary) -> None: ... - @overload - def add_with_ids(self, x: torch.Tensor, ids: torch.Tensor) -> None: ... - @overload - def add_with_ids( - self, x: npt.NDArray[np.uint8], ids: npt.NDArray[np.int64] - ) -> None: ... - def check_consistency(self) -> None: ... - def construct_rev_map(self) -> None: ... - def merge_from(self, other_index: IndexBinary, add_id: int = 0) -> None: ... - def remove_ids(self, sel: IDSelector) -> int: ... - @overload - def reconstruct(self, key: int) -> torch.Tensor: ... - @overload - def reconstruct(self, key: int) -> npt.NDArray[np.uint8]: ... - -# IndexNSG (Neural Sparse Graph) -class IndexNSG(Index): - nsg: Any # faiss::nsg::Graph - base_index: Index - GK: int - build_type: int - verbose: bool - - def __init__( - self, d: int, R: int, metric: MetricType = METRIC_L2 - ) -> None: ... - @overload - def build(self, x: torch.Tensor, graph: torch.Tensor) -> None: ... - @overload - def build( - self, x: npt.NDArray[np.float32], graph: npt.NDArray[np.int64] - ) -> None: ... - -class IndexNSGFlat(IndexNSG): - def __init__(self, d: int, R: int) -> None: ... - -class IndexNSGPQ(IndexNSG): - pq: ProductQuantizer - - def __init__(self, d: int, pq: ProductQuantizer, R: int) -> None: ... - -class IndexNSGSQ(IndexNSG): - sq: ScalarQuantizer - - def __init__(self, d: int, sq: ScalarQuantizer, R: int) -> None: ... - -# IndexNNDescent -class IndexNNDescent(Index): - nndescent: Any # faiss::nndescent::NNDescent - storage: Index - own_fields: bool - - def __init__( - self, d: int, K: int, metric: MetricType = METRIC_L2 - ) -> None: ... - -class IndexNNDescentFlat(IndexNNDescent): - def __init__( - self, d: int, K: int, metric: MetricType = METRIC_L2 - ) -> None: ... - -# Index2Layer -class Index2Layer(Index): - q1: Quantizer - pq: ProductQuantizer - code_size_1: int - code_size_2: int - code_size: int - codes: UInt8Vector - - def __init__( - self, - quantizer: Index, - nlist: int, - M: int, - nbit: int = 8, - metric: MetricType = METRIC_L2, - ) -> None: ... - def transfer_to_IVFPQ(self, other: IndexIVFPQ) -> None: ... - -# IndexIVFPQR (with Refine) -class IndexIVFPQR(IndexIVFPQ): - k_factor: float - refine_index: Index - own_refine_index: bool - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - M: int, - nbits: int, - M_refine: int, - nbits_refine: int, - metric: MetricType = METRIC_L2, - ) -> None: ... - -# IndexLattice -class IndexLattice(Index): - d: int - dsub: int - scale_nbit: int - r2: float - - def __init__(self, d: int, dsub: int = 0, dsuper: int = 0) -> None: ... - -# IndexRowwiseMinMax -class IndexRowwiseMinMaxBase(Index): - index: Index - own_fields: bool - - @overload - def __init__(self, index: Index) -> None: ... - @overload - def __init__(self) -> None: ... - -class IndexRowwiseMinMax(IndexRowwiseMinMaxBase): - @overload - def __init__(self, index: Index) -> None: ... - @overload - def __init__(self) -> None: ... - -class IndexRowwiseMinMaxFP16(IndexRowwiseMinMaxBase): - @overload - def __init__(self, index: Index) -> None: ... - @overload - def __init__(self) -> None: ... - -# IndexRandom for testing -class IndexRandom(Index): - seed: int - - def __init__( - self, - d: int, - ntotal: int = 0, - seed: int = 1234, - metric: MetricType = METRIC_L2, - ) -> None: ... - -# IndexShards and IndexReplicas (already templated in SWIG) -class IndexShardsIVF(Index): - own_fields: bool - threaded: bool - successive_ids: bool - - def __init__( - self, quantizer: Index, d: int, nlist: int, threaded: bool = False - ) -> None: ... - def add_shard(self, index: Index) -> None: ... - -# Missing IVF Fast Scan variants - -# Additive quantizer base classes -class IndexAdditiveQuantizer(IndexFlatCodes): - aq: AdditiveQuantizer - -class IndexResidualQuantizer(IndexAdditiveQuantizer): - rq: ResidualQuantizer - - @overload - def __init__( - self, - d: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - ) -> None: ... - @overload - def __init__(self) -> None: ... - -class IndexLocalSearchQuantizer(IndexAdditiveQuantizer): - lsq: LocalSearchQuantizer - - @overload - def __init__( - self, - d: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - ) -> None: ... - @overload - def __init__(self) -> None: ... - -class IndexIVFAdditiveQuantizer(IndexIVF): - aq: AdditiveQuantizer - use_precomputed_table: int - -class IndexIVFResidualQuantizer(IndexIVFAdditiveQuantizer): - rq: ResidualQuantizer - code_size: int - by_residual: bool - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - encode_residual: bool = True, - ) -> None: ... - -class IndexIVFLocalSearchQuantizer(IndexIVFAdditiveQuantizer): - lsq: LocalSearchQuantizer - code_size: int - by_residual: bool - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - encode_residual: bool = True, - ) -> None: ... - -# FastScan variants for IVF -class IndexIVFAdditiveQuantizerFastScan(IndexIVFFastScan): - aq: AdditiveQuantizer - rescale_norm: bool - norm_scale: int - max_train_points: int - - def __init__(self) -> None: ... - -class IndexIVFResidualQuantizerFastScan(IndexIVFAdditiveQuantizerFastScan): - rq: ResidualQuantizer - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - bbs: int = 32, - ) -> None: ... - -class IndexIVFLocalSearchQuantizerFastScan(IndexIVFAdditiveQuantizerFastScan): - lsq: LocalSearchQuantizer - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - bbs: int = 32, - ) -> None: ... - -# Product variants -class IndexProductResidualQuantizer(IndexAdditiveQuantizer): - prq: ProductResidualQuantizer - - def __init__( - self, d: int, M: int, nbits: int, metric: MetricType = METRIC_L2 - ) -> None: ... - -class IndexProductLocalSearchQuantizer(IndexAdditiveQuantizer): - plsq: ProductLocalSearchQuantizer - - def __init__( - self, d: int, M: int, nbits: int, metric: MetricType = METRIC_L2 - ) -> None: ... - -# IVF Product variants -class IndexIVFProductResidualQuantizer(IndexIVFAdditiveQuantizer): - prq: ProductResidualQuantizer - code_size: int - by_residual: bool - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - nsplits: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - encode_residual: bool = True, - ) -> None: ... - -class IndexIVFProductLocalSearchQuantizer(IndexIVFAdditiveQuantizer): - plsq: ProductLocalSearchQuantizer - code_size: int - by_residual: bool - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - nsplits: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - encode_residual: bool = True, - ) -> None: ... - -# FastScan product variants -class IndexProductResidualQuantizerFastScan(IndexAdditiveQuantizerFastScan): - prq: ProductResidualQuantizer - - def __init__( - self, - d: int, - nsplits: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - bbs: int = 32, - ) -> None: ... - -class IndexProductLocalSearchQuantizerFastScan(IndexAdditiveQuantizerFastScan): - plsq: ProductLocalSearchQuantizer - - def __init__( - self, - d: int, - nsplits: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - bbs: int = 32, - ) -> None: ... - -class IndexIVFProductResidualQuantizerFastScan( - IndexIVFAdditiveQuantizerFastScan -): - prq: ProductResidualQuantizer - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - nsplits: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - bbs: int = 32, - ) -> None: ... - -class IndexIVFProductLocalSearchQuantizerFastScan( - IndexIVFAdditiveQuantizerFastScan -): - plsq: ProductLocalSearchQuantizer - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - nsplits: int, - M: int, - nbits: int, - metric: MetricType = METRIC_L2, - bbs: int = 32, - ) -> None: ... - -# Additional missing classes -class MultiIndexQuantizer(Index): - pq: ProductQuantizer - - def __init__(self, d: int, M: int, nbits: int) -> None: ... - -# Additive coarse quantizers -class AdditiveCoarseQuantizer(Index): - aq: AdditiveQuantizer - centroid_norms: Float32Vector - - def __init__( - self, - d: int = 0, - aq: AdditiveQuantizer | None = None, - metric: MetricType = METRIC_L2, - ) -> None: ... - -class SearchParametersResidualCoarseQuantizer(SearchParameters): - beam_factor: float - - def __init__(self) -> None: ... - -class ResidualCoarseQuantizer(AdditiveCoarseQuantizer): - rq: ResidualQuantizer - beam_factor: float - - @overload - def __init__(self, d: int, M: int, nbits: int) -> None: ... - @overload - def __init__( - self, d: int, nbits: UInt64Vector, metric: int = METRIC_L2 - ) -> None: ... - @overload - def __init__(self) -> None: ... - def set_beam_factor(self, new_beam_factor: float) -> None: ... - -class LocalSearchCoarseQuantizer(AdditiveCoarseQuantizer): - lsq: LocalSearchQuantizer - - @overload - def __init__(self, d: int, M: int, nbits: int) -> None: ... - @overload - def __init__(self) -> None: ... - -# NeuralNet index -class IndexNeuralNetCodec(Index): - M: int - nbits: int - - def __init__( - self, - d: int = 0, - M: int = 0, - nbits: int = 0, - metric: MetricType = METRIC_L2, - ) -> None: ... - -class IndexQINCo(IndexNeuralNetCodec): - def __init__( - self, - d: int, - M: int, - nbits: int, - L: int, - h: int, - metric: MetricType = METRIC_L2, - ) -> None: ... - -# RaBitQ indices -class RaBitQSearchParameters(SearchParameters): - qb: int # number of bits to quantize a query with (0 = raw fp32) - centered: bool # quantize with zero-centered scalar quantizer - - def __init__(self) -> None: ... - -class IndexRaBitQ(Index): - rabitq: RaBitQuantizer - center: Float32Vector - qb: int # default number of bits to quantize a query with - centered: bool - - def __init__( - self, - d: int, - metric: MetricType = METRIC_L2, - nb_bits: int = 1, - ) -> None: ... - -class IVFRaBitQSearchParameters(IVFSearchParameters): - qb: int - centered: bool - - def __init__(self) -> None: ... - -class IndexIVFRaBitQ(IndexIVF): - rabitq: RaBitQuantizer - qb: int - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - metric: MetricType = METRIC_L2, - own_invlists: bool = True, - nb_bits: int = 1, - ) -> None: ... - -# EDEN indices -class IndexEDEN(IndexFlatCodes): - sq: ScalarQuantizer - scale_type: EDENScaleType - center: Float32Vector - - def __init__( - self, - d: int, - metric: MetricType = METRIC_L2, - nb_bits: int = 1, - scale_type: EDENScaleType = EDENScaleType_UNBIASED, - ) -> None: ... - -class IndexIVFEDEN(IndexIVF): - sq: ScalarQuantizer - scale_type: EDENScaleType - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - metric: MetricType = METRIC_L2, - own_invlists: bool = True, - nb_bits: int = 1, - scale_type: EDENScaleType = EDENScaleType_UNBIASED, - ) -> None: ... - -class IndexRaBitQFastScan(IndexFastScan): - """Fast-scan version of RaBitQ that processes 32 database vectors at a time using SIMD.""" - - rabitq: RaBitQuantizer - center: Float32Vector - qb: int - centered: bool - - @overload - def __init__( - self, - d: int, - metric: MetricType = METRIC_L2, - bbs: int = 32, - nb_bits: int = 1, - ) -> None: ... - @overload - def __init__(self, orig: IndexRaBitQ, bbs: int = 32) -> None: ... - -class IndexIVFRaBitQFastScan(IndexIVFFastScan): - """Fast-scan version of IndexIVFRaBitQ that processes vectors in batches using SIMD.""" - - rabitq: RaBitQuantizer - qb: int - centered: bool - - @overload - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - metric: MetricType = METRIC_L2, - bbs: int = 32, - own_invlists: bool = True, - nb_bits: int = 1, - ) -> None: ... - @overload - def __init__(self, orig: IndexIVFRaBitQ, bbs: int = 32) -> None: ... - -# SVS (Intel Scalable Vector Search) indexes -class IndexSVSFlat(Index): - nlabels: int - def __init__(self, d: int, metric: MetricType = METRIC_L2) -> None: ... - -class IndexSVSVamana(Index): - graph_max_degree: int - prune_to: int - alpha: float - search_window_size: int - search_buffer_capacity: int - construction_window_size: int - max_candidate_pool_size: int - use_full_search_history: bool - is_static: bool - storage_kind: SVSStorageKind - - def __init__( - self, - d: int, - degree: int, - metric: MetricType = METRIC_L2, - storage: SVSStorageKind = SVS_FP32, - is_static: bool = False, - ) -> None: ... - @staticmethod - def is_lvq_leanvec_enabled() -> bool: ... - -class IndexSVSVamanaLVQ(IndexSVSVamana): - def __init__( - self, - d: int, - degree: int, - metric: MetricType = METRIC_L2, - storage: SVSStorageKind = SVS_LVQ4x0, - is_static: bool = False, - ) -> None: ... - -class IndexSVSVamanaLeanVec(IndexSVSVamana): - leanvec_d: int - - def __init__( - self, - d: int, - degree: int, - metric: MetricType = METRIC_L2, - leanvec_dims: int = 0, - storage: SVSStorageKind = SVS_LeanVec4x4, - is_static: bool = False, - ) -> None: ... - -class IndexSVSIVF(Index): - num_centroids: int - minibatch_size: int - num_iterations: int - is_hierarchical: bool - training_fraction: float - hierarchical_level1_clusters: int - seed: int - n_probes: int - k_reorder: float - num_threads: int - intra_query_threads: int - is_static: bool - storage_kind: SVSStorageKind - - def __init__( - self, - d: int, - nlist: int, - metric: MetricType = METRIC_L2, - storage: SVSStorageKind = SVS_FP32, - is_static: bool = False, - ) -> None: ... - @staticmethod - def is_lvq_leanvec_enabled() -> bool: ... - -class IndexSVSIVFLVQ(IndexSVSIVF): - def __init__( - self, - d: int, - nlist: int, - metric: MetricType = METRIC_L2, - storage: SVSStorageKind = SVS_LVQ4x0, - is_static: bool = False, - ) -> None: ... - -class IndexSVSIVFLeanVec(IndexSVSIVF): - leanvec_d: int - - def __init__( - self, - d: int, - nlist: int, - metric: MetricType = METRIC_L2, - leanvec_dims: int = 0, - storage: SVSStorageKind = SVS_LeanVec4x4, - is_static: bool = False, - ) -> None: ... - -# Independent quantizer -class IndexIVFIndependentQuantizer(Index): - quantizer: Index - index_ivf: Index - own_fields: bool - - def __init__(self, quantizer: Index, index_ivf: Index) -> None: ... - -# IVF Spectral Hash -class IndexIVFSpectralHash(IndexIVF): - vt: VectorTransform - threshold_type: int - period: int - trained: UInt8Vector - - def __init__( - self, - quantizer: Index, - d: int, - nlist: int, - nbit: int, - metric: MetricType = METRIC_L2, - period: int = 64, - ) -> None: ... - def replace_vt(self, vt: VectorTransform) -> None: ... - -# Missing Binary index variants -class IndexBinaryHNSWCagra(IndexBinaryHNSW): - def __init__(self, d: int, M: int = 32) -> None: ... - -# AutoTune related classes -class ParameterRange: - name: str - values: Float64Vector - - def __init__(self, name: str = "") -> None: ... - -class OperatingPoint: - perf: float - t: float - key: str - cno: int - - def __init__(self) -> None: ... - -# ParameterSpace from AutoTune.h with complete API -# IndexIVFInterface from IndexIVF.h -class IndexIVFInterface: - nprobe: int # number of probes at query time - max_codes: int # max nb of codes to visit to do a query - - def __init__( - self, quantizer: Index | None = None, nlist: int = 0 - ) -> None: ... - def search_preassigned( - self, - n: int, - x: npt.NDArray[np.float32], - k: int, - assign: npt.NDArray[np.int64], - centroid_dis: npt.NDArray[np.float32], - distances: npt.NDArray[np.float32], - labels: npt.NDArray[np.int64], - store_pairs: bool, - params: IVFSearchParameters | None = None, - stats: IndexIVFStats | None = None, - ) -> None: ... - def range_search_preassigned( - self, - nx: int, - x: npt.NDArray[np.float32], - radius: float, - keys: npt.NDArray[np.int64], - coarse_dis: npt.NDArray[np.float32], - result: RangeSearchResult, - store_pairs: bool = False, - params: IVFSearchParameters | None = None, - stats: IndexIVFStats | None = None, - ) -> None: ... - -class ParameterSpace: - parameter_ranges: ParameterRangeVector - verbose: int - n_experiments: int - batchsize: int - thread_over_batches: bool - min_test_duration: float - - def __init__(self) -> None: ... - def add_range(self, name: str) -> ParameterRange: ... - def n_combinations(self) -> int: ... - def combination_ge(self, c1: int, c2: int) -> bool: ... - def combination_name(self, cno: int) -> str: ... - def display(self) -> None: ... - def initialize(self, index: Index) -> None: ... - def set_index_parameters(self, index: Index, cno: int) -> None: ... - def set_index_parameters(self, index: Index, param_string: str) -> None: ... - def set_index_parameter( - self, index: Index, name: str, val: float - ) -> None: ... - def update_bounds( - self, - cno: int, - op: OperatingPoint, - upper_bound_perf: float, - lower_bound_t: float, - ) -> tuple[float, float]: ... - @overload - def explore( - self, - index: Index, - nq: int, - xq: torch.Tensor, - crit: AutoTuneCriterion, - ops: OperatingPoints, - ) -> None: ... - @overload - def explore( - self, - index: Index, - nq: int, - xq: npt.NDArray[np.float32], - crit: AutoTuneCriterion, - ops: OperatingPoints, - ) -> None: ... - -# GPU IVF base classes -class GpuIndex(Index): - def getDevice(self) -> int: ... - def getResources(self) -> StandardGpuResources: ... - def setMinPagingSize(self, size: int) -> None: ... - def getMinPagingSize(self) -> int: ... - def copyFrom(self, index: Index) -> None: ... - def copyTo(self, index: Index) -> None: ... - -class GpuIndexIVF(GpuIndex, IndexIVFInterface): ... - -class GpuIndexIVFConfig(GpuIndexConfig): - """Configuration for GPU IVF indices""" - - def __init__(self) -> None: ... - -class GpuIndexIVFPQConfig(GpuIndexIVFConfig): - """Configuration for GPU IVFPQ index""" - - useFloat16LookupTables: bool - usePrecomputedTables: bool - interleavedLayout: bool - useMMCodeDistance: bool - - def __init__(self) -> None: ... - -class GpuIndexIVFPQ(GpuIndexIVF): - """GPU IVFPQ index implementation""" - - pq: ProductQuantizer - - @overload - def __init__( - self, - provider: StandardGpuResources, - index: IndexIVFPQ, - config: GpuIndexIVFPQConfig | None = None, - ) -> None: ... - @overload - def __init__( - self, - provider: StandardGpuResources, - dims: int, - nlist: int, - subQuantizers: int, - bitsPerCode: int, - metric: MetricType = METRIC_L2, - config: GpuIndexIVFPQConfig | None = None, - ) -> None: ... - @overload - def __init__( - self, - provider: StandardGpuResources, - coarseQuantizer: Index, - dims: int, - nlist: int, - subQuantizers: int, - bitsPerCode: int, - metric: MetricType = METRIC_L2, - config: GpuIndexIVFPQConfig | None = None, - ) -> None: ... - def copyFrom(self, index: IndexIVFPQ) -> None: ... - def copyTo(self, index: IndexIVFPQ) -> None: ... - def reserveMemory(self, numVecs: int) -> None: ... - def setPrecomputedCodes(self, enable: bool) -> None: ... - def getPrecomputedCodes(self) -> bool: ... - def getNumSubQuantizers(self) -> int: ... - def getBitsPerCode(self) -> int: ... - def getCentroidsPerSubQuantizer(self) -> int: ... - def reclaimMemory(self) -> int: ... - def reset(self) -> None: ... - def updateQuantizer(self) -> None: ... - @overload - def train(self, x: torch.Tensor) -> None: ... - @overload - def train(self, x: npt.NDArray[np.float32]) -> None: ... - -# GPU functions with comprehensive overloaded signatures for tensor support - -class GpuParameterSpace(ParameterSpace): - """GPU-specific parameter space for auto-tuning""" - - def initialize(self, index: Index) -> None: ... - def set_index_parameter( - self, index: Index, name: str, val: float - ) -> None: ... - -class OperatingPoints: - all_pts: OperatingPointVector - optimal_pts: OperatingPointVector - - def __init__(self) -> None: ... - def merge_with(self, other: OperatingPoints, prefix: str = "") -> int: ... - def clear(self) -> None: ... - def add(self, perf: float, t: float, key: str, cno: int = 0) -> bool: ... - def t_for_perf(self, perf: float) -> float: ... - def display(self, only_optimal: bool = True) -> None: ... - def all_to_gnuplot(self, fname: str) -> None: ... - def optimal_to_gnuplot(self, fname: str) -> None: ... - -# Threading support -class IndexSplitVectors(Index): - sub_indexes: list[Index] - own_fields: bool - threaded: bool - - def __init__(self, d: int, threaded: bool = True) -> None: ... - -# Distance functions -def fvec_L2sqr( - x: npt.NDArray[np.float32], y: npt.NDArray[np.float32], d: int -) -> float: ... -def fvec_inner_product( - x: npt.NDArray[np.float32], y: npt.NDArray[np.float32], d: int -) -> float: ... -def fvec_L1( - x: npt.NDArray[np.float32], y: npt.NDArray[np.float32], d: int -) -> float: ... -def fvec_Linf( - x: npt.NDArray[np.float32], y: npt.NDArray[np.float32], d: int -) -> float: ... -def fvec_norm_L2sqr(x: npt.NDArray[np.float32], d: int) -> float: ... -def pairwise_L2sqr( - d: int, - nq: int, - xq: npt.NDArray[np.float32], - nb: int, - xb: npt.NDArray[np.float32], - dis: npt.NDArray[np.float32], - ldq: int = -1, - ldb: int = -1, - ldd: int = -1, -) -> None: ... -def knn_inner_product( - x: npt.NDArray[np.float32], - y: npt.NDArray[np.float32], - d: int, - nx: int, - ny: int, - k: int, - distances: npt.NDArray[np.float32], - labels: npt.NDArray[np.int64], - sel: IDSelector | None = None, -) -> None: ... -def knn_L2sqr( - x: npt.NDArray[np.float32], - y: npt.NDArray[np.float32], - d: int, - nx: int, - ny: int, - k: int, - distances: npt.NDArray[np.float32], - labels: npt.NDArray[np.int64], - y_norm2: npt.NDArray[np.float32] | None = None, - sel: IDSelector | None = None, -) -> None: ... -def range_search_L2sqr( - x: npt.NDArray[np.float32], - y: npt.NDArray[np.float32], - d: int, - nx: int, - ny: int, - radius: float, - result: RangeSearchResult, - sel: IDSelector | None = None, -) -> None: ... -def range_search_inner_product( - x: npt.NDArray[np.float32], - y: npt.NDArray[np.float32], - d: int, - nx: int, - ny: int, - radius: float, - result: RangeSearchResult, - sel: IDSelector | None = None, -) -> None: ... -@overload -def imbalance_factor(k: int, hist: int) -> float: ... -@overload -def imbalance_factor(n: int, k: int, assign: int) -> float: ... - -# Index factory functions -def index_factory( - d: int, - description: str, - metric: MetricType = METRIC_L2, - own_invlists: bool = True, -) -> Index: ... -def index_binary_factory( - d: int, description: str, own_invlists: bool = True -) -> IndexBinary: ... - -# Cloner classes -class Cloner: - def clone_Index(self, index: Index) -> Index: ... - -# GpuResourcesProvider base class -class GpuResourcesProvider: - def __init__(self) -> None: ... - -def clone_index(index: Index) -> Index: ... - -# Utility functions -def omp_set_num_threads(num_threads: int) -> None: ... -def omp_get_max_threads() -> int: ... - -# Version utilities -def swig_version() -> int: ... - -# Hash table implementation -class MapLong2Long: - def __init__(self) -> None: ... - def add(self, n: int, keys: np.ndarray, vals: np.ndarray) -> None: ... - def search(self, key: int) -> int: ... - def search_multiple( - self, n: int, keys: np.ndarray, vals: np.ndarray - ) -> None: ... - -# Additional utilities -def get_num_gpus() -> int: ... -def gpu_profiler_start() -> None: ... -def gpu_profiler_stop() -> None: ... -def gpu_sync_all_devices() -> None: ... - -# Distance computation globals -distance_compute_blas_threshold: int -distance_compute_blas_query_bs: int -distance_compute_blas_database_bs: int -distance_compute_min_k_reservoir: int - -# Index factory verbose flag -index_factory_verbose: int - -# GPU-specific types and functions -class GpuDistanceParams: - metric: MetricType - k: int - dims: int - vectors: Any - vectorsRowMajor: bool - vectorType: int - numVectors: int - queries: Any - queriesRowMajor: bool - queryType: int - numQueries: int - outDistances: Any - outIndices: Any - outIndicesType: int - device: int - use_cuvs: bool - - def __init__(self) -> None: ... - -class StandardGpuResourcesImpl: - """Standard implementation of the GpuResources object that provides for a temporary memory manager""" - - def __init__(self) -> None: ... - def supportsBFloat16(self, device: int) -> bool: ... - def noTempMemory(self) -> None: ... - def setTempMemory(self, size: int) -> None: ... - def setPinnedMemory(self, size: int) -> None: ... - def setDefaultStream( - self, device: int, stream: Any - ) -> None: ... # cudaStream_t - def revertDefaultStream(self, device: int) -> None: ... - def getDefaultStream(self, device: int) -> Any: ... # cudaStream_t - def setDefaultNullStreamAllDevices(self) -> None: ... - def setLogMemoryAllocations(self, enable: bool) -> None: ... - def initializeForDevice(self, device: int) -> None: ... - def getBlasHandle(self, device: int) -> Any: ... # cublasHandle_t - def getAlternateStreams( - self, device: int - ) -> list[Any]: ... # vector - def allocMemory(self, req: Any) -> Any: ... # AllocRequest -> void* - def deallocMemory(self, device: int, ptr: Any) -> None: ... - def getTempMemoryAvailable(self, device: int) -> int: ... - def getMemoryInfo(self) -> dict[int, dict[str, tuple[int, int]]]: ... - def getPinnedMemory(self) -> tuple[Any, int]: ... # (void*, size_t) - def getAsyncCopyStream(self, device: int) -> Any: ... # cudaStream_t - -class StandardGpuResources: - """Default implementation of GpuResources that allocates a cuBLAS stream and 2 streams for use, as well as temporary memory.""" - - def __init__(self) -> None: ... - def getResources(self) -> Any: ... # shared_ptr - def supportsBFloat16(self, device: int) -> bool: ... - def supportsBFloat16CurrentDevice(self) -> bool: ... - def noTempMemory(self) -> None: ... - def setTempMemory(self, size: int) -> None: ... - def setPinnedMemory(self, size: int) -> None: ... - def setDefaultStream( - self, device: int, stream: Any - ) -> None: ... # cudaStream_t - def revertDefaultStream(self, device: int) -> None: ... - def setDefaultNullStreamAllDevices(self) -> None: ... - def getMemoryInfo(self) -> dict[int, dict[str, tuple[int, int]]]: ... - def getDefaultStream(self, device: int) -> Any: ... # cudaStream_t - def getTempMemoryAvailable(self, device: int) -> int: ... - def syncDefaultStreamCurrentDevice(self) -> None: ... - def setLogMemoryAllocations(self, enable: bool) -> None: ... - -class GpuIndexBinaryFlatConfig(GpuIndexConfig): - def __init__(self) -> None: ... - -class GpuIndexBinaryFlat(IndexBinary): - @overload - def __init__( - self, - resources: StandardGpuResources, - index: IndexBinaryFlat, - config: GpuIndexBinaryFlatConfig = GpuIndexBinaryFlatConfig(), - ) -> None: ... - @overload - def __init__( - self, - resources: StandardGpuResources, - dims: int, - config: GpuIndexBinaryFlatConfig = GpuIndexBinaryFlatConfig(), - ) -> None: ... - def getDevice(self) -> int: ... - def getResources(self) -> StandardGpuResources: ... - def copyFrom(self, index: IndexBinaryFlat) -> None: ... - def copyTo(self, index: IndexBinaryFlat) -> None: ... - @overload - def add(self, x: torch.Tensor) -> None: ... - @overload - def add(self, x: npt.NDArray[np.uint8]) -> None: ... - def reset(self) -> None: ... - @overload - def search( - self, - x: torch.Tensor, - k: int, - *, - params: SearchParameters | None = None, - ) -> tuple[torch.Tensor, torch.Tensor]: ... - @overload - def search( - self, - x: npt.NDArray[np.uint8], - k: int, - *, - params: SearchParameters | None = None, - ) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.int64]]: ... - @overload - def reconstruct(self, key: int) -> torch.Tensor: ... - @overload - def reconstruct(self, key: int) -> npt.NDArray[np.uint8]: ... - -class GpuResourcesVector: - def __init__(self) -> None: ... - def push_back(self, res: StandardGpuResources) -> None: ... - def size(self) -> int: ... - -# GPU Cloner Options (from gpu/GpuClonerOptions.h) -class GpuClonerOptions: - indicesOptions: int # IndicesOptions enum - useFloat16CoarseQuantizer: bool - useFloat16: bool - usePrecomputed: bool - reserveVecs: int - storeTransposed: bool - verbose: bool - use_cuvs: bool - allowCpuCoarseQuantizer: bool - - def __init__(self) -> None: ... - -class GpuMultipleClonerOptions(GpuClonerOptions): - shard: bool - shard_type: int - common_ivf_quantizer: bool - - def __init__(self) -> None: ... - -# GPU Cloner classes (from gpu/GpuCloner.h) -class ToCPUCloner(Cloner): - def merge_index( - self, dst: Index, src: Index, successive_ids: bool - ) -> None: ... - def clone_Index(self, index: Index) -> Index: ... - -class ToGpuCloner(Cloner, GpuClonerOptions): - provider: GpuResourcesProvider - device: int - - def __init__( - self, - provider: GpuResourcesProvider, - device: int, - options: GpuClonerOptions, - ) -> None: ... - def clone_Index(self, index: Index) -> Index: ... - -class ToGpuClonerMultiple(Cloner, GpuMultipleClonerOptions): - sub_cloners: list[ToGpuCloner] - - @overload - def __init__( - self, - providers: list[GpuResourcesProvider], - devices: list[int], - options: GpuMultipleClonerOptions, - ) -> None: ... - @overload - def __init__( - self, - sub_cloners: list[ToGpuCloner], - options: GpuMultipleClonerOptions, - ) -> None: ... - def copy_ivf_shard( - self, - index_ivf: IndexIVF, - idx2: IndexIVF, - n: int, - i: int, - ) -> None: ... - def clone_Index_to_shards(self, index: Index) -> Index: ... - def clone_Index(self, index: Index) -> Index: ... - -class GpuProgressiveDimIndexFactory(ProgressiveDimIndexFactory): - options: GpuMultipleClonerOptions - vres: list[GpuResourcesProvider] - devices: list[int] - ncall: int - - def __init__(self, ngpu: int) -> None: ... - def __call__(self, dim: int) -> Index: ... - -# GPU Cloner functions (from gpu/GpuCloner.h) -def index_gpu_to_cpu(gpu_index: Index) -> Index: ... -def index_binary_gpu_to_cpu(gpu_index: IndexBinary) -> IndexBinary: ... -def index_binary_cpu_to_gpu( - provider: GpuResourcesProvider, - device: int, - index: IndexBinary, - options: GpuClonerOptions | None = None, -) -> IndexBinary: ... - -# GPU distance data types -DistanceDataType_F32: int -DistanceDataType_F16: int - -# GPU indices data types -IndicesDataType_I64: int -IndicesDataType_I32: int - -# GPU functions -def bfKnn(res: StandardGpuResources, params: GpuDistanceParams) -> None: ... -def bfKnn_tiling( - res: StandardGpuResources, - params: GpuDistanceParams, - vectorsMemoryLimit: int, - queriesMemoryLimit: int, -) -> None: ... -def index_cpu_to_gpu( - provider: StandardGpuResources, - device: int, - index: Index, - options: GpuClonerOptions | None = None, -) -> GpuIndex: ... -def index_cpu_to_gpu_multiple( - resources: GpuResourcesVector, - devices: Int32Vector, - index: Index, - co: GpuMultipleClonerOptions | None = None, -) -> GpuIndex: ... -def index_binary_cpu_to_gpu_multiple( - resources: GpuResourcesVector, - devices: Int32Vector, - index: IndexBinary, - co: GpuMultipleClonerOptions | None = None, -) -> IndexBinary: ... -def index_cpu_to_gpu_multiple_py( - resources: list[StandardGpuResources], - index: Index, - co: GpuMultipleClonerOptions | None = None, - gpus: list[int] | None = None, -) -> GpuIndex: ... -def index_cpu_to_all_gpus( - index: Index, co: GpuClonerOptions | None = None, ngpu: int = -1 -) -> GpuIndex: ... -def index_cpu_to_gpus_list( - index: Index, - co: GpuClonerOptions | None = None, - gpus: list[int] | None = None, - ngpu: int = -1, -) -> GpuIndex: ... - -class GpuIndexConfig: - device: int - memorySpace: Any # MemorySpace enum - use_cuvs: bool - - def __init__(self) -> None: ... - -class GpuIndexFlatConfig(GpuIndexConfig): - useFloat16: bool - - def __init__(self) -> None: ... - -class GpuIndexFlat(Index): - def __init__( - self, - provider: StandardGpuResources, - d: int, - metric: MetricType = METRIC_L2, - config: GpuIndexFlatConfig | None = None, - ) -> None: ... - -class GpuIndexFlatL2(GpuIndexFlat): - @overload - def __init__( - self, - provider: StandardGpuResources, - d: int, - config: GpuIndexFlatConfig | None = None, - ) -> None: ... - @overload - def __init__( - self, - provider: StandardGpuResources, - index: IndexFlatL2, - config: GpuIndexFlatConfig | None = None, - ) -> None: ... - -class GpuIndexFlatIP(GpuIndexFlat): - @overload - def __init__( - self, - provider: StandardGpuResources, - d: int, - config: GpuIndexFlatConfig | None = None, - ) -> None: ... - @overload - def __init__( - self, - provider: StandardGpuResources, - index: Any, # IndexFlatIP* - config: GpuIndexFlatConfig | None = None, - ) -> None: ... - -# GPU functions with comprehensive overloaded signatures for tensor support - -# knn_gpu overloads: Precise return types based on input types -@overload -def knn_gpu( - res: StandardGpuResources, - xq: torch.Tensor, - xb: torch.Tensor, - k: int, - D: torch.Tensor | None = None, - I: torch.Tensor | None = None, - metric: MetricType = METRIC_L2, - device: int = -1, - use_cuvs: bool = False, - vectorsMemoryLimit: int = 0, - queriesMemoryLimit: int = 0, -) -> tuple[torch.Tensor, torch.Tensor]: ... -@overload -def knn_gpu( - res: StandardGpuResources, - xq: npt.NDArray[np.float32], - xb: npt.NDArray[np.float32], - k: int, - D: npt.NDArray[np.float32] | None = None, - I: npt.NDArray[np.int64] | None = None, - metric: MetricType = METRIC_L2, - device: int = -1, - use_cuvs: bool = False, - vectorsMemoryLimit: int = 0, - queriesMemoryLimit: int = 0, -) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... -@overload -def knn_gpu( - res: StandardGpuResources, - xq: torch.Tensor, - xb: npt.NDArray[np.float32], - k: int, - D: torch.Tensor | npt.NDArray[np.float32] | None = None, - I: torch.Tensor | npt.NDArray[np.int64] | None = None, - metric: MetricType = METRIC_L2, - device: int = -1, - use_cuvs: bool = False, - vectorsMemoryLimit: int = 0, - queriesMemoryLimit: int = 0, -) -> tuple[torch.Tensor, torch.Tensor]: ... -@overload -def knn_gpu( - res: StandardGpuResources, - xq: npt.NDArray[np.float32], - xb: torch.Tensor, - k: int, - D: torch.Tensor | npt.NDArray[np.float32] | None = None, - I: torch.Tensor | npt.NDArray[np.int64] | None = None, - metric: MetricType = METRIC_L2, - device: int = -1, - use_cuvs: bool = False, - vectorsMemoryLimit: int = 0, - queriesMemoryLimit: int = 0, -) -> tuple[torch.Tensor, torch.Tensor]: ... - -# pairwise_distance_gpu overloads: Precise return types based on input types -@overload -def pairwise_distance_gpu( - res: StandardGpuResources, - xq: torch.Tensor, - xb: torch.Tensor, - D: torch.Tensor | None = None, - metric: MetricType = METRIC_L2, - device: int = -1, -) -> torch.Tensor: ... -@overload -def pairwise_distance_gpu( - res: StandardGpuResources, - xq: npt.NDArray[np.float32], - xb: npt.NDArray[np.float32], - D: npt.NDArray[np.float32] | None = None, - metric: MetricType = METRIC_L2, - device: int = -1, -) -> npt.NDArray[np.float32]: ... -@overload -def pairwise_distance_gpu( - res: StandardGpuResources, - xq: torch.Tensor, - xb: npt.NDArray[np.float32], - D: torch.Tensor | npt.NDArray[np.float32] | None = None, - metric: MetricType = METRIC_L2, - device: int = -1, -) -> torch.Tensor: ... -@overload -def pairwise_distance_gpu( - res: StandardGpuResources, - xq: npt.NDArray[np.float32], - xb: torch.Tensor, - D: torch.Tensor | npt.NDArray[np.float32] | None = None, - metric: MetricType = METRIC_L2, - device: int = -1, -) -> torch.Tensor: ... - -# Additional utility functions with tensor support - -# Utility functions from extra_wrappers.py with tensor overloads -@overload -def kmin(array: torch.Tensor, k: int) -> tuple[torch.Tensor, torch.Tensor]: ... -@overload -def kmin( - array: npt.NDArray[np.float32], k: int -) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... -@overload -def kmax(array: torch.Tensor, k: int) -> tuple[torch.Tensor, torch.Tensor]: ... -@overload -def kmax( - array: npt.NDArray[np.float32], k: int -) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... -@overload -def pairwise_distances( - xq: torch.Tensor, - xb: torch.Tensor, - metric: MetricType = METRIC_L2, - metric_arg: float = 0, -) -> torch.Tensor: ... -@overload -def pairwise_distances( - xq: npt.NDArray[np.float32], - xb: npt.NDArray[np.float32], - metric: MetricType = METRIC_L2, - metric_arg: float = 0, -) -> npt.NDArray[np.float32]: ... -def rand( - n: int | tuple[int, ...] | list[int], seed: int = 12345 -) -> npt.NDArray[np.float32]: ... -def randint( - n: int, seed: int = 12345, vmax: int | None = None -) -> npt.NDArray[np.int64]: ... - -# Alias for randint -lrand = randint - -def randn( - n: int | tuple[int, ...] | list[int], seed: int = 12345 -) -> npt.NDArray[np.float32]: ... -@overload -def checksum(a: torch.Tensor) -> Any: ... -@overload -def checksum(a: npt.NDArray[Any]) -> Any: ... -def rand_smooth_vectors( - n: int, d: int, seed: int = 1234 -) -> npt.NDArray[np.float32]: ... -@overload -def eval_intersection(I1: torch.Tensor, I2: torch.Tensor) -> int: ... -@overload -def eval_intersection( - I1: npt.NDArray[np.int64], I2: npt.NDArray[np.int64] -) -> int: ... -@overload -def normalize_L2(x: torch.Tensor) -> None: ... -@overload -def normalize_L2(x: npt.NDArray[np.float32]) -> None: ... -@overload -def bucket_sort( - tab: torch.Tensor, nbucket: int | None = None, nt: int = 0 -) -> tuple[torch.Tensor, torch.Tensor]: ... -@overload -def bucket_sort( - tab: npt.NDArray[np.int64], nbucket: int | None = None, nt: int = 0 -) -> tuple[npt.NDArray[np.int64], npt.NDArray[np.int64]]: ... -@overload -def matrix_bucket_sort_inplace( - tab: torch.Tensor, nbucket: int | None = None, nt: int = 0 -) -> torch.Tensor: ... -@overload -def matrix_bucket_sort_inplace( - tab: npt.NDArray[np.int32] | npt.NDArray[np.int64], - nbucket: int | None = None, - nt: int = 0, -) -> npt.NDArray[np.int64]: ... -@overload -def knn( - xq: torch.Tensor, - xb: torch.Tensor, - k: int, - metric: MetricType = METRIC_L2, - metric_arg: float = 0.0, -) -> tuple[torch.Tensor, torch.Tensor]: ... -@overload -def knn( - xq: npt.NDArray[np.float32], - xb: npt.NDArray[np.float32], - k: int, - metric: MetricType = METRIC_L2, - metric_arg: float = 0.0, -) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... -@overload -def knn_hamming( - xq: torch.Tensor, xb: torch.Tensor, k: int, variant: str = "hc" -) -> tuple[torch.Tensor, torch.Tensor]: ... -@overload -def knn_hamming( - xq: npt.NDArray[np.uint8], - xb: npt.NDArray[np.uint8], - k: int, - variant: str = "hc", -) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.int64]]: ... -@overload -def merge_knn_results( - Dall: torch.Tensor, Iall: torch.Tensor, keep_max: bool = False -) -> tuple[torch.Tensor, torch.Tensor]: ... -@overload -def merge_knn_results( - Dall: npt.NDArray[np.float32], - Iall: npt.NDArray[np.int64], - keep_max: bool = False, -) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... - -# Pack/unpack bitstring functions from extra_wrappers.py -@overload -def pack_bitstrings( - a: npt.NDArray[np.int32], nbit: int -) -> npt.NDArray[np.uint8]: ... -@overload -def pack_bitstrings( - a: npt.NDArray[np.int32], nbit: npt.NDArray[np.int32] -) -> npt.NDArray[np.uint8]: ... -@overload -def unpack_bitstrings( - b: npt.NDArray[np.uint8], M: int, nbit: int -) -> npt.NDArray[np.int32]: ... -@overload -def unpack_bitstrings( - b: npt.NDArray[np.uint8], nbits: npt.NDArray[np.int32] -) -> npt.NDArray[np.int32]: ... - -# Array conversion utilities with tensor support -@overload -def vector_to_array(v: Any) -> npt.NDArray[Any]: ... -@overload -def vector_float_to_array(v: Float32Vector) -> npt.NDArray[np.float32]: ... -@overload -def copy_array_to_vector(a: torch.Tensor, v: Any) -> None: ... -@overload -def copy_array_to_vector(a: npt.NDArray[Any], v: Any) -> None: ... -@overload -def copy_array_to_AlignedTable(a: torch.Tensor, v: Any) -> None: ... -@overload -def copy_array_to_AlignedTable(a: npt.NDArray[Any], v: Any) -> None: ... -@overload -def array_to_AlignedTable(a: torch.Tensor) -> Any: ... -@overload -def array_to_AlignedTable(a: npt.NDArray[Any]) -> Any: ... -@overload -def AlignedTable_to_array(v: Any) -> npt.NDArray[Any]: ... - -# AlignedTable types -class AlignedTableUint8: - def __init__(self, n: int) -> None: ... - def size(self) -> int: ... - def itemsize(self) -> int: ... - def get(self) -> Any: ... - def data(self) -> Any: ... - def resize(self, n: int) -> None: ... - -class AlignedTableUint16: - def __init__(self, n: int) -> None: ... - def size(self) -> int: ... - def itemsize(self) -> int: ... - def get(self) -> Any: ... - def data(self) -> Any: ... - def resize(self, n: int) -> None: ... - -class AlignedTableFloat32: - def __init__(self, n: int) -> None: ... - def size(self) -> int: ... - def itemsize(self) -> int: ... - def get(self) -> Any: ... - def data(self) -> Any: ... - def resize(self, n: int) -> None: ... - -# MaybeOwnedVector types -class MaybeOwnedVectorUInt8: - is_owned: bool - - def __init__(self) -> None: ... - def size(self) -> int: ... - def data(self) -> Any: ... - def resize(self, n: int) -> None: ... - -class MaybeOwnedVectorInt32: - is_owned: bool - - def __init__(self) -> None: ... - def size(self) -> int: ... - def data(self) -> Any: ... - def resize(self, n: int) -> None: ... - -class MaybeOwnedVectorFloat32: - is_owned: bool - - def __init__(self) -> None: ... - def size(self) -> int: ... - def data(self) -> Any: ... - def resize(self, n: int) -> None: ... - -# Memory utilities -def memcpy(dest: Any, src: Any, n: int) -> Any: ... - -# Heap implementations -class float_minheap_array_t: - k: int - nh: int - val: Any - ids: Any - - def __init__(self) -> None: ... - def heapify(self) -> None: ... - def addn(self, n: int, vals: Any) -> None: ... - def addn_with_ids( - self, n: int, vals: Any, ids: Any, id_stride: int - ) -> None: ... - def addn_query_subset_with_ids( - self, - nsubset: int, - subset: Any, - n: int, - vals: Any, - ids: Any, - id_stride: int, - ) -> None: ... - def reorder(self) -> None: ... - -class float_maxheap_array_t: - k: int - nh: int - val: Any - ids: Any - - def __init__(self) -> None: ... - def heapify(self) -> None: ... - def addn(self, n: int, vals: Any) -> None: ... - def addn_with_ids( - self, n: int, vals: Any, ids: Any, id_stride: int - ) -> None: ... - def addn_query_subset_with_ids( - self, - nsubset: int, - subset: Any, - n: int, - vals: Any, - ids: Any, - id_stride: int, - ) -> None: ... - def reorder(self) -> None: ... - -class int_minheap_array_t: - k: int - nh: int - val: Any - ids: Any - - def __init__(self) -> None: ... - def heapify(self) -> None: ... - def addn(self, n: int, vals: Any) -> None: ... - def reorder(self) -> None: ... - -class int_maxheap_array_t: - k: int - nh: int - val: Any - ids: Any - - def __init__(self) -> None: ... - def heapify(self) -> None: ... - def addn(self, n: int, vals: Any) -> None: ... - def reorder(self) -> None: ... - -# Hamming distance functions -def hammings_knn_hc( - heap: int_maxheap_array_t, - xq: npt.NDArray[np.uint8], - xb: npt.NDArray[np.uint8], - nb: int, - ncodes: int, - ordered: int, -) -> None: ... -def hammings_knn_mc( - xq: npt.NDArray[np.uint8], - xb: npt.NDArray[np.uint8], - nq: int, - nb: int, - k: int, - ncodes: int, - distances: npt.NDArray[np.int32], - labels: npt.NDArray[np.int64], -) -> None: ... - -# Supported instruction sets utility -def supported_instruction_sets() -> set[str]: ... - -# Additional merge functions for different types -def merge_knn_results_CMin( - n: int, - k: int, - nshard: int, - all_distances: Any, - all_labels: Any, - distances: Any, - labels: Any, -) -> None: ... -def merge_knn_results_CMax( - n: int, - k: int, - nshard: int, - all_distances: Any, - all_labels: Any, - distances: Any, - labels: Any, -) -> None: ... - -# Efficient ID to ID map class from extra_wrappers.py -class MapInt64ToInt64: - log2_capacity: int - capacity: int - tab: npt.NDArray[np.int64] - - def __init__(self, capacity: int) -> None: ... - def add( - self, keys: npt.NDArray[np.int64], vals: npt.NDArray[np.int64] - ) -> None: ... - def lookup(self, keys: npt.NDArray[np.int64]) -> npt.NDArray[np.int64]: ... - -# Additional hash table functions -def hashtable_int64_to_int64_init(log2_capacity: int, tab: Any) -> None: ... -def hashtable_int64_to_int64_add( - log2_capacity: int, tab: Any, n: int, keys: Any, vals: Any -) -> None: ... -def hashtable_int64_to_int64_lookup( - log2_capacity: int, tab: Any, n: int, keys: Any, vals: Any -) -> None: ... - -# ResultHeap utility class from extra_wrappers.py -class ResultHeap: - I: npt.NDArray[np.int64] - D: npt.NDArray[np.float32] - nq: int - k: int - heaps: Any - - def __init__(self, nq: int, k: int, keep_max: bool = False) -> None: ... - @overload - def add_result(self, D: torch.Tensor, I: torch.Tensor) -> None: ... - @overload - def add_result( - self, D: npt.NDArray[np.float32], I: npt.NDArray[np.int64] - ) -> None: ... - @overload - def add_result_subset( - self, - subset: torch.Tensor, - D: torch.Tensor, - I: torch.Tensor, - ) -> None: ... - @overload - def add_result_subset( - self, - subset: npt.NDArray[np.int64], - D: npt.NDArray[np.float32], - I: npt.NDArray[np.int64], - ) -> None: ... - def finalize(self) -> None: ... - -# Kmeans utility class -class Kmeans: - d: int - k: int - centroids: npt.NDArray[np.float32] | None - obj: npt.NDArray[np.float32] | None - iteration_stats: list[dict[str, Any]] | None - cp: ClusteringParameters - index: Index - gpu: Any - fac: Any - - def __init__(self, d: int, k: int, **kwargs: Any) -> None: ... - def set_index(self) -> None: ... - def reset(self, k: int | None = None) -> None: ... - def train( - self, - x: npt.NDArray[np.float32], - weights: npt.NDArray[np.float32] | None = None, - init_centroids: npt.NDArray[np.float32] | None = None, - ) -> float: ... - def assign( - self, x: npt.NDArray[np.float32] - ) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.int64]]: ... diff --git a/bundle/python-cpu/Lib/site-packages/faiss/_swigfaiss.pyd b/bundle/python-cpu/Lib/site-packages/faiss/_swigfaiss.pyd deleted file mode 100644 index 56e99df665cd5a5e51968ad74a8db7eea89131d2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/_swigfaiss.pyd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:63696e976bb360a01b01b7475b3ed410f6933169e2d83072257a5728dab5c120 -size 5585920 diff --git a/bundle/python-cpu/Lib/site-packages/faiss/array_conversions.py b/bundle/python-cpu/Lib/site-packages/faiss/array_conversions.py deleted file mode 100644 index 14d5d3498d09025dea0bc061a21f931e747130c0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/array_conversions.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# @nolint - -# not linting this file because it imports * from swigfaiss, which -# causes a ton of useless warnings. - -import numpy as np -import array -import warnings - -from faiss.loader import * - -########################################### -# Utility to add a deprecation warning to -# classes from the SWIG interface -########################################### - - -def _make_deprecated_swig_class(deprecated_name, base_name): - """ - Dynamically construct deprecated classes as wrappers around renamed ones - - The deprecation warning added in their __new__-method will trigger upon - construction of an instance of the class, but only once per session. - - We do this here (in __init__.py) because the base classes are defined in - the SWIG interface, making it cumbersome to add the deprecation there. - - Parameters - ---------- - deprecated_name : string - Name of the class to be deprecated; _not_ present in SWIG interface. - base_name : string - Name of the class that is replacing deprecated_name; must already be - imported into the current namespace. - - Returns - ------- - None - However, the deprecated class gets added to the faiss namespace - """ - base_class = globals()[base_name] - - def new_meth(cls, *args, **kwargs): - msg = ( - f"The class faiss.{deprecated_name} is deprecated in favour of " - f"faiss.{base_name}!" - ) - warnings.warn(msg, DeprecationWarning, stacklevel=2) - instance = super(base_class, cls).__new__(cls, *args, **kwargs) - return instance - - # three-argument version of "type" uses (name, tuple-of-bases, - # dict-of-attributes) - klazz = type(deprecated_name, (base_class,), {"__new__": new_meth}) - - # this ends up adding the class to the "faiss" namespace, in a way that it - # is available both through "import faiss" and "from faiss import *" - globals()[deprecated_name] = klazz - - -########################################### -# numpy array / std::vector conversions -########################################### - -sizeof_long = array.array("l").itemsize -deprecated_name_map = { - # deprecated: replacement - "Float": "Float32", - "Double": "Float64", - "Char": "Int8", - "Int": "Int32", - "Long": "Int32" if sizeof_long == 4 else "Int64", - "LongLong": "Int64", - "Byte": "UInt8", - # previously misspelled variant - "Uint64": "UInt64", -} - -for depr_prefix, base_prefix in deprecated_name_map.items(): - _make_deprecated_swig_class(depr_prefix + "Vector", base_prefix + "Vector") - - # same for the three legacy *VectorVector classes - if depr_prefix in ["Float", "Long", "Byte"]: - _make_deprecated_swig_class( - depr_prefix + "VectorVector", base_prefix + "VectorVector" - ) - -# mapping from vector names in swigfaiss.swig and the numpy dtype names -# TODO: once deprecated classes are removed, remove the dict and just use -# .lower() below -vector_name_map = { - "Float32": "float32", - "Float64": "float64", - "Int8": "int8", - "Int16": "int16", - "Int32": "int32", - "Int64": "int64", - "UInt8": "uint8", - "UInt16": "uint16", - "UInt32": "uint32", - "UInt64": "uint64", - **{k: v.lower() for k, v in deprecated_name_map.items()}, -} - - -def vector_to_array(v): - """convert a C++ vector to a numpy array""" - classname = v.__class__.__name__ - if classname.startswith("AlignedTable"): - return AlignedTable_to_array(v) - if classname.startswith("MaybeOwnedVector"): - dtype = np.dtype(vector_name_map[classname[16:]]) - a = np.empty(v.size(), dtype=dtype) - if v.size() > 0: - memcpy(swig_ptr(a), v.data(), a.nbytes) - return a - - assert classname.endswith("Vector") - dtype = np.dtype(vector_name_map[classname[:-6]]) - a = np.empty(v.size(), dtype=dtype) - if v.size() > 0: - memcpy(swig_ptr(a), v.data(), a.nbytes) - return a - - -def vector_float_to_array(v): - return vector_to_array(v) - - -def copy_array_to_vector(a, v): - """copy a numpy array to a vector""" - (n,) = a.shape - classname = v.__class__.__name__ - if classname.startswith("MaybeOwnedVector"): - assert v.is_owned, "cannot copy to an non-owned MaybeOwnedVector" - dtype = np.dtype(vector_name_map[classname[16:]]) - assert ( - dtype == a.dtype - ), "cannot copy a %s array to a %s (should be %s)" % ( - a.dtype, - classname, - dtype, - ) - v.resize(n) - if n > 0: - memcpy(v.data(), swig_ptr(a), a.nbytes) - return - - assert classname.endswith("Vector") - dtype = np.dtype(vector_name_map[classname[:-6]]) - assert dtype == a.dtype, "cannot copy a %s array to a %s (should be %s)" % ( - a.dtype, - classname, - dtype, - ) - v.resize(n) - if n > 0: - memcpy(v.data(), swig_ptr(a), a.nbytes) - - -# same for AlignedTable - - -def copy_array_to_AlignedTable(a, v): - (n,) = a.shape - # TODO check class name - assert v.itemsize() == a.itemsize - v.resize(n) - if n > 0: - memcpy(v.get(), swig_ptr(a), a.nbytes) - - -def array_to_AlignedTable(a): - if a.dtype == "uint16": - v = AlignedTableUint16(a.size) - elif a.dtype == "uint8": - v = AlignedTableUint8(a.size) - else: - assert False - copy_array_to_AlignedTable(a, v) - return v - - -def AlignedTable_to_array(v): - """convert an AlignedTable to a numpy array""" - classname = v.__class__.__name__ - assert classname.startswith("AlignedTable") - dtype = classname[12:].lower() - a = np.empty(v.size(), dtype=dtype) - if a.size > 0: - memcpy(swig_ptr(a), v.data(), a.nbytes) - return a diff --git a/bundle/python-cpu/Lib/site-packages/faiss/class_wrappers.py b/bundle/python-cpu/Lib/site-packages/faiss/class_wrappers.py deleted file mode 100644 index b2c0bc6247afb71d105094b44aa1cdb6208c88fc..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/class_wrappers.py +++ /dev/null @@ -1,1675 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import faiss -import numpy as np - -from faiss.loader import ( - DirectMap, - IDSelector, - IDSelectorArray, - IDSelectorBatch, - OperatingPoints, - RangeSearchResult, - rev_swig_ptr, - swig_ptr, - try_extract_index_ivf, -) - -################################################################## -# The functions below add or replace some methods for classes -# this is to be able to pass in numpy arrays directly -# The C++ version of the classnames will be suffixed with _c -# -# The docstrings in the wrappers are intended to be similar to numpy -# comments, they will appear with help(Class.method) or ?Class.method -# For methods that are not replaced, the C++ documentation will be used if -# swig 4.x is run with -doxygen. -################################################################## - -# For most arrays we force the convesion to the target type with -# np.ascontiguousarray, but for uint8 codes, we raise a type error -# because it is unclear how the conversion should occur: with a view -# (= cast) or conversion? - - -def _check_dtype_uint8(codes): - if codes.dtype != "uint8": - raise TypeError( - "Input argument %s must be ndarray of dtype " - " uint8, but found %s" % ("codes", codes.dtype) - ) - return np.ascontiguousarray(codes) - - -def _numeric_to_str(numeric_type): - if numeric_type == faiss.Float32: - return "float32" - elif numeric_type == faiss.Float16: - return "float16" - elif numeric_type == faiss.Int8: - return "int8" - else: - raise ValueError( - "numeric type must be either faiss.Float32, faiss.Float16, " - "or faiss.Int8" - ) - - -def replace_method(the_class, name, replacement, ignore_missing=False): - """Replaces a method in a class with another version. The old method - is renamed to method_name_c (because presumably it was implemented in C)""" - try: - orig_method = getattr(the_class, name) - except AttributeError: - if ignore_missing: - return - raise - if orig_method.__name__ == "replacement_" + name: - # replacement was done in parent class - return - setattr(the_class, name + "_c", orig_method) - setattr(the_class, name, replacement) - - -def handle_Clustering(the_class): - - def replacement_train(self, x, index, weights=None): - """Perform clustering on a set of vectors. The index is used for - assignment. - - Parameters - ---------- - x : array_like - Training vectors, shape (n, self.d). `dtype` must be float32. - index : faiss.Index - Index used for assignment. The dimension of the index - should be `self.d`. - weights : array_like, optional - Per training sample weight (size n) used when computing - the weighted average to obtain the centroid (default is - 1 for all training vectors). - """ - n, d = x.shape - x = np.ascontiguousarray(x, dtype="float32") - assert d == self.d - if weights is not None: - weights = np.ascontiguousarray(weights, dtype="float32") - assert weights.shape == (n,) - self.train_c(n, swig_ptr(x), index, swig_ptr(weights)) - else: - self.train_c(n, swig_ptr(x), index) - - def replacement_train_encoded(self, x, codec, index, weights=None): - """Perform clustering on a set of compressed vectors. The index is - used for assignment. - The decompression is performed on-the-fly. - - Parameters - ---------- - x : array_like - Training vectors, shape (n, codec.code_size()). `dtype` must - be `uint8`. - codec : faiss.Index - Index used to decode the vectors. Should have dimension `self.d`. - index : faiss.Index - Index used for assignment. The dimension of the index - should be `self.d`. - weights : array_like, optional - Per training sample weight (size n) used when computing - the weighted average to obtain the centroid (default is - 1 for all training vectors). - """ - n, d = x.shape - x = _check_dtype_uint8(x) - assert d == codec.sa_code_size() - assert codec.d == index.d - if weights is not None: - weights = np.ascontiguousarray(weights, dtype="float32") - assert weights.shape == (n,) - self.train_encoded_c( - n, swig_ptr(x), codec, index, swig_ptr(weights) - ) - else: - self.train_encoded_c(n, swig_ptr(x), codec, index) - - replace_method(the_class, "train", replacement_train) - replace_method(the_class, "train_encoded", replacement_train_encoded) - - -def handle_SuperKMeans(the_class): - - def replacement_train(self, x): - """Perform SuperKMeans clustering on a set of vectors. - - Parameters - ---------- - x : array_like - Training vectors, shape (n, self.d). `dtype` must be float32. - """ - n, d = x.shape - assert d == self.d - x = np.ascontiguousarray(x, dtype="float32") - self.train_c(n, swig_ptr(x)) - - replace_method(the_class, "train", replacement_train) - - -def handle_Clustering1D(the_class): - - def replacement_train_exact(self, x): - """Perform clustering on a set of 1D vectors. - - Parameters - ---------- - x : array_like - Training vectors, shape (n, 1). `dtype` must be float32. - """ - n, d = x.shape - x = np.ascontiguousarray(x, dtype="float32") - assert d == self.d - self.train_exact_c(n, swig_ptr(x)) - - replace_method(the_class, "train_exact", replacement_train_exact) - - -def handle_Quantizer(the_class): - - def replacement_train(self, x): - """Train the quantizer on a set of training vectors. - - Parameters - ---------- - x : array_like - Training vectors, shape (n, self.d). `dtype` must be float32. - """ - n, d = x.shape - x = np.ascontiguousarray(x, dtype="float32") - assert d == self.d - self.train_c(n, swig_ptr(x)) - - def replacement_compute_codes(self, x): - """Compute the codes corresponding to a set of vectors. - - Parameters - ---------- - x : array_like - Vectors to encode, shape (n, self.d). `dtype` must be float32. - - Returns - ------- - codes : array_like - Corresponding code for each vector, shape (n, self.code_size) - and `dtype` uint8. - """ - n, d = x.shape - x = np.ascontiguousarray(x, dtype="float32") - assert d == self.d - codes = np.empty((n, self.code_size), dtype="uint8") - self.compute_codes_c(swig_ptr(x), swig_ptr(codes), n) - return codes - - def replacement_decode(self, codes): - """Reconstruct an approximation of vectors given their codes. - - Parameters - ---------- - codes : array_like - Codes to decode, shape (n, self.code_size). `dtype` must be uint8. - - Returns - ------- - Reconstructed vectors for each code, shape `(n, d)` and - `dtype` float32. - """ - n, cs = codes.shape - codes = _check_dtype_uint8(codes) - assert cs == self.code_size - x = np.empty((n, self.d), dtype="float32") - self.decode_c(swig_ptr(codes), swig_ptr(x), n) - return x - - replace_method(the_class, "train", replacement_train) - replace_method(the_class, "compute_codes", replacement_compute_codes) - replace_method(the_class, "decode", replacement_decode) - - -def handle_NSG(the_class): - - def replacement_build(self, x, graph): - n, d = x.shape - assert d == self.d - assert graph.ndim == 2 - assert graph.shape[0] == n - K = graph.shape[1] - x = np.ascontiguousarray(x, dtype="float32") - graph = np.ascontiguousarray(graph, dtype="int64") - self.build_c(n, swig_ptr(x), swig_ptr(graph), K) - - replace_method(the_class, "build", replacement_build) - - -def handle_Index(the_class): - - def replacement_setattr(self, name, value): - # Prevent silent failures when setting attributes that don't exist - # as described in GitHub issue 3766 - - # Allow SWIG internal attributes that are essential for object - # functionality - if name in ["this", "thisown"]: - return original_setattr(self, name, value) - - # Allow internal Faiss attributes used during construction/operation - if name in ["referenced_objects"]: - return original_setattr(self, name, value) - - # Check if the attribute already exists (valid attribute) - try: - # Check if it exists on the instance or class - if hasattr(self, name) or hasattr(self.__class__, name): - return original_setattr(self, name, value) - except (AttributeError, TypeError, SystemError): - # During object construction, hasattr might fail, so be permissive - return original_setattr(self, name, value) - - # If we reach here, the attribute doesn't exist on the object - # This is the core issue: SWIG classes silently accept unknown - # attributes - # We should generally block this to prevent silent failures - - # Block unknown attributes to prevent silent failures - # This is the general solution that doesn't rely on hardcoded names - raise AttributeError( - f"'{self.__class__.__name__}' object has no attribute '{name}'." - ) - - def replacement_add(self, x, numeric_type=faiss.Float32): - """Adds vectors to the index. - The index must be trained before vectors can be added to it. - The vectors are implicitly numbered in sequence. When `n` - vectors are added to the index, they are given ids `ntotal`, - `ntotal + 1`, ..., `ntotal + n - 1`. - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate for the index. - `dtype` must be float32. - """ - - n, d = x.shape - assert d == self.d - x = np.ascontiguousarray(x, dtype=_numeric_to_str(numeric_type)) - if numeric_type == faiss.Float32: - self.add_c(n, swig_ptr(x)) - else: - self.add_ex(n, swig_ptr(x), numeric_type) - - def replacement_add_with_ids(self, x, ids, numeric_type=faiss.Float32): - """Adds vectors with arbitrary ids to the index (not all indexes - support this). - The index must be trained before vectors can be added to it. - Vector `i` is stored in `x[i]` and has id `ids[i]`. - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate for the index. - `dtype` must be float32. - ids : array_like - Array if ids of size n. The ids must be of type `int64`. - Note that `-1` is reserved in result lists to mean "not - found" so it's better to not use it as an id. - """ - n, d = x.shape - assert d == self.d - assert ids.shape == (n,), "not same nb of vectors as ids" - x = np.ascontiguousarray(x, dtype=_numeric_to_str(numeric_type)) - ids = np.ascontiguousarray(ids, dtype="int64") - if numeric_type == faiss.Float32: - self.add_with_ids_c(n, swig_ptr(x), swig_ptr(ids)) - else: - self.add_with_ids_ex(n, swig_ptr(x), numeric_type, swig_ptr(ids)) - - def replacement_assign(self, x, k, labels=None): - """Find the k nearest neighbors of the set of vectors x in the index. - This is the same as the `search` method, but discards the distances. - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate for the index. - `dtype` must be float32. - k : int - Number of nearest neighbors. - labels : array_like, optional - Labels array to store the results. - - Returns - ------- - labels: array_like - Labels of the nearest neighbors, shape (n, k). - When not enough results are found, the label is set to -1 - """ - n, d = x.shape - assert d == self.d - x = np.ascontiguousarray(x, dtype="float32") - - if labels is None: - labels = np.empty((n, k), dtype=np.int64) - else: - assert labels.shape == (n, k) - - self.assign_c(n, swig_ptr(x), swig_ptr(labels), k) - return labels - - def replacement_train( - self, x, *, numeric_type=faiss.Float32, xq_train=None - ): - """Trains the index on a representative set of vectors. - The index must be trained before vectors can be added to it. - Optionally accepts numeric_type to specify the type of - input vectors. - Optionally accepts a set of training query vectors for - out-of-distribution training. - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate - for the index. `dtype` must be float32. - numeric_type : type - Numeric type of the input vectors. - xq_train : array_like, optional - Training query vectors, shape (n_train_q, d) where - d is appropriate for the index. - `dtype` must be float32. - """ - # Prepare training data - n, d = x.shape - assert d == self.d - x = np.ascontiguousarray(x, dtype=_numeric_to_str(numeric_type)) - - # Prepare training queries if provided - n_train_q, train_q = 0, None - if xq_train is not None: - if numeric_type != faiss.Float32: - raise TypeError( - "xq_train is only supported for numeric_type faiss.Float32" - ) - n_train_q, d_train = xq_train.shape - assert d_train == self.d - train_q = swig_ptr( - np.ascontiguousarray( - xq_train, - dtype=_numeric_to_str(numeric_type), - ) - ) - - # Dispatch to train_c / train_with_queries / train_ex - if numeric_type == faiss.Float32: - if train_q is not None: - self.train_with_queries(n, swig_ptr(x), n_train_q, train_q) - else: - self.train_c(n, swig_ptr(x)) - else: - self.train_ex(n, swig_ptr(x), numeric_type) - - def replacement_search( - self, x, k, *, params=None, D=None, I=None, numeric_type=faiss.Float32 - ): - """Find the k nearest neighbors of the set of vectors x in the index. - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate for the index. - `dtype` must be float32. - k : int - Number of nearest neighbors. - params : SearchParameters - Search parameters of the current search (overrides the - class-level params) - D : array_like, optional - Distance array to store the result. - I : array_like, optional - Labels array to store the results. - - Returns - ------- - D : array_like - Distances of the nearest neighbors, shape (n, k). When - not enough results are found the label is set to +Inf or - -Inf. - I : array_like - Labels of the nearest neighbors, shape (n, k). - When not enough results are found, the label is set to -1 - """ - - n, d = x.shape - x = np.ascontiguousarray(x, _numeric_to_str(numeric_type)) - assert d == self.d - - assert k > 0 - - if D is None: - D = np.empty((n, k), dtype=np.float32) - else: - assert D.shape == (n, k) - - if I is None: - I = np.empty((n, k), dtype=np.int64) - else: - assert I.shape == (n, k) - - if numeric_type == faiss.Float32: - self.search_c(n, swig_ptr(x), k, swig_ptr(D), swig_ptr(I), params) - else: - self.search_ex( - n, - swig_ptr(x), - numeric_type, - k, - swig_ptr(D), - swig_ptr(I), - params, - ) - return D, I - - def replacement_search_and_reconstruct( - self, x, k, *, params=None, D=None, I=None, R=None - ): - """Find the k nearest neighbors of the set of vectors x in the index, - and return an approximation of these vectors. - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate for the index. - `dtype` must be float32. - k : int - Number of nearest neighbors. - params : SearchParameters - Search parameters of the current search (overrides the - class-level params) - D : array_like, optional - Distance array to store the result. - I : array_like, optional - Labels array to store the result. - R : array_like, optional - reconstruction array to store - - Returns - ------- - D : array_like - Distances of the nearest neighbors, shape (n, k). When - not enough results are found the label is set to +Inf or - -Inf. - I : array_like - Labels of the nearest neighbors, shape (n, k). When not - enough results are found, the label is set to -1 - R : array_like - Approximate (reconstructed) nearest neighbor vectors, - shape (n, k, d). - """ - n, d = x.shape - assert d == self.d - x = np.ascontiguousarray(x, dtype="float32") - - assert k > 0 - - if D is None: - D = np.empty((n, k), dtype=np.float32) - else: - assert D.shape == (n, k) - - if I is None: - I = np.empty((n, k), dtype=np.int64) - else: - assert I.shape == (n, k) - - if R is None: - R = np.empty((n, k, d), dtype=np.float32) - else: - assert R.shape == (n, k, d) - - self.search_and_reconstruct_c( - n, swig_ptr(x), k, swig_ptr(D), swig_ptr(I), swig_ptr(R), params - ) - return D, I, R - - def replacement_search_and_return_codes( - self, - x, - k, - *, - include_listnos=False, - params=None, - D=None, - I=None, - codes=None, - ): - """Find the k nearest neighbors of the set of vectors x in the index, - and return the codes stored for these vectors - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate for the index. - `dtype` must be float32. - k : int - Number of nearest neighbors. - params : SearchParameters - Search parameters of the current search (overrides the - class-level params) - include_listnos : bool, optional - whether to include the list ids in the first bytes of each code - D : array_like, optional - Distance array to store the result. - I : array_like, optional - Labels array to store the result. - codes : array_like, optional - codes array to store - - Returns - ------- - D : array_like - Distances of the nearest neighbors, shape (n, k). When - not enough results are found the label is set to +Inf or - -Inf. - I : array_like - Labels of the nearest neighbors, shape (n, k). When not - enough results are found, the label is set to -1 - R : array_like - Approximate (reconstructed) nearest neighbor vectors, - shape (n, k, d). - """ - n, d = x.shape - assert d == self.d - x = np.ascontiguousarray(x, dtype="float32") - - assert k > 0 - - if D is None: - D = np.empty((n, k), dtype=np.float32) - else: - assert D.shape == (n, k) - - if I is None: - I = np.empty((n, k), dtype=np.int64) - else: - assert I.shape == (n, k) - - code_size_1 = self.code_size - if include_listnos: - code_size_1 += self.coarse_code_size() - - if codes is None: - codes = np.empty((n, k, code_size_1), dtype=np.uint8) - else: - assert codes.shape == (n, k, code_size_1) - - self.search_and_return_codes_c( - n, - swig_ptr(x), - k, - swig_ptr(D), - swig_ptr(I), - swig_ptr(codes), - include_listnos, - params, - ) - return D, I, codes - - def replacement_remove_ids(self, x): - """Remove some ids from the index. - This is a O(ntotal) operation by default, so could be expensive. - - Parameters - ---------- - x : array_like or faiss.IDSelector - Either an IDSelector that returns True for vectors to remove, or a - list of ids to reomove (1D array of int64). When `x` is a list, - it is wrapped into an IDSelector. - - Returns - ------- - n_remove: int - number of vectors that were removed - """ - if isinstance(x, IDSelector): - sel = x - else: - assert x.ndim == 1 - index_ivf = try_extract_index_ivf(self) - x = np.ascontiguousarray(x, dtype="int64") - if index_ivf and index_ivf.direct_map.type == DirectMap.Hashtable: - sel = IDSelectorArray(x.size, swig_ptr(x)) - else: - sel = IDSelectorBatch(x.size, swig_ptr(x)) - return self.remove_ids_c(sel) - - def replacement_reconstruct(self, key, x=None): - """Approximate reconstruction of one vector from the index. - - Parameters - ---------- - key : int - Id of the vector to reconstruct - x : array_like, optional - pre-allocated array to store the results - - Returns - ------- - x : array_like reconstructed vector, size `self.d`, `dtype`=float32 - """ - if x is None: - x = np.empty(self.d, dtype=np.float32) - else: - assert x.shape == (self.d,) - - self.reconstruct_c(key, swig_ptr(x)) - return x - - def replacement_reconstruct_batch(self, key, x=None): - """Approximate reconstruction of several vectors from the index. - - Parameters - ---------- - key : array of ints - Ids of the vectors to reconstruct - x : array_like, optional - pre-allocated array to store the results - - Returns - ------- - x : array_like - reconstrcuted vectors, size `len(key), self.d` - """ - key = np.ascontiguousarray(key, dtype="int64") - (n,) = key.shape - if x is None: - x = np.empty((n, self.d), dtype=np.float32) - else: - assert x.shape == (n, self.d) - self.reconstruct_batch_c(n, swig_ptr(key), swig_ptr(x)) - return x - - def replacement_reconstruct_n(self, n0=0, ni=-1, x=None): - """Approximate reconstruction of vectors `n0` ... `n0 + ni - 1` - from the index. - Missing vectors trigger an exception. - - Parameters - ---------- - n0 : int - Id of the first vector to reconstruct (default 0) - ni : int - Number of vectors to reconstruct (-1 = default = ntotal) - x : array_like, optional - pre-allocated array to store the results - - Returns - ------- - x : array_like - Reconstructed vectors, size (`ni`, `self.d`), `dtype`=float32 - """ - if ni == -1: - ni = self.ntotal - n0 - if x is None: - x = np.empty((ni, self.d), dtype=np.float32) - else: - assert x.shape == (ni, self.d) - - self.reconstruct_n_c(n0, ni, swig_ptr(x)) - return x - - def replacement_update_vectors(self, keys, x): - n = keys.size - assert keys.shape == (n,) - assert x.shape == (n, self.d) - x = np.ascontiguousarray(x, dtype="float32") - keys = np.ascontiguousarray(keys, dtype="int64") - self.update_vectors_c(n, swig_ptr(keys), swig_ptr(x)) - - # No support passed-in for output buffers - def replacement_range_search(self, x, thresh, *, params=None): - """Search vectors that are within a distance of the query vectors. - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate for the index. - `dtype` must be float32. - thresh : float - Threshold to select neighbors. All elements within this - radius are returned, except for maximum inner product - indexes, where the elements above the threshold are - returned - params : SearchParameters - Search parameters of the current search (overrides the - class-level params) - - - Returns - ------- - lims: array_like - Starting index of the results for each query vector, size n+1. - D : array_like - Distances of the nearest neighbors, shape `lims[n]`. The - distances for query i are in `D[lims[i]:lims[i+1]]`. - I : array_like - Labels of nearest neighbors, shape `lims[n]`. The labels for query i - are in `I[lims[i]:lims[i+1]]`. - - """ - n, d = x.shape - assert d == self.d - x = np.ascontiguousarray(x, dtype="float32") - thresh = float(thresh) - - res = RangeSearchResult(n) - self.range_search_c(n, swig_ptr(x), thresh, res, params) - # get pointers and copy them - lims = rev_swig_ptr(res.lims, n + 1).copy() - nd = int(lims[-1]) - D = rev_swig_ptr(res.distances, nd).copy() - I = rev_swig_ptr(res.labels, nd).copy() - return lims, D, I - - def replacement_search_preassigned( - self, x, k, Iq, Dq, *, params=None, D=None, I=None - ): - """Find the k nearest neighbors of the set of vectors x in an IVF index, - with precalculated coarse quantization assignment. - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate for the index. - `dtype` must be float32. - k : int - Number of nearest neighbors. - Dq : array_like, optional - Distance array to the centroids, size (n, nprobe) - Iq : array_like, optional - Nearest centroids, size (n, nprobe) - - params : SearchParameters - Search parameters of the current search (overrides the - class-level params) - D : array_like, optional - Distance array to store the result. - I : array_like, optional - Labels array to store the results. - - Returns - ------- - D : array_like - Distances of the nearest neighbors, shape (n, k). When - not enough results are found the label is set to +Inf or - -Inf. - I : array_like - Labels of the nearest neighbors, shape (n, k). - When not enough results are found, the label is set to -1 - """ - n, d = x.shape - x = np.ascontiguousarray(x, dtype="float32") - assert d == self.d - assert k > 0 - - if D is None: - D = np.empty((n, k), dtype=np.float32) - else: - assert D.shape == (n, k) - - if I is None: - I = np.empty((n, k), dtype=np.int64) - else: - assert I.shape == (n, k) - - Iq = np.ascontiguousarray(Iq, dtype="int64") - assert params is None, "params not supported" - assert Iq.shape == (n, self.nprobe) - - if Dq is not None: - Dq = np.ascontiguousarray(Dq, dtype="float32") - assert Dq.shape == Iq.shape - else: - Dq = np.zeros(Iq.shape, dtype="float32") - - self.search_preassigned_c( - n, - swig_ptr(x), - k, - swig_ptr(Iq), - swig_ptr(Dq), - swig_ptr(D), - swig_ptr(I), - False, - ) - return D, I - - def replacement_range_search_preassigned( - self, x, thresh, Iq, Dq, *, params=None - ): - """Search vectors that are within a distance of the query vectors. - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate for the index. - `dtype` must be float32. - thresh : float - Threshold to select neighbors. All elements within this - radius are returned, except for maximum inner product - indexes, where the elements above the threshold are - returned - Iq : array_like, optional - Nearest centroids, size (n, nprobe) - Dq : array_like, optional - Distance array to the centroids, size (n, nprobe) - params : SearchParameters - Search parameters of the current search (overrides the - class-level params) - - - Returns - ------- - lims: array_like - Starting index of the results for each query vector, size n+1. - D : array_like - Distances of the nearest neighbors, shape `lims[n]`. The - distances for query i are in `D[lims[i]:lims[i+1]]`. - I : array_like - Labels of nearest neighbors, shape `lims[n]`. The labels for query i - are in `I[lims[i]:lims[i+1]]`. - - """ - n, d = x.shape - assert d == self.d - x = np.ascontiguousarray(x, dtype="float32") - - Iq = np.ascontiguousarray(Iq, dtype="int64") - assert params is None, "params not supported" - assert Iq.shape == (n, self.nprobe) - - if Dq is not None: - Dq = np.ascontiguousarray(Dq, dtype="float32") - assert Dq.shape == Iq.shape - else: - Dq = np.zeros(Iq.shape, dtype="float32") - - thresh = float(thresh) - res = RangeSearchResult(n) - self.range_search_preassigned_c( - n, swig_ptr(x), thresh, swig_ptr(Iq), swig_ptr(Dq), res - ) - # get pointers and copy them - lims = rev_swig_ptr(res.lims, n + 1).copy() - nd = int(lims[-1]) - D = rev_swig_ptr(res.distances, nd).copy() - I = rev_swig_ptr(res.labels, nd).copy() - return lims, D, I - - def replacement_sa_encode(self, x, codes=None): - n, d = x.shape - assert d == self.d - x = np.ascontiguousarray(x, dtype="float32") - code_size = self.sa_code_size() - - if codes is None: - codes = np.empty((n, code_size), dtype=np.uint8) - else: - assert codes.shape == (n, code_size) - - self.sa_encode_c(n, swig_ptr(x), swig_ptr(codes)) - return codes - - def replacement_sa_decode(self, codes, x=None): - n, cs = codes.shape - assert cs == self.sa_code_size() - codes = _check_dtype_uint8(codes) - - if x is None: - x = np.empty((n, self.d), dtype=np.float32) - else: - assert x.shape == (n, self.d) - - self.sa_decode_c(n, swig_ptr(codes), swig_ptr(x)) - return x - - def replacement_add_sa_codes(self, codes, ids=None): - n, cs = codes.shape - assert cs == self.sa_code_size() - codes = _check_dtype_uint8(codes) - - ids_ptr = None - if ids is not None: - assert ids.shape == (n,) - ids = np.ascontiguousarray(ids, dtype="int64") - ids_ptr = swig_ptr(ids) - self.add_sa_codes_c(n, swig_ptr(codes), ids_ptr) - - def replacement_permute_entries(self, perm): - (n,) = perm.shape - assert n == self.ntotal - perm = np.ascontiguousarray(perm, dtype="int64") - self.permute_entries_c(faiss.swig_ptr(perm)) - - replace_method(the_class, "add", replacement_add) - replace_method(the_class, "add_with_ids", replacement_add_with_ids) - replace_method(the_class, "assign", replacement_assign) - replace_method(the_class, "train", replacement_train) - replace_method(the_class, "search", replacement_search) - replace_method(the_class, "remove_ids", replacement_remove_ids) - replace_method(the_class, "reconstruct", replacement_reconstruct) - replace_method( - the_class, "reconstruct_batch", replacement_reconstruct_batch - ) - replace_method(the_class, "reconstruct_n", replacement_reconstruct_n) - replace_method(the_class, "range_search", replacement_range_search) - replace_method( - the_class, - "update_vectors", - replacement_update_vectors, - ignore_missing=True, - ) - replace_method( - the_class, - "search_and_reconstruct", - replacement_search_and_reconstruct, - ignore_missing=True, - ) - replace_method( - the_class, - "search_and_return_codes", - replacement_search_and_return_codes, - ignore_missing=True, - ) - - # these ones are IVF-specific - replace_method( - the_class, - "search_preassigned", - replacement_search_preassigned, - ignore_missing=True, - ) - replace_method( - the_class, - "range_search_preassigned", - replacement_range_search_preassigned, - ignore_missing=True, - ) - replace_method(the_class, "sa_encode", replacement_sa_encode) - replace_method(the_class, "sa_decode", replacement_sa_decode) - replace_method(the_class, "add_sa_codes", replacement_add_sa_codes) - replace_method( - the_class, - "permute_entries", - replacement_permute_entries, - ignore_missing=True, - ) - - # Store the original __setattr__ method - original_setattr = ( - the_class.__setattr__ - if hasattr(the_class, "__setattr__") - else object.__setattr__ - ) - - the_class.__setattr__ = replacement_setattr - - # get/set state for pickle - # the data is serialized to std::vector -> numpy array -> python bytes - # so not very efficient for now. - - def index_getstate(self): - return {"this": faiss.serialize_index(self).tobytes()} - - def index_setstate(self, st): - index2 = faiss.deserialize_index( - np.frombuffer(st["this"], dtype="uint8") - ) - self.this = index2.this - - the_class.__getstate__ = index_getstate - the_class.__setstate__ = index_setstate - - -def handle_IndexBinary(the_class): - - def replacement_add(self, x): - n, d = x.shape - x = _check_dtype_uint8(x) - assert d == self.code_size - self.add_c(n, swig_ptr(x)) - - def replacement_add_with_ids(self, x, ids): - n, d = x.shape - x = _check_dtype_uint8(x) - ids = np.ascontiguousarray(ids, dtype="int64") - assert d == self.code_size - assert ids.shape == (n,), "not same nb of vectors as ids" - self.add_with_ids_c(n, swig_ptr(x), swig_ptr(ids)) - - def replacement_train(self, x): - n, d = x.shape - x = _check_dtype_uint8(x) - assert d == self.code_size - self.train_c(n, swig_ptr(x)) - - def replacement_reconstruct(self, key): - x = np.empty(self.code_size, dtype=np.uint8) - self.reconstruct_c(key, swig_ptr(x)) - return x - - def replacement_reconstruct_n(self, n0=0, ni=-1, x=None): - if ni == -1: - ni = self.ntotal - n0 - if x is None: - x = np.empty((ni, self.code_size), dtype=np.uint8) - else: - assert x.shape == (ni, self.code_size) - - self.reconstruct_n_c(n0, ni, swig_ptr(x)) - return x - - def replacement_search(self, x, k, *, params=None): - x = _check_dtype_uint8(x) - n, d = x.shape - assert d == self.code_size - assert k > 0 - distances = np.empty((n, k), dtype=np.int32) - labels = np.empty((n, k), dtype=np.int64) - self.search_c( - n, swig_ptr(x), k, swig_ptr(distances), swig_ptr(labels), params - ) - return distances, labels - - def replacement_search_preassigned(self, x, k, Iq, Dq): - n, d = x.shape - x = _check_dtype_uint8(x) - assert d == self.code_size - assert k > 0 - - D = np.empty((n, k), dtype=np.int32) - I = np.empty((n, k), dtype=np.int64) - - Iq = np.ascontiguousarray(Iq, dtype="int64") - assert Iq.shape == (n, self.nprobe) - - if Dq is not None: - Dq = np.ascontiguousarray(Dq, dtype="int32") - assert Dq.shape == Iq.shape - else: - Dq = np.zeros(Iq.shape, dtype="int32") - - self.search_preassigned_c( - n, - swig_ptr(x), - k, - swig_ptr(Iq), - swig_ptr(Dq), - swig_ptr(D), - swig_ptr(I), - False, - ) - return D, I - - def replacement_range_search(self, x, thresh, *, params=None): - n, d = x.shape - x = _check_dtype_uint8(x) - assert d == self.code_size - res = RangeSearchResult(n) - self.range_search_c(n, swig_ptr(x), thresh, res, params=params) - # get pointers and copy them - lims = rev_swig_ptr(res.lims, n + 1).copy() - nd = int(lims[-1]) - D = rev_swig_ptr(res.distances, nd).copy() - I = rev_swig_ptr(res.labels, nd).copy() - return lims, D, I - - def replacement_range_search_preassigned( - self, x, thresh, Iq, Dq, *, params=None - ): - n, d = x.shape - x = _check_dtype_uint8(x) - assert d == self.code_size - - Iq = np.ascontiguousarray(Iq, dtype="int64") - assert params is None, "params not supported" - assert Iq.shape == (n, self.nprobe) - - if Dq is not None: - Dq = np.ascontiguousarray(Dq, dtype="int32") - assert Dq.shape == Iq.shape - else: - Dq = np.zeros(Iq.shape, dtype="int32") - - thresh = int(thresh) - res = RangeSearchResult(n) - self.range_search_preassigned_c( - n, swig_ptr(x), thresh, swig_ptr(Iq), swig_ptr(Dq), res - ) - # get pointers and copy them - lims = rev_swig_ptr(res.lims, n + 1).copy() - nd = int(lims[-1]) - D = rev_swig_ptr(res.distances, nd).copy() - I = rev_swig_ptr(res.labels, nd).copy() - return lims, D, I - - def replacement_remove_ids(self, x): - if isinstance(x, IDSelector): - sel = x - else: - assert x.ndim == 1 - x = np.ascontiguousarray(x, dtype="int64") - sel = IDSelectorBatch(x.size, swig_ptr(x)) - return self.remove_ids_c(sel) - - def replacement_assign(self, x, k, labels=None): - """Find the k nearest neighbors of the set of vectors x in the index. - This is the same as the `search` method, but discards the distances. - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate for the index. - `dtype` must be uint8. - k : int - Number of nearest neighbors. - labels : array_like, optional - Labels array to store the results. - - Returns - ------- - labels: array_like - Labels of the nearest neighbors, shape (n, k). - When not enough results are found, the label is set to -1 - """ - n, d = x.shape - x = _check_dtype_uint8(x) - assert d == self.code_size - assert k > 0 - - if labels is None: - labels = np.empty((n, k), dtype=np.int64) - else: - assert labels.shape == (n, k) - - self.assign_c(n, swig_ptr(x), swig_ptr(labels), k) - return labels - - replace_method(the_class, "add", replacement_add) - replace_method(the_class, "add_with_ids", replacement_add_with_ids) - replace_method(the_class, "train", replacement_train) - replace_method(the_class, "search", replacement_search) - replace_method(the_class, "assign", replacement_assign) - replace_method(the_class, "range_search", replacement_range_search) - replace_method(the_class, "reconstruct", replacement_reconstruct) - replace_method(the_class, "reconstruct_n", replacement_reconstruct_n) - replace_method(the_class, "remove_ids", replacement_remove_ids) - replace_method( - the_class, - "search_preassigned", - replacement_search_preassigned, - ignore_missing=True, - ) - replace_method( - the_class, - "range_search_preassigned", - replacement_range_search_preassigned, - ignore_missing=True, - ) - - -def handle_VectorTransform(the_class): - - def apply_method(self, x): - n, d = x.shape - x = np.ascontiguousarray(x, dtype="float32") - assert d == self.d_in - y = np.empty((n, self.d_out), dtype=np.float32) - self.apply_noalloc(n, swig_ptr(x), swig_ptr(y)) - return y - - def replacement_reverse_transform(self, x): - n, d = x.shape - x = np.ascontiguousarray(x, dtype="float32") - assert d == self.d_out - y = np.empty((n, self.d_in), dtype=np.float32) - self.reverse_transform_c(n, swig_ptr(x), swig_ptr(y)) - return y - - def replacement_vt_train(self, x): - n, d = x.shape - x = np.ascontiguousarray(x, dtype="float32") - assert d == self.d_in - self.train_c(n, swig_ptr(x)) - - replace_method(the_class, "train", replacement_vt_train) - # apply is reserved in Python... - the_class.apply_py = apply_method - the_class.apply = apply_method - replace_method( - the_class, "reverse_transform", replacement_reverse_transform - ) - - -def handle_AutoTuneCriterion(the_class): - def replacement_set_groundtruth(self, D, I): - if D is not None: - assert I.shape == D.shape - self.nq, self.gt_nnn = I.shape - self.set_groundtruth_c( - self.gt_nnn, swig_ptr(D) if D is not None else None, swig_ptr(I) - ) - - def replacement_evaluate(self, D, I): - assert I.shape == D.shape - assert I.shape == (self.nq, self.nnn) - return self.evaluate_c(swig_ptr(D), swig_ptr(I)) - - replace_method(the_class, "set_groundtruth", replacement_set_groundtruth) - replace_method(the_class, "evaluate", replacement_evaluate) - - -def handle_ParameterSpace(the_class): - def replacement_explore(self, index, xq, crit): - assert xq.shape == (crit.nq, index.d) - xq = np.ascontiguousarray(xq, dtype="float32") - ops = OperatingPoints() - self.explore_c(index, crit.nq, swig_ptr(xq), crit, ops) - return ops - - replace_method(the_class, "explore", replacement_explore) - - -def handle_MatrixStats(the_class): - original_init = the_class.__init__ - - def replacement_init(self, m): - assert len(m.shape) == 2 - m = np.ascontiguousarray(m, dtype="float32") - original_init(self, m.shape[0], m.shape[1], swig_ptr(m)) - - the_class.__init__ = replacement_init - - -def handle_IOWriter(the_class): - """add a write_bytes method""" - - def write_bytes(self, b): - return self(swig_ptr(b), 1, len(b)) - - the_class.write_bytes = write_bytes - - -def handle_IOReader(the_class): - """add a read_bytes method""" - - def read_bytes(self, totsz): - buf = bytearray(totsz) - was_read = self(swig_ptr(buf), 1, len(buf)) - return bytes(buf[:was_read]) - - the_class.read_bytes = read_bytes - - -def handle_IndexRowwiseMinMax(the_class): - def replacement_train_inplace(self, x): - """Trains the index on a representative set of vectors inplace. - The index must be trained before vectors can be added to it. - - This call WILL change the values in the input array, because - of two scaling procedures being performed inplace. - - Parameters - ---------- - x : array_like - Query vectors, shape (n, d) where d is appropriate for the index. - `dtype` must be float32. - """ - n, d = x.shape - assert d == self.d - x = np.ascontiguousarray(x, dtype="float32") - self.train_inplace_c(n, swig_ptr(x)) - - replace_method(the_class, "train_inplace", replacement_train_inplace) - - -def handle_CodePacker(the_class): - - def replacement_pack_1(self, x, offset, block): - assert x.shape == (self.code_size,) - nblock, block_size = block.shape - assert block_size == self.block_size - assert 0 <= offset < block_size * self.nvec - self.pack_1_c(swig_ptr(x), offset, faiss.swig_ptr(block)) - - def replacement_unpack_1(self, block, offset): - nblock, block_size = block.shape - assert block_size == self.block_size - assert 0 <= offset < block_size * self.nvec - x = np.zeros(self.code_size, dtype="uint8") - self.unpack_1_c(faiss.swig_ptr(block), offset, swig_ptr(x)) - return x - - replace_method(the_class, "pack_1", replacement_pack_1) - replace_method(the_class, "unpack_1", replacement_unpack_1) - - -###################################################### -# MapLong2Long interface -###################################################### - - -def handle_MapLong2Long(the_class): - - def replacement_map_add(self, keys, vals): - (n,) = keys.shape - assert (n,) == vals.shape - keys = np.ascontiguousarray(keys, dtype="int64") - vals = np.ascontiguousarray(vals, dtype="int64") - self.add_c(n, swig_ptr(keys), swig_ptr(vals)) - - def replacement_map_search_multiple(self, keys): - (n,) = keys.shape - keys = np.ascontiguousarray(keys, dtype="int64") - vals = np.empty(n, dtype="int64") - self.search_multiple_c(n, swig_ptr(keys), swig_ptr(vals)) - return vals - - replace_method(the_class, "add", replacement_map_add) - replace_method( - the_class, "search_multiple", replacement_map_search_multiple - ) - - -###################################################### -# SearchParameters and related interface -###################################################### - - -def add_to_referenced_objects(self, ref): - if not hasattr(self, "referenced_objects"): - self.referenced_objects = [ref] - else: - self.referenced_objects.append(ref) - - -class RememberSwigOwnership: - """ - SWIG's seattr transfers ownership of SWIG wrapped objects to the class - (btw this seems to contradict - https://www.swig.org/Doc1.3/Python.html#Python_nn22 - 31.4.2) - This interferes with how we manage ownership: with the referenced_objects - table. Therefore, we reset the thisown field in this context manager. - """ - - def __init__(self, obj): - self.obj = obj - - def __enter__(self): - if hasattr(self.obj, "thisown"): - self.old_thisown = self.obj.thisown - else: - self.old_thisown = None - - def __exit__(self, *ignored): - if self.old_thisown is not None: - self.obj.thisown = self.old_thisown - - -def handle_SearchParameters(the_class): - """Protect SearchParameters from leaking SWIG-owned sub-objects assigned - via either kwargs construction (SearchParametersXX(sel=x)) or bare - attribute assignment (params.sel = x). - """ - the_class.original_init = the_class.__init__ - - def replacement_init(self, **args): - self.original_init() - for k, v in args.items(): - assert hasattr(self, k) - setattr(self, k, v) - - the_class.__init__ = replacement_init - - # Install __setattr__ once per hierarchy; subclasses inherit via MRO. - if getattr(the_class, "_protected_setattr", False): - return - parent_setattr = the_class.__setattr__ - - def replacement_setattr(self, k, v): - # Per-field ref dict. Reassigning the same field drops the prior - # ref instead of accumulating, so a long-lived SearchParameters - # with repeated `params.sel = ...` does not leak. - if v is not None and hasattr(v, "thisown"): - if not hasattr(self, "_sp_field_refs"): - parent_setattr(self, "_sp_field_refs", {}) - with RememberSwigOwnership(v): - parent_setattr(self, k, v) - self._sp_field_refs[k] = v - else: - parent_setattr(self, k, v) - if hasattr(self, "_sp_field_refs"): - self._sp_field_refs.pop(k, None) - - the_class.__setattr__ = replacement_setattr - the_class._protected_setattr = True - - -def handle_IDSelectorSubset(the_class, class_owns, force_int64=True): - the_class.original_init = the_class.__init__ - - def replacement_init(self, *args): - if len(args) == 1: - # assume it's an array - (subset,) = args - if force_int64: - subset = np.ascontiguousarray(subset, dtype="int64") - args = (len(subset), faiss.swig_ptr(subset)) - if not class_owns: - add_to_referenced_objects(self, subset) - self.original_init(*args) - - the_class.__init__ = replacement_init - - -def handle_CodeSet(the_class): - - def replacement_insert(self, codes, inserted=None): - n, d = codes.shape - assert d == self.d - codes = np.ascontiguousarray(codes, dtype=np.uint8) - - if inserted is None: - inserted = np.empty(n, dtype=bool) - else: - assert inserted.shape == (n,) - - self.insert_c(n, swig_ptr(codes), swig_ptr(inserted)) - return inserted - - replace_method(the_class, "insert", replacement_insert) - - -###################################################### -# Syntactic sugar for NeuralNet classes -###################################################### - - -def handle_Tensor2D(the_class): - the_class.original_init = the_class.__init__ - - def replacement_init(self, *args): - if len(args) == 1: - (array,) = args - n, d = array.shape - self.original_init(n, d) - faiss.copy_array_to_vector( - np.ascontiguousarray(array).ravel(), self.v - ) - else: - self.original_init(*args) - - def numpy(self): - shape = np.zeros(2, dtype=np.int64) - faiss.memcpy(faiss.swig_ptr(shape), self.shape, shape.nbytes) - return faiss.vector_to_array(self.v).reshape(shape[0], shape[1]) - - the_class.__init__ = replacement_init - the_class.numpy = numpy - - -def handle_Embedding(the_class): - the_class.original_init = the_class.__init__ - - def replacement_init(self, *args): - if len(args) != 1 or args[0].__class__ == the_class: - self.original_init(*args) - return - # assume it's a torch.Embedding - emb = args[0] - self.original_init(emb.num_embeddings, emb.embedding_dim) - self.from_torch(emb) - - def from_torch(self, emb): - """copy weights from torch.Embedding""" - assert emb.weight.shape == (self.num_embeddings, self.embedding_dim) - faiss.copy_array_to_vector( - np.ascontiguousarray(emb.weight.data).ravel(), self.weight - ) - - def from_array(self, array): - """copy weights from numpy array""" - assert array.shape == (self.num_embeddings, self.embedding_dim) - faiss.copy_array_to_vector( - np.ascontiguousarray(array).ravel(), self.weight - ) - - the_class.from_array = from_array - the_class.from_torch = from_torch - the_class.__init__ = replacement_init - - -def handle_Linear(the_class): - the_class.original_init = the_class.__init__ - - def replacement_init(self, *args): - if len(args) != 1 or args[0].__class__ == the_class: - self.original_init(*args) - return - # assume it's a torch.Linear - linear = args[0] - bias = linear.bias is not None - self.original_init(linear.in_features, linear.out_features, bias) - self.from_torch(linear) - - def from_torch(self, linear): - """copy weights from torch.Linear""" - assert linear.weight.shape == (self.out_features, self.in_features) - faiss.copy_array_to_vector( - linear.weight.data.numpy().ravel(), self.weight - ) - if linear.bias is not None: - assert linear.bias.shape == (self.out_features,) - faiss.copy_array_to_vector(linear.bias.data.numpy(), self.bias) - - def from_array(self, array, bias=None): - """copy weights from numpy array""" - assert array.shape == (self.out_features, self.in_features) - faiss.copy_array_to_vector( - np.ascontiguousarray(array).ravel(), self.weight - ) - if bias is not None: - assert bias.shape == (self.out_features,) - faiss.copy_array_to_vector(bias, self.bias) - - the_class.__init__ = replacement_init - the_class.from_array = from_array - the_class.from_torch = from_torch - - -###################################################### -# Syntactic sugar for QINCo and QINCoStep -###################################################### - - -def handle_QINCoStep(the_class): - the_class.original_init = the_class.__init__ - - def replacement_init(self, *args): - if len(args) != 1 or args[0].__class__ == the_class: - self.original_init(*args) - return - step = args[0] - # assume it's a Torch QINCoStep - self.original_init(step.d, step.K, step.L, step.h) - self.from_torch(step) - - def from_torch(self, step): - """copy weights from torch.QINCoStep""" - assert (step.d, step.K, step.L, step.h) == ( - self.d, - self.K, - self.L, - self.h, - ) - self.codebook.from_torch(step.codebook) - self.MLPconcat.from_torch(step.MLPconcat) - - for l in range(step.L): - src = step.residual_blocks[l] - dest = self.get_residual_block(l) - dest.linear1.from_torch(src[0]) - dest.linear2.from_torch(src[2]) - - the_class.__init__ = replacement_init - the_class.from_torch = from_torch - - -def handle_QINCo(the_class): - the_class.original_init = the_class.__init__ - - def replacement_init(self, *args): - if len(args) != 1 or args[0].__class__ == the_class: - self.original_init(*args) - return - - # assume it's a Torch QINCo - qinco = args[0] - self.original_init(qinco.d, qinco.K, qinco.L, qinco.M, qinco.h) - self.from_torch(qinco) - - def from_torch(self, qinco): - """copy weights from torch.QINCo""" - assert (qinco.d, qinco.K, qinco.L, qinco.M, qinco.h) == ( - self.d, - self.K, - self.L, - self.M, - self.h, - ) - self.codebook0.from_torch(qinco.codebook0) - for m in range(qinco.M - 1): - self.get_step(m).from_torch(qinco.steps[m]) - - the_class.__init__ = replacement_init - the_class.from_torch = from_torch - - -def handle_shard_ivf_index_centroids(func): - def wrapper(*args, **kwargs): - args = list(args) - if len(args) > 3 and args[3] is not None: - args[3] = faiss.PyCallbackShardingFunction(args[3]) - return func(*args, **kwargs) - - return wrapper diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/__init__.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/big_batch_search.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/big_batch_search.py deleted file mode 100644 index ab067590c407bc2c99f7ddea73cf33f8665a4cd4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/big_batch_search.py +++ /dev/null @@ -1,542 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import time -import pickle -import os -import logging -from multiprocessing.pool import ThreadPool -import threading -import _thread -from queue import Queue -import traceback -import datetime - -import numpy as np -import faiss - -from faiss.contrib.inspect_tools import get_invlist - - -class BigBatchSearcher: - """ - Object that manages all the data related to the computation - except the actual within-bucket matching and the organization of the - computation (parallel or not) - """ - - def __init__(self, index, xq, k, verbose=0, use_float16=False): - - # verbosity - self.verbose = verbose - self.tictoc = [] - - self.xq = xq - self.index = index - self.use_float16 = use_float16 - keep_max = faiss.is_similarity_metric(index.metric_type) - self.rh = faiss.ResultHeap(len(xq), k, keep_max=keep_max) - self.t_accu = [0] * 6 - self.t_display = self.t0 = time.time() - - def start_t_accu(self): - self.t_accu_t0 = time.time() - - def stop_t_accu(self, n): - self.t_accu[n] += time.time() - self.t_accu_t0 - - def tic(self, name): - self.tictoc = (name, time.time()) - if self.verbose > 0: - print(name, end="\r", flush=True) - - def toc(self): - name, t0 = self.tictoc - dt = time.time() - t0 - if self.verbose > 0: - print(f"{name}: {dt:.3f} s") - return dt - - def report(self, l): - if self.verbose == 1 or ( - self.verbose == 2 - and (l > 1000 and time.time() < self.t_display + 1.0) - ): - return - t = time.time() - self.t0 - print( - f"[{t:.1f} s] list {l}/{self.index.nlist} " - f"times prep q {self.t_accu[0]:.3f} prep b {self.t_accu[1]:.3f} " - f"comp {self.t_accu[2]:.3f} res {self.t_accu[3]:.3f} " - f"wait in {self.t_accu[4]:.3f} " - f"wait out {self.t_accu[5]:.3f} " - f"eta {datetime.timedelta(seconds=t*self.index.nlist/(l+1)-t)} " - f"mem {faiss.get_mem_usage_kb()}", - end="\r" if self.verbose <= 2 else "\n", - flush=True, - ) - self.t_display = time.time() - - def coarse_quantization(self): - self.tic("coarse quantization") - bs = 65536 - nq = len(self.xq) - q_assign = np.empty((nq, self.index.nprobe), dtype="int32") - for i0 in range(0, nq, bs): - i1 = min(nq, i0 + bs) - q_dis_i, q_assign_i = self.index.quantizer.search( - self.xq[i0:i1], self.index.nprobe - ) - # q_dis[i0:i1] = q_dis_i - q_assign[i0:i1] = q_assign_i - self.toc() - self.q_assign = q_assign - - def reorder_assign(self): - self.tic("bucket sort") - q_assign = self.q_assign - q_assign += 1 # move -1 -> 0 - self.bucket_lims = faiss.matrix_bucket_sort_inplace( - self.q_assign, nbucket=self.index.nlist + 1, nt=16 - ) - self.query_ids = self.q_assign.ravel() - if self.verbose > 0: - print(" number of -1s:", self.bucket_lims[1]) - self.bucket_lims = self.bucket_lims[1:] # shift back to ignore -1s - del self.q_assign # inplace so let's forget about the old version... - self.toc() - - def prepare_bucket(self, l): - """prepare the queries and database items for bucket l""" - t0 = time.time() - index = self.index - # prepare queries - i0, i1 = self.bucket_lims[l], self.bucket_lims[l + 1] - q_subset = self.query_ids[i0:i1] - xq_l = self.xq[q_subset] - if self.by_residual: - xq_l = xq_l - index.quantizer.reconstruct(l) - t1 = time.time() - # prepare database side - list_ids, xb_l = get_invlist(index.invlists, l) - - if self.decode_func is None: - xb_l = xb_l.ravel() - else: - xb_l = self.decode_func(xb_l) - - if self.use_float16: - xb_l = xb_l.astype("float16") - xq_l = xq_l.astype("float16") - - t2 = time.time() - self.t_accu[0] += t1 - t0 - self.t_accu[1] += t2 - t1 - return q_subset, xq_l, list_ids, xb_l - - def add_results_to_heap(self, q_subset, D, list_ids, I): - """add the bucket results to the heap structure""" - if D is None: - return - t0 = time.time() - if I is None: - I = list_ids - else: - I = list_ids[I] - self.rh.add_result_subset(q_subset, D, I) - self.t_accu[3] += time.time() - t0 - - def sizes_in_checkpoint(self): - return (self.xq.shape, self.index.nprobe, self.index.nlist) - - def write_checkpoint(self, fname, completed): - # write to temp file then move to final file - tmpname = fname + ".tmp" - with open(tmpname, "wb") as f: - pickle.dump( - { - "sizes": self.sizes_in_checkpoint(), - "completed": completed, - "rh": (self.rh.D, self.rh.I), - }, - f, - -1, - ) - os.replace(tmpname, fname) - - def read_checkpoint(self, fname): - with open(fname, "rb") as f: - ckp = pickle.load(f) - assert ckp["sizes"] == self.sizes_in_checkpoint() - self.rh.D[:] = ckp["rh"][0] - self.rh.I[:] = ckp["rh"][1] - return ckp["completed"] - - -class BlockComputer: - """computation within one bucket""" - - def __init__( - self, - index, - method="knn_function", - pairwise_distances=faiss.pairwise_distances, - knn=faiss.knn, - ): - - self.index = index - if index.__class__ == faiss.IndexIVFFlat: - index_help = faiss.IndexFlat(index.d, index.metric_type) - decode_func = lambda x: x.view("float32") - by_residual = False - elif index.__class__ == faiss.IndexIVFPQ: - index_help = faiss.IndexPQ( - index.d, index.pq.M, index.pq.nbits, index.metric_type - ) - index_help.pq = index.pq - decode_func = index_help.pq.decode - index_help.is_trained = True - by_residual = index.by_residual - elif index.__class__ == faiss.IndexIVFScalarQuantizer: - index_help = faiss.IndexScalarQuantizer( - index.d, index.sq.qtype, index.metric_type - ) - index_help.sq = index.sq - decode_func = index_help.sq.decode - index_help.is_trained = True - by_residual = index.by_residual - else: - raise RuntimeError(f"index type {index.__class__} not supported") - self.index_help = index_help - self.decode_func = None if method == "index" else decode_func - self.by_residual = by_residual - self.method = method - self.pairwise_distances = pairwise_distances - self.knn = knn - - def block_search(self, xq_l, xb_l, list_ids, k, **extra_args): - metric_type = self.index.metric_type - if xq_l.size == 0 or xb_l.size == 0: - D = I = None - elif self.method == "index": - faiss.copy_array_to_vector(xb_l, self.index_help.codes) - self.index_help.ntotal = len(list_ids) - D, I = self.index_help.search(xq_l, k) - elif self.method == "pairwise_distances": - # TODO implement blockwise to avoid mem blowup - D = self.pairwise_distances(xq_l, xb_l, metric=metric_type) - I = None - elif self.method == "knn_function": - D, I = self.knn(xq_l, xb_l, k, metric=metric_type, **extra_args) - - return D, I - - -def big_batch_search( - index, - xq, - k, - method="knn_function", - pairwise_distances=faiss.pairwise_distances, - knn=faiss.knn, - verbose=0, - threaded=0, - use_float16=False, - prefetch_threads=1, - computation_threads=1, - q_assign=None, - checkpoint=None, - checkpoint_freq=7200, - start_list=0, - end_list=None, - crash_at=-1, -): - """ - Search queries xq in the IVF index, with a search function that collects - batches of query vectors per inverted list. This can be faster than the - regular search indexes. - Supports IVFFlat, IVFPQ and IVFScalarQuantizer. - - Supports three computation methods: - method = "index": - build a flat index and populate it separately for each index - method = "pairwise_distances": - decompress codes and compute all pairwise distances for the queries - and index and add result to heap - method = "knn_function": - decompress codes and compute knn results for the queries - - threaded=0: sequential execution - threaded=1: prefetch next bucket while computing the current one - threaded=2: prefetch prefetch_threads buckets at a time. - - compute_threads>1: the knn function will get an additional thread_no that - tells which worker should handle this. - - In threaded mode, the computation is tiled with the bucket preparation and - the writeback of results (useful to maximize GPU utilization). - - use_float16: convert all matrices to float16 (faster for GPU gemm) - - q_assign: override coarse assignment, should be a matrix of size nq * nprobe - - checkpointing (only for threaded > 1): - checkpoint: file where the checkpoints are stored - checkpoint_freq: when to perform checkpointing. Should be a multiple of - threaded - - start_list, end_list: process only a subset of invlists - """ - nprobe = index.nprobe - - assert method in ("index", "pairwise_distances", "knn_function") - - mem_queries = xq.nbytes - mem_assign = len(xq) * nprobe * np.dtype("int32").itemsize - mem_res = ( - len(xq) - * k - * (np.dtype("int64").itemsize + np.dtype("float32").itemsize) - ) - mem_tot = mem_queries + mem_assign + mem_res - if verbose > 0: - logging.info( - f"memory: queries {mem_queries} assign {mem_assign} " - f"result {mem_res} total {mem_tot} = {mem_tot / (1<<30):.3f} GiB" - ) - - bbs = BigBatchSearcher( - index, xq, k, verbose=verbose, use_float16=use_float16 - ) - - comp = BlockComputer( - index, method=method, pairwise_distances=pairwise_distances, knn=knn - ) - - bbs.decode_func = comp.decode_func - - bbs.by_residual = comp.by_residual - if q_assign is None: - bbs.coarse_quantization() - else: - bbs.q_assign = q_assign - bbs.reorder_assign() - - if end_list is None: - end_list = index.nlist - - completed = set() - if checkpoint is not None: - assert (start_list, end_list) == (0, index.nlist) - if os.path.exists(checkpoint): - logging.info(f"recovering checkpoint: {checkpoint}") - completed = bbs.read_checkpoint(checkpoint) - logging.info(f" already completed: {len(completed)}") - else: - logging.info("no checkpoint: starting from scratch") - - if threaded == 0: - # simple sequential version - - for l in range(start_list, end_list): - bbs.report(l) - q_subset, xq_l, list_ids, xb_l = bbs.prepare_bucket(l) - t0i = time.time() - D, I = comp.block_search(xq_l, xb_l, list_ids, k) - bbs.t_accu[2] += time.time() - t0i - bbs.add_results_to_heap(q_subset, D, list_ids, I) - - elif threaded == 1: - - # parallel version with granularity 1 - - def add_results_and_prefetch(to_add, l): - """perform the addition for the previous bucket and - prefetch the next (if applicable)""" - if to_add is not None: - bbs.add_results_to_heap(*to_add) - if l < index.nlist: - return bbs.prepare_bucket(l) - - prefetched_bucket = bbs.prepare_bucket(start_list) - to_add = None - pool = ThreadPool(1) - - for l in range(start_list, end_list): - bbs.report(l) - prefetched_bucket_a = pool.apply_async( - add_results_and_prefetch, (to_add, l + 1) - ) - q_subset, xq_l, list_ids, xb_l = prefetched_bucket - bbs.start_t_accu() - D, I = comp.block_search(xq_l, xb_l, list_ids, k) - bbs.stop_t_accu(2) - to_add = q_subset, D, list_ids, I - bbs.start_t_accu() - prefetched_bucket = prefetched_bucket_a.get() - bbs.stop_t_accu(4) - - bbs.add_results_to_heap(*to_add) - pool.close() - else: - - def task_manager_thread( - task, - pool_size, - start_task, - end_task, - completed, - output_queue, - input_queue, - ): - try: - with ThreadPool(pool_size) as pool: - res = [ - pool.apply_async( - task, args=(i, output_queue, input_queue) - ) - for i in range(start_task, end_task) - if i not in completed - ] - for r in res: - r.get() - pool.close() - pool.join() - output_queue.put(None) - except: - traceback.print_exc() - _thread.interrupt_main() - raise - - def task_manager(*args): - task_manager = threading.Thread( - target=task_manager_thread, - args=args, - ) - task_manager.daemon = True - task_manager.start() - return task_manager - - def prepare_task(task_id, output_queue, input_queue=None): - try: - logging.info(f"Prepare start: {task_id}") - q_subset, xq_l, list_ids, xb_l = bbs.prepare_bucket(task_id) - output_queue.put((task_id, q_subset, xq_l, list_ids, xb_l)) - logging.info(f"Prepare end: {task_id}") - except: - traceback.print_exc() - _thread.interrupt_main() - raise - - def compute_task(task_id, output_queue, input_queue): - try: - logging.info(f"Compute start: {task_id}") - t_wait_out = 0 - while True: - t0 = time.time() - logging.info(f"Compute input: task {task_id}") - input_value = input_queue.get() - t_wait_in = time.time() - t0 - if input_value is None: - # signal for other compute tasks - input_queue.put(None) - break - centroid, q_subset, xq_l, list_ids, xb_l = input_value - logging.info( - f"Compute work: task {task_id}, centroid {centroid}" - ) - t0 = time.time() - if computation_threads > 1: - D, I = comp.block_search( - xq_l, xb_l, list_ids, k, thread_id=task_id - ) - else: - D, I = comp.block_search(xq_l, xb_l, list_ids, k) - t_compute = time.time() - t0 - logging.info( - f"Compute output: task {task_id}, centroid {centroid}" - ) - t0 = time.time() - output_queue.put( - ( - centroid, - t_wait_in, - t_wait_out, - t_compute, - q_subset, - D, - list_ids, - I, - ) - ) - t_wait_out = time.time() - t0 - logging.info(f"Compute end: {task_id}") - except: - traceback.print_exc() - _thread.interrupt_main() - raise - - prepare_to_compute_queue = Queue(2) - compute_to_main_queue = Queue(2) - compute_task_manager = task_manager( - compute_task, - computation_threads, - 0, - computation_threads, - set(), - compute_to_main_queue, - prepare_to_compute_queue, - ) - prepare_task_manager = task_manager( - prepare_task, - prefetch_threads, - start_list, - end_list, - completed, - prepare_to_compute_queue, - None, - ) - - t_checkpoint = time.time() - while True: - logging.info("Waiting for result") - value = compute_to_main_queue.get() - if not value: - break - ( - centroid, - t_wait_in, - t_wait_out, - t_compute, - q_subset, - D, - list_ids, - I, - ) = value - # to test checkpointing - if centroid == crash_at: - 1 / 0 - bbs.t_accu[2] += t_compute - bbs.t_accu[4] += t_wait_in - bbs.t_accu[5] += t_wait_out - logging.info(f"Adding to heap start: centroid {centroid}") - bbs.add_results_to_heap(q_subset, D, list_ids, I) - logging.info(f"Adding to heap end: centroid {centroid}") - completed.add(centroid) - bbs.report(centroid) - if checkpoint is not None: - if time.time() - t_checkpoint > checkpoint_freq: - logging.info("writing checkpoint") - bbs.write_checkpoint(checkpoint, completed) - t_checkpoint = time.time() - - prepare_task_manager.join() - compute_task_manager.join() - - bbs.tic("finalize heap") - bbs.rh.finalize() - bbs.toc() - - return bbs.rh.D, bbs.rh.I diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/client_server.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/client_server.py deleted file mode 100644 index d445585783a6800676fa443981d56a8b87800e6c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/client_server.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from multiprocessing.pool import ThreadPool -import faiss -from typing import List, Tuple - -from . import rpc - -############################################################ -# Server implementation -############################################################ - - -class SearchServer(rpc.Server): - """Assign version that can be exposed via RPC""" - - def __init__(self, s: int, index: faiss.Index): - rpc.Server.__init__(self, s) - self.index = index - self.index_ivf = faiss.extract_index_ivf(index) - - def set_nprobe(self, nprobe: int) -> int: - """set nprobe field""" - self.index_ivf.nprobe = nprobe - - def get_ntotal(self) -> int: - return self.index.ntotal - - def __getattr__(self, f): - # all other functions get forwarded to the index - return getattr(self.index, f) - - -def run_index_server(index: faiss.Index, port: int, v6: bool = False): - """serve requests for that index forever""" - rpc.run_server(lambda s: SearchServer(s, index), port, v6=v6) - - -############################################################ -# Client implementation -############################################################ - - -class ClientIndex: - """manages a set of distance sub-indexes. The sub_indexes search a - subset of the inverted lists. Searches are merged afterwards - """ - - def __init__(self, machine_ports: List[Tuple[str, int]], v6: bool = False): - """connect to a series of (host, port) pairs""" - self.sub_indexes = [] - for machine, port in machine_ports: - self.sub_indexes.append(rpc.Client(machine, port, v6)) - - self.ni = len(self.sub_indexes) - # pool of threads. Each thread manages one sub-index. - self.pool = ThreadPool(self.ni) - # test connection... - self.ntotal = self.get_ntotal() - self.verbose = False - - def set_nprobe(self, nprobe: int) -> None: - self.pool.map(lambda idx: idx.set_nprobe(nprobe), self.sub_indexes) - - def set_omp_num_threads(self, nt: int) -> None: - self.pool.map(lambda idx: idx.set_omp_num_threads(nt), self.sub_indexes) - - def get_ntotal(self) -> None: - return sum( - self.pool.map(lambda idx: idx.get_ntotal(), self.sub_indexes) - ) - - def search(self, x, k: int): - - rh = faiss.ResultHeap(x.shape[0], k) - - for Di, Ii in self.pool.imap( - lambda idx: idx.search(x, k), self.sub_indexes - ): - rh.add_result(Di, Ii) - rh.finalize() - return rh.D, rh.I diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/clustering.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/clustering.py deleted file mode 100644 index acdf6b112cc267f2bad18cc67992f097f07a47b2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/clustering.py +++ /dev/null @@ -1,539 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -This contrib module contains a few routines useful to do clustering variants. -""" - -import numpy as np -import faiss -import time -from multiprocessing.pool import ThreadPool - - -try: - import scipy.sparse -except ImportError: - print("scipy not accessible, Python k-means will not work") - - -def print_nop(*arg, **kwargs): - pass - - -def two_level_clustering( - xt, nc1, nc2, rebalance=True, clustering_niter=25, **args -): - """ - perform 2-level clustering on a training set xt - nc1 and nc2 are the number of clusters at each level, the final number of - clusters is nc2. Additional arguments are passed to the Kmeans object. - - Rebalance allocates the number of sub-clusters depending on the number of - first-level assignment. - """ - d = xt.shape[1] - - verbose = args.get("verbose", False) - - log = print if verbose else print_nop - - log( - f"2-level clustering of {xt.shape} nb 1st level clusters = {nc1} " - f"total {nc2}" - ) - log("perform coarse training") - - km = faiss.Kmeans( - d, nc1, niter=clustering_niter, max_points_per_centroid=2000, **args - ) - km.train(xt) - - iteration_stats = [km.iteration_stats] - log() - - log("assigning the training set") - t0 = time.time() - _, assign1 = km.assign(xt) - bc = np.bincount(assign1, minlength=nc1) - log( - f"done in {time.time() - t0:.2f} s. " - f"Sizes of clusters {min(bc)}-{max(bc)}" - ) - o = assign1.argsort() - del km - - if not rebalance: - # make sure the sub-clusters sum up to exactly nc2 - cc = np.arange(nc1 + 1) * nc2 // nc1 - all_nc2 = cc[1:] - cc[:-1] - else: - bc_sum = np.cumsum(bc) - all_nc2 = bc_sum * nc2 // bc_sum[-1] - all_nc2[1:] -= all_nc2[:-1] - assert sum(all_nc2) == nc2 - log(f"nb 2nd-level centroids {min(all_nc2)}-{max(all_nc2)}") - - # train sub-clusters - i0 = 0 - c2 = [] - t0 = time.time() - for c1 in range(nc1): - nc2 = int(all_nc2[c1]) - log( - f"[{time.time() - t0:.2f} s] training sub-cluster " - f"{c1}/{nc1} nc2={nc2}\r", - end="", - flush=True, - ) - i1 = i0 + bc[c1] - subset = o[i0:i1] - assert np.all(assign1[subset] == c1) - km = faiss.Kmeans(d, nc2, **args) - xtsub = xt[subset] - km.train(xtsub) - iteration_stats.append(km.iteration_stats) - c2.append(km.centroids) - del km - i0 = i1 - log(f"done in {time.time() - t0:.2f} s") - return np.vstack(c2), iteration_stats - - -def train_ivf_index_with_2level(index, xt, **args): - """ - Applies 2-level clustering to an index_ivf embedded in an index. - """ - # handle PreTransforms - index = faiss.downcast_index(index) - if isinstance(index, faiss.IndexPreTransform): - for i in range(index.chain.size()): - vt = index.chain.at(i) - vt.train(xt) - xt = vt.apply(xt) - train_ivf_index_with_2level(index.index, xt, **args) - index.is_trained = True - return - assert isinstance(index, faiss.IndexIVF) - assert index.metric_type == faiss.METRIC_L2 - # now do 2-level clustering - nc1 = int(np.sqrt(index.nlist)) - print("REBALANCE=", args) - - centroids, _ = two_level_clustering(xt, nc1, index.nlist, **args) - index.quantizer.train(centroids) - index.quantizer.add(centroids) - # finish training - index.train(xt) - - -def balanced_assignment_with_penalties( - x, centroids, alpha=0.03, num_iter=20, maxk=100 -): - """ - Assign vectors x to centroids with a balance constraint. - - Iteratively adjusts per-cluster penalties so that oversized clusters - become less attractive. At each iteration the penalized distance for - cluster c is ``d(x, c)^2 + penalty_c^2`` and the penalty is updated as - ``penalty_c *= (binsize_c / n_opt) ** alpha`` where ``n_opt = n / nc``. - - A single kNN call (with *maxk* neighbors) is done upfront; subsequent - iterations only re-weight among those candidates, making the routine - fast even for large datasets. - - Reference: "Balancing clusters to reduce response time variability in - large scale image search", Tavenard et al., CBMI 2011. - https://inria.hal.science/inria-00576886/document - See also notebook N10159950. - - Args: - x: (n, d) float32 array of vectors to assign. - centroids: (nc, d) float32 array of cluster centroids. - alpha: exponent that controls how aggressively penalties grow. - Higher values yield more balanced clusters at the cost - of higher MSE. Typical range: 0.01 – 0.1. - num_iter: number of penalty-update iterations. - maxk: number of nearest centroids to consider per vector. - Must be <= nc. - - Returns: - assign: (n,) int64 array of centroid indices. - stats: dict with keys - - - *imf*: imbalance factor (1.0 = perfectly balanced) - - *mse*: mean squared error of the assignment - - *binsize_min*, *binsize_max*: smallest / largest cluster - - *penalty_min*, *penalty_max*: penalty value range - - *alpha*: the alpha value used - """ - - nc = len(centroids) - n = len(x) - nopt = n / nc # targed bin sizes - - # we assign to the top-maxk clusters. The final assignment will pick - # among these clusters. - full_d2, full_assign = faiss.knn(x, centroids, maxk) - - # scalar penalty for each cluster - penalties = np.ones(nc, dtype=np.float32) - - for it in range(num_iter): - # compute penalized assignment - penalties2 = penalties**2 - full_d2_penalized = full_d2 + penalties2[full_assign] - a0 = full_d2_penalized.argmin(axis=1) - assign = np.take_along_axis(full_assign, a0[:, None], axis=1).ravel() - binsizes = np.bincount(assign, minlength=nc) - # print(imbalance_factor(nc, assign), mse, int(binsizes.min()), - # int(binsizes.max())) - penalties *= (binsizes / nopt) ** alpha - - stats = dict( - alpha=alpha, - imf=imbalance_factor(nc, assign), - mse=((x - centroids[assign]) ** 2).sum(1).mean(), # recompute MSE - binsize_min=int(binsizes.min()), - binsize_max=int(binsizes.max()), - penalty_min=penalties.min(), - penalty_max=penalties.max(), - ) - - return assign, stats - - -############################################################################### -# K-means implementation in Python -# -# It relies on DatasetAssign, an abstraction of the training vectors that offers -# the minimal set of operations to perform k-means clustering. -############################################################################### - - -class DatasetAssign: - """Wrapper for a matrix that offers a function to assign the vectors - to centroids. All other implementations offer the same interface""" - - def __init__(self, x): - self.x = np.ascontiguousarray(x, dtype="float32") - - def count(self): - return self.x.shape[0] - - def dim(self): - return self.x.shape[1] - - def get_subset(self, indices): - return self.x[indices] - - def perform_search(self, centroids): - return faiss.knn(self.x, centroids, 1) - - def assign_to(self, centroids, weights=None): - D, I = self.perform_search(centroids) - - I = I.ravel() - D = D.ravel() - nc, d = centroids.shape - sum_per_centroid = np.zeros((nc, d), dtype="float32") - if weights is None: - np.add.at(sum_per_centroid, I, self.x) - else: - np.add.at(sum_per_centroid, I, weights[:, np.newaxis] * self.x) - - return I, D, sum_per_centroid - - -class DatasetAssignGPU(DatasetAssign): - """GPU version of the previous""" - - def __init__(self, x, gpu_id, verbose=False): - DatasetAssign.__init__(self, x) - index = faiss.IndexFlatL2(x.shape[1]) - if gpu_id >= 0: - self.index = faiss.index_cpu_to_gpu( - faiss.StandardGpuResources(), gpu_id, index - ) - else: - # -1 -> assign to all GPUs - self.index = faiss.index_cpu_to_all_gpus(index) - - def perform_search(self, centroids): - self.index.reset() - self.index.add(centroids) - return self.index.search(self.x, 1) - - -def sparse_assign_to_dense(xq, xb, xq_norms=None, xb_norms=None): - """assignment function for xq is sparse, xb is dense - uses a matrix multiplication. The squared norms can be provided if - available. - """ - nq = xq.shape[0] - nb = xb.shape[0] - if xb_norms is None: - xb_norms = (xb**2).sum(1) - if xq_norms is None: - xq_norms = np.array(xq.power(2).sum(1)) - d2 = xb_norms - 2 * xq @ xb.T - I = d2.argmin(axis=1) - D = d2.ravel()[I + np.arange(nq) * nb] + xq_norms.ravel() - return D, I - - -def sparse_assign_to_dense_blocks( - xq, xb, xq_norms=None, xb_norms=None, qbs=16384, bbs=16384, nt=None -): - """ - decomposes the sparse_assign_to_dense function into blocks to avoid a - possible memory blow up. Can be run in multithreaded mode, because scipy's - sparse-dense matrix multiplication is single-threaded. - """ - nq = xq.shape[0] - nb = xb.shape[0] - D = np.empty(nq, dtype="float32") - D.fill(np.inf) - I = -np.ones(nq, dtype=int) - - if xb_norms is None: - xb_norms = (xb**2).sum(1) - - def handle_query_block(i): - xq_block = xq[i : i + qbs] - Iblock = I[i : i + qbs] - Dblock = D[i : i + qbs] - if xq_norms is None: - xq_norms_block = np.array(xq_block.power(2).sum(1)) - else: - xq_norms_block = xq_norms[i : i + qbs] - for j in range(0, nb, bbs): - Di, Ii = sparse_assign_to_dense( - xq_block, - xb[j : j + bbs], - xq_norms=xq_norms_block, - xb_norms=xb_norms[j : j + bbs], - ) - if j == 0: - Iblock[:] = Ii - Dblock[:] = Di - else: - mask = Di < Dblock - Iblock[mask] = Ii[mask] + j - Dblock[mask] = Di[mask] - - if nt == 0 or nt == 1 or nq <= qbs: - list(map(handle_query_block, range(0, nq, qbs))) - else: - pool = ThreadPool(nt) - pool.map(handle_query_block, range(0, nq, qbs)) - - return D, I - - -class DatasetAssignSparse(DatasetAssign): - """Wrapper for a matrix that offers a function to assign the vectors - to centroids. All other implementations offer the same interface""" - - def __init__(self, x): - assert x.__class__ == scipy.sparse.csr_matrix - self.x = x - self.squared_norms = np.array(x.power(2).sum(1)) - - def get_subset(self, indices): - return np.array(self.x[indices].todense()) - - def perform_search(self, centroids): - return sparse_assign_to_dense_blocks( - self.x, centroids, xq_norms=self.squared_norms - ) - - def assign_to(self, centroids, weights=None): - D, I = self.perform_search(centroids) - - I = I.ravel() - D = D.ravel() - n = self.x.shape[0] - if weights is None: - weights = np.ones(n, dtype="float32") - nc = len(centroids) - - m = scipy.sparse.csc_matrix( - (weights, I, np.arange(n + 1)), shape=(nc, n) - ) - sum_per_centroid = np.array((m * self.x).todense()) - - return I, D, sum_per_centroid - - -def imbalance_factor(k, assign): - assign = np.ascontiguousarray(assign, dtype="int64") - return faiss.imbalance_factor(len(assign), k, faiss.swig_ptr(assign)) - - -def check_if_torch(x): - if x.__class__ == np.ndarray: - return False - import torch - - if isinstance(x, torch.Tensor): - return True - raise NotImplementedError(f"Unknown tensor type {type(x)}") - - -def reassign_centroids(hassign, centroids, rs=None): - """reassign centroids when some of them collapse""" - if rs is None: - rs = np.random - k, d = centroids.shape - nsplit = 0 - is_torch = check_if_torch(centroids) - - empty_cents = np.where(hassign == 0)[0] - - if len(empty_cents) == 0: - return 0 - - if is_torch: - import torch - - fac = torch.ones_like(centroids[0]) - else: - fac = np.ones_like(centroids[0]) - fac[::2] += 1 / 1024.0 - fac[1::2] -= 1 / 1024.0 - - # this is a single pass unless there are more than k/2 - # empty centroids - while len(empty_cents) > 0: - # choose which centroids to split (numpy) - probas = hassign.astype("float") - 1 - probas[probas < 0] = 0 - probas /= probas.sum() - nnz = (probas > 0).sum() - - nreplace = min(nnz, empty_cents.size) - cjs = rs.choice(k, size=nreplace, p=probas) - - for ci, cj in zip(empty_cents[:nreplace], cjs): - - c = centroids[cj] - centroids[ci] = c * fac - centroids[cj] = c / fac - - hassign[ci] = hassign[cj] // 2 - hassign[cj] -= hassign[ci] - nsplit += 1 - - empty_cents = empty_cents[nreplace:] - - return nsplit - - -def kmeans( - k, - data, - niter=25, - seed=1234, - checkpoint=None, - verbose=True, - return_stats=False, -): - """Pure python kmeans implementation. Follows the Faiss C++ version - quite closely, but takes a DatasetAssign instead of a training data - matrix. Also redo is not implemented. - - For the torch implementation, the centroids are tensors (possibly on GPU), - but the indices remain numpy on CPU. - """ - n, d = data.count(), data.dim() - log = print if verbose else print_nop - - log( - ( - "Clustering %d points in %dD to %d clusters, " - + "%d iterations seed %d" - ) - % (n, d, k, niter, seed) - ) - - rs = np.random.RandomState(seed) - print("preproc...") - t0 = time.time() - # initialization - perm = rs.choice(n, size=k, replace=False) - centroids = data.get_subset(perm) - is_torch = check_if_torch(centroids) - - iteration_stats = [] - - log(" done") - t_search_tot = 0 - obj = [] - for i in range(niter): - t0s = time.time() - - log("assigning", end="\r", flush=True) - assign, D, sums = data.assign_to(centroids) - - log("compute centroids", end="\r", flush=True) - - t_search_tot += time.time() - t0s - - err = D.sum() - if is_torch: - err = err.item() - obj.append(err) - - hassign = np.bincount(assign, minlength=k) - - fac = hassign.reshape(-1, 1).astype("float32") - fac[fac == 0] = 1 # quiet warning - if is_torch: - import torch - - fac = torch.from_numpy(fac).to(sums.device) - - centroids = sums / fac - - nsplit = reassign_centroids(hassign, centroids, rs) - - s = { - "obj": err, - "time": (time.time() - t0), - "time_search": t_search_tot, - "imbalance_factor": imbalance_factor(k, assign), - "nsplit": nsplit, - } - - log( - ( - " Iteration %d (%.2f s, search %.2f s): " - "objective=%g imbalance=%.3f nsplit=%d" - ) - % ( - i, - s["time"], - s["time_search"], - err, - s["imbalance_factor"], - nsplit, - ) - ) - iteration_stats.append(s) - - if checkpoint is not None: - log("storing centroids in", checkpoint) - if is_torch: - import torch - - torch.save(centroids, checkpoint) - else: - np.save(checkpoint, centroids) - - if return_stats: - return centroids, iteration_stats - else: - return centroids diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/datasets.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/datasets.py deleted file mode 100644 index bab5281b092dfe1d3c8fdcac2b4a3e5d3ab6a30a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/datasets.py +++ /dev/null @@ -1,547 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os -import numpy as np -import faiss -import getpass - - -from .vecs_io import ( - fvecs_read, - ivecs_read, - bvecs_mmap, - fvecs_mmap, - bvecs_iter, - bvecs_iter_chunked, -) -from .exhaustive_search import knn - - -class Dataset: - """Generic abstract class for a test dataset""" - - def __init__(self): - """the constructor should set the following fields:""" - self.d = -1 - self.metric = "L2" # or IP - self.nq = -1 - self.nb = -1 - self.nt = -1 - - def get_queries(self): - """return the queries as a (nq, d) array""" - raise NotImplementedError() - - def get_train(self, maxtrain=None): - """return the queries as a (nt, d) array""" - raise NotImplementedError() - - def get_database(self): - """return the queries as a (nb, d) array""" - raise NotImplementedError() - - def database_iterator(self, bs=128, split=(1, 0)): - """returns an iterator on database vectors. - bs is the number of vectors per batch - split = (nsplit, rank) means the dataset is split in nsplit - shards and we want shard number rank - The default implementation just iterates over the full matrix - returned by get_dataset. - """ - xb = self.get_database() - nsplit, rank = split - i0, i1 = self.nb * rank // nsplit, self.nb * (rank + 1) // nsplit - for j0 in range(i0, i1, bs): - yield xb[j0 : min(j0 + bs, i1)] - - def get_groundtruth(self, k=None): - """return the ground truth for k-nearest neighbor search""" - raise NotImplementedError() - - def get_groundtruth_range(self, thresh=None): - """return the ground truth for range search""" - raise NotImplementedError() - - def __str__(self): - return ( - f"dataset in dimension {self.d}, with metric {self.metric}, " - f"size: Q {self.nq} B {self.nb} T {self.nt}" - ) - - def check_sizes(self): - """runs the previous and checks the sizes of the matrices""" - assert self.get_queries().shape == (self.nq, self.d) - if self.nt > 0: - xt = self.get_train(maxtrain=123) - assert xt.shape == (123, self.d), "shape=%s" % (xt.shape,) - assert self.get_database().shape == (self.nb, self.d) - assert self.get_groundtruth(k=13).shape == (self.nq, 13) - - -class SyntheticDataset(Dataset): - """A dataset that is not completely random but still challenging to - index - """ - - def __init__(self, d, nt, nb, nq, metric="L2", seed=1338): - Dataset.__init__(self) - self.d, self.nt, self.nb, self.nq = d, nt, nb, nq - d1 = 10 # intrinsic dimension (more or less) - n = nb + nt + nq - rs = np.random.RandomState(seed) - x = rs.normal(size=(n, d1)) - x = np.dot(x, rs.rand(d1, d)) - # now we have a d1-dim ellipsoid in d-dimensional space - # higher factor (>4) -> higher frequency -> less linear - x = x * (rs.rand(d) * 4 + 0.1) - x = np.sin(x) - x = x.astype("float32") - self.metric = metric - self.xt = x[:nt] - self.xb = x[nt : nt + nb] - self.xq = x[nt + nb :] - - def get_queries(self): - return self.xq - - def get_train(self, maxtrain=None): - maxtrain = maxtrain if maxtrain is not None else self.nt - return self.xt[:maxtrain] - - def get_database(self): - return self.xb - - def get_groundtruth(self, k=100): - return knn( - self.xq, - self.xb, - k, - ( - faiss.METRIC_L2 - if self.metric == "L2" - else faiss.METRIC_INNER_PRODUCT - ), - )[1] - - -############################################################################ -# The following datasets are a few standard open-source datasets -# they should be stored in a directory, and we start by guessing where -# that directory is -############################################################################ - -username = getpass.getuser() - -for dataset_basedir in ( - "/datasets01/simsearch/041218/", - "/mnt/vol/gfsai-flash3-east/ai-group/datasets/simsearch/", - f"/home/{username}/simsearch/data/", -): - if os.path.exists(dataset_basedir): - break -else: - # users can link their data directory to `./data` - dataset_basedir = "data/" - - -def set_dataset_basedir(path): - global dataset_basedir - dataset_basedir = path - - -class DatasetSIFT1M(Dataset): - """ - The original dataset is available at: http://corpus-texmex.irisa.fr/ - (ANN_SIFT1M) - """ - - def __init__(self): - Dataset.__init__(self) - self.d, self.nt, self.nb, self.nq = 128, 100000, 1000000, 10000 - self.basedir = dataset_basedir + "sift1M/" - - def get_queries(self): - return fvecs_read(self.basedir + "sift_query.fvecs") - - def get_train(self, maxtrain=None): - maxtrain = maxtrain if maxtrain is not None else self.nt - return fvecs_read(self.basedir + "sift_learn.fvecs")[:maxtrain] - - def get_database(self): - return fvecs_read(self.basedir + "sift_base.fvecs") - - def get_groundtruth(self, k=None): - gt = ivecs_read(self.basedir + "sift_groundtruth.ivecs") - if k is not None: - assert k <= 100 - gt = gt[:, :k] - return gt - - -def sanitize(x): - return np.ascontiguousarray(x, dtype="float32") - - -class DatasetBigANN(Dataset): - """ - The original dataset is available at: http://corpus-texmex.irisa.fr/ - (ANN_SIFT1B) - """ - - def __init__(self, nb_M=1000): - Dataset.__init__(self) - assert nb_M in (1, 2, 5, 10, 20, 50, 100, 200, 500, 1000) - self.nb_M = nb_M - nb = nb_M * 10**6 - self.d, self.nt, self.nb, self.nq = 128, 10**8, nb, 10000 - self.basedir = dataset_basedir + "bigann/" - - def get_queries(self): - return sanitize(bvecs_mmap(self.basedir + "bigann_query.bvecs")[:]) - - def get_train(self, maxtrain=None): - maxtrain = maxtrain if maxtrain is not None else self.nt - return sanitize( - bvecs_mmap(self.basedir + "bigann_learn.bvecs")[:maxtrain] - ) - - def get_groundtruth(self, k=None): - gt = ivecs_read(self.basedir + "gnd/idx_%dM.ivecs" % self.nb_M) - if k is not None: - assert k <= 100 - gt = gt[:, :k] - return gt - - def get_database(self): - assert self.nb_M < 100, "dataset too large, use iterator" - return sanitize( - bvecs_mmap(self.basedir + "bigann_base.bvecs")[: self.nb] - ) - - def database_iterator(self, bs=128, split=(1, 0)): - xb = bvecs_mmap(self.basedir + "bigann_base.bvecs") - nsplit, rank = split - i0, i1 = self.nb * rank // nsplit, self.nb * (rank + 1) // nsplit - for j0 in range(i0, i1, bs): - yield sanitize(xb[j0 : min(j0 + bs, i1)]) - - -class DatasetDeep1B(Dataset): - """ - See - https://github.com/facebookresearch/faiss/tree/main/benchs#getting-deep1b - on how to get the data - """ - - def __init__(self, nb=10**9): - Dataset.__init__(self) - nb_to_name = { - 10**5: "100k", - 10**6: "1M", - 10**7: "10M", - 10**8: "100M", - 10**9: "1B", - } - assert nb in nb_to_name - self.d, self.nt, self.nb, self.nq = 96, 358480000, nb, 10000 - self.basedir = dataset_basedir + "deep1b/" - self.gt_fname = "%sdeep%s_groundtruth.ivecs" % ( - self.basedir, - nb_to_name[self.nb], - ) - - def get_queries(self): - return sanitize(fvecs_read(self.basedir + "deep1B_queries.fvecs")) - - def get_train(self, maxtrain=None): - maxtrain = maxtrain if maxtrain is not None else self.nt - return sanitize(fvecs_mmap(self.basedir + "learn.fvecs")[:maxtrain]) - - def get_groundtruth(self, k=None): - gt = ivecs_read(self.gt_fname) - if k is not None: - assert k <= 100 - gt = gt[:, :k] - return gt - - def get_database(self): - assert self.nb <= 10**8, "dataset too large, use iterator" - return sanitize(fvecs_mmap(self.basedir + "base.fvecs")[: self.nb]) - - def database_iterator(self, bs=128, split=(1, 0)): - xb = fvecs_mmap(self.basedir + "base.fvecs") - nsplit, rank = split - i0, i1 = self.nb * rank // nsplit, self.nb * (rank + 1) // nsplit - for j0 in range(i0, i1, bs): - yield sanitize(xb[j0 : min(j0 + bs, i1)]) - - -class DatasetGlove(Dataset): - """ - Data from http://ann-benchmarks.com/glove-100-angular.hdf5 - """ - - def __init__(self, loc=None, download=False): - import h5py - - assert not download, "not implemented" - if not loc: - loc = dataset_basedir + "glove/glove-100-angular.hdf5" - self.glove_h5py = h5py.File(loc, "r") - # IP and L2 are equivalent in this case, but it is traditionally - # seen as an IP dataset - self.metric = "IP" - self.d, self.nt = 100, 0 - self.nb = self.glove_h5py["train"].shape[0] - self.nq = self.glove_h5py["test"].shape[0] - - def get_queries(self): - xq = np.array(self.glove_h5py["test"]) - faiss.normalize_L2(xq) - return xq - - def get_database(self): - xb = np.array(self.glove_h5py["train"]) - faiss.normalize_L2(xb) - return xb - - def get_groundtruth(self, k=None): - gt = self.glove_h5py["neighbors"] - if k is not None: - assert k <= 100 - gt = gt[:, :k] - return gt - - -class DatasetMusic100(Dataset): - """ - get dataset from - https://github.com/stanis-morozov/ip-nsw#dataset - """ - - def __init__(self): - Dataset.__init__(self) - self.d, self.nt, self.nb, self.nq = 100, 0, 10**6, 10000 - self.metric = "IP" - self.basedir = dataset_basedir + "music-100/" - - def get_queries(self): - xq = np.fromfile(self.basedir + "query_music100.bin", dtype="float32") - xq = xq.reshape(-1, 100) - return xq - - def get_database(self): - xb = np.fromfile( - self.basedir + "database_music100.bin", dtype="float32" - ) - xb = xb.reshape(-1, 100) - return xb - - def get_groundtruth(self, k=None): - gt = np.load(self.basedir + "gt.npy") - if k is not None: - assert k <= 100 - gt = gt[:, :k] - return gt - - -class DatasetGIST1M(Dataset): - """ - The original dataset is available at: http://corpus-texmex.irisa.fr/ - (ANN_GIST1M) - """ - - def __init__(self): - Dataset.__init__(self) - self.d, self.nt, self.nb, self.nq = 960, 100000, 1000000, 10000 - self.basedir = dataset_basedir + "gist1M/" - - def get_queries(self): - return fvecs_read(self.basedir + "gist_query.fvecs") - - def get_train(self, maxtrain=None): - maxtrain = maxtrain if maxtrain is not None else self.nt - return fvecs_read(self.basedir + "gist_learn.fvecs")[:maxtrain] - - def get_database(self): - return fvecs_read(self.basedir + "gist_base.fvecs") - - def get_groundtruth(self, k=None): - gt = ivecs_read(self.basedir + "gist_groundtruth.ivecs") - if k is not None: - assert k <= 100 - gt = gt[:, :k] - return gt - - -class DatasetDINO10B(Dataset): - """ - Data from https://dl.fbaipublicfiles.com/large_objects/dino_vitl_10B/ - The dataset contains 10 billion 1024-d vectors extracted from image - patches from the YFCC100M dataset, using a Dino-ViT-L 16 model - (facebook/dinov3-vitl16-pretrain-lvd1689m). - The dataset is sharded in multiple chunked .bvecs files. Downloading - instructions can be obtained with - "wget https://dl.fbaipublicfiles.com/large_objects/dino_vitl_10B/README.md". - Supported sizes : 100k 200k 500k 1M ... 5B 10B listed in supported_nbs - (see __init__). - """ - - def __init__(self, nb, ignore_supported=False): - Dataset.__init__(self) - supported_nbs = [ - 100_000, - 200_000, - 500_000, - 1_000_000, - 2_000_000, - 5_000_000, - 10_000_000, - 20_000_000, - 50_000_000, - 100_000_000, - 200_000_000, - 500_000_000, - 1_000_000_000, - 2_000_000_000, - 5_000_000_000, - 10_000_000_000, - ] - if nb not in supported_nbs and not ignore_supported: - raise ValueError( - f"Unsupported dataset size: {nb}, supported values are: " - f"{supported_nbs}" - ) - if not os.path.exists(dataset_basedir): - raise ValueError( - "Provided dataset base directory does not exist: " - f"{dataset_basedir}" - ) - self.basedir = dataset_basedir + "dino_vitl_10B/" - self.indexdir = self.basedir + "chunked_base_10B" - assert os.path.exists( - self.indexdir - ), f"Index path should exist, check your dataset path: {self.indexdir}" - self.queriesdir = self.basedir + "queries_clean.bvecs" - assert os.path.exists(self.queriesdir), ( - f"Queries path should exist as dataset size {nb} is supported: " - f"{self.queriesdir}" - ) - self.gtsdir = ( - self.basedir - + "gts/" - + "gts_dino_patch_" - + str(nb) - + "_" - + "k10.npy" - ) - self.train_queriesdir = self.basedir + "train_queries_99M.bvecs" - self.nb = nb - self.d = 1024 - self.nq = 100_000 - self.nt = 99_000_000 - self.metric = "L2" - - def get_queries(self): - """Get all vectors as a single array""" - queries = bvecs_mmap(self.queriesdir) - return sanitize(queries) - - def get_train(self, maxtrain=None): - """Get training query vectors as a single array""" - if maxtrain is None or maxtrain > 10_000_000: - raise NotImplementedError( - "The training set is potentially too large to fit in RAM " - "(400 GB of data). Please use train_iterator or use " - "maxtrain parameter below 10_000_000 to get the first " - "maxtrain training vectors." - ) - return sanitize(bvecs_mmap(self.train_queriesdir)[:maxtrain]) - - def get_database(self): - """Get all database vectors as a single array""" - if self.nb > 10_000_000: - raise NotImplementedError( - "The dataset is potentially too large to fit in RAM. " - "Please use database_iterator or use a dataset size equal " - "to or below 10_000_000." - ) - else: - return sanitize( - bvecs_iter_chunked(self.indexdir, batch_size=self.nb).__next__() - ) - - def database_iterator(self, bs=10_000): - """Iterator over the database of size nb, corresponding to the - first nb vectors in the .bvecs file""" - total_read = 0 - for batch in bvecs_iter_chunked(self.indexdir, batch_size=bs): - if total_read + batch.shape[0] > self.nb: - batch = batch[: self.nb - total_read] - yield sanitize(batch) - total_read += batch.shape[0] - if total_read >= self.nb: - break - - def train_iterator(self, bs=10_000): - """Iterator over all training query vectors in the .bvecs file""" - for batch in bvecs_iter(self.train_queriesdir, batch_size=bs): - yield sanitize(batch) - - def get_groundtruth(self, k=10): - """Get ground truth from .npy file""" - if k > 10: - raise NotImplementedError( - "Ground truth files only available for k<=10" - ) - gts = np.load(self.gtsdir) - gts = gts[:, :k] - return gts - - def distance(self): - return "euclidean" - - -def dataset_from_name(dataset="deep1M", download=False): - """converts a string describing a dataset to a Dataset object - Supports sift1M, bigann1M..bigann1B, deep1M..deep1B, music-100 and glove - """ - - if dataset == "sift1M": - return DatasetSIFT1M() - - elif dataset == "gist1M": - return DatasetGIST1M() - - elif dataset.startswith("bigann"): - dbsize = 1000 if dataset == "bigann1B" else int(dataset[6:-1]) - return DatasetBigANN(nb_M=dbsize) - - elif dataset.startswith("deep"): - - szsuf = dataset[4:] - if szsuf[-1] == "M": - dbsize = 10**6 * int(szsuf[:-1]) - elif szsuf == "1B": - dbsize = 10**9 - elif szsuf[-1] == "k": - dbsize = 1000 * int(szsuf[:-1]) - else: - raise AssertionError("did not recognize suffix " + szsuf) - return DatasetDeep1B(nb=dbsize) - - elif dataset == "music-100": - return DatasetMusic100() - - elif dataset == "glove": - return DatasetGlove(download=download) - - elif dataset.startswith("dino"): - dbsize = 10_000_000_000 if dataset == "dino10B" else int(dataset[4:]) - return DatasetDINO10B(nb=dbsize) - - else: - raise RuntimeError("unknown dataset " + dataset) diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/evaluation.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/evaluation.py deleted file mode 100644 index 95d003ae97206d13cdd1e8e1da9293fd9af79090..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/evaluation.py +++ /dev/null @@ -1,502 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -import unittest -import time -import faiss - -from multiprocessing.pool import ThreadPool - -############################################################### -# Simple functions to evaluate knn results - - -def knn_intersection_measure(I1, I2): - """computes the intersection measure of two result tables""" - nq, rank = I1.shape - assert I2.shape == (nq, rank) - ninter = sum(np.intersect1d(I1[i], I2[i]).size for i in range(nq)) - return ninter / I1.size - - -############################################################### -# Range search results can be compared with Precision-Recall - - -def filter_range_results(lims, D, I, thresh): - """select a set of results""" - nq = lims.size - 1 - mask = D < thresh - new_lims = np.zeros_like(lims) - for i in range(nq): - new_lims[i + 1] = new_lims[i] + mask[lims[i] : lims[i + 1]].sum() - return new_lims, D[mask], I[mask] - - -def range_PR(lims_ref, Iref, lims_new, Inew, mode="overall"): - """compute the precision and recall of range search results. The - function does not take the distances into account.""" - - def ref_result_for(i): - return Iref[lims_ref[i] : lims_ref[i + 1]] - - def new_result_for(i): - return Inew[lims_new[i] : lims_new[i + 1]] - - nq = lims_ref.size - 1 - assert lims_new.size - 1 == nq - - ninter = np.zeros(nq, dtype="int64") - - def compute_PR_for(q): - - # ground truth results for this query - gt_ids = ref_result_for(q) - - # results for this query - new_ids = new_result_for(q) - - # there are no set functions in numpy so let's do this - inter = np.intersect1d(gt_ids, new_ids) - - ninter[q] = len(inter) - - # run in a thread pool, which helps in spite of the GIL - pool = ThreadPool(20) - pool.map(compute_PR_for, range(nq)) - - return counts_to_PR( - lims_ref[1:] - lims_ref[:-1], - lims_new[1:] - lims_new[:-1], - ninter, - mode=mode, - ) - - -def counts_to_PR(ngt, nres, ninter, mode="overall"): - """computes a precision-recall for a set of queries. - ngt = nb of GT results per query - nres = nb of found results per query - ninter = nb of correct results per query (smaller than nres of course) - """ - - if mode == "overall": - ngt, nres, ninter = ngt.sum(), nres.sum(), ninter.sum() - - if nres > 0: - precision = ninter / nres - else: - precision = 1.0 - - if ngt > 0: - recall = ninter / ngt - elif nres == 0: - recall = 1.0 - else: - recall = 0.0 - - return precision, recall - - elif mode == "average": - # average precision and recall over queries - - mask = ngt == 0 - ngt[mask] = 1 - - recalls = ninter / ngt - recalls[mask] = (nres[mask] == 0).astype(float) - - # avoid division by 0 - mask = nres == 0 - assert np.all(ninter[mask] == 0) - ninter[mask] = 1 - nres[mask] = 1 - - precisions = ninter / nres - - return precisions.mean(), recalls.mean() - - else: - raise AssertionError() - - -def sort_range_res_2(lims, D, I): - """sort 2 arrays using the first as key""" - I2 = np.empty_like(I) - D2 = np.empty_like(D) - nq = len(lims) - 1 - for i in range(nq): - l0, l1 = lims[i], lims[i + 1] - ii = I[l0:l1] - di = D[l0:l1] - o = di.argsort() - I2[l0:l1] = ii[o] - D2[l0:l1] = di[o] - return I2, D2 - - -def sort_range_res_1(lims, I): - I2 = np.empty_like(I) - nq = len(lims) - 1 - for i in range(nq): - l0, l1 = lims[i], lims[i + 1] - I2[l0:l1] = I[l0:l1] - I2[l0:l1].sort() - return I2 - - -def range_PR_multiple_thresholds( - lims_ref, - Iref, - lims_new, - Dnew, - Inew, - thresholds, - mode="overall", - do_sort="ref,new", -): - """compute precision-recall values for range search results - for several thresholds on the "new" results. - This is to plot PR curves - """ - # ref should be sorted by ids - if "ref" in do_sort: - Iref = sort_range_res_1(lims_ref, Iref) - - # new should be sorted by distances - if "new" in do_sort: - Inew, Dnew = sort_range_res_2(lims_new, Dnew, Inew) - - def ref_result_for(i): - return Iref[lims_ref[i] : lims_ref[i + 1]] - - def new_result_for(i): - l0, l1 = lims_new[i], lims_new[i + 1] - return Inew[l0:l1], Dnew[l0:l1] - - nq = lims_ref.size - 1 - assert lims_new.size - 1 == nq - - nt = len(thresholds) - counts = np.zeros((nq, nt, 3), dtype="int64") - - def compute_PR_for(q): - gt_ids = ref_result_for(q) - res_ids, res_dis = new_result_for(q) - - counts[q, :, 0] = len(gt_ids) - - if res_dis.size == 0: - # the rest remains at 0 - return - - # which offsets we are interested in - nres = np.searchsorted(res_dis, thresholds) - counts[q, :, 1] = nres - - if gt_ids.size == 0: - return - - # find number of TPs at each stage in the result list - ii = np.searchsorted(gt_ids, res_ids) - ii[ii == len(gt_ids)] = -1 - n_ok = np.cumsum(gt_ids[ii] == res_ids) - - # focus on threshold points - n_ok = np.hstack(([0], n_ok)) - counts[q, :, 2] = n_ok[nres] - - pool = ThreadPool(20) - pool.map(compute_PR_for, range(nq)) - # print(counts.transpose(2, 1, 0)) - - precisions = np.zeros(nt) - recalls = np.zeros(nt) - for t in range(nt): - p, r = counts_to_PR( - counts[:, t, 0], counts[:, t, 1], counts[:, t, 2], mode=mode - ) - precisions[t] = p - recalls[t] = r - - return precisions, recalls - - -############################################################### -# Functions that compare search results with a reference result. -# They are intended for use in tests - - -def _cluster_tables_with_tolerance(tab1, tab2, thr): - """for two tables, cluster them by merging values closer than thr. - Returns the cluster ids for each table element""" - tab = np.hstack([tab1, tab2]) - tab.sort() - n = len(tab) - diffs = np.ones(n) - diffs[1:] = tab[1:] - tab[:-1] - unique_vals = tab[diffs > thr] - idx1 = np.searchsorted(unique_vals, tab1, side="right") - 1 - idx2 = np.searchsorted(unique_vals, tab2, side="right") - 1 - return idx1, idx2 - - -def check_ref_knn_with_draws(Dref, Iref, Dnew, Inew, rtol=1e-5, atol=0): - """test that knn search results are identical, with possible ties. - Raise if not.""" - np.testing.assert_allclose(Dref, Dnew, rtol=rtol, atol=atol) - # here we have to be careful because of draws - testcase = unittest.TestCase() # because it makes nice error messages - for i in range(len(Iref)): - if np.all(Iref[i] == Inew[i]): # easy case - continue - - # otherwise collect elements per distance - r = rtol * Dref[i].max() + atol - - DrefC, DnewC = _cluster_tables_with_tolerance(Dref[i], Dnew[i], r) - - for dis in np.unique(DrefC): - if dis == DrefC[-1]: - continue - mask = DrefC == dis - testcase.assertEqual(set(Iref[i, mask]), set(Inew[i, mask])) - - -def check_ref_range_results(Lref, Dref, Iref, Lnew, Dnew, Inew): - """compare range search results wrt. a reference result, - throw if it fails""" - np.testing.assert_array_equal(Lref, Lnew) - nq = len(Lref) - 1 - for i in range(nq): - l0, l1 = Lref[i], Lref[i + 1] - Ii_ref = Iref[l0:l1] - Ii_new = Inew[l0:l1] - Di_ref = Dref[l0:l1] - Di_new = Dnew[l0:l1] - if np.all(Ii_ref == Ii_new): # easy - pass - else: - - def sort_by_ids(I, D): - o = I.argsort() - return I[o], D[o] - - # sort both - (Ii_ref, Di_ref) = sort_by_ids(Ii_ref, Di_ref) - (Ii_new, Di_new) = sort_by_ids(Ii_new, Di_new) - np.testing.assert_array_equal(Ii_ref, Ii_new) - np.testing.assert_array_almost_equal(Di_ref, Di_new, decimal=5) - - -############################################################### -# OperatingPoints functions -# this is the Python version of the AutoTune object in C++ - - -class OperatingPoints: - """ - Manages a set of search parameters with associated performance and time. - Keeps the Pareto optimal points. - """ - - def __init__(self): - # list of (key, perf, t) - self.operating_points = [ - # (self.do_nothing_key(), 0.0, 0.0) - ] - self.suboptimal_points = [] - - def compare_keys(self, k1, k2): - """return -1 if k1 > k2, 1 if k2 > k1, 0 otherwise""" - raise NotImplementedError - - def do_nothing_key(self): - """parameters to say we do nothing, takes 0 time and has 0 - performance""" - raise NotImplementedError - - def is_pareto_optimal(self, perf_new, t_new): - for _, perf, t in self.operating_points: - if perf >= perf_new and t <= t_new: - return False - return True - - def predict_bounds(self, key): - """predicts the bound on time and performance""" - min_time = 0.0 - max_perf = 1.0 - for key2, perf, t in self.operating_points + self.suboptimal_points: - cmp = self.compare_keys(key, key2) - if cmp > 0: # key2 > key - if t > min_time: - min_time = t - if cmp < 0: # key2 < key - if perf < max_perf: - max_perf = perf - return max_perf, min_time - - def should_run_experiment(self, key): - (max_perf, min_time) = self.predict_bounds(key) - return self.is_pareto_optimal(max_perf, min_time) - - def add_operating_point(self, key, perf, t): - if self.is_pareto_optimal(perf, t): - i = 0 - # maybe it shadows some other operating point completely? - while i < len(self.operating_points): - op_Ls, perf2, t2 = self.operating_points[i] - if perf >= perf2 and t <= t2: - self.suboptimal_points.append(self.operating_points.pop(i)) - else: - i += 1 - self.operating_points.append((key, perf, t)) - return True - else: - self.suboptimal_points.append((key, perf, t)) - return False - - -class OperatingPointsWithRanges(OperatingPoints): - """ - Set of parameters that are each picked from a discrete range of values. - An increase of each parameter is assumed to make the operation slower - and more accurate. - A key = int array of indices in the ordered set of parameters. - """ - - def __init__(self): - OperatingPoints.__init__(self) - # list of (name, values) - self.ranges = [] - - def add_range(self, name, values): - self.ranges.append((name, values)) - - def compare_keys(self, k1, k2): - if np.all(k1 >= k2): - return 1 - if np.all(k2 >= k1): - return -1 - return 0 - - def do_nothing_key(self): - return np.zeros(len(self.ranges), dtype=int) - - def num_experiments(self): - return int(np.prod([len(values) for name, values in self.ranges])) - - def sample_experiments(self, n_autotune, rs=np.random): - """sample a set of experiments of max size n_autotune - (run all experiments in random order if n_autotune is 0) - """ - assert n_autotune == 0 or n_autotune >= 2 - totex = self.num_experiments() - rs = np.random.RandomState(123) - if n_autotune == 0 or totex < n_autotune: - experiments = rs.permutation(totex - 2) - else: - experiments = rs.choice( - totex - 2, size=n_autotune - 2, replace=False - ) - - experiments = [0, totex - 1] + [int(cno) + 1 for cno in experiments] - return experiments - - def cno_to_key(self, cno): - """Convert a sequential experiment number to a key""" - k = np.zeros(len(self.ranges), dtype=int) - for i, (name, values) in enumerate(self.ranges): - k[i] = cno % len(values) - cno //= len(values) - assert cno == 0 - return k - - def get_parameters(self, k): - """Convert a key to a dictionary with parameter values""" - return { - name: values[k[i]] for i, (name, values) in enumerate(self.ranges) - } - - def restrict_range(self, name, max_val): - """remove too large values from a range""" - for name2, values in self.ranges: - if name == name2: - val2 = [v for v in values if v < max_val] - values[:] = val2 - return - raise RuntimeError(f"parameter {name} not found") - - -############################################################### -# Timer object - - -class TimerIter: - def __init__(self, timer): - self.ts = [] - self.runs = timer.runs - self.timer = timer - if timer.nt >= 0: - faiss.omp_set_num_threads(timer.nt) - - def __next__(self): - timer = self.timer - self.runs -= 1 - self.ts.append(time.time()) - total_time = self.ts[-1] - self.ts[0] if len(self.ts) >= 2 else 0 - if self.runs == -1 or total_time > timer.max_secs: - if timer.nt >= 0: - faiss.omp_set_num_threads(timer.remember_nt) - ts = np.array(self.ts) - times = ts[1:] - ts[:-1] - if len(times) == timer.runs: - timer.times = times[timer.warmup :] - else: - # if timeout, we use all the runs - timer.times = times[:] - raise StopIteration - - -class RepeatTimer: - """ - This is yet another timer object. It is adapted to Faiss by - taking a number of openmp threads to set on input. It should be called - in an explicit loop as: - - timer = RepeatTimer(warmup=1, nt=1, runs=6) - - for _ in timer: - # perform operation - - print(f"time={timer.get_ms():.1f} ± {timer.get_ms_std():.1f} ms") - - the same timer can be re-used. In that case it is reset each time it - enters a loop. It focuses on ms-scale times because for second scale - it's usually less relevant to repeat the operation. - """ - - def __init__(self, warmup=0, nt=-1, runs=1, max_secs=np.inf): - assert warmup < runs - self.warmup = warmup - self.nt = nt - self.runs = runs - self.max_secs = max_secs - self.remember_nt = faiss.omp_get_max_threads() - - def __iter__(self): - return TimerIter(self) - - def ms(self): - return np.mean(self.times) * 1000 - - def ms_std(self): - return np.std(self.times) * 1000 if len(self.times) > 1 else 0.0 - - def nruns(self): - """effective number of runs (may be lower than runs - warmup due - to timeout)""" - return len(self.times) diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/exhaustive_search.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/exhaustive_search.py deleted file mode 100644 index 87c7e422d707ebf73e043fcdb61f53f49972bf93..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/exhaustive_search.py +++ /dev/null @@ -1,394 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import faiss -import time -import numpy as np - -import logging - -LOG = logging.getLogger(__name__) - - -def knn_ground_truth( - xq, db_iterator, k, metric_type=faiss.METRIC_L2, shard=False, ngpu=-1 -): - """Computes the exact KNN search results for a dataset that possibly - does not fit in RAM but for which we have an iterator that - returns it block by block. - """ - LOG.info("knn_ground_truth queries size %s k=%d" % (xq.shape, k)) - t0 = time.time() - nq, d = xq.shape - keep_max = faiss.is_similarity_metric(metric_type) - rh = faiss.ResultHeap(nq, k, keep_max=keep_max) - - index = faiss.IndexFlat(d, metric_type) - if ngpu == -1: - ngpu = faiss.get_num_gpus() - - if ngpu: - LOG.info("running on %d GPUs" % ngpu) - co = faiss.GpuMultipleClonerOptions() - co.shard = shard - index = faiss.index_cpu_to_all_gpus(index, co=co, ngpu=ngpu) - - # compute ground-truth by blocks, and add to heaps - i0 = 0 - for xbi in db_iterator: - ni = xbi.shape[0] - index.add(xbi) - D, I = index.search(xq, k) - I += i0 - rh.add_result(D, I) - index.reset() - i0 += ni - LOG.info("%d db elements, %.3f s" % (i0, time.time() - t0)) - - rh.finalize() - LOG.info("GT time: %.3f s (%d vectors)" % (time.time() - t0, i0)) - - return rh.D, rh.I - - -# knn function used to be here -knn = faiss.knn - - -def range_search_gpu(xq, r2, index_gpu, index_cpu, gpu_k=1024): - """GPU does not support range search, so we emulate it with - knn search + fallback to CPU index. - - The index_cpu can either be: - - a CPU index that supports range search - - a numpy table, that will be used to construct a Flat index if needed. - - None. In that case, at most gpu_k results will be returned - """ - nq, d = xq.shape - is_binary_index = isinstance(index_gpu, faiss.IndexBinary) - keep_max = faiss.is_similarity_metric(index_gpu.metric_type) - r2 = int(r2) if is_binary_index else float(r2) - k = min(index_gpu.ntotal, gpu_k) - LOG.debug( - f"GPU search {nq} queries with {k=:} {is_binary_index=:} {keep_max=:}" - ) - t0 = time.time() - D, I = index_gpu.search(xq, k) - t1 = time.time() - t0 - if is_binary_index: - assert d * 8 < 32768 # let's compact the distance matrix - D = D.astype("int16") - t2 = 0 - lim_remain = None - if index_cpu is not None: - if not keep_max: - mask = D[:, k - 1] < r2 - else: - mask = D[:, k - 1] > r2 - if mask.sum() > 0: - LOG.debug("CPU search remain %d" % mask.sum()) - t0 = time.time() - if isinstance(index_cpu, np.ndarray): - # then it in fact an array that we have to make flat - xb = index_cpu - if is_binary_index: - index_cpu = faiss.IndexBinaryFlat(d * 8) - else: - index_cpu = faiss.IndexFlat(d, index_gpu.metric_type) - index_cpu.add(xb) - lim_remain, D_remain, I_remain = index_cpu.range_search( - xq[mask], r2 - ) - if is_binary_index: - D_remain = D_remain.astype("int16") - t2 = time.time() - t0 - LOG.debug("combine") - t0 = time.time() - - CombinerRangeKNN = ( - faiss.CombinerRangeKNNint16 - if is_binary_index - else faiss.CombinerRangeKNNfloat - ) - - combiner = CombinerRangeKNN(nq, k, r2, keep_max) - if True: - sp = faiss.swig_ptr - combiner.I = sp(I) - combiner.D = sp(D) - # combiner.set_knn_result(sp(I), sp(D)) - if lim_remain is not None: - combiner.mask = sp(mask) - combiner.D_remain = sp(D_remain) - combiner.lim_remain = sp(lim_remain.view("int64")) - combiner.I_remain = sp(I_remain) - # combiner.set_range_result(sp(mask), - # sp(lim_remain.view("int64")), sp(D_remain), sp(I_remain)) - L_res = np.empty(nq + 1, dtype="int64") - combiner.compute_sizes(sp(L_res)) - nres = L_res[-1] - D_res = np.empty(nres, dtype=D.dtype) - I_res = np.empty(nres, dtype="int64") - combiner.write_result(sp(D_res), sp(I_res)) - else: - D_res, I_res = [], [] - nr = 0 - for i in range(nq): - if not mask[i]: - if index_gpu.metric_type == faiss.METRIC_L2: - nv = (D[i, :] < r2).sum() - else: - nv = (D[i, :] > r2).sum() - D_res.append(D[i, :nv]) - I_res.append(I[i, :nv]) - else: - l0, l1 = lim_remain[nr], lim_remain[nr + 1] - D_res.append(D_remain[l0:l1]) - I_res.append(I_remain[l0:l1]) - nr += 1 - L_res = np.cumsum([0] + [len(di) for di in D_res]) - D_res = np.hstack(D_res) - I_res = np.hstack(I_res) - t3 = time.time() - t0 - LOG.debug(f"times {t1:.3f}s {t2:.3f}s {t3:.3f}s") - return L_res, D_res, I_res - - -def range_ground_truth( - xq, - db_iterator, - threshold, - metric_type=faiss.METRIC_L2, - shard=False, - ngpu=-1, -): - """Computes the range-search search results for a dataset that possibly - does not fit in RAM but for which we have an iterator that - returns it block by block. - """ - nq, d = xq.shape - t0 = time.time() - xq = np.ascontiguousarray(xq, dtype="float32") - - index = faiss.IndexFlat(d, metric_type) - if ngpu == -1: - ngpu = faiss.get_num_gpus() - if ngpu: - LOG.info("running on %d GPUs" % ngpu) - co = faiss.GpuMultipleClonerOptions() - co.shard = shard - index_gpu = faiss.index_cpu_to_all_gpus(index, co=co, ngpu=ngpu) - - # compute ground-truth by blocks - i0 = 0 - D = [[] for _i in range(nq)] - I = [[] for _i in range(nq)] - for xbi in db_iterator: - ni = xbi.shape[0] - if ngpu > 0: - index_gpu.add(xbi) - lims_i, Di, Ii = range_search_gpu(xq, threshold, index_gpu, xbi) - index_gpu.reset() - else: - index.add(xbi) - lims_i, Di, Ii = index.range_search(xq, threshold) - index.reset() - Ii += i0 - for j in range(nq): - l0, l1 = lims_i[j], lims_i[j + 1] - if l1 > l0: - D[j].append(Di[l0:l1]) - I[j].append(Ii[l0:l1]) - i0 += ni - LOG.info("%d db elements, %.3f s" % (i0, time.time() - t0)) - - empty_I = np.zeros(0, dtype="int64") - empty_D = np.zeros(0, dtype="float32") - # import pdb; pdb.set_trace() - D = [(np.hstack(i) if i != [] else empty_D) for i in D] - I = [(np.hstack(i) if i != [] else empty_I) for i in I] - sizes = [len(i) for i in I] - assert len(sizes) == nq - lims = np.zeros(nq + 1, dtype="uint64") - lims[1:] = np.cumsum(sizes) - return lims, np.hstack(D), np.hstack(I) - - -def threshold_radius_nres(nres, dis, ids, thresh, keep_max=False): - """select a set of results""" - if keep_max: - mask = dis > thresh - else: - mask = dis < thresh - new_nres = np.zeros_like(nres) - o = 0 - for i, nr in enumerate(nres): - nr = int(nr) # avoid issues with int64 + uint64 - new_nres[i] = mask[o : o + nr].sum() - o += nr - return new_nres, dis[mask], ids[mask] - - -def threshold_radius(lims, dis, ids, thresh, keep_max=False): - """restrict range-search results to those below a given radius""" - if keep_max: - mask = dis > thresh - else: - mask = dis < thresh - new_lims = np.zeros_like(lims) - n = len(lims) - 1 - for i in range(n): - l0, l1 = lims[i], lims[i + 1] - new_lims[i + 1] = new_lims[i] + mask[l0:l1].sum() - return new_lims, dis[mask], ids[mask] - - -def apply_maxres(res_batches, target_nres, keep_max=False): - """find radius that reduces number of results to target_nres, and - applies it in-place to the result batches used in - range_search_max_results""" - alldis = np.hstack([dis for _, dis, _ in res_batches]) - assert len(alldis) > target_nres - if keep_max: - alldis.partition(len(alldis) - target_nres - 1) - radius = alldis[-1 - target_nres] - else: - alldis.partition(target_nres) - radius = alldis[target_nres] - - if alldis.dtype == "float32": - radius = float(radius) - else: - radius = int(radius) - LOG.debug(" setting radius to %s" % radius) - totres = 0 - for i, (nres, dis, ids) in enumerate(res_batches): - nres, dis, ids = threshold_radius_nres( - nres, dis, ids, radius, keep_max=keep_max - ) - totres += len(dis) - res_batches[i] = nres, dis, ids - LOG.debug(" updated previous results, new nb results %d" % totres) - return radius, totres - - -def range_search_max_results( - index, - query_iterator, - radius, - max_results=None, - min_results=None, - shard=False, - ngpu=0, - clip_to_min=False, -): - """Performs a range search with many queries (given by an iterator) - and adjusts the threshold on-the-fly so that the total results - table does not grow larger than max_results. - - If ngpu != 0, the function moves the index to this many GPUs to - speed up search. - """ - # TODO: all result manipulations are in python, should move to C++ if perf - # critical - is_binary_index = isinstance(index, faiss.IndexBinary) - - if min_results is None: - assert max_results is not None - min_results = int(0.8 * max_results) - - if max_results is None: - assert min_results is not None - max_results = int(min_results * 1.5) - - if ngpu == -1: - ngpu = faiss.get_num_gpus() - - if ngpu: - LOG.info("running on %d GPUs" % ngpu) - co = faiss.GpuMultipleClonerOptions() - co.shard = shard - index_gpu = faiss.index_cpu_to_all_gpus(index, co=co, ngpu=ngpu) - else: - index_gpu = None - - t_start = time.time() - t_search = t_post_process = 0 - qtot = totres = raw_totres = 0 - res_batches = [] - - for xqi in query_iterator: - t0 = time.time() - LOG.debug(f"searching {len(xqi)} vectors") - if index_gpu: - lims_i, Di, Ii = range_search_gpu(xqi, radius, index_gpu, index) - else: - lims_i, Di, Ii = index.range_search(xqi, radius) - - nres_i = lims_i[1:] - lims_i[:-1] - raw_totres += len(Di) - qtot += len(xqi) - - t1 = time.time() - if is_binary_index: - # weird Faiss quirk that returns floats for Hamming distances - Di = Di.astype("int16") - - totres += len(Di) - res_batches.append((nres_i, Di, Ii)) - - if max_results is not None and totres > max_results: - LOG.info( - "too many results %d > %d, scaling back radius" - % (totres, max_results) - ) - radius, totres = apply_maxres( - res_batches, - min_results, - keep_max=index.metric_type == faiss.METRIC_INNER_PRODUCT, - ) - t2 = time.time() - t_search += t1 - t0 - t_post_process += t2 - t1 - LOG.debug( - " [%.3f s] %d queries done, %d results" - % (time.time() - t_start, qtot, totres) - ) - - LOG.info( - "search done in %.3f s + %.3f s, total %d results, end threshold %g" - % (t_search, t_post_process, totres, radius) - ) - - if clip_to_min and totres > min_results: - radius, totres = apply_maxres( - res_batches, - min_results, - keep_max=index.metric_type == faiss.METRIC_INNER_PRODUCT, - ) - - nres = np.hstack([nres_i for nres_i, dis_i, ids_i in res_batches]) - dis = np.hstack([dis_i for nres_i, dis_i, ids_i in res_batches]) - ids = np.hstack([ids_i for nres_i, dis_i, ids_i in res_batches]) - - lims = np.zeros(len(nres) + 1, dtype="uint64") - lims[1:] = np.cumsum(nres) - - return radius, lims, dis, ids - - -def exponential_query_iterator(xq, start_bs=32, max_bs=20000): - """produces batches of progressively increasing sizes. This is useful to - adjust the search radius progressively without overflowing with - intermediate results""" - nq = len(xq) - bs = start_bs - i = 0 - while i < nq: - xqi = xq[i : i + bs] - yield xqi - if bs < max_bs: - bs *= 2 - i += len(xqi) diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/factory_tools.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/factory_tools.py deleted file mode 100644 index 6327d7891633a6ff44baabdb9324741f48a69de6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/factory_tools.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import faiss -import re - - -def get_code_size(d, indexkey): - """size of one vector in an index in dimension d - constructed with factory string indexkey""" - - if indexkey == "Flat": - return d * 4 - - if indexkey.endswith(",RFlat"): - return d * 4 + get_code_size(d, indexkey[: -len(",RFlat")]) - - mo = re.match("IVF\\d+(_HNSW\\d+)?,(.*)$", indexkey) - if mo: - return get_code_size(d, mo.group(2)) - - mo = re.match("IVF\\d+\\(.*\\)?,(.*)$", indexkey) - if mo: - return get_code_size(d, mo.group(1)) - - mo = re.match("IMI\\d+x2,(.*)$", indexkey) - if mo: - return get_code_size(d, mo.group(1)) - - mo = re.match("(.*),Refine\\((.*)\\)$", indexkey) - if mo: - return get_code_size(d, mo.group(1)) + get_code_size(d, mo.group(2)) - - mo = re.match("PQ(\\d+)x(\\d+)(fs|fsr)?$", indexkey) - if mo: - return (int(mo.group(1)) * int(mo.group(2)) + 7) // 8 - - mo = re.match("PQ(\\d+)\\+(\\d+)$", indexkey) - if mo: - return int(mo.group(1)) + int(mo.group(2)) - - mo = re.match("PQ(\\d+)$", indexkey) - if mo: - return int(mo.group(1)) - - mo = re.match("HNSW(\\d+)(,Flat)?$", indexkey) - if mo: - M = int(mo.group(1)) - return d * 4 + M * 2 * 4 # roughly - - if indexkey == "SQ8": - return d - elif indexkey == "SQ4": - return (d + 1) // 2 - elif indexkey == "SQ6": - return (d * 6 + 7) // 8 - elif indexkey == "SQfp16": - return d * 2 - elif indexkey == "SQbf16": - return d * 2 - - mo = re.match("PCAR?(\\d+),(.*)$", indexkey) - if mo: - return get_code_size(int(mo.group(1)), mo.group(2)) - mo = re.match("OPQ\\d+_(\\d+),(.*)$", indexkey) - if mo: - return get_code_size(int(mo.group(1)), mo.group(2)) - mo = re.match("OPQ\\d+,(.*)$", indexkey) - if mo: - return get_code_size(d, mo.group(1)) - mo = re.match("RR(\\d+),(.*)$", indexkey) - if mo: - return get_code_size(int(mo.group(1)), mo.group(2)) - raise RuntimeError("cannot parse " + indexkey) - - -def get_hnsw_M(index): - return index.hnsw.cum_nneighbor_per_level.at(1) // 2 - - -def reverse_index_factory(index): - """ - attempts to get the factory string the index was built with - """ - sq_names = { - faiss.ScalarQuantizer.QT_8bit: "SQ8", - faiss.ScalarQuantizer.QT_4bit: "SQ4", - # QT_8bit_uniform/QT_4bit_uniform have no index_factory string; these - # synthetic names are not round-trippable through index_factory. - faiss.ScalarQuantizer.QT_8bit_uniform: "SQ8u", - faiss.ScalarQuantizer.QT_4bit_uniform: "SQ4u", - faiss.ScalarQuantizer.QT_6bit: "SQ6", - faiss.ScalarQuantizer.QT_fp16: "SQfp16", - faiss.ScalarQuantizer.QT_bf16: "SQbf16", - faiss.ScalarQuantizer.QT_8bit_direct: "SQ8_direct", - faiss.ScalarQuantizer.QT_8bit_direct_signed: "SQ8_direct_signed", - # QT_0bit ("SQ0") is parsed by index_factory; for the IVF path. - faiss.ScalarQuantizer.QT_0bit: "SQ0", - faiss.ScalarQuantizer.QT_1bit_tqmse: "SQtqmse1", - faiss.ScalarQuantizer.QT_2bit_tqmse: "SQtqmse2", - faiss.ScalarQuantizer.QT_3bit_tqmse: "SQtqmse3", - faiss.ScalarQuantizer.QT_4bit_tqmse: "SQtqmse4", - faiss.ScalarQuantizer.QT_8bit_tqmse: "SQtqmse8", - faiss.ScalarQuantizer.QT_2bit_tq: "SQtq2", - faiss.ScalarQuantizer.QT_3bit_tq: "SQtq3", - faiss.ScalarQuantizer.QT_4bit_tq: "SQtq4", - faiss.ScalarQuantizer.QT_5bit_tq: "SQtq5", - } - index = faiss.downcast_index(index) - if isinstance(index, faiss.IndexFlat): - return "Flat" - elif isinstance(index, faiss.IndexIVF): - quantizer = faiss.downcast_index(index.quantizer) - - if isinstance(quantizer, faiss.IndexFlat): - prefix = f"IVF{index.nlist}" - elif isinstance(quantizer, faiss.MultiIndexQuantizer): - prefix = f"IMI{quantizer.pq.M}x{quantizer.pq.nbits}" - elif isinstance(quantizer, faiss.IndexHNSW): - prefix = f"IVF{index.nlist}_HNSW{get_hnsw_M(quantizer)}" - else: - prefix = f"IVF{index.nlist}({reverse_index_factory(quantizer)})" - - if isinstance(index, faiss.IndexIVFFlat): - return prefix + ",Flat" - if isinstance(index, faiss.IndexIVFScalarQuantizer): - return prefix + "," + sq_names[index.sq.qtype] - if isinstance(index, faiss.IndexIVFPQ): - return prefix + f",PQ{index.pq.M}x{index.pq.nbits}" - if isinstance(index, faiss.IndexIVFPQFastScan): - return prefix + f",PQ{index.pq.M}x{index.pq.nbits}fs" - if isinstance(index, faiss.IndexIVFRaBitQ): - nb_bits = index.rabitq.nb_bits - suffix = "RaBitQ" if nb_bits == 1 else f"RaBitQ{nb_bits}" - return prefix + "," + suffix - - elif isinstance(index, faiss.IndexPreTransform): - if index.chain.size() != 1: - raise NotImplementedError() - vt = faiss.downcast_VectorTransform(index.chain.at(0)) - if isinstance(vt, faiss.OPQMatrix): - prefix = f"OPQ{vt.M}_{vt.d_out}" - elif isinstance(vt, faiss.ITQTransform): - prefix = f"ITQ{vt.itq.d_out}" - elif isinstance(vt, faiss.PCAMatrix): - assert vt.eigen_power == 0 - prefix = "PCA" + ("R" if vt.random_rotation else "") + str(vt.d_out) - else: - raise NotImplementedError() - return f"{prefix},{reverse_index_factory(index.index)}" - - elif isinstance(index, faiss.IndexHNSW): - return f"HNSW{get_hnsw_M(index)}" - - elif isinstance(index, faiss.IndexRefine): - return ( - f"{reverse_index_factory(index.base_index)}," - f"Refine({reverse_index_factory(index.refine_index)})" - ) - - elif isinstance(index, faiss.IndexPQFastScan): - return f"PQ{index.pq.M}x{index.pq.nbits}fs" - - elif isinstance(index, faiss.IndexPQ): - return f"PQ{index.pq.M}x{index.pq.nbits}" - - elif isinstance(index, faiss.IndexLSH): - return ( - "LSH" - + ("r" if index.rotate_data else "") - + ("t" if index.train_thresholds else "") - ) - - elif isinstance(index, faiss.IndexScalarQuantizer): - return sq_names[index.sq.qtype] - - elif isinstance(index, faiss.IndexRaBitQ): - nb_bits = index.rabitq.nb_bits - return "RaBitQ" if nb_bits == 1 else f"RaBitQ{nb_bits}" - - # IndexIDMap2 is a subclass of IndexIDMap, so it must be checked first. - elif isinstance(index, faiss.IndexIDMap2): - return f"IDMap2,{reverse_index_factory(index.index)}" - - elif isinstance(index, faiss.IndexIDMap): - return f"IDMap,{reverse_index_factory(index.index)}" - - raise NotImplementedError() diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/inspect_tools.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/inspect_tools.py deleted file mode 100644 index 400e1316625013cc0e5b3216c3582dd0b2c849e4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/inspect_tools.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -import faiss - - -def get_invlist(invlists, l): - """returns the inverted lists content as a pair of (list_ids, list_codes). - The codes are reshaped to a proper size - """ - invlists = faiss.downcast_InvertedLists(invlists) - ls = invlists.list_size(l) - list_ids = np.zeros(ls, dtype="int64") - ids = codes = None - try: - ids = invlists.get_ids(l) - if ls > 0: - faiss.memcpy(faiss.swig_ptr(list_ids), ids, list_ids.nbytes) - codes = invlists.get_codes(l) - if invlists.code_size != faiss.InvertedLists.INVALID_CODE_SIZE: - list_codes = np.zeros((ls, invlists.code_size), dtype="uint8") - else: - # it's a BlockInvertedLists - npb = invlists.n_per_block - bs = invlists.block_size - ls_round = (ls + npb - 1) // npb - list_codes = np.zeros((ls_round, bs // npb, npb), dtype="uint8") - if ls > 0: - faiss.memcpy(faiss.swig_ptr(list_codes), codes, list_codes.nbytes) - finally: - if ids is not None: - invlists.release_ids(l, ids) - if codes is not None: - invlists.release_codes(l, codes) - return list_ids, list_codes - - -def get_invlist_sizes(invlists): - """return the array of sizes of the inverted lists""" - return np.array( - [invlists.list_size(i) for i in range(invlists.nlist)], dtype="int64" - ) - - -def print_object_fields(obj): - """list values all fields of an object known to SWIG""" - - for name in obj.__class__.__swig_getmethods__: - print(f"{name} = {getattr(obj, name)}") - - -def get_pq_centroids(pq): - """return the PQ centroids as an array""" - cen = faiss.vector_to_array(pq.centroids) - return cen.reshape(pq.M, pq.ksub, pq.dsub) - - -def get_LinearTransform_matrix(pca): - """extract matrix + bias from the PCA object - works for any linear transform (OPQ, random rotation, etc.) - """ - b = faiss.vector_to_array(pca.b) - A = faiss.vector_to_array(pca.A).reshape(pca.d_out, pca.d_in) - return A, b - - -def make_LinearTransform_matrix(A, b=None): - """make a linear transform from a matrix and a bias term (optional)""" - d_out, d_in = A.shape - if b is not None: - assert b.shape == (d_out,) - lt = faiss.LinearTransform(d_in, d_out, b is not None) - faiss.copy_array_to_vector(A.ravel(), lt.A) - if b is not None: - faiss.copy_array_to_vector(b, lt.b) - lt.is_trained = True - lt.set_is_orthonormal() - return lt - - -def get_additive_quantizer_codebooks(aq): - """return to codebooks of an additive quantizer""" - codebooks = faiss.vector_to_array(aq.codebooks).reshape(-1, aq.d) - co = faiss.vector_to_array(aq.codebook_offsets) - return [codebooks[co[i] : co[i + 1]] for i in range(aq.M)] - - -def get_flat_data(index): - """copy and return the data matrix in an IndexFlat""" - xb = faiss.vector_to_array(index.codes).view("float32") - return xb.reshape(index.ntotal, index.d) - - -def get_flat_codes(index_flat): - """get the codes from an indexFlatCodes as an array""" - return faiss.vector_to_array(index_flat.codes).reshape( - index_flat.ntotal, index_flat.code_size - ) - - -def get_NSG_neighbors(nsg): - """get the neighbor list for the vectors stored in the NSG structure, as - a N-by-K matrix of indices""" - graph = nsg.get_final_graph() - neighbors = np.zeros((graph.N, graph.K), dtype="int32") - faiss.memcpy(faiss.swig_ptr(neighbors), graph.data, neighbors.nbytes) - return neighbors diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/ivf_tools.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/ivf_tools.py deleted file mode 100644 index 17d32dee16f9f68738d0c07853cab5e1bacadc3e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/ivf_tools.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import numpy as np -import faiss - -from faiss.contrib.inspect_tools import get_invlist_sizes - - -def add_preassigned(index_ivf, x, a, ids=None): - """ - Add elements to an IVF index, where the assignment is already computed - """ - n, d = x.shape - assert a.shape == (n,) - if isinstance(index_ivf, faiss.IndexBinaryIVF): - d *= 8 - assert d == index_ivf.d - if ids is not None: - assert ids.shape == (n,) - ids = faiss.swig_ptr(ids) - index_ivf.add_core(n, faiss.swig_ptr(x), ids, faiss.swig_ptr(a)) - - -def search_preassigned(index_ivf, xq, k, list_nos, coarse_dis=None): - """ - Perform a search in the IVF index, with predefined lists to search into. - Supports indexes with pretransforms (as opposed to the - IndexIVF.search_preassigned, that cannot be applied with pretransform). - """ - if isinstance(index_ivf, faiss.IndexPreTransform): - assert index_ivf.chain.size() == 1, "chain must have only one component" - transform = faiss.downcast_VectorTransform(index_ivf.chain.at(0)) - xq = transform.apply(xq) - index_ivf = faiss.downcast_index(index_ivf.index) - n, d = xq.shape - if isinstance(index_ivf, faiss.IndexBinaryIVF): - d *= 8 - dis_type = "int32" - else: - dis_type = "float32" - - assert d == index_ivf.d - assert list_nos.shape == (n, index_ivf.nprobe) - - # the coarse distances are used in IVFPQ with L2 distance and - # by_residual=True otherwise we provide dummy coarse_dis - if coarse_dis is None: - coarse_dis = np.zeros((n, index_ivf.nprobe), dtype=dis_type) - else: - assert coarse_dis.shape == (n, index_ivf.nprobe) - - return index_ivf.search_preassigned(xq, k, list_nos, coarse_dis) - - -def range_search_preassigned(index_ivf, x, radius, list_nos, coarse_dis=None): - """ - Perform a range search in the IVF index, with predefined lists to - search into - """ - n, d = x.shape - if isinstance(index_ivf, faiss.IndexBinaryIVF): - d *= 8 - dis_type = "int32" - else: - dis_type = "float32" - - # the coarse distances are used in IVFPQ with L2 distance and - # by_residual=True otherwise we provide dummy coarse_dis - if coarse_dis is None: - coarse_dis = np.zeros((n, index_ivf.nprobe), dtype=dis_type) - else: - assert coarse_dis.shape == (n, index_ivf.nprobe) - - assert d == index_ivf.d - assert list_nos.shape == (n, index_ivf.nprobe) - - res = faiss.RangeSearchResult(n) - sp = faiss.swig_ptr - - index_ivf.range_search_preassigned_c( - n, sp(x), radius, sp(list_nos), sp(coarse_dis), res - ) - # get pointers and copy them - lims = faiss.rev_swig_ptr(res.lims, n + 1).copy() - num_results = int(lims[-1]) - dist = faiss.rev_swig_ptr(res.distances, num_results).copy() - indices = faiss.rev_swig_ptr(res.labels, num_results).copy() - return lims, dist, indices - - -def replace_ivf_quantizer(index_ivf, new_quantizer): - """replace the IVF quantizer with a flat quantizer and return the - old quantizer""" - if new_quantizer.ntotal == 0: - centroids = index_ivf.quantizer.reconstruct_n() - new_quantizer.train(centroids) - new_quantizer.add(centroids) - else: - assert new_quantizer.ntotal == index_ivf.nlist - - # cleanly dealloc old quantizer - old_own = index_ivf.own_fields - index_ivf.own_fields = False - old_quantizer = faiss.downcast_index(index_ivf.quantizer) - old_quantizer.this.own(old_own) - index_ivf.quantizer = new_quantizer - - if hasattr(index_ivf, "referenced_objects"): - index_ivf.referenced_objects.append(new_quantizer) - else: - index_ivf.referenced_objects = [new_quantizer] - return old_quantizer - - -def permute_invlists(index_ivf, perm): - """Apply some permutation to the inverted lists, and modify the quantizer - entries accordingly. - Perm is an array of size nlist, where old_index = perm[new_index] - """ - (nlist,) = perm.shape - assert index_ivf.nlist == nlist - quantizer = faiss.downcast_index(index_ivf.quantizer) - assert quantizer.ntotal == index_ivf.nlist - perm = np.ascontiguousarray(perm, dtype="int64") - - # just make sure it's a permutation... - bc = np.bincount(perm, minlength=nlist) - assert np.all(bc == np.ones(nlist, dtype=int)) - - # handle quantizer - quantizer.permute_entries(perm) - - # handle inverted lists - invlists = faiss.downcast_InvertedLists(index_ivf.invlists) - invlists.permute_invlists(faiss.swig_ptr(perm)) - - -def sort_invlists_by_size(index_ivf): - invlist_sizes = get_invlist_sizes(index_ivf.invlists) - perm = np.argsort(invlist_sizes) - permute_invlists(index_ivf, perm) diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/ondisk.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/ondisk.py deleted file mode 100644 index e51341c52811ace12077b04de6291c0d048c7be7..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/ondisk.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from typing import List -import faiss -import logging - -LOG = logging.getLogger(__name__) - - -def merge_ondisk( - trained_index: faiss.Index, - shard_fnames: List[str], - ivfdata_fname: str, - shift_ids=False, -) -> None: - """Add the contents of the indexes stored in shard_fnames into the index - trained_index. The on-disk data is stored in ivfdata_fname""" - assert not isinstance( - trained_index, faiss.IndexIVFPQR - ), "IndexIVFPQR is not supported as an on disk index." - # merge the images into an on-disk index - # first load the inverted lists - ivfs = [] - for fname in shard_fnames: - # the IO_FLAG_MMAP is to avoid actually loading the data thus - # the total size of the inverted lists can exceed the - # available RAM - LOG.info("read " + fname) - index = faiss.read_index(fname, faiss.IO_FLAG_MMAP) - index_ivf = faiss.extract_index_ivf(index) - ivfs.append(index_ivf.invlists) - - # avoid that the invlists get deallocated with the index - index_ivf.own_invlists = False - - # construct the output index - index = trained_index - index_ivf = faiss.extract_index_ivf(index) - - assert index.ntotal == 0, "works only on empty index" - - # prepare the output inverted lists. They will be written - # to merged_index.ivfdata - invlists = faiss.OnDiskInvertedLists( - index_ivf.nlist, index_ivf.code_size, ivfdata_fname - ) - - # merge all the inverted lists - ivf_vector = faiss.InvertedListsPtrVector() - for ivf in ivfs: - ivf_vector.push_back(ivf) - - LOG.info("merge %d inverted lists " % ivf_vector.size()) - ntotal = invlists.merge_from_multiple( - ivf_vector.data(), ivf_vector.size(), shift_ids - ) - - # now replace the inverted lists in the output index - index.ntotal = index_ivf.ntotal = ntotal - index_ivf.replace_invlists(invlists, True) - invlists.this.disown() diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/rpc.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/rpc.py deleted file mode 100644 index 0d9b46af64d61a7d00fab9c0e08f5d03162a2156..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/rpc.py +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -Simplistic RPC implementation. -Exposes all functions of a Server object. - -This code is for demonstration purposes only, and does not include certain -security protections. It is not meant to be run on an untrusted network or -in a production environment. -""" - -import importlib -import os -import pickle -import sys -import _thread -import traceback -import socket -import logging - -LOG = logging.getLogger(__name__) - -# default -PORT = 12032 - -safe_modules = { - "numpy", - "numpy.core.multiarray", -} - - -class RestrictedUnpickler(pickle.Unpickler): - - def find_class(self, module, name): - # Only allow safe modules. - if module in safe_modules: - return getattr(importlib.import_module(module), name) - # Forbid everything else. - raise pickle.UnpicklingError( - "global '%s.%s' is forbidden" % (module, name) - ) - - -class FileSock: - "wraps a socket so that it is usable by pickle/cPickle" - - def __init__(self, sock): - self.sock = sock - self.nr = 0 - - def write(self, buf): - # print("sending %d bytes"%len(buf)) - # self.sock.sendall(buf) - # print("...done") - bs = 512 * 1024 - ns = 0 - while ns < len(buf): - sent = self.sock.send(buf[ns : ns + bs]) - ns += sent - - def read(self, bs=512 * 1024): - # if self.nr==10000: pdb.set_trace() - self.nr += 1 - # print("read bs=%d"%bs) - b = [] - nb = 0 - while len(b) < bs: - # print(' loop') - rb = self.sock.recv(bs - nb) - if not rb: - break - b.append(rb) - nb += len(rb) - return b"".join(b) - - def readline(self): - # print("readline!") - """may be optimized...""" - s = bytes() - while True: - c = self.read(1) - s += c - if len(c) == 0 or chr(c[0]) == "\n": - return s - - -class ClientExit(Exception): - pass - - -class ServerException(Exception): - pass - - -class Server: - """ - server protocol. Methods from classes that subclass Server can be called - transparently from a client - """ - - def __init__(self, s, logf=sys.stderr, log_prefix=""): - self.logf = logf - self.log_prefix = log_prefix - - # connection - - self.conn = s - self.fs = FileSock(s) - - def log(self, s): - self.logf.write("Server log %s: %s\n" % (self.log_prefix, s)) - - def one_function(self): - """ - Executes a single function with associated I/O. - Protocol: - - the arguments and results are serialized with the pickle protocol - - client sends : (fname,args) - fname = method name to call - args = tuple of arguments - - server sends result: (rid,st,ret) - rid = request id - st = None, or exception if there was during execution - ret = return value or None if st!=None - """ - - try: - (fname, args) = RestrictedUnpickler(self.fs).load() - except EOFError: - raise ClientExit("read args") - self.log("executing method %s" % (fname)) - st = None - ret = None - try: - f = getattr(self, fname) - except AttributeError: - st = AttributeError("unknown method " + fname) - self.log("unknown method") - - try: - ret = f(*args) - except Exception as e: - # due to a bug (in mod_python?), ServerException cannot be - # unpickled, so send the string and make the exception on the - # client side - - # st=ServerException( - # "".join(traceback.format_tb(sys.exc_info()[2]))+ - # str(e)) - st = "".join(traceback.format_tb(sys.exc_info()[2])) + str(e) - self.log("exception in method") - traceback.print_exc(50, self.logf) - self.logf.flush() - - LOG.info("return") - try: - pickle.dump((st, ret), self.fs, protocol=4) - except EOFError: - raise ClientExit("function return") - - def exec_loop(self): - """main execution loop. Loops and handles exit states""" - - self.log("in exec_loop") - try: - while True: - self.one_function() - except ClientExit as e: - self.log("ClientExit %s" % e) - except socket.error as e: - self.log("socket error %s" % e) - traceback.print_exc(50, self.logf) - except EOFError: - self.log("EOF during communication") - traceback.print_exc(50, self.logf) - except BaseException: - # unexpected - traceback.print_exc(50, sys.stderr) - sys.exit(1) - - LOG.info("exit server") - - def exec_loop_cleanup(self): - pass - - ################################################################### - # spying stuff - - def get_ps_stats(self): - ret = "" - f = os.popen( - "echo ============ `hostname` uptime:; uptime;" - + "echo ============ self:; " - + "ps -p %d -o pid,vsize,rss,%%cpu,nlwp,psr; " % os.getpid() - + "echo ============ run queue:;" - + "ps ar -o user,pid,%cpu,%mem,ni,nlwp,psr,vsz,rss,cputime,command" - ) - for l in f: - ret += l - return ret - - -class Client: - """ - Methods of the server object can be called transparently. Exceptions are - re-raised. - """ - - def __init__(self, HOST, port=PORT, v6=False): - socktype = socket.AF_INET6 if v6 else socket.AF_INET - - sock = socket.socket(socktype, socket.SOCK_STREAM) - LOG.info("connecting to %s:%d, socket type: %s", HOST, port, socktype) - sock.connect((HOST, port)) - self.sock = sock - self.fs = FileSock(sock) - - def generic_fun(self, fname, args): - # int "gen fun",fname - pickle.dump((fname, args), self.fs, protocol=4) - return self.get_result() - - def get_result(self): - (st, ret) = RestrictedUnpickler(self.fs).load() - if st != None: - raise ServerException(st) - else: - return ret - - def __getattr__(self, name): - return lambda *x: self.generic_fun(name, x) - - -def run_server(new_handler, port=PORT, report_to_file=None, v6=False): - - HOST = "" # Symbolic name meaning the local host - socktype = socket.AF_INET6 if v6 else socket.AF_INET - s = socket.socket(socktype, socket.SOCK_STREAM) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - - LOG.info("bind %s:%d", HOST, port) - s.bind((HOST, port)) - s.listen(5) - - LOG.info("accepting connections") - if report_to_file is not None: - LOG.info("storing host+port in %s", report_to_file) - open(report_to_file, "w").write("%s:%d " % (socket.gethostname(), port)) - - while True: - try: - conn, addr = s.accept() - except socket.error as e: - if e[1] == "Interrupted system call": - continue - raise - - LOG.info("Connected to %s", addr) - - ibs = new_handler(conn) - - tid = _thread.start_new_thread(ibs.exec_loop, ()) - - LOG.debug("Thread ID: %d", tid) diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch/README.md b/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch/README.md deleted file mode 100644 index c8e144b84cef3bbcfc4820b667e6e1b634479ade..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# The Torch contrib - -This contrib directory contains a few Pytorch routines that -are useful for similarity search. They do not necessarily depend on Faiss. - -The code is designed to work with CPU and GPU tensors. diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch/__init__.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch/clustering.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch/clustering.py deleted file mode 100644 index 37e98d9cd2d0d3480a1b88c7322a54a9c85e1390..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch/clustering.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -This contrib module contains Pytorch code for k-means clustering -""" -import faiss -import faiss.contrib.torch_utils -import torch - -# the kmeans can produce both torch and numpy centroids -from faiss.contrib.clustering import kmeans # noqa: F401 some libraries import kmeans from here - - -class DatasetAssign: - """Wrapper for a tensor that offers a function to assign the vectors - to centroids. All other implementations offer the same interface""" - - def __init__(self, x): - self.x = x - - def count(self): - return self.x.shape[0] - - def dim(self): - return self.x.shape[1] - - def get_subset(self, indices): - return self.x[indices] - - def perform_search(self, centroids): - return faiss.knn(self.x, centroids, 1) - - def assign_to(self, centroids, weights=None): - D, I = self.perform_search(centroids) - - I = I.ravel() - D = D.ravel() - nc, d = centroids.shape - - sum_per_centroid = torch.zeros_like(centroids) - if weights is None: - sum_per_centroid.index_add_(0, I, self.x) - else: - sum_per_centroid.index_add_(0, I, self.x * weights[:, None]) - - # the indices are still in numpy. - return I.cpu().numpy(), D, sum_per_centroid - - -class DatasetAssignGPU(DatasetAssign): - - def __init__(self, res, x): - DatasetAssign.__init__(self, x) - self.res = res - - def perform_search(self, centroids): - return faiss.knn_gpu(self.res, self.x, centroids, 1) diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch/quantization.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch/quantization.py deleted file mode 100644 index 34ae9671b015d330bfb08ae11f4d703fbbc93ad0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch/quantization.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -This contrib module contains Pytorch code for quantization. -""" - -import torch -import faiss -import math -from faiss.contrib.torch import clustering - -# the kmeans can produce both torch and numpy centroids - - -class Quantizer: - - def __init__(self, d, code_size): - """ - d: dimension of vectors - code_size: nb of bytes of the code (per vector) - """ - self.d = d - self.code_size = code_size - - def train(self, x): - """ - takes a n-by-d array and performs training - """ - pass - - def encode(self, x): - """ - takes a n-by-d float array, encodes to an n-by-code_size uint8 array - """ - pass - - def decode(self, codes): - """ - takes a n-by-code_size uint8 array, returns a n-by-d array - """ - pass - - -class VectorQuantizer(Quantizer): - - def __init__(self, d, k): - - code_size = int(math.ceil(torch.log2(k) / 8)) - Quantizer.__init__(d, code_size) - self.k = k - - def train(self, x): - pass - - -class ProductQuantizer(Quantizer): - def __init__(self, d, M, nbits): - """M: number of subvectors, d%M == 0 - nbits: number of bits that each vector is encoded into - """ - assert d % M == 0 - assert nbits == 8 # todo: implement other nbits values - code_size = int(math.ceil(M * nbits / 8)) - Quantizer.__init__(self, d, code_size) - self.M = M - self.nbits = nbits - self.code_size = code_size - - def train(self, x): - nc = 2**self.nbits - sd = self.d // self.M - dev = x.device - dtype = x.dtype - self.codebook = torch.zeros((self.M, nc, sd), device=dev, dtype=dtype) - for m in range(self.M): - xsub = x[:, m * self.d // self.M : (m + 1) * self.d // self.M] - data = clustering.DatasetAssign(xsub.contiguous()) - self.codebook[m] = clustering.kmeans(2**self.nbits, data) - - def encode(self, x): - codes = torch.zeros((x.shape[0], self.code_size), dtype=torch.uint8) - for m in range(self.M): - xsub = x[:, m * self.d // self.M : (m + 1) * self.d // self.M] - _, I = faiss.knn(xsub.contiguous(), self.codebook[m], 1) - codes[:, m] = I.ravel() - return codes - - def decode(self, codes): - idxs = [codes[:, m].long() for m in range(self.M)] - vectors = [self.codebook[m, idxs[m], :] for m in range(self.M)] - stacked_vectors = torch.stack(vectors, dim=1) - cbd = self.codebook.shape[-1] - x_rec = stacked_vectors.reshape(-1, cbd * self.M) - return x_rec diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch_utils.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch_utils.py deleted file mode 100644 index 325da8c766f08bdd6109996746e0426da16382d2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/torch_utils.py +++ /dev/null @@ -1,888 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" - -This is a set of function wrappers that override the default numpy versions. - -Interoperability functions for pytorch and Faiss: Importing this will allow -pytorch Tensors (CPU or GPU) to be used as arguments to Faiss indexes and -other functions. Torch GPU tensors can only be used with Faiss GPU indexes. -If this is imported with a package that supports Faiss GPU, the necessary -stream synchronization with the current pytorch stream will be automatically -performed. - -Numpy ndarrays can continue to be used in the Faiss python interface after -importing this file. All arguments must be uniformly either numpy ndarrays -or Torch tensors; no mixing is allowed. - -""" - - -import faiss -import torch -import contextlib -import inspect -import sys -import numpy as np - -################################################################## -# Equivalent of swig_ptr for Torch tensors -################################################################## - - -def swig_ptr_from_UInt8Tensor(x): - """gets a Faiss SWIG pointer from a pytorch tensor (on CPU or GPU)""" - assert x.is_contiguous() - assert x.dtype == torch.uint8 - return faiss.cast_integer_to_uint8_ptr( - x.untyped_storage().data_ptr() + x.storage_offset() - ) - - -def swig_ptr_from_HalfTensor(x): - """gets a Faiss SWIG pointer from a pytorch tensor (on CPU or GPU)""" - assert x.is_contiguous() - assert x.dtype == torch.float16 - # no canonical half type in C/C++ - return faiss.cast_integer_to_void_ptr( - x.untyped_storage().data_ptr() + x.storage_offset() * 2 - ) - - -def swig_ptr_from_FloatTensor(x): - """gets a Faiss SWIG pointer from a pytorch tensor (on CPU or GPU)""" - assert x.is_contiguous() - assert x.dtype == torch.float32 - return faiss.cast_integer_to_float_ptr( - x.untyped_storage().data_ptr() + x.storage_offset() * 4 - ) - - -def swig_ptr_from_BFloat16Tensor(x): - """gets a Faiss SWIG pointer from a pytorch tensor (on CPU or GPU)""" - assert x.is_contiguous() - assert x.dtype == torch.bfloat16 - return faiss.cast_integer_to_void_ptr( - x.untyped_storage().data_ptr() + x.storage_offset() * 2 - ) - - -def swig_ptr_from_IntTensor(x): - """gets a Faiss SWIG pointer from a pytorch tensor (on CPU or GPU)""" - assert x.is_contiguous() - assert x.dtype == torch.int32, "dtype=%s" % x.dtype - return faiss.cast_integer_to_int_ptr( - x.untyped_storage().data_ptr() + x.storage_offset() * 4 - ) - - -def swig_ptr_from_IndicesTensor(x): - """gets a Faiss SWIG pointer from a pytorch tensor (on CPU or GPU)""" - assert x.is_contiguous() - assert x.dtype == torch.int64, "dtype=%s" % x.dtype - return faiss.cast_integer_to_idx_t_ptr( - x.untyped_storage().data_ptr() + x.storage_offset() * 8 - ) - - -################################################################## -# utilities -################################################################## - - -@contextlib.contextmanager -def using_stream(res, pytorch_stream=None): - """Creates a scoping object to make Faiss GPU use the same stream - as pytorch, based on torch.cuda.current_stream(). - Or, a specific pytorch stream can be passed in as a second - argument, in which case we will use that stream. - """ - - if pytorch_stream is None: - pytorch_stream = torch.cuda.current_stream() - - # This is the cudaStream_t that we wish to use - cuda_stream_s = faiss.cast_integer_to_cudastream_t( - pytorch_stream.cuda_stream - ) - - # So we can revert GpuResources stream state upon exit - prior_dev = torch.cuda.current_device() - prior_stream = res.getDefaultStream(torch.cuda.current_device()) - - res.setDefaultStream(torch.cuda.current_device(), cuda_stream_s) - - # Do the user work - try: - yield - finally: - res.setDefaultStream(prior_dev, prior_stream) - - -def torch_replace_method( - the_class, name, replacement, ignore_missing=False, ignore_no_base=False -): - try: - orig_method = getattr(the_class, name) - except AttributeError: - if ignore_missing: - return - raise - if orig_method.__name__ == "torch_replacement_" + name: - # replacement was done in parent class - return - - # We should already have the numpy replacement methods patched - assert ignore_no_base or (orig_method.__name__ == "replacement_" + name) - setattr(the_class, name + "_numpy", orig_method) - setattr(the_class, name, replacement) - - -################################################################## -# Setup wrappers -################################################################## - - -def handle_torch_Index(the_class): - def torch_replacement_add(self, x, numeric_type=faiss.Float32): - if type(x) is np.ndarray: - # forward to faiss __init__.py base method - return self.add_numpy(x) - - assert type(x) is torch.Tensor - n, d = x.shape - assert d == self.d - if numeric_type == faiss.Float32: - x_ptr = swig_ptr_from_FloatTensor(x) - elif numeric_type == faiss.Float16: - x_ptr = swig_ptr_from_HalfTensor(x) - else: - raise ValueError( - "numeric type must be either faiss.Float32 or faiss.Float16" - ) - - if x.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.add_ex(n, x_ptr, numeric_type) - else: - # CPU torch - self.add_ex(n, x_ptr, numeric_type) - - def torch_replacement_add_with_ids( - self, x, ids, numeric_type=faiss.Float32 - ): - if type(x) is np.ndarray: - # forward to faiss __init__.py base method - return self.add_with_ids_numpy(x, ids) - - assert type(x) is torch.Tensor - n, d = x.shape - assert d == self.d - if numeric_type == faiss.Float32: - x_ptr = swig_ptr_from_FloatTensor(x) - elif numeric_type == faiss.Float16: - x_ptr = swig_ptr_from_HalfTensor(x) - else: - raise ValueError( - "numeric type must be either faiss.Float32 or faiss.Float16" - ) - - assert type(ids) is torch.Tensor - assert ids.shape == (n,), "not same number of vectors as ids" - ids_ptr = swig_ptr_from_IndicesTensor(ids) - - if x.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.add_with_ids_ex(n, x_ptr, numeric_type, ids_ptr) - else: - # CPU torch - self.add_with_ids_ex(n, x_ptr, numeric_type, ids_ptr) - - def torch_replacement_assign(self, x, k, labels=None): - if type(x) is np.ndarray: - # forward to faiss __init__.py base method - return self.assign_numpy(x, k, labels) - - assert type(x) is torch.Tensor - n, d = x.shape - assert d == self.d - x_ptr = swig_ptr_from_FloatTensor(x) - - if labels is None: - labels = torch.empty(n, k, device=x.device, dtype=torch.int64) - else: - assert type(labels) is torch.Tensor - assert labels.shape == (n, k) - L_ptr = swig_ptr_from_IndicesTensor(labels) - - if x.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.assign_c(n, x_ptr, L_ptr, k) - else: - # CPU torch - self.assign_c(n, x_ptr, L_ptr, k) - - return labels - - def torch_replacement_train(self, x, numeric_type=faiss.Float32): - if type(x) is np.ndarray: - # forward to faiss __init__.py base method - return self.train_numpy(x) - - assert type(x) is torch.Tensor - n, d = x.shape - assert d == self.d - if numeric_type == faiss.Float32: - x_ptr = swig_ptr_from_FloatTensor(x) - elif numeric_type == faiss.Float16: - x_ptr = swig_ptr_from_HalfTensor(x) - else: - raise ValueError( - "numeric type must be either faiss.Float32 or faiss.Float16" - ) - - if x.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.train_ex(n, x_ptr, numeric_type) - else: - # CPU torch - self.train_ex(n, x_ptr, numeric_type) - - def search_methods_common(x, k, D, I, numeric_type=faiss.Float32): - n, d = x.shape - if numeric_type == faiss.Float32: - x_ptr = swig_ptr_from_FloatTensor(x) - elif numeric_type == faiss.Float16: - x_ptr = swig_ptr_from_HalfTensor(x) - else: - raise ValueError( - "numeric type must be either faiss.Float32 or faiss.Float16" - ) - - if D is None: - D = torch.empty(n, k, device=x.device, dtype=torch.float32) - else: - assert type(D) is torch.Tensor - assert D.shape == (n, k) - D_ptr = swig_ptr_from_FloatTensor(D) - - if I is None: - I = torch.empty(n, k, device=x.device, dtype=torch.int64) - else: - assert type(I) is torch.Tensor - assert I.shape == (n, k) - I_ptr = swig_ptr_from_IndicesTensor(I) - - return x_ptr, D_ptr, I_ptr, D, I - - def torch_replacement_search( - self, x, k, D=None, I=None, numeric_type=faiss.Float32 - ): - if type(x) is np.ndarray: - # forward to faiss __init__.py base method - return self.search_numpy(x, k, D=D, I=I) - - assert type(x) is torch.Tensor - n, d = x.shape - assert d == self.d - - x_ptr, D_ptr, I_ptr, D, I = search_methods_common(x, k, D, I) - - if x.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.search_ex(n, x_ptr, numeric_type, k, D_ptr, I_ptr) - else: - # CPU torch - self.search_ex(n, x_ptr, numeric_type, k, D_ptr, I_ptr) - - return D, I - - def torch_replacement_search_and_reconstruct( - self, x, k, D=None, I=None, R=None - ): - if type(x) is np.ndarray: - # Forward to faiss __init__.py base method - return self.search_and_reconstruct_numpy(x, k, D=D, I=I, R=R) - - assert type(x) is torch.Tensor - n, d = x.shape - assert d == self.d - - x_ptr, D_ptr, I_ptr, D, I = search_methods_common(x, k, D, I) - - if R is None: - R = torch.empty(n, k, d, device=x.device, dtype=torch.float32) - else: - assert type(R) is torch.Tensor - assert R.shape == (n, k, d) - R_ptr = swig_ptr_from_FloatTensor(R) - - if x.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.search_and_reconstruct_c(n, x_ptr, k, D_ptr, I_ptr, R_ptr) - else: - # CPU torch - self.search_and_reconstruct_c(n, x_ptr, k, D_ptr, I_ptr, R_ptr) - - return D, I, R - - def torch_replacement_search_preassigned( - self, x, k, Iq, Dq, *, D=None, I=None - ): - if type(x) is np.ndarray: - # forward to faiss __init__.py base method - return self.search_preassigned_numpy(x, k, Iq, Dq, D=D, I=I) - - assert type(x) is torch.Tensor - n, d = x.shape - assert d == self.d - - x_ptr, D_ptr, I_ptr, D, I = search_methods_common(x, k, D, I) - - assert Iq.shape == (n, self.nprobe) - Iq = Iq.contiguous() - Iq_ptr = swig_ptr_from_IndicesTensor(Iq) - - if Dq is not None: - Dq = Dq.contiguous() - assert Dq.shape == Iq.shape - Dq_ptr = swig_ptr_from_FloatTensor(Dq) - else: - Dq_ptr = None - - if x.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.search_preassigned_c( - n, x_ptr, k, Iq_ptr, Dq_ptr, D_ptr, I_ptr, False - ) - else: - # CPU torch - self.search_preassigned_c( - n, x_ptr, k, Iq_ptr, Dq_ptr, D_ptr, I_ptr, False - ) - - return D, I - - def torch_replacement_remove_ids(self, x): - # Not yet implemented - assert ( - type(x) is not torch.Tensor - ), "remove_ids not yet implemented for torch" - return self.remove_ids_numpy(x) - - def torch_replacement_reconstruct(self, key, x=None): - # No tensor inputs are required, but with importing this module, we - # assume that the default should be torch tensors. If we are passed a - # numpy array, however, assume that the user is overriding this default - if (x is not None) and (type(x) is np.ndarray): - # Forward to faiss __init__.py base method - return self.reconstruct_numpy(key, x) - - # If the index is a CPU index, the default device is CPU, otherwise we - # produce a GPU tensor - device = torch.device("cpu") - if hasattr(self, "getDevice"): - # same device as the index - device = torch.device("cuda", self.getDevice()) - - if x is None: - x = torch.empty(self.d, device=device, dtype=torch.float32) - else: - assert type(x) is torch.Tensor - assert x.shape == (self.d,) - x_ptr = swig_ptr_from_FloatTensor(x) - - if x.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.reconstruct_c(key, x_ptr) - else: - # CPU torch - self.reconstruct_c(key, x_ptr) - - return x - - def torch_replacement_reconstruct_n(self, n0=0, ni=-1, x=None): - if ni == -1: - ni = self.ntotal - - # No tensor inputs are required, but with importing this module, we - # assume that the default should be torch tensors. If we are passed a - # numpy array, however, assume that the user is overriding this default - if (x is not None) and (type(x) is np.ndarray): - # Forward to faiss __init__.py base method - return self.reconstruct_n_numpy(n0, ni, x) - - # If the index is a CPU index, the default device is CPU, otherwise we - # produce a GPU tensor - device = torch.device("cpu") - if hasattr(self, "getDevice"): - # same device as the index - device = torch.device("cuda", self.getDevice()) - - if x is None: - x = torch.empty(ni, self.d, device=device, dtype=torch.float32) - else: - assert type(x) is torch.Tensor - assert x.shape == (ni, self.d) - x_ptr = swig_ptr_from_FloatTensor(x) - - if x.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.reconstruct_n_c(n0, ni, x_ptr) - else: - # CPU torch - self.reconstruct_n_c(n0, ni, x_ptr) - - return x - - def torch_replacement_update_vectors(self, keys, x): - if type(keys) is np.ndarray: - # Forward to faiss __init__.py base method - return self.update_vectors_numpy(keys, x) - - assert type(keys) is torch.Tensor - (n,) = keys.shape - keys_ptr = swig_ptr_from_IndicesTensor(keys) - - assert type(x) is torch.Tensor - assert x.shape == (n, self.d) - x_ptr = swig_ptr_from_FloatTensor(x) - - if x.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.update_vectors_c(n, keys_ptr, x_ptr) - else: - # CPU torch - self.update_vectors_c(n, keys_ptr, x_ptr) - - # Until the GPU version is implemented, we do not support pre-allocated - # output buffers - def torch_replacement_range_search(self, x, thresh): - if type(x) is np.ndarray: - # Forward to faiss __init__.py base method - return self.range_search_numpy(x, thresh) - - assert type(x) is torch.Tensor - n, d = x.shape - assert d == self.d - x_ptr = swig_ptr_from_FloatTensor(x) - - assert ( - not x.is_cuda - ), "Range search using GPU tensor not yet implemented" - assert not hasattr( - self, "getDevice" - ), "Range search on GPU index not yet implemented" - - res = faiss.RangeSearchResult(n) - self.range_search_c(n, x_ptr, thresh, res) - - # get pointers and copy them - # FIXME: no rev_swig_ptr equivalent for torch.Tensor, just convert - # np to torch - # NOTE: torch does not support np.uint64, just np.int64 - lims = torch.from_numpy( - faiss.rev_swig_ptr(res.lims, n + 1).copy().astype("int64") - ) - nd = int(lims[-1]) - D = torch.from_numpy(faiss.rev_swig_ptr(res.distances, nd).copy()) - I = torch.from_numpy(faiss.rev_swig_ptr(res.labels, nd).copy()) - - return lims, D, I - - def torch_replacement_sa_encode(self, x, codes=None): - if type(x) is np.ndarray: - # Forward to faiss __init__.py base method - return self.sa_encode_numpy(x, codes) - - assert type(x) is torch.Tensor - n, d = x.shape - assert d == self.d - x_ptr = swig_ptr_from_FloatTensor(x) - - if codes is None: - codes = torch.empty(n, self.sa_code_size(), dtype=torch.uint8) - else: - assert codes.shape == (n, self.sa_code_size()) - codes_ptr = swig_ptr_from_UInt8Tensor(codes) - - if x.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.sa_encode_c(n, x_ptr, codes_ptr) - else: - # CPU torch - self.sa_encode_c(n, x_ptr, codes_ptr) - - return codes - - def torch_replacement_sa_decode(self, codes, x=None): - if type(codes) is np.ndarray: - # Forward to faiss __init__.py base method - return self.sa_decode_numpy(codes, x) - - assert type(codes) is torch.Tensor - n, cs = codes.shape - assert cs == self.sa_code_size() - codes_ptr = swig_ptr_from_UInt8Tensor(codes) - - if x is None: - x = torch.empty(n, self.d, dtype=torch.float32) - else: - assert type(x) is torch.Tensor - assert x.shape == (n, self.d) - x_ptr = swig_ptr_from_FloatTensor(x) - - if codes.is_cuda: - assert hasattr( - self, "getDevice" - ), "GPU tensor on CPU index not allowed" - - # On the GPU, use proper stream ordering - with using_stream(self.getResources()): - self.sa_decode_c(n, codes_ptr, x_ptr) - else: - # CPU torch - self.sa_decode_c(n, codes_ptr, x_ptr) - - return x - - torch_replace_method(the_class, "add", torch_replacement_add) - torch_replace_method( - the_class, "add_with_ids", torch_replacement_add_with_ids - ) - torch_replace_method(the_class, "assign", torch_replacement_assign) - torch_replace_method(the_class, "train", torch_replacement_train) - torch_replace_method(the_class, "search", torch_replacement_search) - torch_replace_method(the_class, "remove_ids", torch_replacement_remove_ids) - torch_replace_method( - the_class, "reconstruct", torch_replacement_reconstruct - ) - torch_replace_method( - the_class, "reconstruct_n", torch_replacement_reconstruct_n - ) - torch_replace_method( - the_class, "range_search", torch_replacement_range_search - ) - torch_replace_method( - the_class, - "update_vectors", - torch_replacement_update_vectors, - ignore_missing=True, - ) - torch_replace_method( - the_class, - "search_and_reconstruct", - torch_replacement_search_and_reconstruct, - ignore_missing=True, - ) - torch_replace_method( - the_class, - "search_preassigned", - torch_replacement_search_preassigned, - ignore_missing=True, - ) - torch_replace_method(the_class, "sa_encode", torch_replacement_sa_encode) - torch_replace_method(the_class, "sa_decode", torch_replacement_sa_decode) - - -faiss_module = sys.modules["faiss"] - -# Re-patch anything that inherits from faiss.Index to add the torch bindings -for symbol in dir(faiss_module): - obj = getattr(faiss_module, symbol) - if inspect.isclass(obj): - the_class = obj - if issubclass(the_class, faiss.Index): - handle_torch_Index(the_class) - - -# allows torch tensor usage with knn -def torch_replacement_knn(xq, xb, k, metric=faiss.METRIC_L2, metric_arg=0): - if type(xb) is np.ndarray: - # Forward to faiss __init__.py base method - return faiss.knn_numpy(xq, xb, k, metric=metric, metric_arg=metric_arg) - - nb, d = xb.size() - assert xb.is_contiguous() - assert xb.dtype == torch.float32 - assert not xb.is_cuda, "use knn_gpu for GPU tensors" - - nq, d2 = xq.size() - assert d2 == d - assert xq.is_contiguous() - assert xq.dtype == torch.float32 - assert not xq.is_cuda, "use knn_gpu for GPU tensors" - - D = torch.empty(nq, k, device=xb.device, dtype=torch.float32) - I = torch.empty(nq, k, device=xb.device, dtype=torch.int64) - I_ptr = swig_ptr_from_IndicesTensor(I) - D_ptr = swig_ptr_from_FloatTensor(D) - xb_ptr = swig_ptr_from_FloatTensor(xb) - xq_ptr = swig_ptr_from_FloatTensor(xq) - - if metric == faiss.METRIC_L2: - faiss.knn_L2sqr(xq_ptr, xb_ptr, d, nq, nb, k, D_ptr, I_ptr) - elif metric == faiss.METRIC_INNER_PRODUCT: - faiss.knn_inner_product(xq_ptr, xb_ptr, d, nq, nb, k, D_ptr, I_ptr) - else: - faiss.knn_extra_metrics( - xq_ptr, xb_ptr, d, nq, nb, metric, metric_arg, k, D_ptr, I_ptr - ) - - return D, I - - -torch_replace_method(faiss_module, "knn", torch_replacement_knn, True, True) - - -# allows torch tensor usage with bfKnn -def torch_replacement_knn_gpu( - res, - xq, - xb, - k, - D=None, - I=None, - metric=faiss.METRIC_L2, - device=-1, - use_cuvs=False, -): - if type(xb) is np.ndarray: - # Forward to faiss __init__.py base method - return faiss.knn_gpu_numpy(res, xq, xb, k, D, I, metric, device) - - nb, d = xb.size() - if xb.is_contiguous(): - xb_row_major = True - elif xb.t().is_contiguous(): - xb = xb.t() - xb_row_major = False - else: - raise TypeError("matrix should be row or column-major") - - if xb.dtype == torch.float32: - xb_type = faiss.DistanceDataType_F32 - xb_ptr = swig_ptr_from_FloatTensor(xb) - elif xb.dtype == torch.float16: - xb_type = faiss.DistanceDataType_F16 - xb_ptr = swig_ptr_from_HalfTensor(xb) - elif xb.dtype == torch.bfloat16: - xb_type = faiss.DistanceDataType_BF16 - xb_ptr = swig_ptr_from_BFloat16Tensor(xb) - else: - raise TypeError("xq must be float32, float16 or bfloat16") - - nq, d2 = xq.size() - assert d2 == d - if xq.is_contiguous(): - xq_row_major = True - elif xq.t().is_contiguous(): - xq = xq.t() - xq_row_major = False - else: - raise TypeError("matrix should be row or column-major") - - if xq.dtype == torch.float32: - xq_type = faiss.DistanceDataType_F32 - xq_ptr = swig_ptr_from_FloatTensor(xq) - elif xq.dtype == torch.float16: - xq_type = faiss.DistanceDataType_F16 - xq_ptr = swig_ptr_from_HalfTensor(xq) - elif xq.dtype == torch.bfloat16: - xq_type = faiss.DistanceDataType_BF16 - xq_ptr = swig_ptr_from_BFloat16Tensor(xq) - else: - raise TypeError("xq must be float32, float16 or bfloat16") - - if D is None: - D = torch.empty(nq, k, device=xb.device, dtype=torch.float32) - else: - assert D.shape == (nq, k) - # interface takes void*, we need to check this - assert D.dtype == torch.float32 - - if I is None: - I = torch.empty(nq, k, device=xb.device, dtype=torch.int64) - else: - assert I.shape == (nq, k) - - if I.dtype == torch.int64: - I_type = faiss.IndicesDataType_I64 - I_ptr = swig_ptr_from_IndicesTensor(I) - elif I.dtype == I.dtype == torch.int32: - I_type = faiss.IndicesDataType_I32 - I_ptr = swig_ptr_from_IntTensor(I) - else: - raise TypeError("I must be i64 or i32") - - D_ptr = swig_ptr_from_FloatTensor(D) - - args = faiss.GpuDistanceParams() - args.metric = metric - args.k = k - args.dims = d - args.vectors = xb_ptr - args.vectorsRowMajor = xb_row_major - args.vectorType = xb_type - args.numVectors = nb - args.queries = xq_ptr - args.queriesRowMajor = xq_row_major - args.queryType = xq_type - args.numQueries = nq - args.outDistances = D_ptr - args.outIndices = I_ptr - args.outIndicesType = I_type - args.device = device - args.use_cuvs = use_cuvs - - with using_stream(res): - faiss.bfKnn(res, args) - - return D, I - - -torch_replace_method( - faiss_module, "knn_gpu", torch_replacement_knn_gpu, True, True -) - - -# allows torch tensor usage with bfKnn for all pairwise distances -def torch_replacement_pairwise_distance_gpu( - res, xq, xb, D=None, metric=faiss.METRIC_L2, device=-1 -): - if type(xb) is np.ndarray: - # Forward to faiss __init__.py base method - return faiss.pairwise_distance_gpu_numpy(res, xq, xb, D, metric) - - nb, d = xb.size() - if xb.is_contiguous(): - xb_row_major = True - elif xb.t().is_contiguous(): - xb = xb.t() - xb_row_major = False - else: - raise TypeError("xb matrix should be row or column-major") - - if xb.dtype == torch.float32: - xb_type = faiss.DistanceDataType_F32 - xb_ptr = swig_ptr_from_FloatTensor(xb) - elif xb.dtype == torch.float16: - xb_type = faiss.DistanceDataType_F16 - xb_ptr = swig_ptr_from_HalfTensor(xb) - else: - raise TypeError("xb must be float32 or float16") - - nq, d2 = xq.size() - assert d2 == d - if xq.is_contiguous(): - xq_row_major = True - elif xq.t().is_contiguous(): - xq = xq.t() - xq_row_major = False - else: - raise TypeError("xq matrix should be row or column-major") - - if xq.dtype == torch.float32: - xq_type = faiss.DistanceDataType_F32 - xq_ptr = swig_ptr_from_FloatTensor(xq) - elif xq.dtype == torch.float16: - xq_type = faiss.DistanceDataType_F16 - xq_ptr = swig_ptr_from_HalfTensor(xq) - else: - raise TypeError("xq must be float32 or float16") - - if D is None: - D = torch.empty(nq, nb, device=xb.device, dtype=torch.float32) - else: - assert D.shape == (nq, nb) - # interface takes void*, we need to check this - assert D.dtype == torch.float32 - - D_ptr = swig_ptr_from_FloatTensor(D) - - args = faiss.GpuDistanceParams() - args.metric = metric - args.k = -1 # selects all pairwise distance - args.dims = d - args.vectors = xb_ptr - args.vectorsRowMajor = xb_row_major - args.vectorType = xb_type - args.numVectors = nb - args.queries = xq_ptr - args.queriesRowMajor = xq_row_major - args.queryType = xq_type - args.numQueries = nq - args.outDistances = D_ptr - args.device = device - - with using_stream(res): - faiss.bfKnn(res, args) - - return D - - -torch_replace_method( - faiss_module, - "pairwise_distance_gpu", - torch_replacement_pairwise_distance_gpu, - True, - True, -) diff --git a/bundle/python-cpu/Lib/site-packages/faiss/contrib/vecs_io.py b/bundle/python-cpu/Lib/site-packages/faiss/contrib/vecs_io.py deleted file mode 100644 index 230287f2f06d28b8055a2e2612eef2037b630a9f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/contrib/vecs_io.py +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import sys -import numpy as np -import os - -""" -I/O functions in fvecs, bvecs, ivecs formats -definition of the formats here: http://corpus-texmex.irisa.fr/ -""" - - -def ivecs_read(fname): - a = np.fromfile(fname, dtype="int32") - if sys.byteorder == "big": - a.byteswap(inplace=True) - d = a[0] - return a.reshape(-1, d + 1)[:, 1:].copy() - - -def fvecs_read(fname): - return ivecs_read(fname).view("float32") - - -def ivecs_mmap(fname): - assert sys.byteorder != "big" - a = np.memmap(fname, dtype="int32", mode="r") - d = a[0] - return a.reshape(-1, d + 1)[:, 1:] - - -def fvecs_mmap(fname): - return ivecs_mmap(fname).view("float32") - - -def bvecs_mmap(fname): - x = np.memmap(fname, dtype="uint8", mode="r") - if sys.byteorder == "big": - da = x[:4][::-1].copy() - d = da.view("int32")[0] - else: - d = x[:4].view("int32")[0] - return x.reshape(-1, d + 4)[:, 4:] - - -def ivecs_write(fname, m): - n, d = m.shape - m1 = np.empty((n, d + 1), dtype="int32") - m1[:, 0] = d - m1[:, 1:] = m - if sys.byteorder == "big": - m1.byteswap(inplace=True) - m1.tofile(fname) - - -def fvecs_write(fname, m): - m = m.astype("float32") - ivecs_write(fname, m.view("int32")) - - -def bvecs_iter(filepath, batch_size=100_000): - """ - Memory-mapped iterator - only loads requested slices into RAM - """ - - file_size = os.path.getsize(filepath) - with open(filepath, "rb") as f: - dim = np.frombuffer(f.read(4), dtype=" 0: - buffer = vectors[start:].copy() - buffer_size = remainder - - if buffer is not None: - yield buffer diff --git a/bundle/python-cpu/Lib/site-packages/faiss/extra_wrappers.py b/bundle/python-cpu/Lib/site-packages/faiss/extra_wrappers.py deleted file mode 100644 index 59f9584189703e793197c1a35fd9e371278827e3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/extra_wrappers.py +++ /dev/null @@ -1,774 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# @nolint - -# not linting this file because it imports * from swigfaiss, which -# causes a ton of useless warnings. - -import numpy as np - -from faiss.loader import * - -import faiss - -import collections.abc - - -########################################### -# Wrapper for a few functions -########################################### - - -def kmin(array, k): - """return k smallest values (and their indices) of the lines of a - float32 array""" - array = np.ascontiguousarray(array, dtype="float32") - m, n = array.shape - I = np.zeros((m, k), dtype="int64") - D = np.zeros((m, k), dtype="float32") - ha = faiss.float_maxheap_array_t() - ha.ids = swig_ptr(I) - ha.val = swig_ptr(D) - ha.nh = m - ha.k = k - ha.heapify() - ha.addn(n, swig_ptr(array)) - ha.reorder() - return D, I - - -def kmax(array, k): - """return k largest values (and their indices) of the lines of a - float32 array""" - array = np.ascontiguousarray(array, dtype="float32") - m, n = array.shape - I = np.zeros((m, k), dtype="int64") - D = np.zeros((m, k), dtype="float32") - ha = faiss.float_minheap_array_t() - ha.ids = swig_ptr(I) - ha.val = swig_ptr(D) - ha.nh = m - ha.k = k - ha.heapify() - ha.addn(n, swig_ptr(array)) - ha.reorder() - return D, I - - -def pairwise_distances(xq, xb, metric=METRIC_L2, metric_arg=0): - """compute the whole pairwise distance matrix between two sets of - vectors""" - xq = np.ascontiguousarray(xq, dtype="float32") - xb = np.ascontiguousarray(xb, dtype="float32") - nq, d = xq.shape - nb, d2 = xb.shape - assert d == d2 - dis = np.empty((nq, nb), dtype="float32") - if metric == METRIC_L2: - pairwise_L2sqr(d, nq, swig_ptr(xq), nb, swig_ptr(xb), swig_ptr(dis)) - elif metric == METRIC_INNER_PRODUCT: - dis[:] = xq @ xb.T - else: - pairwise_extra_distances( - d, - nq, - swig_ptr(xq), - nb, - swig_ptr(xb), - metric, - metric_arg, - swig_ptr(dis), - ) - return dis - - -def rand(n, seed=12345): - res = np.empty(n, dtype="float32") - float_rand(swig_ptr(res), res.size, seed) - return res - - -def randint(n, seed=12345, vmax=None): - res = np.empty(n, dtype="int64") - if vmax is None: - int64_rand(swig_ptr(res), res.size, seed) - else: - int64_rand_max(swig_ptr(res), res.size, vmax, seed) - return res - - -lrand = randint - - -def randn(n, seed=12345): - res = np.empty(n, dtype="float32") - float_randn(swig_ptr(res), res.size, seed) - return res - - -def checksum(a): - """compute a checksum for quick-and-dirty comparisons of arrays""" - a = a.view("uint8") - if a.ndim == 1: - return bvec_checksum(a.size, swig_ptr(a)) - n, d = a.shape - cs = np.zeros(n, dtype="uint64") - bvecs_checksum(n, d, swig_ptr(a), swig_ptr(cs)) - return cs - - -rand_smooth_vectors_c = rand_smooth_vectors - - -def rand_smooth_vectors(n, d, seed=1234): - res = np.empty((n, d), dtype="float32") - rand_smooth_vectors_c(n, d, swig_ptr(res), seed) - return res - - -def eval_intersection(I1, I2): - """size of intersection between each line of two result tables""" - I1 = np.ascontiguousarray(I1, dtype="int64") - I2 = np.ascontiguousarray(I2, dtype="int64") - n = I1.shape[0] - assert I2.shape[0] == n - k1, k2 = I1.shape[1], I2.shape[1] - ninter = 0 - for i in range(n): - ninter += ranklist_intersection_size( - k1, swig_ptr(I1[i]), k2, swig_ptr(I2[i]) - ) - return ninter - - -def normalize_L2(x): - fvec_renorm_L2(x.shape[1], x.shape[0], swig_ptr(x)) - - -bucket_sort_c = bucket_sort - - -def bucket_sort(tab, nbucket=None, nt=0): - """Perform a bucket sort on a table of integers. - - Parameters - ---------- - tab : array_like - elements to sort, max value nbucket - 1 - nbucket : integer - number of buckets, None if unknown - nt : integer - number of threads to use (0 = use unthreaded codepath) - - Returns - ------- - lims : array_like - cumulative sum of bucket sizes (size vmax + 1) - perm : array_like - perm[lims[i] : lims[i + 1]] contains the indices of bucket #i - (size tab.size) - """ - tab = np.ascontiguousarray(tab, dtype="int64") - if nbucket is None: - nbucket = int(tab.max() + 1) - lims = np.empty(nbucket + 1, dtype="int64") - perm = np.empty(tab.size, dtype="int64") - bucket_sort_c( - tab.size, - faiss.swig_ptr(tab.view("uint64")), - nbucket, - faiss.swig_ptr(lims), - faiss.swig_ptr(perm), - nt, - ) - return lims, perm - - -matrix_bucket_sort_inplace_c = matrix_bucket_sort_inplace - - -def matrix_bucket_sort_inplace(tab, nbucket=None, nt=0): - """Perform a bucket sort on a matrix, recording the original - row of each element. - - Parameters - ---------- - tab : array_like - array of size (N, ncol) that contains the bucket ids, maximum - value nbucket - 1. - On output, it the elements are shuffled such that the flat array - tab.ravel()[lims[i] : lims[i + 1]] contains the row numbers - of each bucket entry. - nbucket : integer - number of buckets (the maximum value in tab should be nbucket - 1) - nt : integer - number of threads to use (0 = use unthreaded codepath) - - Returns - ------- - lims : array_like - cumulative sum of bucket sizes (size vmax + 1) - """ - assert tab.dtype == "int32" or tab.dtype == "int64" - nrow, ncol = tab.shape - if nbucket is None: - nbucket = int(tab.max() + 1) - lims = np.empty(nbucket + 1, dtype="int64") - matrix_bucket_sort_inplace_c( - nrow, ncol, faiss.swig_ptr(tab), nbucket, faiss.swig_ptr(lims), nt - ) - return lims - - -########################################### -# ResultHeap -########################################### - - -class ResultHeap: - """Accumulate query results from a sliced dataset. The final result will - be in self.D, self.I.""" - - def __init__(self, nq, k, keep_max=False): - """ - nq: number of query vectors, - k: number of results per query - keep_max: keep the top-k maximum values instead of the minima - """ - self.I = np.zeros((nq, k), dtype="int64") - self.D = np.zeros((nq, k), dtype="float32") - self.nq, self.k = nq, k - if keep_max: - heaps = float_minheap_array_t() - else: - heaps = float_maxheap_array_t() - heaps.k = k - heaps.nh = nq - heaps.val = swig_ptr(self.D) - heaps.ids = swig_ptr(self.I) - heaps.heapify() - self.heaps = heaps - - def add_result(self, D, I): - """ - Add results for all heaps - D, I should be of size (nh, nres) - D, I do not need to be in a particular order (heap or sorted) - """ - nq, kd = D.shape - D = np.ascontiguousarray(D, dtype="float32") - I = np.ascontiguousarray(I, dtype="int64") - assert I.shape == (nq, kd) - assert nq == self.nq - self.heaps.addn_with_ids(kd, swig_ptr(D), swig_ptr(I), kd) - - def add_result_subset(self, subset, D, I): - """ - Add results for a subset of heaps. - D, I should hold results for all the subset - as a special case, if I is 1D, then all ids are assumed to be the same - """ - nsubset, kd = D.shape - assert nsubset == len(subset) - assert ( - I.ndim == 2 - and D.shape == I.shape - or I.ndim == 1 - and I.shape == (kd,) - ) - D = np.ascontiguousarray(D, dtype="float32") - I = np.ascontiguousarray(I, dtype="int64") - subset = np.ascontiguousarray(subset, dtype="int64") - id_stride = 0 if I.ndim == 1 else kd - self.heaps.addn_query_subset_with_ids( - nsubset, swig_ptr(subset), kd, swig_ptr(D), swig_ptr(I), id_stride - ) - - def finalize(self): - self.heaps.reorder() - - -def merge_knn_results(Dall, Iall, keep_max=False): - """ - Merge a set of sorted knn-results obtained from different shards in a - dataset - Dall and Iall are of size (nshard, nq, k) each D[i, j] should be sorted - returns D, I of size (nq, k) as the merged result set - """ - assert Iall.shape == Dall.shape - nshard, n, k = Dall.shape - Dnew = np.empty((n, k), dtype=Dall.dtype) - Inew = np.empty((n, k), dtype=Iall.dtype) - func = merge_knn_results_CMax if keep_max else merge_knn_results_CMin - func( - n, - k, - nshard, - swig_ptr(Dall), - swig_ptr(Iall), - swig_ptr(Dnew), - swig_ptr(Inew), - ) - return Dnew, Inew - - -###################################################### -# Efficient ID to ID map -###################################################### - - -class MapInt64ToInt64: - - def __init__(self, capacity): - self.log2_capacity = int(np.log2(capacity)) - assert capacity == 2**self.log2_capacity, "need power of 2 capacity" - self.capacity = capacity - self.tab = np.empty((capacity, 2), dtype="int64") - faiss.hashtable_int64_to_int64_init( - self.log2_capacity, swig_ptr(self.tab) - ) - - def add(self, keys, vals): - (n,) = keys.shape - assert vals.shape == (n,) - faiss.hashtable_int64_to_int64_add( - self.log2_capacity, - swig_ptr(self.tab), - n, - swig_ptr(keys), - swig_ptr(vals), - ) - - def lookup(self, keys): - (n,) = keys.shape - vals = np.empty((n,), dtype="int64") - faiss.hashtable_int64_to_int64_lookup( - self.log2_capacity, - swig_ptr(self.tab), - n, - swig_ptr(keys), - swig_ptr(vals), - ) - return vals - - -###################################################### -# KNN function -###################################################### - - -def knn(xq, xb, k, metric=METRIC_L2, metric_arg=0.0): - """ - Compute the k nearest neighbors of a vector without constructing an index - - - Parameters - ---------- - xq : array_like - Query vectors, shape (nq, d) where the dimension d is that same as xb - `dtype` must be float32. - xb : array_like - Database vectors, shape (nb, d) where dimension d is the same as xq - `dtype` must be float32. - k : int - Number of nearest neighbors. - metric : MetricType, optional - distance measure to use (either METRIC_L2 or METRIC_INNER_PRODUCT) - - Returns - ------- - D : array_like - Distances of the nearest neighbors, shape (nq, k) - I : array_like - Labels of the nearest neighbors, shape (nq, k) - """ - xq = np.ascontiguousarray(xq, dtype="float32") - xb = np.ascontiguousarray(xb, dtype="float32") - nq, d = xq.shape - nb, d2 = xb.shape - assert d == d2 - - I = np.empty((nq, k), dtype="int64") - D = np.empty((nq, k), dtype="float32") - - if metric == METRIC_L2: - knn_L2sqr( - swig_ptr(xq), swig_ptr(xb), d, nq, nb, k, swig_ptr(D), swig_ptr(I) - ) - elif metric == METRIC_INNER_PRODUCT: - knn_inner_product( - swig_ptr(xq), swig_ptr(xb), d, nq, nb, k, swig_ptr(D), swig_ptr(I) - ) - else: - knn_extra_metrics( - swig_ptr(xq), - swig_ptr(xb), - d, - nq, - nb, - metric, - metric_arg, - k, - swig_ptr(D), - swig_ptr(I), - ) - - return D, I - - -def knn_hamming(xq, xb, k, variant="hc"): - """ - Compute the k nearest neighbors of a set of vectors without constructing - an index. - - Parameters - ---------- - xq : array_like - Query vectors, shape (nq, d) where d is the number of bits / 8 - `dtype` must be uint8. - xb : array_like - Database vectors, shape (nb, d) where d is the number of bits / 8 - `dtype` must be uint8. - k : int - Number of nearest neighbors. - variant : string - Function variant to use, either "mc" (counter) or "hc" (heap) - - Returns - ------- - D : array_like - Distances of the nearest neighbors, shape (nq, k) - I : array_like - Labels of the nearest neighbors, shape (nq, k) - """ - # other variant is "mc" - nq, d = xq.shape - nb, d2 = xb.shape - assert d == d2 - D = np.empty((nq, k), dtype="int32") - I = np.empty((nq, k), dtype="int64") - - if variant == "hc": - heap = faiss.int_maxheap_array_t() - heap.k = k - heap.nh = nq - heap.ids = faiss.swig_ptr(I) - heap.val = faiss.swig_ptr(D) - faiss.hammings_knn_hc( - heap, faiss.swig_ptr(xq), faiss.swig_ptr(xb), nb, d, 1 - ) - elif variant == "mc": - faiss.hammings_knn_mc( - faiss.swig_ptr(xq), - faiss.swig_ptr(xb), - nq, - nb, - k, - d, - faiss.swig_ptr(D), - faiss.swig_ptr(I), - ) - else: - raise NotImplementedError - return D, I - - -########################################### -# Kmeans object -########################################### - - -class Kmeans: - """Object that performs k-means clustering and manages the centroids. - The `Kmeans` class is essentially a wrapper around the C++ `Clustering` - object. - - Parameters - ---------- - d : int - dimension of the vectors to cluster - k : int - number of clusters - gpu: bool or int, optional - False: don't use GPU - True: use all GPUs - number: use this many GPUs - progressive_dim_steps: - use a progressive dimension clustering (with that number of steps) - - Subsequent parameters are fields of the Clustring object. The most - important are: - - niter: int, optional - clustering iterations - nredo: int, optional - redo clustering this many times and keep best - verbose: bool, optional - spherical: bool, optional - do we want normalized centroids? - int_centroids: bool, optional - round centroids coordinates to integer - seed: int, optional - seed for the random number generator - init_method: ClusteringInitMethod, optional - centroid initialization method: - - ClusteringInitMethod_RANDOM: uniform random sampling (default) - - ClusteringInitMethod_KMEANS_PLUS_PLUS: k-means++ D²-weighted sampling, - selects centroids with probability proportional to squared distance - from existing centroids. Better quality but O(nkd) complexity. - - ClusteringInitMethod_AFK_MC2: Assumption-Free K-MC², MCMC-based - approximation using Metropolis-Hastings. Good quality with lower - complexity than k-means++ for large k. - afkmc2_chain_length: int, optional - chain length for AFK-MC² initialization (default 50). Longer chains - give better approximation to k-means++ but are slower. - - """ - - def __init__(self, d, k, **kwargs): - """d: input dimension, k: nb of centroids. Additional - parameters are passed on the ClusteringParameters object, - including niter=25, verbose=False, spherical = False - """ - self.d = d - self.reset(k) - self.gpu = False - if "progressive_dim_steps" in kwargs: - self.cp = ProgressiveDimClusteringParameters() - else: - self.cp = ClusteringParameters() - for k, v in kwargs.items(): - if k == "gpu": - if v == True or v == -1: - v = get_num_gpus() - self.gpu = v - else: - # if this raises an exception, it means that - # it is a non-existent field - getattr(self.cp, k) - setattr(self.cp, k, v) - self.set_index() - - def set_index(self): - d = self.d - if self.cp.__class__ == ClusteringParameters: - if self.cp.spherical: - self.index = IndexFlatIP(d) - else: - self.index = IndexFlatL2(d) - if self.gpu: - self.index = faiss.index_cpu_to_all_gpus( - self.index, ngpu=self.gpu - ) - else: - if self.gpu: - fac = GpuProgressiveDimIndexFactory(ngpu=self.gpu) - else: - fac = ProgressiveDimIndexFactory() - self.fac = fac - - def reset(self, k=None): - """prepare k-means object to perform a new clustering, possibly - with another number of centroids""" - if k is not None: - self.k = int(k) - self.centroids = None - self.obj = None - self.iteration_stats = None - - def train(self, x, weights=None, init_centroids=None): - """Perform k-means clustering. - On output of the function call: - - - the centroids are in the centroids field of size (`k`, `d`). - - - the objective value at each iteration is in the array obj (size - `niter`) - - - detailed optimization statistics are in the array iteration_stats. - - Parameters - ---------- - x : array_like - Training vectors, shape (n, d), `dtype` must be float32 and n should - be larger than the number of clusters `k`. - weights : array_like - weight associated to each vector, shape `n` - init_centroids : array_like - initial set of centroids, shape (n, d) - - Returns - ------- - final_obj: float - final optimization objective - - """ - x = np.ascontiguousarray(x, dtype="float32") - n, d = x.shape - assert d == self.d - - if self.cp.__class__ == ClusteringParameters: - # regular clustering - clus = Clustering(d, self.k, self.cp) - if init_centroids is not None: - nc, d2 = init_centroids.shape - assert d2 == d - faiss.copy_array_to_vector( - init_centroids.ravel(), clus.centroids - ) - clus.train(x, self.index, weights) - else: - # not supported for progressive dim - assert weights is None - assert init_centroids is None - assert not self.cp.spherical - clus = ProgressiveDimClustering(d, self.k, self.cp) - clus.train(n, swig_ptr(x), self.fac) - - centroids = faiss.vector_float_to_array(clus.centroids) - - self.centroids = centroids.reshape(self.k, d) - stats = clus.iteration_stats - stats = [stats.at(i) for i in range(stats.size())] - self.obj = np.array([st.obj for st in stats]) - # copy all the iteration_stats objects to a python array - stat_fields = "obj time time_search imbalance_factor nsplit".split() - self.iteration_stats = [ - {field: getattr(st, field) for field in stat_fields} for st in stats - ] - return self.obj[-1] if self.obj.size > 0 else 0.0 - - def assign(self, x): - x = np.ascontiguousarray(x, dtype="float32") - assert self.centroids is not None, "should train before assigning" - self.index.reset() - self.index.add(self.centroids) - D, I = self.index.search(x, 1) - return D.ravel(), I.ravel() - - -class SuperKmeans(Kmeans): - """Drop-in replacement for `Kmeans` that runs `SuperKMeans` (ADSampling + - PDX progressive pruning) instead of `Clustering`. Same `centroids`, `obj`, - `iteration_stats`, and `assign()` surface; additionally exposes - `gemm_pruning_rates`. - - kwargs are forwarded to `SuperKMeansParameters`. Fields not present on it - (e.g. `spherical`, `int_centroids`, `nredo`, `frozen_centroids`, - `init_method`, `update_index`, `early_stop_threshold`, - `progressive_dim_steps`, `gpu`) raise `AttributeError`. - """ - - def __init__(self, d, k, **kwargs): - self.d = d - self.reset(k) - self.gpu = False - self.cp = SuperKMeansParameters() - for key, v in kwargs.items(): - getattr(self.cp, key) - setattr(self.cp, key, v) - self.set_index() - - def set_index(self): - self.index = IndexFlatL2(self.d) - - def train(self, x, weights=None, init_centroids=None): - assert weights is None, "SuperKmeans does not support weights" - assert ( - init_centroids is None - ), "SuperKmeans does not support init_centroids" - x = np.ascontiguousarray(x, dtype="float32") - n, d = x.shape - assert d == self.d - - sc = SuperKMeans(d, self.k, self.cp) - sc.train(x) - - centroids = faiss.vector_to_array(sc.centroids) - self.centroids = centroids.reshape(self.k, d) - stats = sc.iteration_stats - stats = [stats.at(i) for i in range(stats.size())] - self.obj = np.array([st.obj for st in stats]) - stat_fields = "obj time time_search imbalance_factor nsplit".split() - self.iteration_stats = [ - {field: getattr(st, field) for field in stat_fields} for st in stats - ] - self.gemm_pruning_rates = faiss.vector_to_array(sc.gemm_pruning_rates) - return self.obj[-1] if self.obj.size > 0 else 0.0 - - -########################################### -# Packing and unpacking bitstrings -########################################### - - -def is_sequence(x): - return isinstance(x, collections.abc.Sequence) - - -pack_bitstrings_c = pack_bitstrings - - -def pack_bitstrings(a, nbit): - """ - Pack a set integers (i, j) where i=0:n and j=0:M into - n bitstrings. - Output is an uint8 array of size (n, code_size), where code_size is - such that at most 7 bits per code are wasted. - - If nbit is an integer: all entries takes nbit bits. - If nbit is an array: entry (i, j) takes nbit[j] bits. - """ - n, M = a.shape - a = np.ascontiguousarray(a, dtype="int32") - if is_sequence(nbit): - nbit = np.ascontiguousarray(nbit, dtype="int32") - assert nbit.shape == (M,) - code_size = int((nbit.sum() + 7) // 8) - b = np.empty((n, code_size), dtype="uint8") - pack_bitstrings_c( - n, M, swig_ptr(nbit), swig_ptr(a), swig_ptr(b), code_size - ) - else: - code_size = (M * nbit + 7) // 8 - b = np.empty((n, code_size), dtype="uint8") - pack_bitstrings_c(n, M, nbit, swig_ptr(a), swig_ptr(b), code_size) - return b - - -unpack_bitstrings_c = unpack_bitstrings - - -def unpack_bitstrings(b, M_or_nbits, nbit=None): - """ - Unpack a set integers (i, j) where i=0:n and j=0:M from - n bitstrings (encoded as uint8s). - Input is an uint8 array of size (n, code_size), where code_size is - such that at most 7 bits per code are wasted. - - Two forms: - - when called with (array, M, nbit): there are M entries of size - nbit per row - - when called with (array, nbits): element (i, j) is encoded in - nbits[j] bits - """ - n, code_size = b.shape - if nbit is None: - nbit = np.ascontiguousarray(M_or_nbits, dtype="int32") - M = len(nbit) - min_code_size = int((nbit.sum() + 7) // 8) - assert code_size >= min_code_size - a = np.empty((n, M), dtype="int32") - unpack_bitstrings_c( - n, M, swig_ptr(nbit), swig_ptr(b), code_size, swig_ptr(a) - ) - else: - M = M_or_nbits - min_code_size = (M * nbit + 7) // 8 - assert code_size >= min_code_size - a = np.empty((n, M), dtype="int32") - unpack_bitstrings_c(n, M, nbit, swig_ptr(b), code_size, swig_ptr(a)) - return a diff --git a/bundle/python-cpu/Lib/site-packages/faiss/faiss.dll b/bundle/python-cpu/Lib/site-packages/faiss/faiss.dll deleted file mode 100644 index 76a0a64f61471c284738411374a2b7b84dc26aa9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/faiss.dll +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ee46a94dd1aac967c10091a2385d93340611a89cee2c7e28e581858762b73943 -size 7428608 diff --git a/bundle/python-cpu/Lib/site-packages/faiss/gpu_wrappers.py b/bundle/python-cpu/Lib/site-packages/faiss/gpu_wrappers.py deleted file mode 100644 index 5299b3748e309a8deb9f5084f1ecec38115227bd..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/gpu_wrappers.py +++ /dev/null @@ -1,310 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# @nolint - -# not linting this file because it imports * from swigfaiss, which -# causes a ton of useless warnings. - -import numpy as np - -from faiss.loader import * - - -########################################### -# GPU functions -########################################### - - -def index_cpu_to_gpu_multiple_py(resources, index, co=None, gpus=None): - """builds the C++ vectors for the GPU indices and the - resources. Handles the case where the resources are assigned to - the list of GPUs""" - if gpus is None: - gpus = range(len(resources)) - vres = GpuResourcesVector() - vdev = Int32Vector() - for i, res in zip(gpus, resources): - vdev.push_back(i) - vres.push_back(res) - if isinstance(index, IndexBinary): - return index_binary_cpu_to_gpu_multiple(vres, vdev, index, co) - else: - return index_cpu_to_gpu_multiple(vres, vdev, index, co) - - -def index_cpu_to_all_gpus(index, co=None, ngpu=-1): - index_gpu = index_cpu_to_gpus_list(index, co=co, gpus=None, ngpu=ngpu) - return index_gpu - - -def index_cpu_to_gpus_list(index, co=None, gpus=None, ngpu=-1): - """Here we can pass list of GPU ids as a parameter or ngpu to - use first n GPU's. gpus mut be a list or None. - co is a GpuMultipleClonerOptions - """ - if (gpus is None) and (ngpu == -1): # All blank - gpus = range(get_num_gpus()) - elif (gpus is None) and (ngpu != -1): # Get number of GPU's only - gpus = range(ngpu) - res = [StandardGpuResources() for _ in gpus] - index_gpu = index_cpu_to_gpu_multiple_py(res, index, co, gpus) - return index_gpu - - -# allows numpy ndarray usage with bfKnn - - -def knn_gpu( - res, - xq, - xb, - k, - D=None, - I=None, - metric=METRIC_L2, - device=-1, - use_cuvs=False, - vectorsMemoryLimit=0, - queriesMemoryLimit=0, -): - """ - Compute the k nearest neighbors of a vector on one GPU without constructing - an index - - Parameters - ---------- - res : StandardGpuResources - GPU resources to use during computation - xq : array_like - Query vectors, shape (nq, d) where d is appropriate for the index. - `dtype` must be float32. - xb : array_like - Database vectors, shape (nb, d) where d is appropriate for the index. - `dtype` must be float32. - k : int - Number of nearest neighbors. - D : array_like, optional - Output array for distances of the nearest neighbors, shape (nq, k) - I : array_like, optional - Output array for the nearest neighbors, shape (nq, k) - metric : MetricType, optional - Distance measure to use (either METRIC_L2 or METRIC_INNER_PRODUCT) - device: int, optional - Which CUDA device in the system to run the search on. -1 indicates that - the current thread-local device state (via cudaGetDevice) should be used - (can also be set via torch.cuda.set_device in PyTorch) - Otherwise, an integer 0 <= device < numDevices indicates the GPU on - which - the computation should be run - vectorsMemoryLimit: int, optional - queriesMemoryLimit: int, optional - Memory limits for vectors and queries. - If not 0, the GPU will use at most this amount of memory - for vectors and queries respectively. - Vectors are broken up into chunks of size vectorsMemoryLimit, - and queries are broken up into chunks of size queriesMemoryLimit, - including the memory required for the results. - - Returns - ------- - D : array_like - Distances of the nearest neighbors, shape (nq, k) - I : array_like - Labels of the nearest neighbors, shape (nq, k) - """ - nq, d = xq.shape - if xq.flags.c_contiguous: - xq_row_major = True - elif xq.flags.f_contiguous: - xq = xq.T - xq_row_major = False - else: - xq = np.ascontiguousarray(xq, dtype="float32") - xq_row_major = True - - xq_ptr = swig_ptr(xq) - - if xq.dtype == np.float32: - xq_type = DistanceDataType_F32 - elif xq.dtype == np.float16: - xq_type = DistanceDataType_F16 - else: - raise TypeError("xq must be f32 or f16") - - nb, d2 = xb.shape - assert d2 == d - if xb.flags.c_contiguous: - xb_row_major = True - elif xb.flags.f_contiguous: - xb = xb.T - xb_row_major = False - else: - xb = np.ascontiguousarray(xb, dtype="float32") - xb_row_major = True - - xb_ptr = swig_ptr(xb) - - if xb.dtype == np.float32: - xb_type = DistanceDataType_F32 - elif xb.dtype == np.float16: - xb_type = DistanceDataType_F16 - else: - raise TypeError("xb must be float32 or float16") - - if D is None: - D = np.empty((nq, k), dtype=np.float32) - else: - assert D.shape == (nq, k) - # interface takes void*, we need to check this - assert D.dtype == np.float32 - - D_ptr = swig_ptr(D) - - if I is None: - I = np.empty((nq, k), dtype=np.int64) - else: - assert I.shape == (nq, k) - - I_ptr = swig_ptr(I) - - if I.dtype == np.int64: - I_type = IndicesDataType_I64 - elif I.dtype == I.dtype == np.int32: - I_type = IndicesDataType_I32 - else: - raise TypeError("I must be i64 or i32") - - args = GpuDistanceParams() - args.metric = metric - args.k = k - args.dims = d - args.vectors = xb_ptr - args.vectorsRowMajor = xb_row_major - args.vectorType = xb_type - args.numVectors = nb - args.queries = xq_ptr - args.queriesRowMajor = xq_row_major - args.queryType = xq_type - args.numQueries = nq - args.outDistances = D_ptr - args.outIndices = I_ptr - args.outIndicesType = I_type - args.device = device - args.use_cuvs = use_cuvs - - # no stream synchronization needed, inputs and outputs are guaranteed to - # be on the CPU (numpy arrays) - if vectorsMemoryLimit > 0 or queriesMemoryLimit > 0: - bfKnn_tiling(res, args, vectorsMemoryLimit, queriesMemoryLimit) - else: - bfKnn(res, args) - - return D, I - - -# allows numpy ndarray usage with bfKnn for all pairwise distances - - -def pairwise_distance_gpu(res, xq, xb, D=None, metric=METRIC_L2, device=-1): - """ - Compute all pairwise distances between xq and xb on one GPU without - constructing an index - - Parameters - ---------- - res : StandardGpuResources - GPU resources to use during computation - xq : array_like - Query vectors, shape (nq, d) where d is appropriate for the index. - `dtype` must be float32. - xb : array_like - Database vectors, shape (nb, d) where d is appropriate for the index. - `dtype` must be float32. - D : array_like, optional - Output array for all pairwise distances, shape (nq, nb) - metric : MetricType, optional - Distance measure to use (either METRIC_L2 or METRIC_INNER_PRODUCT) - device: int, optional - Which CUDA device in the system to run the search on. -1 indicates that - the current thread-local device state (via cudaGetDevice) should be used - (can also be set via torch.cuda.set_device in PyTorch) - Otherwise, an integer 0 <= device < numDevices indicates the GPU on - which - the computation should be run - - Returns - ------- - D : array_like - All pairwise distances, shape (nq, nb) - """ - nq, d = xq.shape - if xq.flags.c_contiguous: - xq_row_major = True - elif xq.flags.f_contiguous: - xq = xq.T - xq_row_major = False - else: - raise TypeError("xq matrix should be row (C) or column-major (Fortran)") - - xq_ptr = swig_ptr(xq) - - if xq.dtype == np.float32: - xq_type = DistanceDataType_F32 - elif xq.dtype == np.float16: - xq_type = DistanceDataType_F16 - else: - xq = np.ascontiguousarray(xb, dtype="float32") - xq_row_major = True - - nb, d2 = xb.shape - assert d2 == d - if xb.flags.c_contiguous: - xb_row_major = True - elif xb.flags.f_contiguous: - xb = xb.T - xb_row_major = False - else: - xb = np.ascontiguousarray(xb, dtype="float32") - xb_row_major = True - - xb_ptr = swig_ptr(xb) - - if xb.dtype == np.float32: - xb_type = DistanceDataType_F32 - elif xb.dtype == np.float16: - xb_type = DistanceDataType_F16 - else: - raise TypeError("xb must be float32 or float16") - - if D is None: - D = np.empty((nq, nb), dtype=np.float32) - else: - assert D.shape == (nq, nb) - # interface takes void*, we need to check this - assert D.dtype == np.float32 - - D_ptr = swig_ptr(D) - - args = GpuDistanceParams() - args.metric = metric - args.k = -1 # selects all pairwise distances - args.dims = d - args.vectors = xb_ptr - args.vectorsRowMajor = xb_row_major - args.vectorType = xb_type - args.numVectors = nb - args.queries = xq_ptr - args.queriesRowMajor = xq_row_major - args.queryType = xq_type - args.numQueries = nq - args.outDistances = D_ptr - args.device = device - - # no stream synchronization needed, inputs and outputs are guaranteed to - # be on the CPU (numpy arrays) - bfKnn(res, args) - - return D diff --git a/bundle/python-cpu/Lib/site-packages/faiss/loader.py b/bundle/python-cpu/Lib/site-packages/faiss/loader.py deleted file mode 100644 index aea834fe91127c6c7d075226ebe425efab85f35c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/loader.py +++ /dev/null @@ -1,228 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import platform -import subprocess -import logging -import os -import sys - -from packaging.version import Version - - -def supported_instruction_sets(): - """ - Returns the set of supported CPU features, see - https://github.com/numpy/numpy/blob/master/numpy/core/src/common/npy_cpu_features.h # noqa: E501 - for the list of features that this set may contain per architecture. - - Example: - >>> supported_instruction_sets() # for x86 - {"SSE2", "AVX2", "AVX512", ...} - >>> supported_instruction_sets() # for PPC - {"VSX", "VSX2", ...} - >>> supported_instruction_sets() # for ARM - {"NEON", "ASIMD", ...} - """ - - # Old numpy.core._multiarray_umath.__cpu_features__ doesn't support Arm SVE, - # so let's read Features in numpy.distutils.cpuinfo and search 'sve' entry - def is_sve_supported(): - if platform.machine() != "aarch64": - return False - # Currently SVE is only supported on Linux - if platform.system() != "Linux": - return False - # Numpy 2.0 supports SVE detection by __cpu_features__, so just skip - import numpy - - if Version(numpy.__version__) >= Version("2.0"): - return False - # platform-dependent legacy fallback using numpy.distutils.cpuinfo - try: - import numpy.distutils.cpuinfo - - return ( - "sve" - in numpy.distutils.cpuinfo.cpu.info[0] - .get("Features", "") - .split() - ) - except ImportError: - # check if SVE is supported by checking the auxval - # using values defined as: - # #define AT_HWCAP 16 - # #define HWCAP_SVE (1 << 22) - return bool( - __import__("ctypes").CDLL(None).getauxval(16) & (1 << 22) - ) - - import numpy - - if Version(numpy.__version__) >= Version("1.19"): - # use private API as next-best thing until numpy/numpy#18058 is solved - from numpy._core._multiarray_umath import __cpu_features__ - - # __cpu_features__ is a dictionary with CPU features - # as keys, and True / False as values - supported = {k for k, v in __cpu_features__.items() if v} - if is_sve_supported(): - supported.add("SVE") - for f in os.getenv("FAISS_DISABLE_CPU_FEATURES", "").split(", \t\n\r"): - supported.discard(f) - return supported - - # platform-dependent legacy fallback before numpy 1.19, no windows - if platform.system() == "Darwin": - if ( - subprocess.check_output(["/usr/sbin/sysctl", "hw.optional.avx2_0"])[ - -1 - ] - == "1" - ): - return {"AVX2"} - elif platform.system() == "Linux": - import numpy.distutils.cpuinfo - - result = set() - if "avx2" in numpy.distutils.cpuinfo.cpu.info[0].get("flags", ""): - result.add("AVX2") - if "avx512" in numpy.distutils.cpuinfo.cpu.info[0].get("flags", ""): - result.add("AVX512") - if "avx512_fp16" in numpy.distutils.cpuinfo.cpu.info[0].get( - "flags", "" - ): - # avx512_fp16 is supported starting SPR - result.add("AVX512_SPR") - if is_sve_supported(): - result.add("SVE") - for f in os.getenv("FAISS_DISABLE_CPU_FEATURES", "").split(", \t\n\r"): - result.discard(f) - return result - return set() - - -logger = logging.getLogger(__name__) - -instruction_sets = None - -# try to load optimization level from env variable -opt_env_variable_name = "FAISS_OPT_LEVEL" -opt_level = os.environ.get(opt_env_variable_name, None) - -if opt_level is None: - logger.debug( - f"Environment variable {opt_env_variable_name} is not set, " - "so let's pick the instruction set according to the current CPU" - ) - instruction_sets = supported_instruction_sets() -else: - logger.debug(f"Using {opt_level} as an instruction set.") - instruction_sets = set() - instruction_sets.add(opt_level) - -loaded = False -has_AVX512_SPR = any("AVX512_SPR" in x.upper() for x in instruction_sets) -if has_AVX512_SPR: - try: - logger.info("Loading faiss with AVX512-SPR support.") - from .swigfaiss_avx512_spr import * # noqa: F401,F403 - - logger.info("Successfully loaded faiss with AVX512-SPR support.") - loaded = True - except ImportError as e: - logger.info( - f"Could not load library with AVX512-SPR support due to:\n{e!r}" - ) - # reset so that we load without AVX512 below - loaded = False - -has_AVX512 = any("AVX512" in x.upper() for x in instruction_sets) -if has_AVX512 and not loaded: - try: - logger.info("Loading faiss with AVX512 support.") - from .swigfaiss_avx512 import * # noqa: F401,F403 - - logger.info("Successfully loaded faiss with AVX512 support.") - loaded = True - except ImportError as e: - logger.info( - f"Could not load library with AVX512 support due to:\n{e!r}" - ) - # reset so that we load without AVX512 below - loaded = False - -has_AVX2 = "AVX2" in instruction_sets -if has_AVX2 and not loaded: - try: - logger.info("Loading faiss with AVX2 support.") - from .swigfaiss_avx2 import * # noqa: F401,F403 - - logger.info("Successfully loaded faiss with AVX2 support.") - loaded = True - except ImportError as e: - logger.info(f"Could not load library with AVX2 support due to:\n{e!r}") - # reset so that we load without AVX2 below - loaded = False - -has_SVE = "SVE" in instruction_sets -if has_SVE and not loaded: - try: - logger.info("Loading faiss with SVE support.") - from .swigfaiss_sve import * # noqa: F401,F403 - - logger.info("Successfully loaded faiss with SVE support.") - loaded = True - except ImportError as e: - logger.info(f"Could not load library with SVE support due to:\n{e!r}") - # reset so that we load without SVE below - loaded = False - -if not loaded: - try: - # we import * so that the symbol X can be accessed as faiss.X - logger.info("Loading faiss.") - from .swigfaiss import * # noqa: F401,F403 - - logger.info("Successfully loaded faiss.") - except ModuleNotFoundError: - formatted_ins_sets = ", ".join(supported_instruction_sets()) - - message = ( - f"No module named 'faiss.swigfaiss' found. To fix this, you must " - f"do both of the following:\n" - f"A) Set the correct FAISS_OPT_LEVEL value when executing " - f"'cmake'.\n" - f"B) Build the correct SWIG wrapper.\n\n" - f"These are the supported instruction sets on your system:\n" - f"{formatted_ins_sets}\n" - f"- If 'AVX512_SPR' (case insensitive) is supported on your " - f"system, you can set the FAISS_OPT_LEVEL=avx512_spr " - f"to build the SWIG wrapper with 'AVX512-SPR' support.\n" - f"You will have to build the 'swigfaiss_avx512_spr' " - f"target in this case.\n" - f"- If 'AVX512' (case insensitive) is supported on your system, " - f"you can set the FAISS_OPT_LEVEL=avx512 to build the SWIG wrapper " - f"with 'AVX512' support.\n" - f"You will have to build the 'swigfaiss_avx512' target in this " - f"case.\n" - f"- If 'AVX2' (case sensitive) is supported on your system, you " - f"can set the FAISS_OPT_LEVEL=AVX2 to build the SWIG wrapper " - f"with 'AVX2' support.\n" - f"You will have to build the 'swigfaiss_avx2' target in this " - f"case.\n" - f"- If 'SVE' (case sensitive) is supported on your system, you can " - f"set the FAISS_OPT_LEVEL=SVE to build the SWIG wrapper with " - f"'SVE' support.\n" - f"You will have to build the 'swigfaiss_sve' target in this " - f"case.\n" - f"- If none of the above instruction sets are supported on your " - f"system, you can execute 'cmake' without setting the " - f"FAISS_OPT_LEVEL variable and build the 'swigfaiss' target." - ) - - logger.error(message) - - sys.exit(1) diff --git a/bundle/python-cpu/Lib/site-packages/faiss/py.typed b/bundle/python-cpu/Lib/site-packages/faiss/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/faiss/swigfaiss.py b/bundle/python-cpu/Lib/site-packages/faiss/swigfaiss.py deleted file mode 100644 index 73686e78c43a926bf23aaff6660bba4ff7b41e6d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss/swigfaiss.py +++ /dev/null @@ -1,14858 +0,0 @@ -# This file was automatically generated by SWIG (https://www.swig.org). -# Version 4.4.1 -# -# Do not make changes to this file unless you know what you are doing - modify -# the SWIG interface file instead. - -from sys import version_info as _swig_python_version_info -# Import the low-level C/C++ module -if getattr(globals().get("__spec__"), "parent", None) or __package__ or "." in __name__: - from . import _swigfaiss -else: - import _swigfaiss - -try: - import builtins as __builtin__ -except ImportError: - import __builtin__ - -def _swig_repr(self): - try: - strthis = "proxy of " + self.this.__repr__() - except __builtin__.Exception: - strthis = "" - return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,) - - -def _swig_setattr_nondynamic_instance_variable(set): - def set_instance_attr(self, name, value): - if name == "this": - set(self, name, value) - elif name == "thisown": - self.this.own(value) - elif hasattr(self, name) and isinstance(getattr(type(self), name), property): - set(self, name, value) - else: - raise AttributeError("You cannot add instance attributes to %s" % self) - return set_instance_attr - - -def _swig_setattr_nondynamic_class_variable(set): - def set_class_attr(cls, name, value): - if hasattr(cls, name) and not isinstance(getattr(cls, name), property): - set(cls, name, value) - else: - raise AttributeError("You cannot add class attributes to %s" % cls) - return set_class_attr - - -def _swig_add_metaclass(metaclass): - """Class decorator for adding a metaclass to a SWIG wrapped class - a slimmed down version of six.add_metaclass""" - def wrapper(cls): - return metaclass(cls.__name__, cls.__bases__, cls.__dict__.copy()) - return wrapper - - -class _SwigNonDynamicMeta(type): - """Meta class to enforce nondynamic attributes (no new attributes) for a class""" - __setattr__ = _swig_setattr_nondynamic_class_variable(type.__setattr__) - - -class SwigPyIterator(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - __swig_destroy__ = _swigfaiss.delete_SwigPyIterator - - def value(self): - return _swigfaiss.SwigPyIterator_value(self) - - def incr(self, n=1): - return _swigfaiss.SwigPyIterator_incr(self, n) - - def decr(self, n=1): - return _swigfaiss.SwigPyIterator_decr(self, n) - - def distance(self, x): - return _swigfaiss.SwigPyIterator_distance(self, x) - - def equal(self, x): - return _swigfaiss.SwigPyIterator_equal(self, x) - - def copy(self): - return _swigfaiss.SwigPyIterator_copy(self) - - def next(self): - return _swigfaiss.SwigPyIterator_next(self) - - def __next__(self): - return _swigfaiss.SwigPyIterator___next__(self) - - def previous(self): - return _swigfaiss.SwigPyIterator_previous(self) - - def advance(self, n): - return _swigfaiss.SwigPyIterator_advance(self, n) - - def __eq__(self, x): - return _swigfaiss.SwigPyIterator___eq__(self, x) - - def __ne__(self, x): - return _swigfaiss.SwigPyIterator___ne__(self, x) - - def __iadd__(self, n): - return _swigfaiss.SwigPyIterator___iadd__(self, n) - - def __isub__(self, n): - return _swigfaiss.SwigPyIterator___isub__(self, n) - - def __add__(self, n): - return _swigfaiss.SwigPyIterator___add__(self, n) - - def __sub__(self, *args): - return _swigfaiss.SwigPyIterator___sub__(self, *args) - def __iter__(self): - return self - -# Register SwigPyIterator in _swigfaiss: -_swigfaiss.SwigPyIterator_swigregister(SwigPyIterator) -SHARED_PTR_DISOWN = _swigfaiss.SHARED_PTR_DISOWN -class Float32Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.Float32Vector_swiginit(self, _swigfaiss.new_Float32Vector()) - - def push_back(self, arg2): - return _swigfaiss.Float32Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.Float32Vector_clear(self) - - def data(self): - return _swigfaiss.Float32Vector_data(self) - - def size(self): - return _swigfaiss.Float32Vector_size(self) - - def at(self, n): - return _swigfaiss.Float32Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.Float32Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.Float32Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_Float32Vector - -# Register Float32Vector in _swigfaiss: -_swigfaiss.Float32Vector_swigregister(Float32Vector) -class Float64Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.Float64Vector_swiginit(self, _swigfaiss.new_Float64Vector()) - - def push_back(self, arg2): - return _swigfaiss.Float64Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.Float64Vector_clear(self) - - def data(self): - return _swigfaiss.Float64Vector_data(self) - - def size(self): - return _swigfaiss.Float64Vector_size(self) - - def at(self, n): - return _swigfaiss.Float64Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.Float64Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.Float64Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_Float64Vector - -# Register Float64Vector in _swigfaiss: -_swigfaiss.Float64Vector_swigregister(Float64Vector) -class Int8Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.Int8Vector_swiginit(self, _swigfaiss.new_Int8Vector()) - - def push_back(self, arg2): - return _swigfaiss.Int8Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.Int8Vector_clear(self) - - def data(self): - return _swigfaiss.Int8Vector_data(self) - - def size(self): - return _swigfaiss.Int8Vector_size(self) - - def at(self, n): - return _swigfaiss.Int8Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.Int8Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.Int8Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_Int8Vector - -# Register Int8Vector in _swigfaiss: -_swigfaiss.Int8Vector_swigregister(Int8Vector) -class Int16Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.Int16Vector_swiginit(self, _swigfaiss.new_Int16Vector()) - - def push_back(self, arg2): - return _swigfaiss.Int16Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.Int16Vector_clear(self) - - def data(self): - return _swigfaiss.Int16Vector_data(self) - - def size(self): - return _swigfaiss.Int16Vector_size(self) - - def at(self, n): - return _swigfaiss.Int16Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.Int16Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.Int16Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_Int16Vector - -# Register Int16Vector in _swigfaiss: -_swigfaiss.Int16Vector_swigregister(Int16Vector) -class Int32Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.Int32Vector_swiginit(self, _swigfaiss.new_Int32Vector()) - - def push_back(self, arg2): - return _swigfaiss.Int32Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.Int32Vector_clear(self) - - def data(self): - return _swigfaiss.Int32Vector_data(self) - - def size(self): - return _swigfaiss.Int32Vector_size(self) - - def at(self, n): - return _swigfaiss.Int32Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.Int32Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.Int32Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_Int32Vector - -# Register Int32Vector in _swigfaiss: -_swigfaiss.Int32Vector_swigregister(Int32Vector) -class Int64Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.Int64Vector_swiginit(self, _swigfaiss.new_Int64Vector()) - - def push_back(self, arg2): - return _swigfaiss.Int64Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.Int64Vector_clear(self) - - def data(self): - return _swigfaiss.Int64Vector_data(self) - - def size(self): - return _swigfaiss.Int64Vector_size(self) - - def at(self, n): - return _swigfaiss.Int64Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.Int64Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.Int64Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_Int64Vector - -# Register Int64Vector in _swigfaiss: -_swigfaiss.Int64Vector_swigregister(Int64Vector) -class UInt8Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.UInt8Vector_swiginit(self, _swigfaiss.new_UInt8Vector()) - - def push_back(self, arg2): - return _swigfaiss.UInt8Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.UInt8Vector_clear(self) - - def data(self): - return _swigfaiss.UInt8Vector_data(self) - - def size(self): - return _swigfaiss.UInt8Vector_size(self) - - def at(self, n): - return _swigfaiss.UInt8Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.UInt8Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.UInt8Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_UInt8Vector - -# Register UInt8Vector in _swigfaiss: -_swigfaiss.UInt8Vector_swigregister(UInt8Vector) -class UInt16Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.UInt16Vector_swiginit(self, _swigfaiss.new_UInt16Vector()) - - def push_back(self, arg2): - return _swigfaiss.UInt16Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.UInt16Vector_clear(self) - - def data(self): - return _swigfaiss.UInt16Vector_data(self) - - def size(self): - return _swigfaiss.UInt16Vector_size(self) - - def at(self, n): - return _swigfaiss.UInt16Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.UInt16Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.UInt16Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_UInt16Vector - -# Register UInt16Vector in _swigfaiss: -_swigfaiss.UInt16Vector_swigregister(UInt16Vector) -class UInt32Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.UInt32Vector_swiginit(self, _swigfaiss.new_UInt32Vector()) - - def push_back(self, arg2): - return _swigfaiss.UInt32Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.UInt32Vector_clear(self) - - def data(self): - return _swigfaiss.UInt32Vector_data(self) - - def size(self): - return _swigfaiss.UInt32Vector_size(self) - - def at(self, n): - return _swigfaiss.UInt32Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.UInt32Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.UInt32Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_UInt32Vector - -# Register UInt32Vector in _swigfaiss: -_swigfaiss.UInt32Vector_swigregister(UInt32Vector) -class UInt64Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.UInt64Vector_swiginit(self, _swigfaiss.new_UInt64Vector()) - - def push_back(self, arg2): - return _swigfaiss.UInt64Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.UInt64Vector_clear(self) - - def data(self): - return _swigfaiss.UInt64Vector_data(self) - - def size(self): - return _swigfaiss.UInt64Vector_size(self) - - def at(self, n): - return _swigfaiss.UInt64Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.UInt64Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.UInt64Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_UInt64Vector - -# Register UInt64Vector in _swigfaiss: -_swigfaiss.UInt64Vector_swigregister(UInt64Vector) -class Float32VectorVector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.Float32VectorVector_swiginit(self, _swigfaiss.new_Float32VectorVector()) - - def push_back(self, arg2): - return _swigfaiss.Float32VectorVector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.Float32VectorVector_clear(self) - - def data(self): - return _swigfaiss.Float32VectorVector_data(self) - - def size(self): - return _swigfaiss.Float32VectorVector_size(self) - - def at(self, n): - return _swigfaiss.Float32VectorVector_at(self, n) - - def resize(self, n): - return _swigfaiss.Float32VectorVector_resize(self, n) - - def swap(self, other): - return _swigfaiss.Float32VectorVector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_Float32VectorVector - -# Register Float32VectorVector in _swigfaiss: -_swigfaiss.Float32VectorVector_swigregister(Float32VectorVector) -class UInt8VectorVector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.UInt8VectorVector_swiginit(self, _swigfaiss.new_UInt8VectorVector()) - - def push_back(self, arg2): - return _swigfaiss.UInt8VectorVector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.UInt8VectorVector_clear(self) - - def data(self): - return _swigfaiss.UInt8VectorVector_data(self) - - def size(self): - return _swigfaiss.UInt8VectorVector_size(self) - - def at(self, n): - return _swigfaiss.UInt8VectorVector_at(self, n) - - def resize(self, n): - return _swigfaiss.UInt8VectorVector_resize(self, n) - - def swap(self, other): - return _swigfaiss.UInt8VectorVector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_UInt8VectorVector - -# Register UInt8VectorVector in _swigfaiss: -_swigfaiss.UInt8VectorVector_swigregister(UInt8VectorVector) -class Int32VectorVector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.Int32VectorVector_swiginit(self, _swigfaiss.new_Int32VectorVector()) - - def push_back(self, arg2): - return _swigfaiss.Int32VectorVector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.Int32VectorVector_clear(self) - - def data(self): - return _swigfaiss.Int32VectorVector_data(self) - - def size(self): - return _swigfaiss.Int32VectorVector_size(self) - - def at(self, n): - return _swigfaiss.Int32VectorVector_at(self, n) - - def resize(self, n): - return _swigfaiss.Int32VectorVector_resize(self, n) - - def swap(self, other): - return _swigfaiss.Int32VectorVector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_Int32VectorVector - -# Register Int32VectorVector in _swigfaiss: -_swigfaiss.Int32VectorVector_swigregister(Int32VectorVector) -class Int64VectorVector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.Int64VectorVector_swiginit(self, _swigfaiss.new_Int64VectorVector()) - - def push_back(self, arg2): - return _swigfaiss.Int64VectorVector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.Int64VectorVector_clear(self) - - def data(self): - return _swigfaiss.Int64VectorVector_data(self) - - def size(self): - return _swigfaiss.Int64VectorVector_size(self) - - def at(self, n): - return _swigfaiss.Int64VectorVector_at(self, n) - - def resize(self, n): - return _swigfaiss.Int64VectorVector_resize(self, n) - - def swap(self, other): - return _swigfaiss.Int64VectorVector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_Int64VectorVector - -# Register Int64VectorVector in _swigfaiss: -_swigfaiss.Int64VectorVector_swigregister(Int64VectorVector) -class VectorTransformVector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.VectorTransformVector_swiginit(self, _swigfaiss.new_VectorTransformVector()) - - def push_back(self, arg2): - return _swigfaiss.VectorTransformVector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.VectorTransformVector_clear(self) - - def data(self): - return _swigfaiss.VectorTransformVector_data(self) - - def size(self): - return _swigfaiss.VectorTransformVector_size(self) - - def at(self, n): - return _swigfaiss.VectorTransformVector_at(self, n) - - def resize(self, n): - return _swigfaiss.VectorTransformVector_resize(self, n) - - def swap(self, other): - return _swigfaiss.VectorTransformVector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_VectorTransformVector - -# Register VectorTransformVector in _swigfaiss: -_swigfaiss.VectorTransformVector_swigregister(VectorTransformVector) -class OperatingPointVector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.OperatingPointVector_swiginit(self, _swigfaiss.new_OperatingPointVector()) - - def push_back(self, arg2): - return _swigfaiss.OperatingPointVector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.OperatingPointVector_clear(self) - - def data(self): - return _swigfaiss.OperatingPointVector_data(self) - - def size(self): - return _swigfaiss.OperatingPointVector_size(self) - - def at(self, n): - return _swigfaiss.OperatingPointVector_at(self, n) - - def resize(self, n): - return _swigfaiss.OperatingPointVector_resize(self, n) - - def swap(self, other): - return _swigfaiss.OperatingPointVector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_OperatingPointVector - -# Register OperatingPointVector in _swigfaiss: -_swigfaiss.OperatingPointVector_swigregister(OperatingPointVector) -class InvertedListsPtrVector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.InvertedListsPtrVector_swiginit(self, _swigfaiss.new_InvertedListsPtrVector()) - - def push_back(self, arg2): - return _swigfaiss.InvertedListsPtrVector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.InvertedListsPtrVector_clear(self) - - def data(self): - return _swigfaiss.InvertedListsPtrVector_data(self) - - def size(self): - return _swigfaiss.InvertedListsPtrVector_size(self) - - def at(self, n): - return _swigfaiss.InvertedListsPtrVector_at(self, n) - - def resize(self, n): - return _swigfaiss.InvertedListsPtrVector_resize(self, n) - - def swap(self, other): - return _swigfaiss.InvertedListsPtrVector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_InvertedListsPtrVector - -# Register InvertedListsPtrVector in _swigfaiss: -_swigfaiss.InvertedListsPtrVector_swigregister(InvertedListsPtrVector) -class RepeatVector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.RepeatVector_swiginit(self, _swigfaiss.new_RepeatVector()) - - def push_back(self, arg2): - return _swigfaiss.RepeatVector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.RepeatVector_clear(self) - - def data(self): - return _swigfaiss.RepeatVector_data(self) - - def size(self): - return _swigfaiss.RepeatVector_size(self) - - def at(self, n): - return _swigfaiss.RepeatVector_at(self, n) - - def resize(self, n): - return _swigfaiss.RepeatVector_resize(self, n) - - def swap(self, other): - return _swigfaiss.RepeatVector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_RepeatVector - -# Register RepeatVector in _swigfaiss: -_swigfaiss.RepeatVector_swigregister(RepeatVector) -class ClusteringIterationStatsVector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.ClusteringIterationStatsVector_swiginit(self, _swigfaiss.new_ClusteringIterationStatsVector()) - - def push_back(self, arg2): - return _swigfaiss.ClusteringIterationStatsVector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.ClusteringIterationStatsVector_clear(self) - - def data(self): - return _swigfaiss.ClusteringIterationStatsVector_data(self) - - def size(self): - return _swigfaiss.ClusteringIterationStatsVector_size(self) - - def at(self, n): - return _swigfaiss.ClusteringIterationStatsVector_at(self, n) - - def resize(self, n): - return _swigfaiss.ClusteringIterationStatsVector_resize(self, n) - - def swap(self, other): - return _swigfaiss.ClusteringIterationStatsVector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_ClusteringIterationStatsVector - -# Register ClusteringIterationStatsVector in _swigfaiss: -_swigfaiss.ClusteringIterationStatsVector_swigregister(ClusteringIterationStatsVector) -class ParameterRangeVector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.ParameterRangeVector_swiginit(self, _swigfaiss.new_ParameterRangeVector()) - - def push_back(self, arg2): - return _swigfaiss.ParameterRangeVector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.ParameterRangeVector_clear(self) - - def data(self): - return _swigfaiss.ParameterRangeVector_data(self) - - def size(self): - return _swigfaiss.ParameterRangeVector_size(self) - - def at(self, n): - return _swigfaiss.ParameterRangeVector_at(self, n) - - def resize(self, n): - return _swigfaiss.ParameterRangeVector_resize(self, n) - - def swap(self, other): - return _swigfaiss.ParameterRangeVector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_ParameterRangeVector - -# Register ParameterRangeVector in _swigfaiss: -_swigfaiss.ParameterRangeVector_swigregister(ParameterRangeVector) -class MaybeOwnedVectorUInt8Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.MaybeOwnedVectorUInt8Vector_swiginit(self, _swigfaiss.new_MaybeOwnedVectorUInt8Vector()) - - def push_back(self, arg2): - return _swigfaiss.MaybeOwnedVectorUInt8Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.MaybeOwnedVectorUInt8Vector_clear(self) - - def data(self): - return _swigfaiss.MaybeOwnedVectorUInt8Vector_data(self) - - def size(self): - return _swigfaiss.MaybeOwnedVectorUInt8Vector_size(self) - - def at(self, n): - return _swigfaiss.MaybeOwnedVectorUInt8Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.MaybeOwnedVectorUInt8Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.MaybeOwnedVectorUInt8Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_MaybeOwnedVectorUInt8Vector - -# Register MaybeOwnedVectorUInt8Vector in _swigfaiss: -_swigfaiss.MaybeOwnedVectorUInt8Vector_swigregister(MaybeOwnedVectorUInt8Vector) -class MaybeOwnedVectorInt32Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.MaybeOwnedVectorInt32Vector_swiginit(self, _swigfaiss.new_MaybeOwnedVectorInt32Vector()) - - def push_back(self, arg2): - return _swigfaiss.MaybeOwnedVectorInt32Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.MaybeOwnedVectorInt32Vector_clear(self) - - def data(self): - return _swigfaiss.MaybeOwnedVectorInt32Vector_data(self) - - def size(self): - return _swigfaiss.MaybeOwnedVectorInt32Vector_size(self) - - def at(self, n): - return _swigfaiss.MaybeOwnedVectorInt32Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.MaybeOwnedVectorInt32Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.MaybeOwnedVectorInt32Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_MaybeOwnedVectorInt32Vector - -# Register MaybeOwnedVectorInt32Vector in _swigfaiss: -_swigfaiss.MaybeOwnedVectorInt32Vector_swigregister(MaybeOwnedVectorInt32Vector) -class MaybeOwnedVectorFloat32Vector(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self): - _swigfaiss.MaybeOwnedVectorFloat32Vector_swiginit(self, _swigfaiss.new_MaybeOwnedVectorFloat32Vector()) - - def push_back(self, arg2): - return _swigfaiss.MaybeOwnedVectorFloat32Vector_push_back(self, arg2) - - def clear(self): - return _swigfaiss.MaybeOwnedVectorFloat32Vector_clear(self) - - def data(self): - return _swigfaiss.MaybeOwnedVectorFloat32Vector_data(self) - - def size(self): - return _swigfaiss.MaybeOwnedVectorFloat32Vector_size(self) - - def at(self, n): - return _swigfaiss.MaybeOwnedVectorFloat32Vector_at(self, n) - - def resize(self, n): - return _swigfaiss.MaybeOwnedVectorFloat32Vector_resize(self, n) - - def swap(self, other): - return _swigfaiss.MaybeOwnedVectorFloat32Vector_swap(self, other) - __swig_destroy__ = _swigfaiss.delete_MaybeOwnedVectorFloat32Vector - -# Register MaybeOwnedVectorFloat32Vector in _swigfaiss: -_swigfaiss.MaybeOwnedVectorFloat32Vector_swigregister(MaybeOwnedVectorFloat32Vector) - -def simd_histogram_8(data, n, min, shift, hist): - r""" - low level SIMD histogramming functions 8-bin histogram of (x - min) >> shift - values outside the range are ignored. - the data table should be aligned on 32 bytes - """ - return _swigfaiss.simd_histogram_8(data, n, min, shift, hist) - -def simd_histogram_16(data, n, min, shift, hist): - r"""same for 16-bin histogram""" - return _swigfaiss.simd_histogram_16(data, n, min, shift, hist) -class PartitionStats(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - bisect_cycles = property(_swigfaiss.PartitionStats_bisect_cycles_get, _swigfaiss.PartitionStats_bisect_cycles_set) - compress_cycles = property(_swigfaiss.PartitionStats_compress_cycles_get, _swigfaiss.PartitionStats_compress_cycles_set) - - def __init__(self): - _swigfaiss.PartitionStats_swiginit(self, _swigfaiss.new_PartitionStats()) - - def reset(self): - return _swigfaiss.PartitionStats_reset(self) - __swig_destroy__ = _swigfaiss.delete_PartitionStats - -# Register PartitionStats in _swigfaiss: -_swigfaiss.PartitionStats_swigregister(PartitionStats) - -def popcount32(x): - return _swigfaiss.popcount32(x) - -def popcount64(x): - return _swigfaiss.popcount64(x) - -def bitvec_print(b, d): - return _swigfaiss.bitvec_print(b, d) - -def fvecs2bitvecs(x, b, d, n): - return _swigfaiss.fvecs2bitvecs(x, b, d, n) - -def bitvecs2fvecs(b, x, d, n): - return _swigfaiss.bitvecs2fvecs(b, x, d, n) - -def fvec2bitvec(x, b, d): - return _swigfaiss.fvec2bitvec(x, b, d) - -def bitvec_shuffle(n, da, db, order, a, b): - r"""Shuffle the bits from b(i, j) := a(i, order[j])""" - return _swigfaiss.bitvec_shuffle(n, da, db, order, a, b) -class BitstringWriter(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - code = property(_swigfaiss.BitstringWriter_code_get, _swigfaiss.BitstringWriter_code_set) - code_size = property(_swigfaiss.BitstringWriter_code_size_get, _swigfaiss.BitstringWriter_code_size_set) - i = property(_swigfaiss.BitstringWriter_i_get, _swigfaiss.BitstringWriter_i_set) - - def __init__(self, code, code_size): - _swigfaiss.BitstringWriter_swiginit(self, _swigfaiss.new_BitstringWriter(code, code_size)) - - def write(self, x, nbit): - return _swigfaiss.BitstringWriter_write(self, x, nbit) - __swig_destroy__ = _swigfaiss.delete_BitstringWriter - -# Register BitstringWriter in _swigfaiss: -_swigfaiss.BitstringWriter_swigregister(BitstringWriter) -cvar = _swigfaiss.cvar - -class BitstringReader(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - code = property(_swigfaiss.BitstringReader_code_get, _swigfaiss.BitstringReader_code_set) - code_size = property(_swigfaiss.BitstringReader_code_size_get, _swigfaiss.BitstringReader_code_size_set) - i = property(_swigfaiss.BitstringReader_i_get, _swigfaiss.BitstringReader_i_set) - - def __init__(self, code, code_size): - _swigfaiss.BitstringReader_swiginit(self, _swigfaiss.new_BitstringReader(code, code_size)) - - def read(self, nbit): - return _swigfaiss.BitstringReader_read(self, nbit) - __swig_destroy__ = _swigfaiss.delete_BitstringReader - -# Register BitstringReader in _swigfaiss: -_swigfaiss.BitstringReader_swigregister(BitstringReader) - -def hammings(a, b, na, nb, nbytespercode, dis): - r""" - Compute a set of Hamming distances between na and nb binary vectors - - :type a: uint8_t - :param a: size na * nbytespercode - :type b: uint8_t - :param b: size nb * nbytespercode - :type nbytespercode: int - :param nbytespercode: should be multiple of 8 - :type dis: int - :param dis: output distances, size na * nb - """ - return _swigfaiss.hammings(a, b, na, nb, nbytespercode, dis) - -def hammings_knn_hc(*args): - r""" - Return the k smallest Hamming distances for a set of binary query vectors, - using a max heap. - :type a: uint8_t - :param a: queries, size ha->nh * ncodes - :type b: uint8_t - :param b: database, size nb * ncodes - :type nb: int - :param nb: number of database vectors - :type ncodes: int - :param ncodes: size of the binary codes (bytes) - :type ordered: int - :param ordered: if != 0: order the results by decreasing distance - (may be bottleneck for k/n > 0.01) - :type approx_topk_mode: int, optional - :param approx_topk_mode: allows to use approximate top-k facilities - to speedup heap - """ - return _swigfaiss.hammings_knn_hc(*args) - -def hammings_knn(ha, a, b, nb, ncodes, ordered): - return _swigfaiss.hammings_knn(ha, a, b, nb, ncodes, ordered) - -def hammings_knn_mc(a, b, na, nb, k, ncodes, distances, labels, sel=None): - r""" - Return the k smallest Hamming distances for a set of binary query vectors, - using counting max. - :type a: uint8_t - :param a: queries, size na * ncodes - :type b: uint8_t - :param b: database, size nb * ncodes - :type na: int - :param na: number of query vectors - :type nb: int - :param nb: number of database vectors - :type k: int - :param k: number of vectors/distances to return - :type ncodes: int - :param ncodes: size of the binary codes (bytes) - :type distances: int - :param distances: output distances from each query vector to its k nearest - neighbors - :type labels: int - :param labels: output ids of the k nearest neighbors to each query vector - """ - return _swigfaiss.hammings_knn_mc(a, b, na, nb, k, ncodes, distances, labels, sel) - -def hamming_range_search(a, b, na, nb, radius, ncodes, result, sel=None): - r"""same as hammings_knn except we are doing a range search with radius""" - return _swigfaiss.hamming_range_search(a, b, na, nb, radius, ncodes, result, sel) - -def hamming_count_thres(bs1, bs2, n1, n2, ht, ncodes, nptr): - return _swigfaiss.hamming_count_thres(bs1, bs2, n1, n2, ht, ncodes, nptr) - -def match_hamming_thres(bs1, bs2, n1, n2, ht, ncodes, idx, dis): - return _swigfaiss.match_hamming_thres(bs1, bs2, n1, n2, ht, ncodes, idx, dis) - -def crosshamming_count_thres(dbs, n, ht, ncodes, nptr): - return _swigfaiss.crosshamming_count_thres(dbs, n, ht, ncodes, nptr) - -def generalized_hammings_knn_hc(ha, a, b, nb, code_size, ordered=1): - r""" - generalized Hamming distances (= count number of code bytes that - are the same) - """ - return _swigfaiss.generalized_hammings_knn_hc(ha, a, b, nb, code_size, ordered) - -def pack_bitstrings(*args): - r""" - *Overload 1:* - Pack a set of n codes of size M * nbit - - :type n: int - :param n: number of codes to pack - :type M: int - :param M: number of elementary codes per code - :type nbit: int - :param nbit: number of bits per elementary code - :type unpacked: int - :param unpacked: input unpacked codes, size (n, M) - :type packed: uint8_t - :param packed: output packed codes, size (n, code_size) - :type code_size: int - :param code_size: should be >= ceil(M * nbit / 8) - - | - - *Overload 2:* - Pack a set of n codes of variable sizes - - :param nbit: number of bits per entry (size M) - """ - return _swigfaiss.pack_bitstrings(*args) - -def unpack_bitstrings(*args): - r""" - *Overload 1:* - Unpack a set of n codes of size M * nbit - - :type n: int - :param n: number of codes to pack - :type M: int - :param M: number of elementary codes per code - :type nbit: int - :param nbit: number of bits per elementary code - :type unpacked: int - :param unpacked: input unpacked codes, size (n, M) - :type packed: uint8_t - :param packed: output packed codes, size (n, code_size) - :type code_size: int - :param code_size: should be >= ceil(M * nbit / 8) - - | - - *Overload 2:* - Unpack a set of n codes of variable sizes - - :param nbit: number of bits per entry (size M) - """ - return _swigfaiss.unpack_bitstrings(*args) - -def generalized_hamming_64(a): - return _swigfaiss.generalized_hamming_64(a) - -def get_num_gpus(): - return _swigfaiss.get_num_gpus() - -def gpu_profiler_start(): - return _swigfaiss.gpu_profiler_start() - -def gpu_profiler_stop(): - return _swigfaiss.gpu_profiler_stop() - -def gpu_sync_all_devices(): - return _swigfaiss.gpu_sync_all_devices() - -def get_compile_options(): - r"""get compile options""" - return _swigfaiss.get_compile_options() - -def get_version(): - return _swigfaiss.get_version() - -def getmillisecs(): - r"""ms elapsed since some arbitrary epoch""" - return _swigfaiss.getmillisecs() - -def get_mem_usage_kb(): - r"""get current RSS usage in kB""" - return _swigfaiss.get_mem_usage_kb() - -def get_cycles(): - return _swigfaiss.get_cycles() - -def reflection(u, x, n, d, nu): - return _swigfaiss.reflection(u, x, n, d, nu) - -def matrix_qr(m, n, a): - r""" - compute the Q of the QR decomposition for m > n - :type a: float - :param a: size n * m: input matrix and output Q - """ - return _swigfaiss.matrix_qr(m, n, a) - -def ranklist_handle_ties(k, idx, dis): - r"""distances are supposed to be sorted. Sorts indices with same distance""" - return _swigfaiss.ranklist_handle_ties(k, idx, dis) - -def ranklist_intersection_size(k1, v1, k2, v2): - r""" - count the number of common elements between v1 and v2 - algorithm = sorting + bisection to avoid double-counting duplicates - """ - return _swigfaiss.ranklist_intersection_size(k1, v1, k2, v2) - -def merge_result_table_with(n, k, I0, D0, I1, D1, keep_min=True, translation=0): - r""" - merge a result table into another one - - :type I0: int - :param I0:, D0 first result table, size (n, k) - :type I1: int - :param I1:, D1 second result table, size (n, k) - :type keep_min: boolean, optional - :param keep_min: if true, keep min values, otherwise keep max - :type translation: int, optional - :param translation: add this value to all I1's indexes - :rtype: int - :return: nb of values that were taken from the second table - """ - return _swigfaiss.merge_result_table_with(n, k, I0, D0, I1, D1, keep_min, translation) - -def imbalance_factor(*args): - r""" - *Overload 1:* - a balanced assignment has a IF of 1, a completely unbalanced assignment has - an IF = k. - - | - - *Overload 2:* - same, takes a histogram as input - """ - return _swigfaiss.imbalance_factor(*args) - -def ivec_hist(n, v, vmax, hist): - r"""compute histogram on v""" - return _swigfaiss.ivec_hist(n, v, vmax, hist) - -def bincode_hist(n, nbits, codes, hist): - r""" - Compute histogram of bits on a code array - - :type codes: uint8_t - :param codes: size(n, nbits / 8) - :type hist: int - :param hist: size(nbits): nb of 1s in the array of codes - """ - return _swigfaiss.bincode_hist(n, nbits, codes, hist) - -def ivec_checksum(n, a): - r"""compute a checksum on a table.""" - return _swigfaiss.ivec_checksum(n, a) - -def bvec_checksum(n, a): - r"""compute a checksum on a table.""" - return _swigfaiss.bvec_checksum(n, a) - -def bvecs_checksum(n, d, a, cs): - r""" - compute checksums for the rows of a matrix - - :type n: int - :param n: number of rows - :type d: int - :param d: size per row - :type a: uint8_t - :param a: matrix to handle, size n * d - :type cs: int - :param cs: output checksums, size n - """ - return _swigfaiss.bvecs_checksum(n, d, a, cs) - -def fvecs_maybe_subsample(d, n, nmax, x, verbose=False, seed=1234): - r""" - random subsamples a set of vectors if there are too many of them - - :type d: int - :param d: dimension of the vectors - :type n: int - :param n: on input: nb of input vectors, output: nb of output vectors - :type nmax: int - :param nmax: max nb of vectors to keep - :type x: float - :param x: input array, size *n-by-d - :type seed: int, optional - :param seed: random seed to use for sampling - :rtype: float - :return: x or an array allocated with new [] with *n vectors - """ - return _swigfaiss.fvecs_maybe_subsample(d, n, nmax, x, verbose, seed) - -def binary_to_real(d, x_in, x_out): - r""" - Convert binary vector to +1/-1 valued float vector. - - :type d: int - :param d: dimension of the vector (multiple of 8) - :type x_in: uint8_t - :param x_in: input binary vector (uint8_t table of size d / 8) - :type x_out: float - :param x_out: output float vector (float table of size d) - """ - return _swigfaiss.binary_to_real(d, x_in, x_out) - -def real_to_binary(d, x_in, x_out): - r""" - Convert float vector to binary vector. Components > 0 are converted to 1, - others to 0. - - :type d: int - :param d: dimension of the vector (multiple of 8) - :type x_in: float - :param x_in: input float vector (float table of size d) - :type x_out: uint8_t - :param x_out: output binary vector (uint8_t table of size d / 8) - """ - return _swigfaiss.real_to_binary(d, x_in, x_out) - -def hash_bytes(bytes, n): - r"""A reasonable hashing function""" - return _swigfaiss.hash_bytes(bytes, n) - -def check_openmp(): - r"""Whether OpenMP annotations were respected.""" - return _swigfaiss.check_openmp() -class CodeSet(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - d = property(_swigfaiss.CodeSet_d_get, _swigfaiss.CodeSet_d_set) - s = property(_swigfaiss.CodeSet_s_get, _swigfaiss.CodeSet_s_set) - - def __init__(self, d_in): - _swigfaiss.CodeSet_swiginit(self, _swigfaiss.new_CodeSet(d_in)) - - def insert(self, n, codes, inserted): - return _swigfaiss.CodeSet_insert(self, n, codes, inserted) - __swig_destroy__ = _swigfaiss.delete_CodeSet - -# Register CodeSet in _swigfaiss: -_swigfaiss.CodeSet_swigregister(CodeSet) -hamdis_tab_ham_bytes = cvar.hamdis_tab_ham_bytes - -class CombinerRangeKNNfloat(object): - r""" - This class is used to combine range and knn search results - in contrib.exhaustive_search.range_search_gpu - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nq = property(_swigfaiss.CombinerRangeKNNfloat_nq_get, _swigfaiss.CombinerRangeKNNfloat_nq_set) - k = property(_swigfaiss.CombinerRangeKNNfloat_k_get, _swigfaiss.CombinerRangeKNNfloat_k_set, doc=r"""nb of queries""") - r2 = property(_swigfaiss.CombinerRangeKNNfloat_r2_get, _swigfaiss.CombinerRangeKNNfloat_r2_set, doc=r"""number of neighbors for the knn search part""") - keep_max = property(_swigfaiss.CombinerRangeKNNfloat_keep_max_get, _swigfaiss.CombinerRangeKNNfloat_keep_max_set, doc=r"""range search radius""") - - def __init__(self, nq_in, k_in, r2_in, keep_max_in): - r"""whether to keep max values instead of min.""" - _swigfaiss.CombinerRangeKNNfloat_swiginit(self, _swigfaiss.new_CombinerRangeKNNfloat(nq_in, k_in, r2_in, keep_max_in)) - I = property(_swigfaiss.CombinerRangeKNNfloat_I_get, _swigfaiss.CombinerRangeKNNfloat_I_set, doc=r"""Knn search results""") - D = property(_swigfaiss.CombinerRangeKNNfloat_D_get, _swigfaiss.CombinerRangeKNNfloat_D_set, doc=r"""size nq * k""") - mask = property(_swigfaiss.CombinerRangeKNNfloat_mask_get, _swigfaiss.CombinerRangeKNNfloat_mask_set, doc=r""" - size nq * k - optional: range search results (ignored if mask is NULL) - """) - lim_remain = property(_swigfaiss.CombinerRangeKNNfloat_lim_remain_get, _swigfaiss.CombinerRangeKNNfloat_lim_remain_set, doc=r"""mask for where knn results are valid, size nq""") - D_remain = property(_swigfaiss.CombinerRangeKNNfloat_D_remain_get, _swigfaiss.CombinerRangeKNNfloat_D_remain_set, doc=r"""size nrange + 1""") - I_remain = property(_swigfaiss.CombinerRangeKNNfloat_I_remain_get, _swigfaiss.CombinerRangeKNNfloat_I_remain_set, doc=r"""size lim_remain[nrange]""") - L_res = property(_swigfaiss.CombinerRangeKNNfloat_L_res_get, _swigfaiss.CombinerRangeKNNfloat_L_res_set, doc=r"""size lim_remain[nrange]""") - - def compute_sizes(self, L_res): - r"""size nq + 1""" - return _swigfaiss.CombinerRangeKNNfloat_compute_sizes(self, L_res) - - def write_result(self, D_res, I_res): - r""" - Phase 2: caller allocates D_res and I_res (size L_res[nq]) - Phase 3: fill in D_res and I_res - """ - return _swigfaiss.CombinerRangeKNNfloat_write_result(self, D_res, I_res) - __swig_destroy__ = _swigfaiss.delete_CombinerRangeKNNfloat - -# Register CombinerRangeKNNfloat in _swigfaiss: -_swigfaiss.CombinerRangeKNNfloat_swigregister(CombinerRangeKNNfloat) -class CombinerRangeKNNint16(object): - r""" - This class is used to combine range and knn search results - in contrib.exhaustive_search.range_search_gpu - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nq = property(_swigfaiss.CombinerRangeKNNint16_nq_get, _swigfaiss.CombinerRangeKNNint16_nq_set) - k = property(_swigfaiss.CombinerRangeKNNint16_k_get, _swigfaiss.CombinerRangeKNNint16_k_set, doc=r"""nb of queries""") - r2 = property(_swigfaiss.CombinerRangeKNNint16_r2_get, _swigfaiss.CombinerRangeKNNint16_r2_set, doc=r"""number of neighbors for the knn search part""") - keep_max = property(_swigfaiss.CombinerRangeKNNint16_keep_max_get, _swigfaiss.CombinerRangeKNNint16_keep_max_set, doc=r"""range search radius""") - - def __init__(self, nq_in, k_in, r2_in, keep_max_in): - r"""whether to keep max values instead of min.""" - _swigfaiss.CombinerRangeKNNint16_swiginit(self, _swigfaiss.new_CombinerRangeKNNint16(nq_in, k_in, r2_in, keep_max_in)) - I = property(_swigfaiss.CombinerRangeKNNint16_I_get, _swigfaiss.CombinerRangeKNNint16_I_set, doc=r"""Knn search results""") - D = property(_swigfaiss.CombinerRangeKNNint16_D_get, _swigfaiss.CombinerRangeKNNint16_D_set, doc=r"""size nq * k""") - mask = property(_swigfaiss.CombinerRangeKNNint16_mask_get, _swigfaiss.CombinerRangeKNNint16_mask_set, doc=r""" - size nq * k - optional: range search results (ignored if mask is NULL) - """) - lim_remain = property(_swigfaiss.CombinerRangeKNNint16_lim_remain_get, _swigfaiss.CombinerRangeKNNint16_lim_remain_set, doc=r"""mask for where knn results are valid, size nq""") - D_remain = property(_swigfaiss.CombinerRangeKNNint16_D_remain_get, _swigfaiss.CombinerRangeKNNint16_D_remain_set, doc=r"""size nrange + 1""") - I_remain = property(_swigfaiss.CombinerRangeKNNint16_I_remain_get, _swigfaiss.CombinerRangeKNNint16_I_remain_set, doc=r"""size lim_remain[nrange]""") - L_res = property(_swigfaiss.CombinerRangeKNNint16_L_res_get, _swigfaiss.CombinerRangeKNNint16_L_res_set, doc=r"""size lim_remain[nrange]""") - - def compute_sizes(self, L_res): - r"""size nq + 1""" - return _swigfaiss.CombinerRangeKNNint16_compute_sizes(self, L_res) - - def write_result(self, D_res, I_res): - r""" - Phase 2: caller allocates D_res and I_res (size L_res[nq]) - Phase 3: fill in D_res and I_res - """ - return _swigfaiss.CombinerRangeKNNint16_write_result(self, D_res, I_res) - __swig_destroy__ = _swigfaiss.delete_CombinerRangeKNNint16 - -# Register CombinerRangeKNNint16 in _swigfaiss: -_swigfaiss.CombinerRangeKNNint16_swigregister(CombinerRangeKNNint16) -SIMDLevel_NONE = _swigfaiss.SIMDLevel_NONE -SIMDLevel_AVX2 = _swigfaiss.SIMDLevel_AVX2 -SIMDLevel_AVX512 = _swigfaiss.SIMDLevel_AVX512 -SIMDLevel_AVX512_SPR = _swigfaiss.SIMDLevel_AVX512_SPR -SIMDLevel_ARM_NEON = _swigfaiss.SIMDLevel_ARM_NEON -SIMDLevel_ARM_SVE = _swigfaiss.SIMDLevel_ARM_SVE -SIMDLevel_RISCV_RVV = _swigfaiss.SIMDLevel_RISCV_RVV -SIMDLevel_COUNT = _swigfaiss.SIMDLevel_COUNT - -def to_string(level): - r"""Convert SIMDLevel to string. Throws FaissException for invalid level.""" - return _swigfaiss.to_string(level) - -def to_simd_level(level_str): - r"""Parse string to SIMDLevel. Throws FaissException for invalid strings.""" - return _swigfaiss.to_simd_level(level_str) -class SIMDConfig(object): - r""" - Current SIMD configuration. - - This class provides a uniform API for querying and setting the SIMD level, - regardless of whether faiss was built with Dynamic Dispatch (DD) or static - SIMD selection. - - In DD mode: - - get_level() returns the runtime-detected or user-set level - - set_level() changes the runtime level (if level is supported) - - supported_simd_levels() returns bitmask of all compiled-in levels - - In static mode: - - get_level() returns the compiled-in level - - set_level() succeeds only if level matches compiled-in level - - supported_simd_levels() returns bitmask with single level - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - level = property(_swigfaiss.SIMDConfig_level_get, _swigfaiss.SIMDConfig_level_set) - supported_simd_levels = property(_swigfaiss.SIMDConfig_supported_simd_levels_get, _swigfaiss.SIMDConfig_supported_simd_levels_set, doc=r"""Returns bitmask of supported SIMD levels (1 << SIMDLevel).""") - avx512_split = property(_swigfaiss.SIMDConfig_avx512_split_get, _swigfaiss.SIMDConfig_avx512_split_set, doc=r""" - CPU implements AVX-512 by splitting over a 256-bit datapath - (AMD Zen 4 / Zen 4c "Bergamo", family 0x19). On such CPUs 512-bit - ops give no throughput gain, so the fast-scan QBS path prefers the - 256-bit kernel. - """) - - @staticmethod - def auto_detect_simd_level(): - return _swigfaiss.SIMDConfig_auto_detect_simd_level() - - @staticmethod - def has_dynamic_dispatch(): - return _swigfaiss.SIMDConfig_has_dynamic_dispatch() - - def __init__(self, faiss_simd_level_env=None): - _swigfaiss.SIMDConfig_swiginit(self, _swigfaiss.new_SIMDConfig(faiss_simd_level_env)) - - @staticmethod - def set_level(level): - r"""Set the SIMD level. Throws FaissException if level is not supported.""" - return _swigfaiss.SIMDConfig_set_level(level) - - @staticmethod - def get_level(): - return _swigfaiss.SIMDConfig_get_level() - - @staticmethod - def get_level_name(): - return _swigfaiss.SIMDConfig_get_level_name() - - @staticmethod - def is_simd_level_available(level): - r"""Check if a SIMD level is available (compiled in).""" - return _swigfaiss.SIMDConfig_is_simd_level_available(level) - - @staticmethod - def get_dispatched_level(): - r""" - Returns the SIMD level via the dispatch mechanism. - In DD mode, uses with_simd_level internally. - In static mode, returns the compiled-in level. - Useful for verification: get_level() == get_dispatched_level() - """ - return _swigfaiss.SIMDConfig_get_dispatched_level() - __swig_destroy__ = _swigfaiss.delete_SIMDConfig - -# Register SIMDConfig in _swigfaiss: -_swigfaiss.SIMDConfig_swigregister(SIMDConfig) -SINGLE_SIMD_LEVEL = cvar.SINGLE_SIMD_LEVEL -SINGLE_SIMD_LEVEL_256 = cvar.SINGLE_SIMD_LEVEL_256 -SINGLE_SIMD_LEVEL_512 = cvar.SINGLE_SIMD_LEVEL_512 - - -def fvec_L2sqr(x, y, d): - r"""Squared L2 distance between two vectors""" - return _swigfaiss.fvec_L2sqr(x, y, d) - -def fvec_inner_product(x, y, d): - r"""inner product""" - return _swigfaiss.fvec_inner_product(x, y, d) - -def fvec_L1(x, y, d): - r"""L1 distance""" - return _swigfaiss.fvec_L1(x, y, d) - -def fvec_Linf(x, y, d): - r"""infinity distance""" - return _swigfaiss.fvec_Linf(x, y, d) - -def fvec_inner_product_batch_4(x, y0, y1, y2, y3, d, dis0, dis1, dis2, dis3): - r""" - Special version of inner product that computes 4 distances - between x and yi, which is performance oriented. - """ - return _swigfaiss.fvec_inner_product_batch_4(x, y0, y1, y2, y3, d, dis0, dis1, dis2, dis3) - -def fvec_L2sqr_batch_4(x, y0, y1, y2, y3, d, dis0, dis1, dis2, dis3): - r""" - Special version of L2sqr that computes 4 distances - between x and yi, which is performance oriented. - """ - return _swigfaiss.fvec_L2sqr_batch_4(x, y0, y1, y2, y3, d, dis0, dis1, dis2, dis3) - -def pairwise_L2sqr(d, nq, xq, nb, xb, dis, ldq=-1, ldb=-1, ldd=-1): - r""" - Compute pairwise distances between sets of vectors - - :type d: int - :param d: dimension of the vectors - :type nq: int - :param nq: nb of query vectors - :type nb: int - :param nb: nb of database vectors - :type xq: float - :param xq: query vectors (size nq * d) - :type xb: float - :param xb: database vectors (size nb * d) - :type dis: float - :param dis: output distances (size nq * nb) - :param ldq,ldb:, ldd strides for the matrices - """ - return _swigfaiss.pairwise_L2sqr(d, nq, xq, nb, xb, dis, ldq, ldb, ldd) - -def fvec_inner_products_ny(ip, x, y, d, ny): - return _swigfaiss.fvec_inner_products_ny(ip, x, y, d, ny) - -def fvec_L2sqr_ny(dis, x, y, d, ny): - return _swigfaiss.fvec_L2sqr_ny(dis, x, y, d, ny) - -def fvec_L2sqr_ny_transposed(dis, x, y, y_sqlen, d, d_offset, ny): - return _swigfaiss.fvec_L2sqr_ny_transposed(dis, x, y, y_sqlen, d, d_offset, ny) - -def fvec_L2sqr_ny_nearest(distances_tmp_buffer, x, y, d, ny): - return _swigfaiss.fvec_L2sqr_ny_nearest(distances_tmp_buffer, x, y, d, ny) - -def fvec_L2sqr_ny_nearest_y_transposed(distances_tmp_buffer, x, y, y_sqlen, d, d_offset, ny): - return _swigfaiss.fvec_L2sqr_ny_nearest_y_transposed(distances_tmp_buffer, x, y, y_sqlen, d, d_offset, ny) - -def fvec_norm_L2sqr(x, d): - r"""squared norm of a vector""" - return _swigfaiss.fvec_norm_L2sqr(x, d) - -def fvec_norms_L2(norms, x, d, nx): - r""" - compute the L2 norms for a set of vectors - - :type norms: float - :param norms: output norms, size nx - :type x: float - :param x: set of vectors, size nx * d - """ - return _swigfaiss.fvec_norms_L2(norms, x, d, nx) - -def fvec_norms_L2sqr(norms, x, d, nx): - r"""same as fvec_norms_L2, but computes squared norms""" - return _swigfaiss.fvec_norms_L2sqr(norms, x, d, nx) - -def fvec_renorm_L2(d, nx, x): - return _swigfaiss.fvec_renorm_L2(d, nx, x) - -def inner_product_to_L2sqr(dis, nr1, nr2, n1, n2): - return _swigfaiss.inner_product_to_L2sqr(dis, nr1, nr2, n1, n2) - -def fvec_add(*args): - r""" - *Overload 1:* - compute c := a + b for vectors - - c and a can overlap, c and b can overlap - - :type a: float - :param a: size d - :type b: float - :param b: size d - :type c: float - :param c: size d - - | - - *Overload 2:* - compute c := a + b for a, c vectors and b a scalar - - c and a can overlap - - :type a: float - :param a: size d - :type c: float - :param c: size d - """ - return _swigfaiss.fvec_add(*args) - -def fvec_sub(d, a, b, c): - r""" - compute c := a - b for vectors - - c and a can overlap, c and b can overlap - - :type a: float - :param a: size d - :type b: float - :param b: size d - :type c: float - :param c: size d - """ - return _swigfaiss.fvec_sub(d, a, b, c) - -def fvec_inner_products_by_idx(ip, x, y, ids, d, nx, ny): - r""" - compute the inner product between x and a subset y of ny vectors defined by - ids - - ip(i, j) = inner_product(x(i, :), y(ids(i, j), :)) - - :type ip: float - :param ip: output array, size nx * ny - :type x: float - :param x: first-term vector, size nx * d - :type y: float - :param y: second-term vector, size (max(ids) + 1) * d - :type ids: int - :param ids: ids to sample from y, size nx * ny - """ - return _swigfaiss.fvec_inner_products_by_idx(ip, x, y, ids, d, nx, ny) - -def fvec_L2sqr_by_idx(dis, x, y, ids, d, nx, ny): - r""" - compute the squared L2 distances between x and a subset y of ny vectors - defined by ids - - dis(i, j) = inner_product(x(i, :), y(ids(i, j), :)) - - :type dis: float - :param dis: output array, size nx * ny - :type x: float - :param x: first-term vector, size nx * d - :type y: float - :param y: second-term vector, size (max(ids) + 1) * d - :type ids: int - :param ids: ids to sample from y, size nx * ny - """ - return _swigfaiss.fvec_L2sqr_by_idx(dis, x, y, ids, d, nx, ny) - -def pairwise_indexed_L2sqr(d, n, x, ix, y, iy, dis): - r""" - compute dis[j] = L2sqr(x[ix[j]], y[iy[j]]) forall j=0..n-1 - - :type x: float - :param x: size (max(ix) + 1, d) - :type y: float - :param y: size (max(iy) + 1, d) - :type ix: int - :param ix: size n - :type iy: int - :param iy: size n - :type dis: float - :param dis: size n - """ - return _swigfaiss.pairwise_indexed_L2sqr(d, n, x, ix, y, iy, dis) - -def pairwise_indexed_inner_product(d, n, x, ix, y, iy, dis): - r""" - compute dis[j] = inner_product(x[ix[j]], y[iy[j]]) forall j=0..n-1 - - :type x: float - :param x: size (max(ix) + 1, d) - :type y: float - :param y: size (max(iy) + 1, d) - :type ix: int - :param ix: size n - :type iy: int - :param iy: size n - :type dis: float - :param dis: size n - """ - return _swigfaiss.pairwise_indexed_inner_product(d, n, x, ix, y, iy, dis) - -def knn_inner_product(*args): - r""" - *Overload 1:* - Return the k nearest neighbors of each of the nx vectors x among the ny - vector y, w.r.t to max inner product. - - :type x: float - :param x: query vectors, size nx * d - :type y: float - :param y: database vectors, size ny * d - :type res: :py:class:`float_minheap_array_t` - :param res: result heap structure, which also provides k. Sorted on output - - | - - *Overload 2:* - Return the k nearest neighbors of each of the nx vectors x among the ny - vector y, for the inner product metric. - - :type x: float - :param x: query vectors, size nx * d - :type y: float - :param y: database vectors, size ny * d - :type distances: float - :param distances: output distances, size nq * k - :type indexes: int - :param indexes: output vector ids, size nq * k - - | - - *Overload 3:* - Return the k nearest neighbors of each of the nx vectors x among the ny - vector y, for the inner product metric. - - :type x: float - :param x: query vectors, size nx * d - :type y: float - :param y: database vectors, size ny * d - :type distances: float - :param distances: output distances, size nq * k - :type indexes: int - :param indexes: output vector ids, size nq * k - """ - return _swigfaiss.knn_inner_product(*args) - -def knn_L2sqr(*args): - r""" - *Overload 1:* - Return the k nearest neighbors of each of the nx vectors x among the ny - vector y, for the L2 distance - :type x: float - :param x: query vectors, size nx * d - :type y: float - :param y: database vectors, size ny * d - :type res: :py:class:`float_maxheap_array_t` - :param res: result heap structure, which also provides k. Sorted on output - :type y_norm2: float, optional - :param y_norm2: (optional) norms for the y vectors (nullptr or size ny) - :type sel: :py:class:`IDSelector`, optional - :param sel: search in this subset of vectors - - | - - *Overload 2:* - Return the k nearest neighbors of each of the nx vectors x among the ny - vector y, for the L2 distance - - :type x: float - :param x: query vectors, size nx * d - :type y: float - :param y: database vectors, size ny * d - :type distances: float - :param distances: output distances, size nq * k - :type indexes: int - :param indexes: output vector ids, size nq * k - :type y_norm2: float, optional - :param y_norm2: (optional) norms for the y vectors (nullptr or size ny) - :type sel: :py:class:`IDSelector`, optional - :param sel: search in this subset of vectors - - | - - *Overload 3:* - Return the k nearest neighbors of each of the nx vectors x among the ny - vector y, for the L2 distance - - :type x: float - :param x: query vectors, size nx * d - :type y: float - :param y: database vectors, size ny * d - :type distances: float - :param distances: output distances, size nq * k - :type indexes: int - :param indexes: output vector ids, size nq * k - :type y_norm2: float, optional - :param y_norm2: (optional) norms for the y vectors (nullptr or size ny) - :param sel: search in this subset of vectors - - | - - *Overload 4:* - Return the k nearest neighbors of each of the nx vectors x among the ny - vector y, for the L2 distance - - :type x: float - :param x: query vectors, size nx * d - :type y: float - :param y: database vectors, size ny * d - :type distances: float - :param distances: output distances, size nq * k - :type indexes: int - :param indexes: output vector ids, size nq * k - :param y_norm2: (optional) norms for the y vectors (nullptr or size ny) - :param sel: search in this subset of vectors - """ - return _swigfaiss.knn_L2sqr(*args) - -def knn_inner_products_by_idx(x, y, subset, d, nx, ny, nsubset, k, vals, ids, ld_ids=-1): - r""" - Find the max inner product neighbors for nx queries in a set of ny vectors - indexed by ids. May be useful for re-ranking a pre-selected vector list - - :type x: float - :param x: query vectors, size nx * d - :type y: float - :param y: database vectors, size (max(ids) + 1) * d - :type ids: int - :param ids: subset of database vectors to consider, size (nx, nsubset) - :param res: result structure - :type ld_ids: int, optional - :param ld_ids: stride for the ids array. -1: use nsubset, 0: all queries - process the same subset - """ - return _swigfaiss.knn_inner_products_by_idx(x, y, subset, d, nx, ny, nsubset, k, vals, ids, ld_ids) - -def knn_L2sqr_by_idx(x, y, subset, d, nx, ny, nsubset, k, vals, ids, ld_subset=-1): - r""" - Find the nearest neighbors for nx queries in a set of ny vectors - indexed by ids. May be useful for re-ranking a pre-selected vector list - - :type x: float - :param x: query vectors, size nx * d - :type y: float - :param y: database vectors, size (max(ids) + 1) * d - :type subset: int - :param subset: subset of database vectors to consider, size (nx, nsubset) - :param res: result structure - :type ld_subset: int, optional - :param ld_subset: stride for the subset array. -1: use nsubset, 0: all queries - process the same subset - """ - return _swigfaiss.knn_L2sqr_by_idx(x, y, subset, d, nx, ny, nsubset, k, vals, ids, ld_subset) - -def range_search_L2sqr(x, y, d, nx, ny, radius, result, sel=None): - r""" - Return the k nearest neighbors of each of the nx vectors x among the ny - vector y, w.r.t to max inner product - - :type x: float - :param x: query vectors, size nx * d - :type y: float - :param y: database vectors, size ny * d - :type radius: float - :param radius: search radius around the x vectors - :type result: :py:class:`RangeSearchResult` - :param result: result structure - """ - return _swigfaiss.range_search_L2sqr(x, y, d, nx, ny, radius, result, sel) - -def range_search_inner_product(x, y, d, nx, ny, radius, result, sel=None): - r"""same as range_search_L2sqr for the inner product similarity""" - return _swigfaiss.range_search_inner_product(x, y, d, nx, ny, radius, result, sel) - -def compute_PQ_dis_tables_dsub2(d, ksub, centroids, nx, x, is_inner_product, dis_tables): - r"""specialized function for PQ2""" - return _swigfaiss.compute_PQ_dis_tables_dsub2(d, ksub, centroids, nx, x, is_inner_product, dis_tables) - -def fvec_madd(n, a, bf, b, c): - r""" - compute c := a + bf * b for a, b and c tables - - :type n: int - :param n: size of the tables - :type a: float - :param a: size n - :type b: float - :param b: size n - :type c: float - :param c: result table, size n - """ - return _swigfaiss.fvec_madd(n, a, bf, b, c) - -def fvec_madd_and_argmin(n, a, bf, b, c): - r""" - same as fvec_madd, also return index of the min of the result table - :rtype: int - :return: index of the min of table c - """ - return _swigfaiss.fvec_madd_and_argmin(n, a, bf, b, c) -class RandomGenerator(object): - r"""random generator that can be used in multithreaded contexts""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - mt = property(_swigfaiss.RandomGenerator_mt_get, _swigfaiss.RandomGenerator_mt_set) - - def rand_int64(self): - r"""random int64_t""" - return _swigfaiss.RandomGenerator_rand_int64(self) - - def rand_int(self, *args): - r""" - *Overload 1:* - random positive integer - - | - - *Overload 2:* - generate random integer between 0 and max-1 - """ - return _swigfaiss.RandomGenerator_rand_int(self, *args) - - def rand_float(self): - r"""between 0 and 1""" - return _swigfaiss.RandomGenerator_rand_float(self) - - def rand_double(self): - return _swigfaiss.RandomGenerator_rand_double(self) - - def __init__(self, seed=1234): - _swigfaiss.RandomGenerator_swiginit(self, _swigfaiss.new_RandomGenerator(seed)) - __swig_destroy__ = _swigfaiss.delete_RandomGenerator - -# Register RandomGenerator in _swigfaiss: -_swigfaiss.RandomGenerator_swigregister(RandomGenerator) -class SplitMix64RandomGenerator(object): - r""" - fast random generator that cannot be used in multithreaded contexts. - based on https://prng.di.unimi.it/ - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - state = property(_swigfaiss.SplitMix64RandomGenerator_state_get, _swigfaiss.SplitMix64RandomGenerator_state_set) - - def rand_int64(self): - r"""random int64_t""" - return _swigfaiss.SplitMix64RandomGenerator_rand_int64(self) - - def rand_int(self, *args): - r""" - *Overload 1:* - random positive integer - - | - - *Overload 2:* - generate random integer between 0 and max-1 - """ - return _swigfaiss.SplitMix64RandomGenerator_rand_int(self, *args) - - def rand_float(self): - r"""between 0 and 1""" - return _swigfaiss.SplitMix64RandomGenerator_rand_float(self) - - def rand_double(self): - return _swigfaiss.SplitMix64RandomGenerator_rand_double(self) - - def __init__(self, seed=1234): - _swigfaiss.SplitMix64RandomGenerator_swiginit(self, _swigfaiss.new_SplitMix64RandomGenerator(seed)) - - def next(self): - return _swigfaiss.SplitMix64RandomGenerator_next(self) - __swig_destroy__ = _swigfaiss.delete_SplitMix64RandomGenerator - -# Register SplitMix64RandomGenerator in _swigfaiss: -_swigfaiss.SplitMix64RandomGenerator_swigregister(SplitMix64RandomGenerator) - -def float_rand(x, n, seed): - return _swigfaiss.float_rand(x, n, seed) - -def float_randn(x, n, seed): - return _swigfaiss.float_randn(x, n, seed) - -def int64_rand(x, n, seed): - return _swigfaiss.int64_rand(x, n, seed) - -def byte_rand(x, n, seed): - return _swigfaiss.byte_rand(x, n, seed) - -def int64_rand_max(x, n, max, seed): - return _swigfaiss.int64_rand_max(x, n, max, seed) - -def rand_perm(perm, n, seed): - return _swigfaiss.rand_perm(perm, n, seed) - -def rand_perm_splitmix64(perm, n, seed): - return _swigfaiss.rand_perm_splitmix64(perm, n, seed) - -def rand_smooth_vectors(n, d, x, seed): - return _swigfaiss.rand_smooth_vectors(n, d, x, seed) - -def fvec_argsort(n, vals, perm): - r""" - Indirect sort of a floating-point array - - :type n: int - :param n: size of the array - :type vals: float - :param vals: array to sort, size n - :type perm: int - :param perm: output: permutation of [0..n-1], st. - vals[perm[i + 1]] >= vals[perm[i]] - """ - return _swigfaiss.fvec_argsort(n, vals, perm) - -def fvec_argsort_parallel(n, vals, perm): - r"""Same as fvec_argsort, parallelized""" - return _swigfaiss.fvec_argsort_parallel(n, vals, perm) - -def bucket_sort(nval, vals, nbucket, lims, perm, nt=0): - r""" - Bucket sort of a list of values - - :type vals: int - :param vals: values to sort, size nval, max value nbucket - 1 - :type lims: int - :param lims: output limits of buckets, size nbucket + 1 - :type perm: int - :param perm: output buckets, the elements of bucket - i are in perm[lims[i]:lims[i + 1]] - :type nt: int, optional - :param nt: number of threads (0 = pure sequential code) - """ - return _swigfaiss.bucket_sort(nval, vals, nbucket, lims, perm, nt) - -def matrix_bucket_sort_inplace(*args): - r""" - *Overload 1:* - in-place bucket sort (with attention to memory=>int32) - on input the values are in a nrow * col matrix - we want to store the row numbers in the output. - - :type vals: int - :param vals: positive values to sort, size nrow * ncol, - max value nbucket - 1 - :type lims: int - :param lims: output limits of buckets, size nbucket + 1 - :type nt: int, optional - :param nt: number of threads (0 = pure sequential code) - - | - - *Overload 2:* - same with int64 elements - - | - - *Overload 3:* - same with int64 elements - """ - return _swigfaiss.matrix_bucket_sort_inplace(*args) - -def hashtable_int64_to_int64_init(log2_capacity, tab): - r""" - Hashtable implementation for int64 -> int64 with external storage - implemented for fast batch add and lookup. - - tab is of size 2 * (1 << log2_capacity) - n is the number of elements to add or search - - adding several values in a same batch: an arbitrary one gets added - in different batches: the newer batch overwrites. - raises an exception if capacity is exhausted. - """ - return _swigfaiss.hashtable_int64_to_int64_init(log2_capacity, tab) - -def hashtable_int64_to_int64_add(log2_capacity, tab, n, keys, vals): - return _swigfaiss.hashtable_int64_to_int64_add(log2_capacity, tab, n, keys, vals) - -def hashtable_int64_to_int64_lookup(log2_capacity, tab, n, keys, vals): - return _swigfaiss.hashtable_int64_to_int64_lookup(log2_capacity, tab, n, keys, vals) -METRIC_INNER_PRODUCT = _swigfaiss.METRIC_INNER_PRODUCT -r"""maximum inner product search""" -METRIC_L2 = _swigfaiss.METRIC_L2 -r"""squared L2 search""" -METRIC_L1 = _swigfaiss.METRIC_L1 -r"""L1 (aka cityblock)""" -METRIC_Linf = _swigfaiss.METRIC_Linf -r"""infinity distance""" -METRIC_Lp = _swigfaiss.METRIC_Lp -r"""L_p distance, p is given by a faiss::Index""" -METRIC_Canberra = _swigfaiss.METRIC_Canberra -r""" - metric_arg - some additional metrics defined in scipy.spatial.distance - """ -METRIC_BrayCurtis = _swigfaiss.METRIC_BrayCurtis -METRIC_JensenShannon = _swigfaiss.METRIC_JensenShannon -METRIC_Jaccard = _swigfaiss.METRIC_Jaccard -r"""sum_i(min(a_i, b_i)) / sum_i(max(a_i, b_i)) where a_i, b_i > 0""" -METRIC_NaNEuclidean = _swigfaiss.METRIC_NaNEuclidean -r"""Squared Euclidean distance, ignoring NaNs""" -METRIC_GOWER = _swigfaiss.METRIC_GOWER -r""" - Gower's distance - numeric dimensions are in [0,1] and categorical - dimensions are negative integers - """ - -def is_similarity_metric(metric_type): - r""" - this function is used to distinguish between min and max indexes since - we need to support similarity and dis-similarity metrics in a flexible way - """ - return _swigfaiss.is_similarity_metric(metric_type) - -def metric_type_from_int(x): - r""" - Convert an integer to MetricType with range validation. - Throws FaissException if the value is not a valid MetricType. - """ - return _swigfaiss.metric_type_from_int(x) - -def metric_type_count(): - r"""Count of entries in the MetricType enum.""" - return _swigfaiss.metric_type_count() -FAISS_VERSION_MAJOR = _swigfaiss.FAISS_VERSION_MAJOR -FAISS_VERSION_MINOR = _swigfaiss.FAISS_VERSION_MINOR -FAISS_VERSION_PATCH = _swigfaiss.FAISS_VERSION_PATCH -VERSION_STRING = _swigfaiss.VERSION_STRING -Float32 = _swigfaiss.Float32 -Float16 = _swigfaiss.Float16 -UInt8 = _swigfaiss.UInt8 -Int8 = _swigfaiss.Int8 - -def get_numeric_type_size(numeric_type): - return _swigfaiss.get_numeric_type_size(numeric_type) -class SearchParameters(object): - r""" - Parent class for the optional search parameters. - - Sub-classes with additional search parameters should inherit this class. - Ownership of the object fields is always to the caller. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - sel = property(_swigfaiss.SearchParameters_sel_get, _swigfaiss.SearchParameters_sel_set, doc=r"""if non-null, only these IDs will be considered during search.""") - __swig_destroy__ = _swigfaiss.delete_SearchParameters - - def __init__(self): - _swigfaiss.SearchParameters_swiginit(self, _swigfaiss.new_SearchParameters()) - -# Register SearchParameters in _swigfaiss: -_swigfaiss.SearchParameters_swigregister(SearchParameters) -class Index(object): - r""" - Abstract structure for an index, supports adding vectors and searching - them. - - All vectors provided at add or search time are 32-bit float arrays, - although the internal representation may vary. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - d = property(_swigfaiss.Index_d_get, _swigfaiss.Index_d_set, doc=r"""vector dimension""") - ntotal = property(_swigfaiss.Index_ntotal_get, _swigfaiss.Index_ntotal_set, doc=r"""total nb of indexed vectors""") - verbose = property(_swigfaiss.Index_verbose_get, _swigfaiss.Index_verbose_set, doc=r"""verbosity level""") - is_trained = property(_swigfaiss.Index_is_trained_get, _swigfaiss.Index_is_trained_set, doc=r""" - set if the Index does not require training, or if training is - done already - """) - metric_type = property(_swigfaiss.Index_metric_type_get, _swigfaiss.Index_metric_type_set, doc=r"""type of metric this index uses for search""") - metric_arg = property(_swigfaiss.Index_metric_arg_get, _swigfaiss.Index_metric_arg_set, doc=r"""argument of the metric type""") - __swig_destroy__ = _swigfaiss.delete_Index - - def train(self, n, x): - r""" - Perform training on a representative set of vectors - - :type n: int - :param n: nb of training vectors - :type x: float - :param x: training vectors, size n * d - """ - return _swigfaiss.Index_train(self, n, x) - - def train_with_queries(self, n, x, n_train_q, xq_train): - r""" - Perform training on a representative set of vectors and a representative - set of queries - - :type n: int - :param n: nb of training vectors - :type x: float - :param x: training vectors, size n * d - :type n_train_q: int - :param n_train_q: nb of training queries - :type xq_train: float - :param xq_train: training queries, size n_train_q * d - """ - return _swigfaiss.Index_train_with_queries(self, n, x, n_train_q, xq_train) - - def train_ex(self, n, x, numeric_type): - return _swigfaiss.Index_train_ex(self, n, x, numeric_type) - - def add(self, n, x): - r""" - Add n vectors of dimension d to the index. - - Vectors are implicitly assigned labels ntotal .. ntotal + n - 1 - This function slices the input vectors in chunks smaller than - blocksize_add and calls add_core. - :type n: int - :param n: number of vectors - :type x: float - :param x: input matrix, size n * d - """ - return _swigfaiss.Index_add(self, n, x) - - def add_ex(self, n, x, numeric_type): - return _swigfaiss.Index_add_ex(self, n, x, numeric_type) - - def add_with_ids(self, n, x, xids): - r""" - Same as add, but stores xids instead of sequential ids. - - The default implementation fails with an assertion, as it is - not supported by all indexes. - - :type n: int - :param n: number of vectors - :type x: float - :param x: input vectors, size n * d - :type xids: int - :param xids: if non-null, ids to store for the vectors (size n) - """ - return _swigfaiss.Index_add_with_ids(self, n, x, xids) - - def add_with_ids_ex(self, n, x, numeric_type, xids): - return _swigfaiss.Index_add_with_ids_ex(self, n, x, numeric_type, xids) - - def search(self, n, x, k, distances, labels, params=None): - r""" - query n vectors of dimension d to the index. - - return at most k vectors. If there are not enough results for a - query, the result array is padded with -1s. - - :type n: int - :param n: number of vectors - :type x: float - :param x: input vectors to search, size n * d - :type k: int - :param k: number of extracted vectors - :type distances: float - :param distances: output pairwise distances, size n*k - :type labels: int - :param labels: output labels of the NNs, size n*k - """ - return _swigfaiss.Index_search(self, n, x, k, distances, labels, params) - - def search_ex(self, n, x, numeric_type, k, distances, labels, params=None): - return _swigfaiss.Index_search_ex(self, n, x, numeric_type, k, distances, labels, params) - - def search1(self, x, handler, params=None): - r"""search one vector with a custom result handler""" - return _swigfaiss.Index_search1(self, x, handler, params) - - def range_search(self, n, x, radius, result, params=None): - r""" - query n vectors of dimension d to the index. - - return all vectors with distance < radius. Note that many - indexes do not implement the range_search (only the k-NN search - is mandatory). - - :type n: int - :param n: number of vectors - :type x: float - :param x: input vectors to search, size n * d - :type radius: float - :param radius: search radius - :type result: :py:class:`RangeSearchResult` - :param result: result table - """ - return _swigfaiss.Index_range_search(self, n, x, radius, result, params) - - def assign(self, n, x, labels, k=1): - r""" - return the indexes of the k vectors closest to the query x. - - This function is identical as search but only return labels of - neighbors. - :type n: int - :param n: number of vectors - :type x: float - :param x: input vectors to search, size n * d - :type labels: int - :param labels: output labels of the NNs, size n*k - :type k: int, optional - :param k: number of nearest neighbours - """ - return _swigfaiss.Index_assign(self, n, x, labels, k) - - def reset(self): - r"""removes all elements from the database.""" - return _swigfaiss.Index_reset(self) - - def remove_ids(self, sel): - r""" - removes IDs from the index. Not supported by all - indexes. Returns the number of elements removed. - """ - return _swigfaiss.Index_remove_ids(self, sel) - - def reconstruct(self, key, recons): - r""" - Reconstruct a stored vector (or an approximation if lossy coding) - - this function may not be defined for some indexes - :type key: int - :param key: id of the vector to reconstruct - :type recons: float - :param recons: reconstructed vector (size d) - """ - return _swigfaiss.Index_reconstruct(self, key, recons) - - def reconstruct_batch(self, n, keys, recons): - r""" - Reconstruct several stored vectors (or an approximation if lossy - coding) - - this function may not be defined for some indexes - :type n: int - :param n: number of vectors to reconstruct - :type keys: int - :param keys: ids of the vectors to reconstruct (size n) - :type recons: float - :param recons: reconstructed vector (size n * d) - """ - return _swigfaiss.Index_reconstruct_batch(self, n, keys, recons) - - def reconstruct_n(self, i0, ni, recons): - r""" - Reconstruct vectors i0 to i0 + ni - 1 - - this function may not be defined for some indexes - :type i0: int - :param i0: index of the first vector in the sequence - :type ni: int - :param ni: number of vectors in the sequence - :type recons: float - :param recons: reconstructed vector (size ni * d) - """ - return _swigfaiss.Index_reconstruct_n(self, i0, ni, recons) - - def search_and_reconstruct(self, n, x, k, distances, labels, recons, params=None): - r""" - Similar to search, but also reconstructs the stored vectors (or an - approximation in the case of lossy coding) for the search results. - - If there are not enough results for a query, the resulting arrays - is padded with -1s. - - :type n: int - :param n: number of vectors - :type x: float - :param x: input vectors to search, size n * d - :type k: int - :param k: number of extracted vectors - :type distances: float - :param distances: output pairwise distances, size n*k - :type labels: int - :param labels: output labels of the NNs, size n*k - :type recons: float - :param recons: reconstructed vectors size (n, k, d) - """ - return _swigfaiss.Index_search_and_reconstruct(self, n, x, k, distances, labels, recons, params) - - def search_subset(self, n, x, k_base, base_labels, k, distances, labels): - r""" - Similar to search, but operates on a potentially different subset - of the dataset for each query. - - The default implementation fails with an assertion, as it is - not supported by all indexes. - - :type n: int - :param n: number of vectors - :type x: float - :param x: input vectors, size n * d - :type k_base: int - :param k_base: number of vectors to search from - :type base_labels: int - :param base_labels: ids of the vectors to search from - :type k: int - :param k: desired number of results per query - :type distances: float - :param distances: output pairwise distances, size n*k - :type labels: int - :param labels: output labels of the NNs, size n*k - """ - return _swigfaiss.Index_search_subset(self, n, x, k_base, base_labels, k, distances, labels) - - def compute_residual(self, x, residual, key): - r""" - Computes a residual vector after indexing encoding. - - The residual vector is the difference between a vector and the - reconstruction that can be decoded from its representation in - the index. The residual can be used for multiple-stage indexing - methods, like IndexIVF's methods. - - :type x: float - :param x: input vector, size d - :type residual: float - :param residual: output residual vector, size d - :type key: int - :param key: encoded index, as returned by search and assign - """ - return _swigfaiss.Index_compute_residual(self, x, residual, key) - - def compute_residual_n(self, n, xs, residuals, keys): - r""" - Computes a residual vector after indexing encoding (batch form). - Equivalent to calling compute_residual for each vector. - - The residual vector is the difference between a vector and the - reconstruction that can be decoded from its representation in - the index. The residual can be used for multiple-stage indexing - methods, like IndexIVF's methods. - - :type n: int - :param n: number of vectors - :type xs: float - :param xs: input vectors, size (n x d) - :type residuals: float - :param residuals: output residual vectors, size (n x d) - :type keys: int - :param keys: encoded index, as returned by search and assign - """ - return _swigfaiss.Index_compute_residual_n(self, n, xs, residuals, keys) - - def get_distance_computer(self): - r""" - Get a DistanceComputer (defined in AuxIndexStructures) object - for this kind of index. - - DistanceComputer is implemented for indexes that support random - access of their vectors. - """ - return _swigfaiss.Index_get_distance_computer(self) - - def sa_code_size(self): - r"""size of the produced codes in bytes""" - return _swigfaiss.Index_sa_code_size(self) - - def sa_encode(self, n, x, bytes): - r""" - encode a set of vectors - - :type n: int - :param n: number of vectors - :type x: float - :param x: input vectors, size n * d - :type bytes: uint8_t - :param bytes: output encoded vectors, size n * sa_code_size() - """ - return _swigfaiss.Index_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - r""" - decode a set of vectors - - :type n: int - :param n: number of vectors - :type bytes: uint8_t - :param bytes: input encoded vectors, size n * sa_code_size() - :type x: float - :param x: output vectors, size n * d - """ - return _swigfaiss.Index_sa_decode(self, n, bytes, x) - - def merge_from(self, otherIndex, add_id=0): - r""" - moves the entries from another dataset to self. - On output, other is empty. - add_id is added to all moved ids - (for sequential ids, this would be this->ntotal) - """ - return _swigfaiss.Index_merge_from(self, otherIndex, add_id) - - def check_compatible_for_merge(self, otherIndex): - r""" - check that the two indexes are compatible (ie, they are - trained in the same way and have the same - parameters). Otherwise throw. - """ - return _swigfaiss.Index_check_compatible_for_merge(self, otherIndex) - - def add_sa_codes(self, n, codes, xids): - r""" - Add vectors that are computed with the standalone codec - - :type codes: uint8_t - :param codes: codes to add size n * sa_code_size() - :type xids: int - :param xids: corresponding ids, size n - """ - return _swigfaiss.Index_add_sa_codes(self, n, codes, xids) - -# Register Index in _swigfaiss: -_swigfaiss.Index_swigregister(Index) -class DistanceComputer(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - - def set_query(self, x): - r""" - called before computing distances. Pointer x should remain valid - while operator () is called - """ - return _swigfaiss.DistanceComputer_set_query(self, x) - - def __call__(self, i): - r"""compute distance of vector i to current query""" - return _swigfaiss.DistanceComputer___call__(self, i) - - def distances_batch_4(self, idx0, idx1, idx2, idx3, dis0, dis1, dis2, dis3): - r""" - compute distances of current query to 4 stored vectors. - certain DistanceComputer implementations may benefit - heavily from this. - """ - return _swigfaiss.DistanceComputer_distances_batch_4(self, idx0, idx1, idx2, idx3, dis0, dis1, dis2, dis3) - - def symmetric_dis(self, i, j): - r"""compute distance between two stored vectors""" - return _swigfaiss.DistanceComputer_symmetric_dis(self, i, j) - __swig_destroy__ = _swigfaiss.delete_DistanceComputer - -# Register DistanceComputer in _swigfaiss: -_swigfaiss.DistanceComputer_swigregister(DistanceComputer) -class NegativeDistanceComputer(DistanceComputer): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - basedis = property(_swigfaiss.NegativeDistanceComputer_basedis_get, _swigfaiss.NegativeDistanceComputer_basedis_set, doc=r"""owned by this""") - - def __init__(self, basedis_): - _swigfaiss.NegativeDistanceComputer_swiginit(self, _swigfaiss.new_NegativeDistanceComputer(basedis_)) - - def set_query(self, x): - return _swigfaiss.NegativeDistanceComputer_set_query(self, x) - - def __call__(self, i): - r"""compute distance of vector i to current query""" - return _swigfaiss.NegativeDistanceComputer___call__(self, i) - - def distances_batch_4(self, idx0, idx1, idx2, idx3, dis0, dis1, dis2, dis3): - return _swigfaiss.NegativeDistanceComputer_distances_batch_4(self, idx0, idx1, idx2, idx3, dis0, dis1, dis2, dis3) - - def symmetric_dis(self, i, j): - r"""compute distance between two stored vectors""" - return _swigfaiss.NegativeDistanceComputer_symmetric_dis(self, i, j) - __swig_destroy__ = _swigfaiss.delete_NegativeDistanceComputer - -# Register NegativeDistanceComputer in _swigfaiss: -_swigfaiss.NegativeDistanceComputer_swigregister(NegativeDistanceComputer) -class FlatCodesDistanceComputer(DistanceComputer): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - codes = property(_swigfaiss.FlatCodesDistanceComputer_codes_get, _swigfaiss.FlatCodesDistanceComputer_codes_set) - code_size = property(_swigfaiss.FlatCodesDistanceComputer_code_size_get, _swigfaiss.FlatCodesDistanceComputer_code_size_set) - q = property(_swigfaiss.FlatCodesDistanceComputer_q_get, _swigfaiss.FlatCodesDistanceComputer_q_set) - - def __call__(self, i): - return _swigfaiss.FlatCodesDistanceComputer___call__(self, i) - - def distances_batch_4(self, idx0, idx1, idx2, idx3, dis0, dis1, dis2, dis3): - return _swigfaiss.FlatCodesDistanceComputer_distances_batch_4(self, idx0, idx1, idx2, idx3, dis0, dis1, dis2, dis3) - - def partial_dot_product(self, arg2, arg3, arg4): - r""" - Computes a partial dot product over a slice of the query vector. - The slice is defined by the following parameters: - — `offset`: the starting index of the first component to include - — `num_components`: the number of consecutive components to include - - Components refer to raw dimensions of the flat (uncompressed) query - vector. - - By default, this method throws an error, as it is only implemented - in specific subclasses such as `FlatL2Dis`. Other flat distance - computers may override this when partial dot product support is needed. - - Over time, this method might be changed to a pure virtual function (`= - 0`) to enforce implementation in subclasses that require this - functionality. - - This method is not part of the generic `DistanceComputer` interface - because for compressed representations (e.g., product quantization), - calling `partial_dot_product` repeatedly is often less efficient than - computing the full distance at once. - - Supporting efficient partial scans generally requires a different memory - layout, such as interleaved blocks that keep SIMD lanes full. This is a - non-trivial change and not supported in the current flat layout. - - For more details on partial (or chunked) dot product computations and - the performance trade-offs involved, refer to the Panorama paper: - https://arxiv.org/pdf/2510.00566 - """ - return _swigfaiss.FlatCodesDistanceComputer_partial_dot_product(self, arg2, arg3, arg4) - - def distance_to_code(self, code): - r"""compute distance of current query to an encoded vector""" - return _swigfaiss.FlatCodesDistanceComputer_distance_to_code(self, code) - - def distance_to_code_batch_4(self, c1, c2, c3, c4, d1, d2, d3, d4): - return _swigfaiss.FlatCodesDistanceComputer_distance_to_code_batch_4(self, c1, c2, c3, c4, d1, d2, d3, d4) - - def partial_dot_product_batch_4(self, idx0, idx1, idx2, idx3, dp0, dp1, dp2, dp3, offset, num_components): - r""" - Compute partial dot products of current query to 4 stored vectors. - See `partial_dot_product` for more details. - """ - return _swigfaiss.FlatCodesDistanceComputer_partial_dot_product_batch_4(self, idx0, idx1, idx2, idx3, dp0, dp1, dp2, dp3, offset, num_components) - __swig_destroy__ = _swigfaiss.delete_FlatCodesDistanceComputer - -# Register FlatCodesDistanceComputer in _swigfaiss: -_swigfaiss.FlatCodesDistanceComputer_swigregister(FlatCodesDistanceComputer) -class IOReader(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - name = property(_swigfaiss.IOReader_name_get, _swigfaiss.IOReader_name_set) - - def __call__(self, ptr, size, nitems): - return _swigfaiss.IOReader___call__(self, ptr, size, nitems) - - def filedescriptor(self): - return _swigfaiss.IOReader_filedescriptor(self) - __swig_destroy__ = _swigfaiss.delete_IOReader - -# Register IOReader in _swigfaiss: -_swigfaiss.IOReader_swigregister(IOReader) -class IOWriter(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - name = property(_swigfaiss.IOWriter_name_get, _swigfaiss.IOWriter_name_set) - - def __call__(self, ptr, size, nitems): - return _swigfaiss.IOWriter___call__(self, ptr, size, nitems) - - def filedescriptor(self): - return _swigfaiss.IOWriter_filedescriptor(self) - __swig_destroy__ = _swigfaiss.delete_IOWriter - -# Register IOWriter in _swigfaiss: -_swigfaiss.IOWriter_swigregister(IOWriter) -class VectorIOReader(IOReader): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - data = property(_swigfaiss.VectorIOReader_data_get, _swigfaiss.VectorIOReader_data_set) - rp = property(_swigfaiss.VectorIOReader_rp_get, _swigfaiss.VectorIOReader_rp_set) - - def __call__(self, ptr, size, nitems): - return _swigfaiss.VectorIOReader___call__(self, ptr, size, nitems) - - def __init__(self): - _swigfaiss.VectorIOReader_swiginit(self, _swigfaiss.new_VectorIOReader()) - __swig_destroy__ = _swigfaiss.delete_VectorIOReader - -# Register VectorIOReader in _swigfaiss: -_swigfaiss.VectorIOReader_swigregister(VectorIOReader) -class VectorIOWriter(IOWriter): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - data = property(_swigfaiss.VectorIOWriter_data_get, _swigfaiss.VectorIOWriter_data_set) - - def __call__(self, ptr, size, nitems): - return _swigfaiss.VectorIOWriter___call__(self, ptr, size, nitems) - - def __init__(self): - _swigfaiss.VectorIOWriter_swiginit(self, _swigfaiss.new_VectorIOWriter()) - __swig_destroy__ = _swigfaiss.delete_VectorIOWriter - -# Register VectorIOWriter in _swigfaiss: -_swigfaiss.VectorIOWriter_swigregister(VectorIOWriter) -class FileIOReader(IOReader): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - f = property(_swigfaiss.FileIOReader_f_get, _swigfaiss.FileIOReader_f_set) - need_close = property(_swigfaiss.FileIOReader_need_close_get, _swigfaiss.FileIOReader_need_close_set) - - def __init__(self, *args): - _swigfaiss.FileIOReader_swiginit(self, _swigfaiss.new_FileIOReader(*args)) - __swig_destroy__ = _swigfaiss.delete_FileIOReader - - def __call__(self, ptr, size, nitems): - return _swigfaiss.FileIOReader___call__(self, ptr, size, nitems) - - def filedescriptor(self): - return _swigfaiss.FileIOReader_filedescriptor(self) - -# Register FileIOReader in _swigfaiss: -_swigfaiss.FileIOReader_swigregister(FileIOReader) -class FileIOWriter(IOWriter): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - f = property(_swigfaiss.FileIOWriter_f_get, _swigfaiss.FileIOWriter_f_set) - need_close = property(_swigfaiss.FileIOWriter_need_close_get, _swigfaiss.FileIOWriter_need_close_set) - - def __init__(self, *args): - _swigfaiss.FileIOWriter_swiginit(self, _swigfaiss.new_FileIOWriter(*args)) - __swig_destroy__ = _swigfaiss.delete_FileIOWriter - - def __call__(self, ptr, size, nitems): - return _swigfaiss.FileIOWriter___call__(self, ptr, size, nitems) - - def filedescriptor(self): - return _swigfaiss.FileIOWriter_filedescriptor(self) - -# Register FileIOWriter in _swigfaiss: -_swigfaiss.FileIOWriter_swigregister(FileIOWriter) -class BufferedIOReader(IOReader): - r"""wraps an ioreader to make buffered reads to avoid too small reads""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - reader = property(_swigfaiss.BufferedIOReader_reader_get, _swigfaiss.BufferedIOReader_reader_set) - bsz = property(_swigfaiss.BufferedIOReader_bsz_get, _swigfaiss.BufferedIOReader_bsz_set) - ofs = property(_swigfaiss.BufferedIOReader_ofs_get, _swigfaiss.BufferedIOReader_ofs_set, doc=r"""offset in input stream""") - ofs2 = property(_swigfaiss.BufferedIOReader_ofs2_get, _swigfaiss.BufferedIOReader_ofs2_set, doc=r"""number of bytes returned to caller""") - b0 = property(_swigfaiss.BufferedIOReader_b0_get, _swigfaiss.BufferedIOReader_b0_set, doc=r"""range of available bytes in the buffer""") - b1 = property(_swigfaiss.BufferedIOReader_b1_get, _swigfaiss.BufferedIOReader_b1_set) - buffer = property(_swigfaiss.BufferedIOReader_buffer_get, _swigfaiss.BufferedIOReader_buffer_set) - - def __init__(self, *args): - r""" - :type bsz: int, optional - :param bsz: buffer size (bytes). Reads will be done by batched of - this size - """ - _swigfaiss.BufferedIOReader_swiginit(self, _swigfaiss.new_BufferedIOReader(*args)) - - def __call__(self, ptr, size, nitems): - return _swigfaiss.BufferedIOReader___call__(self, ptr, size, nitems) - __swig_destroy__ = _swigfaiss.delete_BufferedIOReader - -# Register BufferedIOReader in _swigfaiss: -_swigfaiss.BufferedIOReader_swigregister(BufferedIOReader) -class BufferedIOWriter(IOWriter): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - writer = property(_swigfaiss.BufferedIOWriter_writer_get, _swigfaiss.BufferedIOWriter_writer_set) - bsz = property(_swigfaiss.BufferedIOWriter_bsz_get, _swigfaiss.BufferedIOWriter_bsz_set) - ofs = property(_swigfaiss.BufferedIOWriter_ofs_get, _swigfaiss.BufferedIOWriter_ofs_set) - ofs2 = property(_swigfaiss.BufferedIOWriter_ofs2_get, _swigfaiss.BufferedIOWriter_ofs2_set, doc=r"""number of bytes received from caller""") - b0 = property(_swigfaiss.BufferedIOWriter_b0_get, _swigfaiss.BufferedIOWriter_b0_set, doc=r"""amount of data in buffer""") - buffer = property(_swigfaiss.BufferedIOWriter_buffer_get, _swigfaiss.BufferedIOWriter_buffer_set) - - def __init__(self, *args): - _swigfaiss.BufferedIOWriter_swiginit(self, _swigfaiss.new_BufferedIOWriter(*args)) - - def __call__(self, ptr, size, nitems): - return _swigfaiss.BufferedIOWriter___call__(self, ptr, size, nitems) - __swig_destroy__ = _swigfaiss.delete_BufferedIOWriter - -# Register BufferedIOWriter in _swigfaiss: -_swigfaiss.BufferedIOWriter_swigregister(BufferedIOWriter) - -def fourcc(*args): - r"""cast a 4-character string to a uint32_t that can be written and read easily""" - return _swigfaiss.fourcc(*args) - -def fourcc_inv(*args): - return _swigfaiss.fourcc_inv(*args) - -def fourcc_inv_printable(x): - return _swigfaiss.fourcc_inv_printable(x) -class MaybeOwnedVectorOwner(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - __swig_destroy__ = _swigfaiss.delete_MaybeOwnedVectorOwner - - def __init__(self): - _swigfaiss.MaybeOwnedVectorOwner_swiginit(self, _swigfaiss.new_MaybeOwnedVectorOwner()) - -# Register MaybeOwnedVectorOwner in _swigfaiss: -_swigfaiss.MaybeOwnedVectorOwner_swigregister(MaybeOwnedVectorOwner) -class MmappedFileMappingOwner(MaybeOwnedVectorOwner): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.MmappedFileMappingOwner_swiginit(self, _swigfaiss.new_MmappedFileMappingOwner(*args)) - __swig_destroy__ = _swigfaiss.delete_MmappedFileMappingOwner - - def data(self): - return _swigfaiss.MmappedFileMappingOwner_data(self) - - def size(self): - return _swigfaiss.MmappedFileMappingOwner_size(self) - -# Register MmappedFileMappingOwner in _swigfaiss: -_swigfaiss.MmappedFileMappingOwner_swigregister(MmappedFileMappingOwner) -class MappedFileIOReader(IOReader): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - mmap_owner = property(_swigfaiss.MappedFileIOReader_mmap_owner_get, _swigfaiss.MappedFileIOReader_mmap_owner_set) - pos = property(_swigfaiss.MappedFileIOReader_pos_get, _swigfaiss.MappedFileIOReader_pos_set) - - def __init__(self, owner): - _swigfaiss.MappedFileIOReader_swiginit(self, _swigfaiss.new_MappedFileIOReader(owner)) - - def __call__(self, ptr, size, nitems): - return _swigfaiss.MappedFileIOReader___call__(self, ptr, size, nitems) - - def mmap(self, ptr, size, nitems): - return _swigfaiss.MappedFileIOReader_mmap(self, ptr, size, nitems) - - def filedescriptor(self): - return _swigfaiss.MappedFileIOReader_filedescriptor(self) - __swig_destroy__ = _swigfaiss.delete_MappedFileIOReader - -# Register MappedFileIOReader in _swigfaiss: -_swigfaiss.MappedFileIOReader_swigregister(MappedFileIOReader) -class ZeroCopyIOReader(IOReader): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - data_ = property(_swigfaiss.ZeroCopyIOReader_data__get, _swigfaiss.ZeroCopyIOReader_data__set) - rp_ = property(_swigfaiss.ZeroCopyIOReader_rp__get, _swigfaiss.ZeroCopyIOReader_rp__set) - total_ = property(_swigfaiss.ZeroCopyIOReader_total__get, _swigfaiss.ZeroCopyIOReader_total__set) - - def __init__(self, data, size): - _swigfaiss.ZeroCopyIOReader_swiginit(self, _swigfaiss.new_ZeroCopyIOReader(data, size)) - __swig_destroy__ = _swigfaiss.delete_ZeroCopyIOReader - - def reset(self): - return _swigfaiss.ZeroCopyIOReader_reset(self) - - def get_data_view(self, ptr, size, nitems): - return _swigfaiss.ZeroCopyIOReader_get_data_view(self, ptr, size, nitems) - - def __call__(self, ptr, size, nitems): - return _swigfaiss.ZeroCopyIOReader___call__(self, ptr, size, nitems) - - def filedescriptor(self): - return _swigfaiss.ZeroCopyIOReader_filedescriptor(self) - -# Register ZeroCopyIOReader in _swigfaiss: -_swigfaiss.ZeroCopyIOReader_swigregister(ZeroCopyIOReader) -class IndexFlatCodes(Index): - r""" - Index that encodes all vectors as fixed-size codes (size code_size). Storage - is in the codes vector - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - code_size = property(_swigfaiss.IndexFlatCodes_code_size_get, _swigfaiss.IndexFlatCodes_code_size_set) - codes = property(_swigfaiss.IndexFlatCodes_codes_get, _swigfaiss.IndexFlatCodes_codes_set, doc=r"""encoded dataset, size ntotal * code_size""") - - def __init__(self, *args): - _swigfaiss.IndexFlatCodes_swiginit(self, _swigfaiss.new_IndexFlatCodes(*args)) - - def add(self, n, x): - r"""default add uses sa_encode""" - return _swigfaiss.IndexFlatCodes_add(self, n, x) - - def reset(self): - return _swigfaiss.IndexFlatCodes_reset(self) - - def reconstruct_n(self, i0, ni, recons): - return _swigfaiss.IndexFlatCodes_reconstruct_n(self, i0, ni, recons) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexFlatCodes_reconstruct(self, key, recons) - - def sa_code_size(self): - return _swigfaiss.IndexFlatCodes_sa_code_size(self) - - def remove_ids(self, sel): - r""" - remove some ids. NB that because of the structure of the - index, the semantics of this operation are - different from the usual ones: the new ids are shifted - """ - return _swigfaiss.IndexFlatCodes_remove_ids(self, sel) - - def get_FlatCodesDistanceComputer(self): - r""" - a FlatCodesDistanceComputer offers a distance_to_code method - - The default implementation explicitly decodes the vector with sa_decode. - """ - return _swigfaiss.IndexFlatCodes_get_FlatCodesDistanceComputer(self) - - def get_distance_computer(self): - return _swigfaiss.IndexFlatCodes_get_distance_computer(self) - - def search(self, n, x, k, distances, labels, params=None): - r""" - Search implemented by decoding (most index types will have a faster - implementation) - """ - return _swigfaiss.IndexFlatCodes_search(self, n, x, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexFlatCodes_range_search(self, n, x, radius, result, params) - - def search1(self, x, handler, params=None): - return _swigfaiss.IndexFlatCodes_search1(self, x, handler, params) - - def get_CodePacker(self): - return _swigfaiss.IndexFlatCodes_get_CodePacker(self) - - def check_compatible_for_merge(self, otherIndex): - return _swigfaiss.IndexFlatCodes_check_compatible_for_merge(self, otherIndex) - - def merge_from(self, otherIndex, add_id=0): - return _swigfaiss.IndexFlatCodes_merge_from(self, otherIndex, add_id) - - def add_sa_codes(self, n, x, xids): - return _swigfaiss.IndexFlatCodes_add_sa_codes(self, n, x, xids) - - def permute_entries(self, perm): - return _swigfaiss.IndexFlatCodes_permute_entries(self, perm) - __swig_destroy__ = _swigfaiss.delete_IndexFlatCodes - -# Register IndexFlatCodes in _swigfaiss: -_swigfaiss.IndexFlatCodes_swigregister(IndexFlatCodes) -class IndexFlat(IndexFlatCodes): - r"""Index that stores the full vectors and performs exhaustive search""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexFlat_search(self, n, x, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexFlat_range_search(self, n, x, radius, result, params) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexFlat_reconstruct(self, key, recons) - - def compute_distance_subset(self, n, x, k, distances, labels): - r""" - compute distance with a subset of vectors - - :type x: float - :param x: query vectors, size n * d - :type labels: int - :param labels: indices of the vectors that should be compared - for each query vector, size n * k - :type distances: float - :param distances: - corresponding output distances, size n * k - """ - return _swigfaiss.IndexFlat_compute_distance_subset(self, n, x, k, distances, labels) - - def get_xb(self, *args): - return _swigfaiss.IndexFlat_get_xb(self, *args) - - def __init__(self, *args): - _swigfaiss.IndexFlat_swiginit(self, _swigfaiss.new_IndexFlat(*args)) - - def get_FlatCodesDistanceComputer(self): - return _swigfaiss.IndexFlat_get_FlatCodesDistanceComputer(self) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexFlat_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexFlat_sa_decode(self, n, bytes, x) - __swig_destroy__ = _swigfaiss.delete_IndexFlat - -# Register IndexFlat in _swigfaiss: -_swigfaiss.IndexFlat_swigregister(IndexFlat) -class IndexFlatIP(IndexFlat): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexFlatIP_swiginit(self, _swigfaiss.new_IndexFlatIP(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexFlatIP - -# Register IndexFlatIP in _swigfaiss: -_swigfaiss.IndexFlatIP_swigregister(IndexFlatIP) -class IndexFlatL2(IndexFlat): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - cached_l2norms = property(_swigfaiss.IndexFlatL2_cached_l2norms_get, _swigfaiss.IndexFlatL2_cached_l2norms_set) - - def __init__(self, *args): - r""":param d: dimensionality of the input vectors""" - _swigfaiss.IndexFlatL2_swiginit(self, _swigfaiss.new_IndexFlatL2(*args)) - - def get_FlatCodesDistanceComputer(self): - return _swigfaiss.IndexFlatL2_get_FlatCodesDistanceComputer(self) - - def sync_l2norms(self): - return _swigfaiss.IndexFlatL2_sync_l2norms(self) - - def clear_l2norms(self): - return _swigfaiss.IndexFlatL2_clear_l2norms(self) - __swig_destroy__ = _swigfaiss.delete_IndexFlatL2 - -# Register IndexFlatL2 in _swigfaiss: -_swigfaiss.IndexFlatL2_swigregister(IndexFlatL2) -class IndexFlatPanorama(IndexFlat): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - batch_size = property(_swigfaiss.IndexFlatPanorama_batch_size_get) - n_levels = property(_swigfaiss.IndexFlatPanorama_n_levels_get) - cum_sums = property(_swigfaiss.IndexFlatPanorama_cum_sums_get, _swigfaiss.IndexFlatPanorama_cum_sums_set) - - def __init__(self, d_in, metric, n_levels_in, batch_size_in): - r""" - :param d: dimensionality of the input vectors - :type metric: int - :param metric: metric type - :param n_levels: number of Panorama levels - :param batch_size: batch size for Panorama storage - """ - _swigfaiss.IndexFlatPanorama_swiginit(self, _swigfaiss.new_IndexFlatPanorama(d_in, metric, n_levels_in, batch_size_in)) - - def add(self, n, x): - return _swigfaiss.IndexFlatPanorama_add(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexFlatPanorama_search(self, n, x, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexFlatPanorama_range_search(self, n, x, radius, result, params) - - def search_subset(self, n, x, k_base, base_labels, k, distances, labels): - return _swigfaiss.IndexFlatPanorama_search_subset(self, n, x, k_base, base_labels, k, distances, labels) - - def reset(self): - return _swigfaiss.IndexFlatPanorama_reset(self) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexFlatPanorama_reconstruct(self, key, recons) - - def reconstruct_n(self, i, n, recons): - return _swigfaiss.IndexFlatPanorama_reconstruct_n(self, i, n, recons) - - def remove_ids(self, sel): - return _swigfaiss.IndexFlatPanorama_remove_ids(self, sel) - - def merge_from(self, otherIndex, add_id): - return _swigfaiss.IndexFlatPanorama_merge_from(self, otherIndex, add_id) - - def add_sa_codes(self, n, codes_in, xids): - return _swigfaiss.IndexFlatPanorama_add_sa_codes(self, n, codes_in, xids) - - def permute_entries(self, perm): - return _swigfaiss.IndexFlatPanorama_permute_entries(self, perm) - __swig_destroy__ = _swigfaiss.delete_IndexFlatPanorama - -# Register IndexFlatPanorama in _swigfaiss: -_swigfaiss.IndexFlatPanorama_swigregister(IndexFlatPanorama) -class IndexFlatL2Panorama(IndexFlatPanorama): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, d_in, n_levels_in, batch_size_in=512): - r""" - :param d: dimensionality of the input vectors - :param n_levels: number of Panorama levels - :param batch_size: batch size for Panorama storage - """ - _swigfaiss.IndexFlatL2Panorama_swiginit(self, _swigfaiss.new_IndexFlatL2Panorama(d_in, n_levels_in, batch_size_in)) - __swig_destroy__ = _swigfaiss.delete_IndexFlatL2Panorama - -# Register IndexFlatL2Panorama in _swigfaiss: -_swigfaiss.IndexFlatL2Panorama_swigregister(IndexFlatL2Panorama) -class IndexFlatIPPanorama(IndexFlatPanorama): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, d_in, n_levels_in, batch_size_in=512): - r""" - :param d: dimensionality of the input vectors - :param n_levels: number of Panorama levels - :param batch_size: batch size for Panorama storage - """ - _swigfaiss.IndexFlatIPPanorama_swiginit(self, _swigfaiss.new_IndexFlatIPPanorama(d_in, n_levels_in, batch_size_in)) - __swig_destroy__ = _swigfaiss.delete_IndexFlatIPPanorama - -# Register IndexFlatIPPanorama in _swigfaiss: -_swigfaiss.IndexFlatIPPanorama_swigregister(IndexFlatIPPanorama) -class IndexFlat1D(IndexFlatL2): - r"""optimized version for 1D "vectors".""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - continuous_update = property(_swigfaiss.IndexFlat1D_continuous_update_get, _swigfaiss.IndexFlat1D_continuous_update_set, doc=r"""is the permutation updated continuously?""") - perm = property(_swigfaiss.IndexFlat1D_perm_get, _swigfaiss.IndexFlat1D_perm_set, doc=r"""sorted database indices""") - - def __init__(self, continuous_update=True): - _swigfaiss.IndexFlat1D_swiginit(self, _swigfaiss.new_IndexFlat1D(continuous_update)) - - def update_permutation(self): - r""" - if not continuous_update, call this between the last add and - the first search - """ - return _swigfaiss.IndexFlat1D_update_permutation(self) - - def add(self, n, x): - return _swigfaiss.IndexFlat1D_add(self, n, x) - - def reset(self): - return _swigfaiss.IndexFlat1D_reset(self) - - def search(self, n, x, k, distances, labels, params=None): - r"""Warn: the distances returned are L1 not L2""" - return _swigfaiss.IndexFlat1D_search(self, n, x, k, distances, labels, params) - __swig_destroy__ = _swigfaiss.delete_IndexFlat1D - -# Register IndexFlat1D in _swigfaiss: -_swigfaiss.IndexFlat1D_swigregister(IndexFlat1D) -ClusteringInitMethod_RANDOM = _swigfaiss.ClusteringInitMethod_RANDOM -r""" - Random sampling: select k random points uniformly from the dataset. - Time complexity: O(k) - """ -ClusteringInitMethod_KMEANS_PLUS_PLUS = _swigfaiss.ClusteringInitMethod_KMEANS_PLUS_PLUS -r""" - k-means++: select centroids with probability proportional to D(x)², - where D(x) is the distance to the nearest existing centroid. - Reference: Arthur, D., & Vassilvitskii, S. (2006). k-means++: - The advantages of careful seeding. Stanford. - Time complexity: O(nkd) - """ -ClusteringInitMethod_AFK_MC2 = _swigfaiss.ClusteringInitMethod_AFK_MC2 -r""" - AFK-MC²: Assumption-Free K-MC² using Markov Chain Monte Carlo. - Provides theoretical guarantees without assumptions on data - distribution. - Uses a non-uniform proposal distribution based on D²-sampling from - the first center, combined with uniform sampling for regularization. - Reference: Bachem, O., Lucic, M., Hassani, H., & Krause, A. (2016). - Fast and provably good seedings for k-means. Advances in neural - information processing systems, 29. - Time complexity: O(nd) preprocessing + O(mk²d) main loop - """ -class ClusteringInitialization(object): - r""" - Centroid initialization for k-means clustering. - - This class provides different algorithms for selecting initial centroids - before running k-means iterations. Good initialization can significantly - improve clustering quality and convergence speed. - - Example usage: - - .. code-block:: c++ - - ClusteringInitialization init(128, 1000); // d=128, k=1000 - init.method = ClusteringInitMethod::KMEANS_PLUS_PLUS; - init.seed = 42; - - std::vector centroids(128 * 1000); - init.init_centroids(n, x, centroids.data()); - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - d = property(_swigfaiss.ClusteringInitialization_d_get, _swigfaiss.ClusteringInitialization_d_set, doc=r"""vector dimension""") - k = property(_swigfaiss.ClusteringInitialization_k_get, _swigfaiss.ClusteringInitialization_k_set, doc=r"""number of centroids to initialize""") - method = property(_swigfaiss.ClusteringInitialization_method_get, _swigfaiss.ClusteringInitialization_method_set, doc=r"""Initialization method to use""") - seed = property(_swigfaiss.ClusteringInitialization_seed_get, _swigfaiss.ClusteringInitialization_seed_set, doc=r"""Random seed.""") - afkmc2_chain_length = property(_swigfaiss.ClusteringInitialization_afkmc2_chain_length_get, _swigfaiss.ClusteringInitialization_afkmc2_chain_length_set, doc=r""" - Chain length for AFK-MC² (only used when method = AFK_MC2). - Longer chains give better approximation to k-means++ but are slower. - """) - - def __init__(self, d, k): - _swigfaiss.ClusteringInitialization_swiginit(self, _swigfaiss.new_ClusteringInitialization(d, k)) - - def init_centroids(self, n, x, centroids, n_existing_centroids=0, existing_centroids=None): - r""" - Initialize k centroids from n input vectors. - - :type n: int - :param n: number of input vectors - :type x: float - :param x: input vectors, size (n, d), row-major - :type centroids: float - :param centroids: output centroids, size (k, d), row-major - :type n_existing_centroids: int, optional - :param n_existing_centroids: number of pre-existing centroids to - consider - when computing distances (for k-means++ and - AFK-MC²). These centroids are not modified. - :type existing_centroids: float, optional - :param existing_centroids: pre-existing centroids, size - (n_existing_centroids, d), row-major. - New centroids will be selected to be far - from these existing ones. - """ - return _swigfaiss.ClusteringInitialization_init_centroids(self, n, x, centroids, n_existing_centroids, existing_centroids) - __swig_destroy__ = _swigfaiss.delete_ClusteringInitialization - -# Register ClusteringInitialization in _swigfaiss: -_swigfaiss.ClusteringInitialization_swigregister(ClusteringInitialization) -class ClusteringParameters(object): - r""" - Class for the clustering parameters. Can be passed to the - constructor of the Clustering object. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - niter = property(_swigfaiss.ClusteringParameters_niter_get, _swigfaiss.ClusteringParameters_niter_set, doc=r"""number of clustering iterations""") - nredo = property(_swigfaiss.ClusteringParameters_nredo_get, _swigfaiss.ClusteringParameters_nredo_set, doc=r""" - redo clustering this many times and keep the clusters with the best - objective - """) - verbose = property(_swigfaiss.ClusteringParameters_verbose_get, _swigfaiss.ClusteringParameters_verbose_set) - spherical = property(_swigfaiss.ClusteringParameters_spherical_get, _swigfaiss.ClusteringParameters_spherical_set, doc=r""" - whether to normalize centroids after each iteration (useful for inner - product clustering) - """) - int_centroids = property(_swigfaiss.ClusteringParameters_int_centroids_get, _swigfaiss.ClusteringParameters_int_centroids_set, doc=r"""round centroids coordinates to integer after each iteration?""") - update_index = property(_swigfaiss.ClusteringParameters_update_index_get, _swigfaiss.ClusteringParameters_update_index_set, doc=r"""re-train index after each iteration?""") - frozen_centroids = property(_swigfaiss.ClusteringParameters_frozen_centroids_get, _swigfaiss.ClusteringParameters_frozen_centroids_set, doc=r""" - Use the subset of centroids provided as input and do not change them - during iterations - """) - min_points_per_centroid = property(_swigfaiss.ClusteringParameters_min_points_per_centroid_get, _swigfaiss.ClusteringParameters_min_points_per_centroid_set, doc=r""" - If fewer than this number of training vectors per centroid are provided, - writes a warning. Note that fewer than 1 point per centroid raises an - exception. - """) - max_points_per_centroid = property(_swigfaiss.ClusteringParameters_max_points_per_centroid_get, _swigfaiss.ClusteringParameters_max_points_per_centroid_set, doc=r"""to limit size of dataset, otherwise the training set is subsampled""") - seed = property(_swigfaiss.ClusteringParameters_seed_get, _swigfaiss.ClusteringParameters_seed_set, doc=r""" - seed for the random number generator. - negative values lead to seeding an internal rng with - std::high_resolution_clock. - """) - decode_block_size = property(_swigfaiss.ClusteringParameters_decode_block_size_get, _swigfaiss.ClusteringParameters_decode_block_size_set, doc=r"""when the training set is encoded, batch size of the codec decoder""") - check_input_data_for_NaNs = property(_swigfaiss.ClusteringParameters_check_input_data_for_NaNs_get, _swigfaiss.ClusteringParameters_check_input_data_for_NaNs_set, doc=r"""whether to check for NaNs in an input data""") - use_faster_subsampling = property(_swigfaiss.ClusteringParameters_use_faster_subsampling_get, _swigfaiss.ClusteringParameters_use_faster_subsampling_set, doc=r""" - Whether to use splitmix64-based random number generator for subsampling, - which is faster, but may pick duplicate points. - """) - init_method = property(_swigfaiss.ClusteringParameters_init_method_get, _swigfaiss.ClusteringParameters_init_method_set, doc=r""" - Initialization method for centroids. - RANDOM: uniform random sampling (default, current behavior) - KMEANS_PLUS_PLUS: k-means++ (O(nkd), better quality) - AFK_MC2: Assumption-Free K-MC² (O(nd) + O(mk²d), fast approximation) - """) - afkmc2_chain_length = property(_swigfaiss.ClusteringParameters_afkmc2_chain_length_get, _swigfaiss.ClusteringParameters_afkmc2_chain_length_set, doc=r""" - Chain length for AFK-MC² initialization. - Only used when init_method = AFK_MC2. - Longer chains give better approximation but are slower. - """) - early_stop_threshold = property(_swigfaiss.ClusteringParameters_early_stop_threshold_get, _swigfaiss.ClusteringParameters_early_stop_threshold_set, doc=r""" - Early stop threshold, the range is [0, 1]. - The value of 0 implies a default Faiss behavior, - so the training process stops only if an error - is unchanged from the previous iteration. - """) - - def __init__(self): - _swigfaiss.ClusteringParameters_swiginit(self, _swigfaiss.new_ClusteringParameters()) - __swig_destroy__ = _swigfaiss.delete_ClusteringParameters - -# Register ClusteringParameters in _swigfaiss: -_swigfaiss.ClusteringParameters_swigregister(ClusteringParameters) -class ClusteringIterationStats(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - obj = property(_swigfaiss.ClusteringIterationStats_obj_get, _swigfaiss.ClusteringIterationStats_obj_set, doc=r"""objective values (sum of distances reported by index)""") - time = property(_swigfaiss.ClusteringIterationStats_time_get, _swigfaiss.ClusteringIterationStats_time_set, doc=r"""seconds for iteration""") - time_search = property(_swigfaiss.ClusteringIterationStats_time_search_get, _swigfaiss.ClusteringIterationStats_time_search_set, doc=r"""seconds for just search""") - imbalance_factor = property(_swigfaiss.ClusteringIterationStats_imbalance_factor_get, _swigfaiss.ClusteringIterationStats_imbalance_factor_set, doc=r"""imbalance factor of iteration""") - nsplit = property(_swigfaiss.ClusteringIterationStats_nsplit_get, _swigfaiss.ClusteringIterationStats_nsplit_set, doc=r"""number of cluster splits""") - - def __init__(self): - _swigfaiss.ClusteringIterationStats_swiginit(self, _swigfaiss.new_ClusteringIterationStats()) - __swig_destroy__ = _swigfaiss.delete_ClusteringIterationStats - -# Register ClusteringIterationStats in _swigfaiss: -_swigfaiss.ClusteringIterationStats_swigregister(ClusteringIterationStats) -class Clustering(ClusteringParameters): - r""" - K-means clustering based on assignment - centroid update iterations - - The clustering is based on an Index object that assigns training - points to the centroids. Therefore, at each iteration the centroids - are added to the index. - - On output, the centroids table is set to the latest version - of the centroids and they are also added to the index. If the - centroids table it is not empty on input, it is also used for - initialization. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - d = property(_swigfaiss.Clustering_d_get, _swigfaiss.Clustering_d_set, doc=r"""dimension of the vectors""") - k = property(_swigfaiss.Clustering_k_get, _swigfaiss.Clustering_k_set, doc=r"""nb of centroids""") - centroids = property(_swigfaiss.Clustering_centroids_get, _swigfaiss.Clustering_centroids_set, doc=r""" - centroids (k * d) - if centroids are set on input to train, they will be used as - initialization - """) - iteration_stats = property(_swigfaiss.Clustering_iteration_stats_get, _swigfaiss.Clustering_iteration_stats_set, doc=r"""stats at every iteration of clustering""") - - def __init__(self, *args): - _swigfaiss.Clustering_swiginit(self, _swigfaiss.new_Clustering(*args)) - - def train(self, n, x, index, x_weights=None): - r""" - run k-means training - - :type x: float - :param x: training vectors, size n * d - :type index: :py:class:`Index` - :param index: index used for assignment - :type x_weights: float, optional - :param x_weights: weight associated to each vector: NULL or size n - """ - return _swigfaiss.Clustering_train(self, n, x, index, x_weights) - - def train_encoded(self, nx, x_in, codec, index, weights=None): - r""" - run with encoded vectors - - in addition to train()'s parameters takes a codec as parameter - to decode the input vectors. - - :type codec: :py:class:`Index` - :param codec: codec used to decode the vectors (nullptr = - vectors are in fact floats) - """ - return _swigfaiss.Clustering_train_encoded(self, nx, x_in, codec, index, weights) - - def post_process_centroids(self): - r""" - Post-process the centroids after each centroid update. - includes optional L2 normalization and nearest integer rounding - """ - return _swigfaiss.Clustering_post_process_centroids(self) - __swig_destroy__ = _swigfaiss.delete_Clustering - -# Register Clustering in _swigfaiss: -_swigfaiss.Clustering_swigregister(Clustering) -class Clustering1D(Clustering): - r""" - Exact 1D clustering algorithm - - Since it does not use an index, it does not overload the train() function - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.Clustering1D_swiginit(self, _swigfaiss.new_Clustering1D(*args)) - - def train_exact(self, n, x): - return _swigfaiss.Clustering1D_train_exact(self, n, x) - __swig_destroy__ = _swigfaiss.delete_Clustering1D - -# Register Clustering1D in _swigfaiss: -_swigfaiss.Clustering1D_swigregister(Clustering1D) -class ProgressiveDimClusteringParameters(ClusteringParameters): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - progressive_dim_steps = property(_swigfaiss.ProgressiveDimClusteringParameters_progressive_dim_steps_get, _swigfaiss.ProgressiveDimClusteringParameters_progressive_dim_steps_set, doc=r"""number of incremental steps""") - apply_pca = property(_swigfaiss.ProgressiveDimClusteringParameters_apply_pca_get, _swigfaiss.ProgressiveDimClusteringParameters_apply_pca_set, doc=r"""apply PCA on input""") - - def __init__(self): - _swigfaiss.ProgressiveDimClusteringParameters_swiginit(self, _swigfaiss.new_ProgressiveDimClusteringParameters()) - __swig_destroy__ = _swigfaiss.delete_ProgressiveDimClusteringParameters - -# Register ProgressiveDimClusteringParameters in _swigfaiss: -_swigfaiss.ProgressiveDimClusteringParameters_swigregister(ProgressiveDimClusteringParameters) -class ProgressiveDimIndexFactory(object): - r"""generates an index suitable for clustering when called""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __call__(self, dim): - r"""ownership transferred to caller""" - return _swigfaiss.ProgressiveDimIndexFactory___call__(self, dim) - __swig_destroy__ = _swigfaiss.delete_ProgressiveDimIndexFactory - - def __init__(self): - _swigfaiss.ProgressiveDimIndexFactory_swiginit(self, _swigfaiss.new_ProgressiveDimIndexFactory()) - -# Register ProgressiveDimIndexFactory in _swigfaiss: -_swigfaiss.ProgressiveDimIndexFactory_swigregister(ProgressiveDimIndexFactory) -class ProgressiveDimClustering(ProgressiveDimClusteringParameters): - r""" - K-means clustering with progressive dimensions used - - The clustering first happens in dim 1, then with exponentially increasing - dimension until d (I steps). This is typically applied after a PCA - transformation (optional). Reference: - - "Improved Residual Vector Quantization for High-dimensional Approximate - Nearest Neighbor Search" - - Shicong Liu, Hongtao Lu, Junru Shao, AAAI'15 - - https://arxiv.org/abs/1509.05195 - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - d = property(_swigfaiss.ProgressiveDimClustering_d_get, _swigfaiss.ProgressiveDimClustering_d_set, doc=r"""dimension of the vectors""") - k = property(_swigfaiss.ProgressiveDimClustering_k_get, _swigfaiss.ProgressiveDimClustering_k_set, doc=r"""nb of centroids""") - centroids = property(_swigfaiss.ProgressiveDimClustering_centroids_get, _swigfaiss.ProgressiveDimClustering_centroids_set, doc=r"""centroids (k * d)""") - iteration_stats = property(_swigfaiss.ProgressiveDimClustering_iteration_stats_get, _swigfaiss.ProgressiveDimClustering_iteration_stats_set, doc=r"""stats at every iteration of clustering""") - - def __init__(self, *args): - _swigfaiss.ProgressiveDimClustering_swiginit(self, _swigfaiss.new_ProgressiveDimClustering(*args)) - - def train(self, n, x, factory): - return _swigfaiss.ProgressiveDimClustering_train(self, n, x, factory) - __swig_destroy__ = _swigfaiss.delete_ProgressiveDimClustering - -# Register ProgressiveDimClustering in _swigfaiss: -_swigfaiss.ProgressiveDimClustering_swigregister(ProgressiveDimClustering) - -def kmeans_clustering(d, n, k, x, centroids): - r""" - simplified interface - - :type d: int - :param d: dimension of the data - :type n: int - :param n: nb of training vectors - :type k: int - :param k: nb of output centroids - :type x: float - :param x: training set (size n * d) - :type centroids: float - :param centroids: output centroids (size k * d) - :rtype: float - :return: final quantization error - """ - return _swigfaiss.kmeans_clustering(d, n, k, x, centroids) -class SuperKMeansParameters(ClusteringParameters): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - d_prime_fraction = property(_swigfaiss.SuperKMeansParameters_d_prime_fraction_get, _swigfaiss.SuperKMeansParameters_d_prime_fraction_set, doc=r"""Initial d_prime as a fraction of d (GEMM/PRUNING split).""") - pdx_block_size = property(_swigfaiss.SuperKMeansParameters_pdx_block_size_get, _swigfaiss.SuperKMeansParameters_pdx_block_size_set, doc=r"""Matches block_l2 SIMD width.""") - ad_epsilon_factor = property(_swigfaiss.SuperKMeansParameters_ad_epsilon_factor_get, _swigfaiss.SuperKMeansParameters_ad_epsilon_factor_set, doc=r"""ADSampling significance: epsilon = ad_epsilon_factor / d.""") - pruning_target_low = property(_swigfaiss.SuperKMeansParameters_pruning_target_low_get, _swigfaiss.SuperKMeansParameters_pruning_target_low_set, doc=r""" - Adaptive d_prime stay-in-band controller. Per iteration: - pruning > high → shrink d_prime (over-pruning, cheaper threshold) - pruning < low → grow d_prime (under-pruning, more GEMM work) - else: hold. - """) - pruning_target_high = property(_swigfaiss.SuperKMeansParameters_pruning_target_high_get, _swigfaiss.SuperKMeansParameters_pruning_target_high_set) - d_prime_adjust = property(_swigfaiss.SuperKMeansParameters_d_prime_adjust_get, _swigfaiss.SuperKMeansParameters_d_prime_adjust_set, doc=r"""Relative step size for d_prime adjustments.""") - d_prime_min = property(_swigfaiss.SuperKMeansParameters_d_prime_min_get, _swigfaiss.SuperKMeansParameters_d_prime_min_set, doc=r"""Floor on d_prime; below this the chi-squared bound is unvalidated.""") - x_batch = property(_swigfaiss.SuperKMeansParameters_x_batch_get, _swigfaiss.SuperKMeansParameters_x_batch_set) - y_batch = property(_swigfaiss.SuperKMeansParameters_y_batch_get, _swigfaiss.SuperKMeansParameters_y_batch_set) - omp_chunk = property(_swigfaiss.SuperKMeansParameters_omp_chunk_get, _swigfaiss.SuperKMeansParameters_omp_chunk_set, doc=r"""OpenMP dynamic-schedule chunk size for the pruning loop.""") - - def __init__(self): - _swigfaiss.SuperKMeansParameters_swiginit(self, _swigfaiss.new_SuperKMeansParameters()) - __swig_destroy__ = _swigfaiss.delete_SuperKMeansParameters - -# Register SuperKMeansParameters in _swigfaiss: -_swigfaiss.SuperKMeansParameters_swigregister(SuperKMeansParameters) -class SuperKMeans(object): - r""" - Drop-in faster k-means: same interface as faiss::Clustering. Per iteration: - iter 0: full GEMM over all d dims (vanilla Lloyd's). - iter 1..niter-1: GEMM over front d_prime dims, then ADSampling - progressive pruning over PDX-laid trailing dims. - - Trains in a randomly-rotated space; centroids are un-rotated before return. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - cp = property(_swigfaiss.SuperKMeans_cp_get, _swigfaiss.SuperKMeans_cp_set) - d = property(_swigfaiss.SuperKMeans_d_get, _swigfaiss.SuperKMeans_d_set) - k = property(_swigfaiss.SuperKMeans_k_get, _swigfaiss.SuperKMeans_k_set) - centroids = property(_swigfaiss.SuperKMeans_centroids_get, _swigfaiss.SuperKMeans_centroids_set, doc=r"""Output: k * d floats, row-major, un-rotated.""") - iteration_stats = property(_swigfaiss.SuperKMeans_iteration_stats_get, _swigfaiss.SuperKMeans_iteration_stats_set, doc=r""" - Per-iter stats. `obj`, `time`, and `nsplit` are populated faithfully. - `time_search` mirrors `time` (phases not timed separately) and - `imbalance_factor` is set to NaN (not computed). - """) - gemm_pruning_rates = property(_swigfaiss.SuperKMeans_gemm_pruning_rates_get, _swigfaiss.SuperKMeans_gemm_pruning_rates_set, doc=r""" - Per-iter fraction of (vector, centroid) pairs pruned at the d_prime - GEMM-boundary chi-squared check. Per-PDX-block early-exits are NOT - counted. iter 0 uses full GEMM, so gemm_pruning_rates[0] == 0.0f. - """) - - def __init__(self, *args): - _swigfaiss.SuperKMeans_swiginit(self, _swigfaiss.new_SuperKMeans(*args)) - - def train(self, n, x): - r""" - Train on `n` row-major vectors of dimension `d`. Honors the applicable - ClusteringParameters fields (niter, seed, verbose, - max_points_per_centroid, use_faster_subsampling, - check_input_data_for_NaNs). - """ - return _swigfaiss.SuperKMeans_train(self, n, x) - __swig_destroy__ = _swigfaiss.delete_SuperKMeans - -# Register SuperKMeans in _swigfaiss: -_swigfaiss.SuperKMeans_swigregister(SuperKMeans) -class SuperKMeansAssignScratch(object): - r""" - Reusable scratch for super_kmeans_assign_iteration; pass one instance across - a loop of calls to avoid reallocating (buffers grow only as needed). - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - Y_trail = property(_swigfaiss.SuperKMeansAssignScratch_Y_trail_get, _swigfaiss.SuperKMeansAssignScratch_Y_trail_set) - Y_pdx = property(_swigfaiss.SuperKMeansAssignScratch_Y_pdx_get, _swigfaiss.SuperKMeansAssignScratch_Y_pdx_set) - x_norms_partial = property(_swigfaiss.SuperKMeansAssignScratch_x_norms_partial_get, _swigfaiss.SuperKMeansAssignScratch_x_norms_partial_set) - y_norms_partial = property(_swigfaiss.SuperKMeansAssignScratch_y_norms_partial_get, _swigfaiss.SuperKMeansAssignScratch_y_norms_partial_set) - partial_ip = property(_swigfaiss.SuperKMeansAssignScratch_partial_ip_get, _swigfaiss.SuperKMeansAssignScratch_partial_ip_set) - - def __init__(self): - _swigfaiss.SuperKMeansAssignScratch_swiginit(self, _swigfaiss.new_SuperKMeansAssignScratch()) - __swig_destroy__ = _swigfaiss.delete_SuperKMeansAssignScratch - -# Register SuperKMeansAssignScratch in _swigfaiss: -_swigfaiss.SuperKMeansAssignScratch_swigregister(SuperKMeansAssignScratch) - -def super_kmeans_assign_iteration(X_tilde, n, d, Y_tilde, k, tau, assignments, d_prime, ad_coeff, cp, total_pairs=None, pruned_at_gemm=None, scratch=None): - r""" - One SuperKMeans iter-1+ assignment pass: partial GEMM over the front - `d_prime` dims + ADSampling progressive pruning over the PDX-laid-out - trailing block. Updates `tau` and `assignments` in place. - """ - return _swigfaiss.super_kmeans_assign_iteration(X_tilde, n, d, Y_tilde, k, tau, assignments, d_prime, ad_coeff, cp, total_pairs, pruned_at_gemm, scratch) - -def normal_quantile(p): - r""" - Inverse standard normal CDF. Three-branch rational polynomial, - absolute error < 1.15e-9 over `p in (0, 1)`. Behavior at the boundaries - (p <= 0 or p >= 1) is unspecified — returns NaN or +/-inf. - """ - return _swigfaiss.normal_quantile(p) - -def chi2_quantile_wh(p, alpha): - r""" - Chi-squared quantile via cube-root approximation. Validated to within - 2% of scipy for `p in [16, d]` and `alpha <= 1 - 1e-6`. Accuracy - degrades for smaller `p` or for `alpha` near 1. - """ - return _swigfaiss.chi2_quantile_wh(p, alpha) - -def precompute_ad_thresholds(d, epsilon): - r""" - Build ADSampling threshold table of size `d + 1`: - coeff[p] = chi2_quantile_wh(p, 1 - epsilon) / d. - - Indexing: coeff[0] is reserved (left at 0.0f). coeff[1..15] are - computed but NOT accuracy-bounded — callers requiring the 2% scipy - tolerance must consume only coeff[16..d]. SuperKMeans enforces - this via its `d_prime_min = 16` parameter. - """ - return _swigfaiss.precompute_ad_thresholds(d, epsilon) - -def pairwise_extra_distances(d, nq, xq, nb, xb, mt, metric_arg, dis, ldq=-1, ldb=-1, ldd=-1): - return _swigfaiss.pairwise_extra_distances(d, nq, xq, nb, xb, mt, metric_arg, dis, ldq, ldb, ldd) - -def knn_extra_metrics(x, y, d, nx, ny, mt, metric_arg, k, distances, indexes, sel=None): - return _swigfaiss.knn_extra_metrics(x, y, d, nx, ny, mt, metric_arg, k, distances, indexes, sel) - -def get_extra_distance_computer(d, mt, metric_arg, xb): - r""" - get a DistanceComputer that refers to this type of distance and - indexes a flat array - """ - return _swigfaiss.get_extra_distance_computer(d, mt, metric_arg, xb) -class Quantizer(object): - r"""General interface for quantizer objects""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - d = property(_swigfaiss.Quantizer_d_get, _swigfaiss.Quantizer_d_set, doc=r"""size of the input vectors""") - code_size = property(_swigfaiss.Quantizer_code_size_get, _swigfaiss.Quantizer_code_size_set, doc=r"""bytes per indexed vector""") - - def train(self, n, x): - r""" - Train the quantizer - - :type x: float - :param x: training vectors, size n * d - """ - return _swigfaiss.Quantizer_train(self, n, x) - - def compute_codes(self, x, codes, n): - r""" - Quantize a set of vectors - - :type x: float - :param x: input vectors, size n * d - :type codes: uint8_t - :param codes: output codes, size n * code_size - """ - return _swigfaiss.Quantizer_compute_codes(self, x, codes, n) - - def decode(self, code, x, n): - r""" - Decode a set of vectors - - :param codes: input codes, size n * code_size - :type x: float - :param x: output vectors, size n * d - """ - return _swigfaiss.Quantizer_decode(self, code, x, n) - __swig_destroy__ = _swigfaiss.delete_Quantizer - -# Register Quantizer in _swigfaiss: -_swigfaiss.Quantizer_swigregister(Quantizer) -class ProductQuantizer(Quantizer): - r""" - Product Quantizer. - PQ is trained using k-means, minimizing the L2 distance to centroids. - PQ supports L2 and Inner Product search, however the quantization error is - biased towards L2 distance. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - M = property(_swigfaiss.ProductQuantizer_M_get, _swigfaiss.ProductQuantizer_M_set, doc=r"""number of subquantizers""") - nbits = property(_swigfaiss.ProductQuantizer_nbits_get, _swigfaiss.ProductQuantizer_nbits_set, doc=r"""number of bits per quantization index""") - dsub = property(_swigfaiss.ProductQuantizer_dsub_get, _swigfaiss.ProductQuantizer_dsub_set, doc=r"""dimensionality of each subvector""") - ksub = property(_swigfaiss.ProductQuantizer_ksub_get, _swigfaiss.ProductQuantizer_ksub_set, doc=r"""number of centroids for each subquantizer""") - verbose = property(_swigfaiss.ProductQuantizer_verbose_get, _swigfaiss.ProductQuantizer_verbose_set, doc=r"""verbose during training?""") - Train_default = _swigfaiss.ProductQuantizer_Train_default - Train_hot_start = _swigfaiss.ProductQuantizer_Train_hot_start - r"""the centroids are already initialized""" - Train_shared = _swigfaiss.ProductQuantizer_Train_shared - r"""share dictionary across PQ segments""" - Train_hypercube = _swigfaiss.ProductQuantizer_Train_hypercube - r"""initialize centroids with nbits-D hypercube""" - Train_hypercube_pca = _swigfaiss.ProductQuantizer_Train_hypercube_pca - r"""initialize centroids with nbits-D hypercube""" - train_type = property(_swigfaiss.ProductQuantizer_train_type_get, _swigfaiss.ProductQuantizer_train_type_set) - cp = property(_swigfaiss.ProductQuantizer_cp_get, _swigfaiss.ProductQuantizer_cp_set, doc=r"""parameters used during clustering""") - assign_index = property(_swigfaiss.ProductQuantizer_assign_index_get, _swigfaiss.ProductQuantizer_assign_index_set, doc=r""" - if non-NULL, use this index for assignment (should be of size - d / M) - """) - centroids = property(_swigfaiss.ProductQuantizer_centroids_get, _swigfaiss.ProductQuantizer_centroids_set, doc=r""" - Centroid table, size M * ksub * dsub. - Layout: (M, ksub, dsub) - """) - transposed_centroids = property(_swigfaiss.ProductQuantizer_transposed_centroids_get, _swigfaiss.ProductQuantizer_transposed_centroids_set, doc=r""" - Transposed centroid table, size M * ksub * dsub. - Layout: (dsub, M, ksub) - """) - centroids_sq_lengths = property(_swigfaiss.ProductQuantizer_centroids_sq_lengths_get, _swigfaiss.ProductQuantizer_centroids_sq_lengths_set, doc=r""" - Squared lengths of centroids, size M * ksub - Layout: (M, ksub) - """) - - def get_centroids(self, m, i): - r"""return the centroids associated with subvector m""" - return _swigfaiss.ProductQuantizer_get_centroids(self, m, i) - - def train(self, n, x): - return _swigfaiss.ProductQuantizer_train(self, n, x) - - def __init__(self, *args): - _swigfaiss.ProductQuantizer_swiginit(self, _swigfaiss.new_ProductQuantizer(*args)) - - def set_derived_values(self): - r"""compute derived values when d, M and nbits have been set""" - return _swigfaiss.ProductQuantizer_set_derived_values(self) - - def set_params(self, centroids, m): - r"""Define the centroids for subquantizer m""" - return _swigfaiss.ProductQuantizer_set_params(self, centroids, m) - - def compute_code(self, x, code): - r"""Quantize one vector with the product quantizer""" - return _swigfaiss.ProductQuantizer_compute_code(self, x, code) - - def compute_codes(self, x, codes, n): - r"""same as compute_code for several vectors""" - return _swigfaiss.ProductQuantizer_compute_codes(self, x, codes, n) - - def compute_codes_with_assign_index(self, x, codes, n): - r""" - speed up code assignment using assign_index - (non-const because the index is changed) - """ - return _swigfaiss.ProductQuantizer_compute_codes_with_assign_index(self, x, codes, n) - - def decode(self, *args): - r"""decode a vector from a given code (or n vectors if third argument)""" - return _swigfaiss.ProductQuantizer_decode(self, *args) - - def compute_code_from_distance_table(self, tab, code): - r""" - If we happen to have the distance tables precomputed, this is - more efficient to compute the codes. - """ - return _swigfaiss.ProductQuantizer_compute_code_from_distance_table(self, tab, code) - - def compute_distance_table(self, x, dis_table): - r""" - Compute distance table for one vector. - - The distance table for x = [x_0 x_1 .. x_(M-1)] is a M * ksub - matrix that contains - - dis_table (m, j) = || x_m - c_(m, j)||^2 - for m = 0..M-1 and j = 0 .. ksub - 1 - - where c_(m, j) is the centroid no j of sub-quantizer m. - - :type x: float - :param x: input vector size d - :type dis_table: float - :param dis_table: output table, size M * ksub - """ - return _swigfaiss.ProductQuantizer_compute_distance_table(self, x, dis_table) - - def compute_inner_prod_table(self, x, dis_table): - return _swigfaiss.ProductQuantizer_compute_inner_prod_table(self, x, dis_table) - - def compute_distance_tables(self, nx, x, dis_tables): - r""" - compute distance table for several vectors - :type nx: int - :param nx: nb of input vectors - :type x: float - :param x: input vector size nx * d - :param dis_table: output table, size nx * M * ksub - """ - return _swigfaiss.ProductQuantizer_compute_distance_tables(self, nx, x, dis_tables) - - def compute_inner_prod_tables(self, nx, x, dis_tables): - return _swigfaiss.ProductQuantizer_compute_inner_prod_tables(self, nx, x, dis_tables) - - def search(self, x, nx, codes, ncodes, res, init_finalize_heap=True): - r""" - perform a search (L2 distance) - :type x: float - :param x: query vectors, size nx * d - :type nx: int - :param nx: nb of queries - :type codes: uint8_t - :param codes: database codes, size ncodes * code_size - :type ncodes: int - :param ncodes: nb of nb vectors - :type res: :py:class:`float_maxheap_array_t` - :param res: heap array to store results (nh == nx) - :type init_finalize_heap: boolean, optional - :param init_finalize_heap: initialize heap (input) and sort (output)? - """ - return _swigfaiss.ProductQuantizer_search(self, x, nx, codes, ncodes, res, init_finalize_heap) - - def search_ip(self, x, nx, codes, ncodes, res, init_finalize_heap=True): - r"""same as search, but with inner product similarity""" - return _swigfaiss.ProductQuantizer_search_ip(self, x, nx, codes, ncodes, res, init_finalize_heap) - sdc_table = property(_swigfaiss.ProductQuantizer_sdc_table_get, _swigfaiss.ProductQuantizer_sdc_table_set, doc=r"""Symmetric Distance Table""") - - def compute_sdc_table(self): - return _swigfaiss.ProductQuantizer_compute_sdc_table(self) - - def search_sdc(self, qcodes, nq, bcodes, ncodes, res, init_finalize_heap=True): - return _swigfaiss.ProductQuantizer_search_sdc(self, qcodes, nq, bcodes, ncodes, res, init_finalize_heap) - - def sync_transposed_centroids(self): - r""" - Sync transposed centroids with regular centroids. This call - is needed if centroids were edited directly. - """ - return _swigfaiss.ProductQuantizer_sync_transposed_centroids(self) - - def clear_transposed_centroids(self): - r"""Clear transposed centroids table so ones are no longer used.""" - return _swigfaiss.ProductQuantizer_clear_transposed_centroids(self) - __swig_destroy__ = _swigfaiss.delete_ProductQuantizer - -# Register ProductQuantizer in _swigfaiss: -_swigfaiss.ProductQuantizer_swigregister(ProductQuantizer) -class PQEncoderGeneric(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - code = property(_swigfaiss.PQEncoderGeneric_code_get, _swigfaiss.PQEncoderGeneric_code_set, doc=r"""code for this vector""") - offset = property(_swigfaiss.PQEncoderGeneric_offset_get, _swigfaiss.PQEncoderGeneric_offset_set) - nbits = property(_swigfaiss.PQEncoderGeneric_nbits_get, doc=r"""number of bits per subquantizer index""") - reg = property(_swigfaiss.PQEncoderGeneric_reg_get, _swigfaiss.PQEncoderGeneric_reg_set) - - def __init__(self, code, nbits, offset=0): - _swigfaiss.PQEncoderGeneric_swiginit(self, _swigfaiss.new_PQEncoderGeneric(code, nbits, offset)) - - def encode(self, x): - return _swigfaiss.PQEncoderGeneric_encode(self, x) - __swig_destroy__ = _swigfaiss.delete_PQEncoderGeneric - -# Register PQEncoderGeneric in _swigfaiss: -_swigfaiss.PQEncoderGeneric_swigregister(PQEncoderGeneric) -class PQEncoder8(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - code = property(_swigfaiss.PQEncoder8_code_get, _swigfaiss.PQEncoder8_code_set) - - def __init__(self, code, nbits): - _swigfaiss.PQEncoder8_swiginit(self, _swigfaiss.new_PQEncoder8(code, nbits)) - - def encode(self, x): - return _swigfaiss.PQEncoder8_encode(self, x) - __swig_destroy__ = _swigfaiss.delete_PQEncoder8 - -# Register PQEncoder8 in _swigfaiss: -_swigfaiss.PQEncoder8_swigregister(PQEncoder8) -class PQEncoder16(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - code = property(_swigfaiss.PQEncoder16_code_get, _swigfaiss.PQEncoder16_code_set) - - def __init__(self, code, nbits): - _swigfaiss.PQEncoder16_swiginit(self, _swigfaiss.new_PQEncoder16(code, nbits)) - - def encode(self, x): - return _swigfaiss.PQEncoder16_encode(self, x) - __swig_destroy__ = _swigfaiss.delete_PQEncoder16 - -# Register PQEncoder16 in _swigfaiss: -_swigfaiss.PQEncoder16_swigregister(PQEncoder16) -class PQDecoderGeneric(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - code = property(_swigfaiss.PQDecoderGeneric_code_get, _swigfaiss.PQDecoderGeneric_code_set) - offset = property(_swigfaiss.PQDecoderGeneric_offset_get, _swigfaiss.PQDecoderGeneric_offset_set) - nbits = property(_swigfaiss.PQDecoderGeneric_nbits_get) - mask = property(_swigfaiss.PQDecoderGeneric_mask_get) - reg = property(_swigfaiss.PQDecoderGeneric_reg_get, _swigfaiss.PQDecoderGeneric_reg_set) - - def __init__(self, code, nbits): - _swigfaiss.PQDecoderGeneric_swiginit(self, _swigfaiss.new_PQDecoderGeneric(code, nbits)) - - def decode(self): - return _swigfaiss.PQDecoderGeneric_decode(self) - __swig_destroy__ = _swigfaiss.delete_PQDecoderGeneric - -# Register PQDecoderGeneric in _swigfaiss: -_swigfaiss.PQDecoderGeneric_swigregister(PQDecoderGeneric) -class PQDecoder8(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nbits = _swigfaiss.PQDecoder8_nbits - code = property(_swigfaiss.PQDecoder8_code_get, _swigfaiss.PQDecoder8_code_set) - - def __init__(self, code, nbits): - _swigfaiss.PQDecoder8_swiginit(self, _swigfaiss.new_PQDecoder8(code, nbits)) - - def decode(self): - return _swigfaiss.PQDecoder8_decode(self) - __swig_destroy__ = _swigfaiss.delete_PQDecoder8 - -# Register PQDecoder8 in _swigfaiss: -_swigfaiss.PQDecoder8_swigregister(PQDecoder8) -class PQDecoder16(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nbits = _swigfaiss.PQDecoder16_nbits - code = property(_swigfaiss.PQDecoder16_code_get, _swigfaiss.PQDecoder16_code_set) - - def __init__(self, code, nbits): - _swigfaiss.PQDecoder16_swiginit(self, _swigfaiss.new_PQDecoder16(code, nbits)) - - def decode(self): - return _swigfaiss.PQDecoder16_decode(self) - __swig_destroy__ = _swigfaiss.delete_PQDecoder16 - -# Register PQDecoder16 in _swigfaiss: -_swigfaiss.PQDecoder16_swigregister(PQDecoder16) -class AdditiveQuantizer(Quantizer): - r""" - Abstract structure for additive quantizers - - Different from the product quantizer in which the decoded vector is the - concatenation of M sub-vectors, additive quantizers sum M sub-vectors - to get the decoded vector. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - M = property(_swigfaiss.AdditiveQuantizer_M_get, _swigfaiss.AdditiveQuantizer_M_set, doc=r"""number of codebooks""") - nbits = property(_swigfaiss.AdditiveQuantizer_nbits_get, _swigfaiss.AdditiveQuantizer_nbits_set, doc=r"""bits for each step""") - codebooks = property(_swigfaiss.AdditiveQuantizer_codebooks_get, _swigfaiss.AdditiveQuantizer_codebooks_set, doc=r"""codebooks""") - codebook_offsets = property(_swigfaiss.AdditiveQuantizer_codebook_offsets_get, _swigfaiss.AdditiveQuantizer_codebook_offsets_set, doc=r""" - codebook #1 is stored in rows codebook_offsets[i]:codebook_offsets[i+1] - in the codebooks table of size total_codebook_size by d - """) - tot_bits = property(_swigfaiss.AdditiveQuantizer_tot_bits_get, _swigfaiss.AdditiveQuantizer_tot_bits_set, doc=r"""total number of bits (indexes + norms)""") - norm_bits = property(_swigfaiss.AdditiveQuantizer_norm_bits_get, _swigfaiss.AdditiveQuantizer_norm_bits_set, doc=r"""bits allocated for the norms""") - total_codebook_size = property(_swigfaiss.AdditiveQuantizer_total_codebook_size_get, _swigfaiss.AdditiveQuantizer_total_codebook_size_set, doc=r"""size of the codebook in vectors""") - only_8bit = property(_swigfaiss.AdditiveQuantizer_only_8bit_get, _swigfaiss.AdditiveQuantizer_only_8bit_set, doc=r"""are all nbits = 8 (use faster decoder)""") - verbose = property(_swigfaiss.AdditiveQuantizer_verbose_get, _swigfaiss.AdditiveQuantizer_verbose_set, doc=r"""verbose during training?""") - is_trained = property(_swigfaiss.AdditiveQuantizer_is_trained_get, _swigfaiss.AdditiveQuantizer_is_trained_set, doc=r"""is trained or not""") - norm_tabs = property(_swigfaiss.AdditiveQuantizer_norm_tabs_get, _swigfaiss.AdditiveQuantizer_norm_tabs_set, doc=r""" - auxiliary data for ST_norm_lsq2x4 and ST_norm_rq2x4 - store norms of codebook entries for 4-bit fastscan - """) - qnorm = property(_swigfaiss.AdditiveQuantizer_qnorm_get, _swigfaiss.AdditiveQuantizer_qnorm_set, doc=r"""store and search norms""") - - def compute_codebook_tables(self): - return _swigfaiss.AdditiveQuantizer_compute_codebook_tables(self) - centroid_norms = property(_swigfaiss.AdditiveQuantizer_centroid_norms_get, _swigfaiss.AdditiveQuantizer_centroid_norms_set, doc=r"""norms of all codebook entries (size total_codebook_size)""") - codebook_cross_products = property(_swigfaiss.AdditiveQuantizer_codebook_cross_products_get, _swigfaiss.AdditiveQuantizer_codebook_cross_products_set, doc=r""" - dot products of all codebook entries with the previous codebooks - size sum(codebook_offsets[m] * 2^nbits[m], m=0..M-1) - """) - max_mem_distances = property(_swigfaiss.AdditiveQuantizer_max_mem_distances_get, _swigfaiss.AdditiveQuantizer_max_mem_distances_set, doc=r""" - norms and distance matrixes with beam search can get large, so use this - to control for the amount of memory that can be allocated - """) - - def encode_norm(self, norm): - r"""encode a norm into norm_bits bits""" - return _swigfaiss.AdditiveQuantizer_encode_norm(self, norm) - - def encode_qcint(self, x): - r"""encode norm by non-uniform scalar quantization""" - return _swigfaiss.AdditiveQuantizer_encode_qcint(self, x) - - def decode_qcint(self, c): - r"""decode norm by non-uniform scalar quantization""" - return _swigfaiss.AdditiveQuantizer_decode_qcint(self, c) - ST_decompress = _swigfaiss.AdditiveQuantizer_ST_decompress - r"""decompress database vector""" - ST_LUT_nonorm = _swigfaiss.AdditiveQuantizer_ST_LUT_nonorm - r""" - use a LUT, don't include norms (OK for IP or - normalized vectors) - """ - ST_norm_from_LUT = _swigfaiss.AdditiveQuantizer_ST_norm_from_LUT - r""" - compute the norms from the look-up tables (cost - is in O(M^2)) - """ - ST_norm_float = _swigfaiss.AdditiveQuantizer_ST_norm_float - r"""use a LUT, and store float32 norm with the vectors""" - ST_norm_qint8 = _swigfaiss.AdditiveQuantizer_ST_norm_qint8 - r"""use a LUT, and store 8bit-quantized norm""" - ST_norm_qint4 = _swigfaiss.AdditiveQuantizer_ST_norm_qint4 - ST_norm_cqint8 = _swigfaiss.AdditiveQuantizer_ST_norm_cqint8 - r"""use a LUT, and store non-uniform quantized norm""" - ST_norm_cqint4 = _swigfaiss.AdditiveQuantizer_ST_norm_cqint4 - ST_norm_lsq2x4 = _swigfaiss.AdditiveQuantizer_ST_norm_lsq2x4 - r""" - use a 2x4 bits lsq as norm quantizer (for fast - scan) - """ - ST_norm_rq2x4 = _swigfaiss.AdditiveQuantizer_ST_norm_rq2x4 - r"""use a 2x4 bits rq as norm quantizer (for fast scan)""" - ST_count = _swigfaiss.AdditiveQuantizer_ST_count - - def set_derived_values(self): - r"""Train the norm quantizer""" - return _swigfaiss.AdditiveQuantizer_set_derived_values(self) - - def train_norm(self, n, norms): - return _swigfaiss.AdditiveQuantizer_train_norm(self, n, norms) - - def compute_codes(self, x, codes, n): - return _swigfaiss.AdditiveQuantizer_compute_codes(self, x, codes, n) - - def compute_codes_add_centroids(self, x, codes, n, centroids=None): - r""" - Encode a set of vectors - - :type x: float - :param x: vectors to encode, size n * d - :type codes: uint8_t - :param codes: output codes, size n * code_size - :type centroids: float, optional - :param centroids: centroids to be added to x, size n * d - """ - return _swigfaiss.AdditiveQuantizer_compute_codes_add_centroids(self, x, codes, n, centroids) - - def pack_codes(self, n, codes, packed_codes, ld_codes=-1, norms=None, centroids=None): - r""" - pack a series of code to bit-compact format - - :type codes: int - :param codes: codes to be packed, size n * code_size - :type packed_codes: uint8_t - :param packed_codes: output bit-compact codes - :type ld_codes: int, optional - :param ld_codes: leading dimension of codes - :type norms: float, optional - :param norms: norms of the vectors (size n). Will be computed if - needed but not provided - :type centroids: float, optional - :param centroids: centroids to be added to x, size n * d - """ - return _swigfaiss.AdditiveQuantizer_pack_codes(self, n, codes, packed_codes, ld_codes, norms, centroids) - - def decode(self, codes, x, n): - r""" - Decode a set of vectors - - :type codes: uint8_t - :param codes: codes to decode, size n * code_size - :type x: float - :param x: output vectors, size n * d - """ - return _swigfaiss.AdditiveQuantizer_decode(self, codes, x, n) - - def decode_unpacked(self, codes, x, n, ld_codes=-1): - r""" - Decode a set of vectors in non-packed format - - :type codes: int - :param codes: codes to decode, size n * ld_codes - :type x: float - :param x: output vectors, size n * d - """ - return _swigfaiss.AdditiveQuantizer_decode_unpacked(self, codes, x, n, ld_codes) - search_type = property(_swigfaiss.AdditiveQuantizer_search_type_get, _swigfaiss.AdditiveQuantizer_search_type_set, doc=r"""Also determines what's in the codes""") - norm_min = property(_swigfaiss.AdditiveQuantizer_norm_min_get, _swigfaiss.AdditiveQuantizer_norm_min_set, doc=r"""min/max for quantization of norms""") - norm_max = property(_swigfaiss.AdditiveQuantizer_norm_max_get, _swigfaiss.AdditiveQuantizer_norm_max_set) - - def decode_64bit(self, n, x): - r"""decoding function for a code in a 64-bit word""" - return _swigfaiss.AdditiveQuantizer_decode_64bit(self, n, x) - - def compute_LUT(self, n, xq, LUT, alpha=1.0, ld_lut=-1): - r""" - Compute inner-product look-up tables. Used in the centroid search - functions. - - :type xq: float - :param xq: query vector, size (n, d) - :type LUT: float - :param LUT: look-up table, size (n, total_codebook_size) - :type alpha: float, optional - :param alpha: compute alpha * inner-product - :type ld_lut: int, optional - :param ld_lut: leading dimension of LUT - """ - return _swigfaiss.AdditiveQuantizer_compute_LUT(self, n, xq, LUT, alpha, ld_lut) - - def knn_centroids_inner_product(self, n, xq, k, distances, labels): - r"""exact IP search""" - return _swigfaiss.AdditiveQuantizer_knn_centroids_inner_product(self, n, xq, k, distances, labels) - - def compute_centroid_norms(self, norms): - r""" - For L2 search we need the L2 norms of the centroids - - :type norms: float - :param norms: output norms table, size total_codebook_size - """ - return _swigfaiss.AdditiveQuantizer_compute_centroid_norms(self, norms) - - def knn_centroids_L2(self, n, xq, k, distances, labels, centroid_norms): - r"""Exact L2 search, with precomputed norms""" - return _swigfaiss.AdditiveQuantizer_knn_centroids_L2(self, n, xq, k, distances, labels, centroid_norms) - __swig_destroy__ = _swigfaiss.delete_AdditiveQuantizer - -# Register AdditiveQuantizer in _swigfaiss: -_swigfaiss.AdditiveQuantizer_swigregister(AdditiveQuantizer) - -def beam_search_encode_step(*args): - r""" - Encode a residual by sampling from a centroid table. - - This is a single encoding step the residual quantizer. - It allows low-level access to the encoding function, exposed mainly for unit - tests. - - :type n: int - :param n: number of vectors to handle - :type residuals: float - :param residuals: vectors to encode, size (n, beam_size, d) - :type cent: float - :param cent: centroids, size (K, d) - :type beam_size: int - :param beam_size: input beam size - :type m: int - :param m: size of the codes for the previous encoding steps - :type codes: int - :param codes: code array for the previous steps of the beam (n, - beam_size, m) - :type new_beam_size: int - :param new_beam_size: output beam size (should be <= K * beam_size) - :type new_codes: int - :param new_codes: output codes, size (n, new_beam_size, m + 1) - :type new_residuals: float - :param new_residuals: output residuals, size (n, new_beam_size, d) - :type new_distances: float - :param new_distances: output distances, size (n, new_beam_size) - :type assign_index: :py:class:`Index`, optional - :param assign_index: if non-NULL, will be used to perform assignment - """ - return _swigfaiss.beam_search_encode_step(*args) - -def beam_search_encode_step_tab(*args): - r""" - Encode a set of vectors using their dot products with the codebooks - - :type K: int - :param K: number of vectors in the codebook - :type n: int - :param n: nb of vectors to encode - :type beam_size: int - :param beam_size: input beam size - :type codebook_cross_norms: float - :param codebook_cross_norms: inner product of this codebook with the m - previously encoded codebooks - :type codebook_offsets: int - :param codebook_offsets: offsets into codebook_cross_norms for each - previous codebook - :type query_cp: float - :param query_cp: dot products of query vectors with ??? - :type cent_norms_i: float - :param cent_norms_i: norms of centroids - """ - return _swigfaiss.beam_search_encode_step_tab(*args) -class RefineBeamMemoryPool(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - new_codes = property(_swigfaiss.RefineBeamMemoryPool_new_codes_get, _swigfaiss.RefineBeamMemoryPool_new_codes_set) - new_residuals = property(_swigfaiss.RefineBeamMemoryPool_new_residuals_get, _swigfaiss.RefineBeamMemoryPool_new_residuals_set) - residuals = property(_swigfaiss.RefineBeamMemoryPool_residuals_get, _swigfaiss.RefineBeamMemoryPool_residuals_set) - codes = property(_swigfaiss.RefineBeamMemoryPool_codes_get, _swigfaiss.RefineBeamMemoryPool_codes_set) - distances = property(_swigfaiss.RefineBeamMemoryPool_distances_get, _swigfaiss.RefineBeamMemoryPool_distances_set) - - def __init__(self): - _swigfaiss.RefineBeamMemoryPool_swiginit(self, _swigfaiss.new_RefineBeamMemoryPool()) - __swig_destroy__ = _swigfaiss.delete_RefineBeamMemoryPool - -# Register RefineBeamMemoryPool in _swigfaiss: -_swigfaiss.RefineBeamMemoryPool_swigregister(RefineBeamMemoryPool) - -def refine_beam_mp(rq, n, beam_size, x, out_beam_size, out_codes, out_residuals, out_distances, pool): - return _swigfaiss.refine_beam_mp(rq, n, beam_size, x, out_beam_size, out_codes, out_residuals, out_distances, pool) -class RefineBeamLUTMemoryPool(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - new_codes = property(_swigfaiss.RefineBeamLUTMemoryPool_new_codes_get, _swigfaiss.RefineBeamLUTMemoryPool_new_codes_set) - new_distances = property(_swigfaiss.RefineBeamLUTMemoryPool_new_distances_get, _swigfaiss.RefineBeamLUTMemoryPool_new_distances_set) - codes = property(_swigfaiss.RefineBeamLUTMemoryPool_codes_get, _swigfaiss.RefineBeamLUTMemoryPool_codes_set) - distances = property(_swigfaiss.RefineBeamLUTMemoryPool_distances_get, _swigfaiss.RefineBeamLUTMemoryPool_distances_set) - - def __init__(self): - _swigfaiss.RefineBeamLUTMemoryPool_swiginit(self, _swigfaiss.new_RefineBeamLUTMemoryPool()) - __swig_destroy__ = _swigfaiss.delete_RefineBeamLUTMemoryPool - -# Register RefineBeamLUTMemoryPool in _swigfaiss: -_swigfaiss.RefineBeamLUTMemoryPool_swigregister(RefineBeamLUTMemoryPool) - -def refine_beam_LUT_mp(rq, n, query_norms, query_cp, out_beam_size, out_codes, out_distances, pool): - return _swigfaiss.refine_beam_LUT_mp(rq, n, query_norms, query_cp, out_beam_size, out_codes, out_distances, pool) -class ComputeCodesAddCentroidsLUT0MemoryPool(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - codes = property(_swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_codes_get, _swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_codes_set) - norms = property(_swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_norms_get, _swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_norms_set) - distances = property(_swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_distances_get, _swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_distances_set) - residuals = property(_swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_residuals_get, _swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_residuals_set) - refine_beam_pool = property(_swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_refine_beam_pool_get, _swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_refine_beam_pool_set) - - def __init__(self): - _swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_swiginit(self, _swigfaiss.new_ComputeCodesAddCentroidsLUT0MemoryPool()) - __swig_destroy__ = _swigfaiss.delete_ComputeCodesAddCentroidsLUT0MemoryPool - -# Register ComputeCodesAddCentroidsLUT0MemoryPool in _swigfaiss: -_swigfaiss.ComputeCodesAddCentroidsLUT0MemoryPool_swigregister(ComputeCodesAddCentroidsLUT0MemoryPool) - -def compute_codes_add_centroids_mp_lut0(rq, x, codes_out, n, centroids, pool): - return _swigfaiss.compute_codes_add_centroids_mp_lut0(rq, x, codes_out, n, centroids, pool) -class ComputeCodesAddCentroidsLUT1MemoryPool(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - codes = property(_swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_codes_get, _swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_codes_set) - distances = property(_swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_distances_get, _swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_distances_set) - query_norms = property(_swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_query_norms_get, _swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_query_norms_set) - query_cp = property(_swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_query_cp_get, _swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_query_cp_set) - residuals = property(_swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_residuals_get, _swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_residuals_set) - refine_beam_lut_pool = property(_swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_refine_beam_lut_pool_get, _swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_refine_beam_lut_pool_set) - - def __init__(self): - _swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_swiginit(self, _swigfaiss.new_ComputeCodesAddCentroidsLUT1MemoryPool()) - __swig_destroy__ = _swigfaiss.delete_ComputeCodesAddCentroidsLUT1MemoryPool - -# Register ComputeCodesAddCentroidsLUT1MemoryPool in _swigfaiss: -_swigfaiss.ComputeCodesAddCentroidsLUT1MemoryPool_swigregister(ComputeCodesAddCentroidsLUT1MemoryPool) - -def compute_codes_add_centroids_mp_lut1(rq, x, codes_out, n, centroids, pool): - return _swigfaiss.compute_codes_add_centroids_mp_lut1(rq, x, codes_out, n, centroids, pool) -class ResidualQuantizer(AdditiveQuantizer): - r""" - Residual quantizer with variable number of bits per sub-quantizer - - The residual centroids are stored in a big cumulative centroid table. - The codes are represented either as a non-compact table of size (n, M) or - as the compact output (n, code_size). - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - train_type = property(_swigfaiss.ResidualQuantizer_train_type_get, _swigfaiss.ResidualQuantizer_train_type_set, doc=r"""Binary or of the Train_* flags below""") - Train_default = _swigfaiss.ResidualQuantizer_Train_default - r"""regular k-means (minimal amount of computation)""" - Train_progressive_dim = _swigfaiss.ResidualQuantizer_Train_progressive_dim - r"""progressive dim clustering (set by default)""" - Train_refine_codebook = _swigfaiss.ResidualQuantizer_Train_refine_codebook - r"""do a few iterations of codebook refinement after first level estimation""" - niter_codebook_refine = property(_swigfaiss.ResidualQuantizer_niter_codebook_refine_get, _swigfaiss.ResidualQuantizer_niter_codebook_refine_set, doc=r"""number of iterations for codebook refinement.""") - Train_top_beam = _swigfaiss.ResidualQuantizer_Train_top_beam - r""" - set this bit on train_type if beam is to be trained only on the - first element of the beam (faster but less accurate) - """ - Skip_codebook_tables = _swigfaiss.ResidualQuantizer_Skip_codebook_tables - r""" - set this bit to *not* automatically compute the codebook tables - after training - """ - max_beam_size = property(_swigfaiss.ResidualQuantizer_max_beam_size_get, _swigfaiss.ResidualQuantizer_max_beam_size_set, doc=r"""beam size used for training and for encoding""") - use_beam_LUT = property(_swigfaiss.ResidualQuantizer_use_beam_LUT_get, _swigfaiss.ResidualQuantizer_use_beam_LUT_set, doc=r"""use LUT for beam search""") - approx_topk_mode = property(_swigfaiss.ResidualQuantizer_approx_topk_mode_get, _swigfaiss.ResidualQuantizer_approx_topk_mode_set, doc=r""" - Currently used mode of approximate min-k computations. - Default value is EXACT_TOPK. - """) - cp = property(_swigfaiss.ResidualQuantizer_cp_get, _swigfaiss.ResidualQuantizer_cp_set, doc=r"""clustering parameters""") - assign_index_factory = property(_swigfaiss.ResidualQuantizer_assign_index_factory_get, _swigfaiss.ResidualQuantizer_assign_index_factory_set, doc=r"""if non-NULL, use this index for assignment""") - - def __init__(self, *args): - _swigfaiss.ResidualQuantizer_swiginit(self, _swigfaiss.new_ResidualQuantizer(*args)) - - def train(self, n, x): - r"""Train the residual quantizer""" - return _swigfaiss.ResidualQuantizer_train(self, n, x) - - def initialize_from(self, other, skip_M=0): - r"""Copy the M codebook levels from other, starting from skip_M""" - return _swigfaiss.ResidualQuantizer_initialize_from(self, other, skip_M) - - def retrain_AQ_codebook(self, n, x): - r""" - Encode the vectors and compute codebook that minimizes the quantization - error on these codes - - :type x: float - :param x: training vectors, size n * d - :type n: int - :param n: nb of training vectors, n >= total_codebook_size - :rtype: float - :return: returns quantization error for the new codebook with old - codes - """ - return _swigfaiss.ResidualQuantizer_retrain_AQ_codebook(self, n, x) - - def compute_codes_add_centroids(self, x, codes, n, centroids=None): - r""" - Encode a set of vectors - - :type x: float - :param x: vectors to encode, size n * d - :type codes: uint8_t - :param codes: output codes, size n * code_size - :type centroids: float, optional - :param centroids: centroids to be added to x, size n * d - """ - return _swigfaiss.ResidualQuantizer_compute_codes_add_centroids(self, x, codes, n, centroids) - - def refine_beam(self, n, beam_size, residuals, new_beam_size, new_codes, new_residuals=None, new_distances=None): - r""" - lower-level encode function - - :type n: int - :param n: number of vectors to handle - :type residuals: float - :param residuals: vectors to encode, size (n, beam_size, d) - :type beam_size: int - :param beam_size: input beam size - :type new_beam_size: int - :param new_beam_size: output beam size (should be <= K * beam_size) - :type new_codes: int - :param new_codes: output codes, size (n, new_beam_size, m + 1) - :type new_residuals: float, optional - :param new_residuals: output residuals, size (n, new_beam_size, d) - :type new_distances: float, optional - :param new_distances: output distances, size (n, new_beam_size) - """ - return _swigfaiss.ResidualQuantizer_refine_beam(self, n, beam_size, residuals, new_beam_size, new_codes, new_residuals, new_distances) - - def refine_beam_LUT(self, n, query_norms, query_cp, new_beam_size, new_codes, new_distances=None): - return _swigfaiss.ResidualQuantizer_refine_beam_LUT(self, n, query_norms, query_cp, new_beam_size, new_codes, new_distances) - - def memory_per_point(self, beam_size=-1): - r""" - Beam search can consume a lot of memory. This function estimates the - amount of mem used by refine_beam to adjust the batch size - - :type beam_size: int, optional - :param beam_size: if != -1, override the beam size - """ - return _swigfaiss.ResidualQuantizer_memory_per_point(self, beam_size) - __swig_destroy__ = _swigfaiss.delete_ResidualQuantizer - -# Register ResidualQuantizer in _swigfaiss: -_swigfaiss.ResidualQuantizer_swigregister(ResidualQuantizer) -class LocalSearchQuantizer(AdditiveQuantizer): - r""" - Implementation of LSQ/LSQ++ described in the following two papers: - - Revisiting additive quantization - Julieta Martinez, et al. ECCV 2016 - - LSQ++: Lower running time and higher recall in multi-codebook quantization - Julieta Martinez, et al. ECCV 2018 - - This implementation is mostly translated from the Julia implementations - by Julieta Martinez: - (https://github.com/una-dinosauria/local-search-quantization, - https://github.com/una-dinosauria/Rayuela.jl) - - The trained codes are stored in `codebooks` which is called - `centroids` in PQ and RQ. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - K = property(_swigfaiss.LocalSearchQuantizer_K_get, _swigfaiss.LocalSearchQuantizer_K_set, doc=r"""number of codes per codebook""") - train_iters = property(_swigfaiss.LocalSearchQuantizer_train_iters_get, _swigfaiss.LocalSearchQuantizer_train_iters_set, doc=r"""number of iterations in training""") - encode_ils_iters = property(_swigfaiss.LocalSearchQuantizer_encode_ils_iters_get, _swigfaiss.LocalSearchQuantizer_encode_ils_iters_set, doc=r"""iterations of local search in encoding""") - train_ils_iters = property(_swigfaiss.LocalSearchQuantizer_train_ils_iters_get, _swigfaiss.LocalSearchQuantizer_train_ils_iters_set, doc=r"""iterations of local search in training""") - icm_iters = property(_swigfaiss.LocalSearchQuantizer_icm_iters_get, _swigfaiss.LocalSearchQuantizer_icm_iters_set, doc=r"""number of iterations in icm""") - p = property(_swigfaiss.LocalSearchQuantizer_p_get, _swigfaiss.LocalSearchQuantizer_p_set, doc=r"""temperature factor""") - lambd = property(_swigfaiss.LocalSearchQuantizer_lambd_get, _swigfaiss.LocalSearchQuantizer_lambd_set, doc=r"""regularization factor""") - chunk_size = property(_swigfaiss.LocalSearchQuantizer_chunk_size_get, _swigfaiss.LocalSearchQuantizer_chunk_size_set, doc=r"""nb of vectors to encode at a time""") - random_seed = property(_swigfaiss.LocalSearchQuantizer_random_seed_get, _swigfaiss.LocalSearchQuantizer_random_seed_set, doc=r"""seed for random generator""") - nperts = property(_swigfaiss.LocalSearchQuantizer_nperts_get, _swigfaiss.LocalSearchQuantizer_nperts_set, doc=r""" - number of perturbation in each code - if non-NULL, use this encoder to encode (owned by the object) - """) - icm_encoder_factory = property(_swigfaiss.LocalSearchQuantizer_icm_encoder_factory_get, _swigfaiss.LocalSearchQuantizer_icm_encoder_factory_set) - update_codebooks_with_double = property(_swigfaiss.LocalSearchQuantizer_update_codebooks_with_double_get, _swigfaiss.LocalSearchQuantizer_update_codebooks_with_double_set) - - def __init__(self, *args): - _swigfaiss.LocalSearchQuantizer_swiginit(self, _swigfaiss.new_LocalSearchQuantizer(*args)) - __swig_destroy__ = _swigfaiss.delete_LocalSearchQuantizer - - def train(self, n, x): - return _swigfaiss.LocalSearchQuantizer_train(self, n, x) - - def compute_codes_add_centroids(self, x, codes, n, centroids=None): - r""" - Encode a set of vectors - - :type x: float - :param x: vectors to encode, size n * d - :type codes: uint8_t - :param codes: output codes, size n * code_size - :type n: int - :param n: number of vectors - :type centroids: float, optional - :param centroids: centroids to be added to x, size n * d - """ - return _swigfaiss.LocalSearchQuantizer_compute_codes_add_centroids(self, x, codes, n, centroids) - - def update_codebooks(self, x, codes, n): - r""" - Update codebooks given encodings - - :type x: float - :param x: training vectors, size n * d - :type codes: int - :param codes: encoded training vectors, size n * M - :type n: int - :param n: number of vectors - """ - return _swigfaiss.LocalSearchQuantizer_update_codebooks(self, x, codes, n) - - def icm_encode(self, codes, x, n, ils_iters, gen): - r""" - Encode vectors given codebooks using iterative conditional mode (icm). - - :type codes: int - :param codes: output codes, size n * M - :type x: float - :param x: vectors to encode, size n * d - :type n: int - :param n: number of vectors - :type ils_iters: int - :param ils_iters: number of iterations of iterative local search - """ - return _swigfaiss.LocalSearchQuantizer_icm_encode(self, codes, x, n, ils_iters, gen) - - def icm_encode_impl(self, codes, x, unaries, gen, n, ils_iters, verbose): - return _swigfaiss.LocalSearchQuantizer_icm_encode_impl(self, codes, x, unaries, gen, n, ils_iters, verbose) - - def icm_encode_step(self, codes, unaries, binaries, n, n_iters): - return _swigfaiss.LocalSearchQuantizer_icm_encode_step(self, codes, unaries, binaries, n, n_iters) - - def perturb_codes(self, codes, n, gen): - r""" - Add some perturbation to codes - - :type codes: int - :param codes: codes to be perturbed, size n * M - :type n: int - :param n: number of vectors - """ - return _swigfaiss.LocalSearchQuantizer_perturb_codes(self, codes, n, gen) - - def perturb_codebooks(self, T, stddev, gen): - r""" - Add some perturbation to codebooks - - :type T: float - :param T: temperature of simulated annealing - :type stddev: std::vector< float > - :param stddev: standard deviations of each dimension in training data - """ - return _swigfaiss.LocalSearchQuantizer_perturb_codebooks(self, T, stddev, gen) - - def compute_binary_terms(self, binaries): - r""" - Compute binary terms - - :type binaries: float - :param binaries: binary terms, size M * M * K * K - """ - return _swigfaiss.LocalSearchQuantizer_compute_binary_terms(self, binaries) - - def compute_unary_terms(self, x, unaries, n): - r""" - Compute unary terms - - :type n: int - :param n: number of vectors - :type x: float - :param x: vectors to encode, size n * d - :type unaries: float - :param unaries: unary terms, size n * M * K - """ - return _swigfaiss.LocalSearchQuantizer_compute_unary_terms(self, x, unaries, n) - - def evaluate(self, codes, x, n, objs=None): - r""" - Helper function to compute reconstruction error - - :type codes: int - :param codes: encoded codes, size n * M - :type x: float - :param x: vectors to encode, size n * d - :type n: int - :param n: number of vectors - :type objs: float, optional - :param objs: if it is not null, store reconstruction - error of each vector into it, size n - """ - return _swigfaiss.LocalSearchQuantizer_evaluate(self, codes, x, n, objs) - -# Register LocalSearchQuantizer in _swigfaiss: -_swigfaiss.LocalSearchQuantizer_swigregister(LocalSearchQuantizer) -class IcmEncoder(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - binaries = property(_swigfaiss.IcmEncoder_binaries_get, _swigfaiss.IcmEncoder_binaries_set) - verbose = property(_swigfaiss.IcmEncoder_verbose_get, _swigfaiss.IcmEncoder_verbose_set) - lsq = property(_swigfaiss.IcmEncoder_lsq_get, _swigfaiss.IcmEncoder_lsq_set) - - def __init__(self, lsq): - _swigfaiss.IcmEncoder_swiginit(self, _swigfaiss.new_IcmEncoder(lsq)) - __swig_destroy__ = _swigfaiss.delete_IcmEncoder - - def set_binary_term(self): - return _swigfaiss.IcmEncoder_set_binary_term(self) - - def encode(self, codes, x, gen, n, ils_iters): - r""" - Encode vectors given codebooks - - :type codes: int - :param codes: output codes, size n * M - :type x: float - :param x: vectors to encode, size n * d - :type gen: std::mt19937 - :param gen: random generator - :type n: int - :param n: number of vectors - :type ils_iters: int - :param ils_iters: number of iterations of iterative local search - """ - return _swigfaiss.IcmEncoder_encode(self, codes, x, gen, n, ils_iters) - -# Register IcmEncoder in _swigfaiss: -_swigfaiss.IcmEncoder_swigregister(IcmEncoder) -class IcmEncoderFactory(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def get(self, lsq): - return _swigfaiss.IcmEncoderFactory_get(self, lsq) - __swig_destroy__ = _swigfaiss.delete_IcmEncoderFactory - - def __init__(self): - _swigfaiss.IcmEncoderFactory_swiginit(self, _swigfaiss.new_IcmEncoderFactory()) - -# Register IcmEncoderFactory in _swigfaiss: -_swigfaiss.IcmEncoderFactory_swigregister(IcmEncoderFactory) -class LSQTimer(object): - r""" - A helper struct to count consuming time during training. - It is NOT thread-safe. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - t = property(_swigfaiss.LSQTimer_t_get, _swigfaiss.LSQTimer_t_set) - - def __init__(self): - _swigfaiss.LSQTimer_swiginit(self, _swigfaiss.new_LSQTimer()) - - def get(self, name): - return _swigfaiss.LSQTimer_get(self, name) - - def add(self, name, delta): - return _swigfaiss.LSQTimer_add(self, name, delta) - - def reset(self): - return _swigfaiss.LSQTimer_reset(self) - __swig_destroy__ = _swigfaiss.delete_LSQTimer - -# Register LSQTimer in _swigfaiss: -_swigfaiss.LSQTimer_swigregister(LSQTimer) -class LSQTimerScope(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - t0 = property(_swigfaiss.LSQTimerScope_t0_get, _swigfaiss.LSQTimerScope_t0_set) - timer = property(_swigfaiss.LSQTimerScope_timer_get, _swigfaiss.LSQTimerScope_timer_set) - name = property(_swigfaiss.LSQTimerScope_name_get, _swigfaiss.LSQTimerScope_name_set) - finished = property(_swigfaiss.LSQTimerScope_finished_get, _swigfaiss.LSQTimerScope_finished_set) - - def __init__(self, timer, name): - _swigfaiss.LSQTimerScope_swiginit(self, _swigfaiss.new_LSQTimerScope(timer, name)) - - def finish(self): - return _swigfaiss.LSQTimerScope_finish(self) - __swig_destroy__ = _swigfaiss.delete_LSQTimerScope - -# Register LSQTimerScope in _swigfaiss: -_swigfaiss.LSQTimerScope_swigregister(LSQTimerScope) -class ProductAdditiveQuantizer(AdditiveQuantizer): - r""" - Product Additive Quantizers - - The product additive quantizer is a variant of AQ and PQ. - It first splits the vector space into multiple orthogonal sub-spaces - just like PQ does. And then it quantizes each sub-space by an independent - additive quantizer. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nsplits = property(_swigfaiss.ProductAdditiveQuantizer_nsplits_get, _swigfaiss.ProductAdditiveQuantizer_nsplits_set, doc=r"""number of sub-vectors we split a vector into""") - quantizers = property(_swigfaiss.ProductAdditiveQuantizer_quantizers_get, _swigfaiss.ProductAdditiveQuantizer_quantizers_set) - - def __init__(self, *args): - r""" - Construct a product additive quantizer. - - The additive quantizers passed in will be cloned into the - ProductAdditiveQuantizer object. - - :type d: int - :param d: dimensionality of the input vectors - :type aqs: std::vector< faiss::AdditiveQuantizer * > - :param aqs: sub-additive quantizers - :type search_type: int, optional - :param search_type: AQ search type - """ - _swigfaiss.ProductAdditiveQuantizer_swiginit(self, _swigfaiss.new_ProductAdditiveQuantizer(*args)) - __swig_destroy__ = _swigfaiss.delete_ProductAdditiveQuantizer - - def init(self, d, aqs, search_type): - return _swigfaiss.ProductAdditiveQuantizer_init(self, d, aqs, search_type) - - def subquantizer(self, m): - r"""Train the product additive quantizer""" - return _swigfaiss.ProductAdditiveQuantizer_subquantizer(self, m) - - def train(self, n, x): - return _swigfaiss.ProductAdditiveQuantizer_train(self, n, x) - - def compute_codes_add_centroids(self, x, codes, n, centroids=None): - r""" - Encode a set of vectors - - :type x: float - :param x: vectors to encode, size n * d - :type codes: uint8_t - :param codes: output codes, size n * code_size - :type centroids: float, optional - :param centroids: centroids to be added to x, size n * d - """ - return _swigfaiss.ProductAdditiveQuantizer_compute_codes_add_centroids(self, x, codes, n, centroids) - - def compute_unpacked_codes(self, x, codes, n, centroids=None): - return _swigfaiss.ProductAdditiveQuantizer_compute_unpacked_codes(self, x, codes, n, centroids) - - def decode_unpacked(self, codes, x, n, ld_codes=-1): - r""" - Decode a set of vectors in non-packed format - - :type codes: int - :param codes: codes to decode, size n * ld_codes - :type x: float - :param x: output vectors, size n * d - """ - return _swigfaiss.ProductAdditiveQuantizer_decode_unpacked(self, codes, x, n, ld_codes) - - def decode(self, codes, x, n): - r""" - Decode a set of vectors - - :type codes: uint8_t - :param codes: codes to decode, size n * code_size - :type x: float - :param x: output vectors, size n * d - """ - return _swigfaiss.ProductAdditiveQuantizer_decode(self, codes, x, n) - - def compute_LUT(self, n, xq, LUT, alpha=1.0, ld_lut=-1): - r""" - Compute inner-product look-up tables. Used in the search functions. - - :type xq: float - :param xq: query vector, size (n, d) - :type LUT: float - :param LUT: look-up table, size (n, total_codebook_size) - :type alpha: float, optional - :param alpha: compute alpha * inner-product - :type ld_lut: int, optional - :param ld_lut: leading dimension of LUT - """ - return _swigfaiss.ProductAdditiveQuantizer_compute_LUT(self, n, xq, LUT, alpha, ld_lut) - -# Register ProductAdditiveQuantizer in _swigfaiss: -_swigfaiss.ProductAdditiveQuantizer_swigregister(ProductAdditiveQuantizer) -class ProductLocalSearchQuantizer(ProductAdditiveQuantizer): - r"""Product Local Search Quantizer""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - r""" - Construct a product LSQ object. - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of sub-vectors we split a vector into - :type Msub: int - :param Msub: number of codebooks of each LSQ - :type nbits: int - :param nbits: bits for each step - :type search_type: int, optional - :param search_type: AQ search type - """ - _swigfaiss.ProductLocalSearchQuantizer_swiginit(self, _swigfaiss.new_ProductLocalSearchQuantizer(*args)) - __swig_destroy__ = _swigfaiss.delete_ProductLocalSearchQuantizer - -# Register ProductLocalSearchQuantizer in _swigfaiss: -_swigfaiss.ProductLocalSearchQuantizer_swigregister(ProductLocalSearchQuantizer) -class ProductResidualQuantizer(ProductAdditiveQuantizer): - r"""Product Residual Quantizer""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - r""" - Construct a product RQ object. - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of sub-vectors we split a vector into - :type Msub: int - :param Msub: number of codebooks of each RQ - :type nbits: int - :param nbits: bits for each step - :type search_type: int, optional - :param search_type: AQ search type - """ - _swigfaiss.ProductResidualQuantizer_swiginit(self, _swigfaiss.new_ProductResidualQuantizer(*args)) - __swig_destroy__ = _swigfaiss.delete_ProductResidualQuantizer - -# Register ProductResidualQuantizer in _swigfaiss: -_swigfaiss.ProductResidualQuantizer_swigregister(ProductResidualQuantizer) -class CodePacker(object): - r""" - Packing consists in combining a fixed number of codes of constant size - (code_size) into a block of data where they may (or may not) be interleaved - for efficient consumption by distance computation kernels. This exists for - the "fast_scan" indexes on CPU and for some GPU kernels. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - code_size = property(_swigfaiss.CodePacker_code_size_get, _swigfaiss.CodePacker_code_size_set) - nvec = property(_swigfaiss.CodePacker_nvec_get, _swigfaiss.CodePacker_nvec_set) - block_size = property(_swigfaiss.CodePacker_block_size_get, _swigfaiss.CodePacker_block_size_set) - - def pack_1(self, flat_code, offset, block): - return _swigfaiss.CodePacker_pack_1(self, flat_code, offset, block) - - def unpack_1(self, block, offset, flat_code): - return _swigfaiss.CodePacker_unpack_1(self, block, offset, flat_code) - - def pack_all(self, flat_codes, block): - return _swigfaiss.CodePacker_pack_all(self, flat_codes, block) - - def unpack_all(self, block, flat_codes): - return _swigfaiss.CodePacker_unpack_all(self, block, flat_codes) - - def clone(self): - return _swigfaiss.CodePacker_clone(self) - __swig_destroy__ = _swigfaiss.delete_CodePacker - -# Register CodePacker in _swigfaiss: -_swigfaiss.CodePacker_swigregister(CodePacker) -class CodePackerFlat(CodePacker): - r"""Trivial code packer where codes are stored one by one""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, code_size): - _swigfaiss.CodePackerFlat_swiginit(self, _swigfaiss.new_CodePackerFlat(code_size)) - - def clone(self): - return _swigfaiss.CodePackerFlat_clone(self) - - def pack_1(self, flat_code, offset, block): - return _swigfaiss.CodePackerFlat_pack_1(self, flat_code, offset, block) - - def unpack_1(self, block, offset, flat_code): - return _swigfaiss.CodePackerFlat_unpack_1(self, block, offset, flat_code) - - def pack_all(self, flat_codes, block): - return _swigfaiss.CodePackerFlat_pack_all(self, flat_codes, block) - - def unpack_all(self, block, flat_codes): - return _swigfaiss.CodePackerFlat_unpack_all(self, block, flat_codes) - __swig_destroy__ = _swigfaiss.delete_CodePackerFlat - -# Register CodePackerFlat in _swigfaiss: -_swigfaiss.CodePackerFlat_swigregister(CodePackerFlat) -class Panorama(object): - r""" - Implements the core logic of Panorama-based refinement. - arXiv: https://arxiv.org/abs/2510.00566 - - Panorama partitions the dimensions of all vectors into L contiguous levels. - During the refinement stage of ANNS, it computes distances between the query - and its candidates level-by-level. After processing each level, it prunes the - candidates whose lower bound exceeds the k-th best distance. - - In order to enable speedups, the dimensions (or codes) of each vector are - stored in a batched, level-major manner. Within each batch of b vectors, the - dimensions corresponding to level 1 will be stored first (for all elements in - that batch), followed by level 2, and so on. This allows for efficient memory - access patterns. - - Coupled with the appropriate orthogonal PreTransform (e.g. PCA, Cayley, - etc.), Panorama can prune the vast majority of dimensions, greatly - accelerating the refinement stage. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - kDefaultBatchSize = _swigfaiss.Panorama_kDefaultBatchSize - d = property(_swigfaiss.Panorama_d_get, _swigfaiss.Panorama_d_set) - code_size = property(_swigfaiss.Panorama_code_size_get, _swigfaiss.Panorama_code_size_set) - n_levels = property(_swigfaiss.Panorama_n_levels_get, _swigfaiss.Panorama_n_levels_set) - level_width = property(_swigfaiss.Panorama_level_width_get, _swigfaiss.Panorama_level_width_set) - level_width_floats = property(_swigfaiss.Panorama_level_width_floats_get, _swigfaiss.Panorama_level_width_floats_set) - batch_size = property(_swigfaiss.Panorama_batch_size_get, _swigfaiss.Panorama_batch_size_set) - - def __init__(self, code_size, n_levels, batch_size): - _swigfaiss.Panorama_swiginit(self, _swigfaiss.new_Panorama(code_size, n_levels, batch_size)) - - def set_derived_values(self): - return _swigfaiss.Panorama_set_derived_values(self) - - def copy_codes_to_level_layout(self, codes, offset, n_entry, code): - r""" - Helper method to copy codes into level-oriented batch layout at a given - offset in the list. - """ - return _swigfaiss.Panorama_copy_codes_to_level_layout(self, codes, offset, n_entry, code) - - def compute_cumulative_sums(self, cumsum_base, offset, n_entry, vectors): - r""" - Helper method to compute the cumulative sums of the codes. - The cumsums also follow the level-oriented batch layout to minimize the - number of random memory accesses. - """ - return _swigfaiss.Panorama_compute_cumulative_sums(self, cumsum_base, offset, n_entry, vectors) - - def compute_query_cum_sums(self, query, query_cum_sums): - r"""Compute the cumulative sums of the query vector.""" - return _swigfaiss.Panorama_compute_query_cum_sums(self, query, query_cum_sums) - - def copy_entry(self, dest_codes, src_codes, dest_cum_sums, src_cum_sums, dest_idx, src_idx): - r"""Copy single entry (code and cum_sum) from one location to another.""" - return _swigfaiss.Panorama_copy_entry(self, dest_codes, src_codes, dest_cum_sums, src_cum_sums, dest_idx, src_idx) - - def reconstruct(self, key, recons, codes_base): - r""" - Panorama's core progressive filtering algorithm: - Process vectors in batches for cache efficiency. For each batch: - 1. Apply ID selection filter and initialize distances - (||y||^2 + ||x||^2). - 2. Maintain an "active set" of candidate indices that haven't been - pruned yet. - 3. For each level, refine distances incrementally and compact the active - set: - - Compute dot product for current level: exact_dist -= 2*. - - Use Cauchy-Schwarz bound on remaining levels to get lower bound - - Prune candidates whose lower bound exceeds k-th best distance. - - Compact active_indices to remove pruned candidates (branchless) - 4. After all levels, survivors are exact distances; update heap. - This achieves early termination while maintaining SIMD-friendly - sequential access patterns in the level-oriented storage layout. - """ - return _swigfaiss.Panorama_reconstruct(self, key, recons, codes_base) - __swig_destroy__ = _swigfaiss.delete_Panorama - -# Register Panorama in _swigfaiss: -_swigfaiss.Panorama_swigregister(Panorama) -class PanoramaStats(object): - r""" - Statistics are not robust to internal threading nor to - concurrent Panorama searches. Use these values in a - single-threaded context to accurately gauge Panorama's - pruning effectiveness. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - total_dims_scanned = property(_swigfaiss.PanoramaStats_total_dims_scanned_get, _swigfaiss.PanoramaStats_total_dims_scanned_set) - total_dims = property(_swigfaiss.PanoramaStats_total_dims_get, _swigfaiss.PanoramaStats_total_dims_set) - ratio_dims_scanned = property(_swigfaiss.PanoramaStats_ratio_dims_scanned_get, _swigfaiss.PanoramaStats_ratio_dims_scanned_set) - - def __init__(self): - _swigfaiss.PanoramaStats_swiginit(self, _swigfaiss.new_PanoramaStats()) - - def reset(self): - return _swigfaiss.PanoramaStats_reset(self) - - def add(self, other): - return _swigfaiss.PanoramaStats_add(self, other) - __swig_destroy__ = _swigfaiss.delete_PanoramaStats - -# Register PanoramaStats in _swigfaiss: -_swigfaiss.PanoramaStats_swigregister(PanoramaStats) -class VectorTransform(object): - r"""Any transformation applied on a set of vectors""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - d_in = property(_swigfaiss.VectorTransform_d_in_get, _swigfaiss.VectorTransform_d_in_set) - d_out = property(_swigfaiss.VectorTransform_d_out_get, _swigfaiss.VectorTransform_d_out_set, doc=r"""input dimension""") - is_trained = property(_swigfaiss.VectorTransform_is_trained_get, _swigfaiss.VectorTransform_is_trained_set, doc=r""" - set if the VectorTransform does not require training, or if - training is done already - """) - - def train(self, n, x): - r""" - Perform training on a representative set of vectors. Does - nothing by default. - - :type n: int - :param n: nb of training vectors - :type x: float - :param x: training vectors, size n * d - """ - return _swigfaiss.VectorTransform_train(self, n, x) - - def apply(self, n, x): - r""" - apply the transformation and return the result in an allocated pointer - :type n: int - :param n: number of vectors to transform - :type x: float - :param x: input vectors, size n * d_in - :rtype: float - :return: output vectors, size n * d_out - """ - return _swigfaiss.VectorTransform_apply(self, n, x) - - def apply_noalloc(self, n, x, xt): - r""" - apply the transformation and return the result in a provided matrix - :type n: int - :param n: number of vectors to transform - :type x: float - :param x: input vectors, size n * d_in - :type xt: float - :param xt: output vectors, size n * d_out - """ - return _swigfaiss.VectorTransform_apply_noalloc(self, n, x, xt) - - def reverse_transform(self, n, xt, x): - r""" - reverse transformation. May not be implemented or may return - approximate result - """ - return _swigfaiss.VectorTransform_reverse_transform(self, n, xt, x) - - def check_identical(self, other): - return _swigfaiss.VectorTransform_check_identical(self, other) - __swig_destroy__ = _swigfaiss.delete_VectorTransform - -# Register VectorTransform in _swigfaiss: -_swigfaiss.VectorTransform_swigregister(VectorTransform) -class LinearTransform(VectorTransform): - r""" - Generic linear transformation, with bias term applied on output - y = A * x + b - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - have_bias = property(_swigfaiss.LinearTransform_have_bias_get, _swigfaiss.LinearTransform_have_bias_set) - is_orthonormal = property(_swigfaiss.LinearTransform_is_orthonormal_get, _swigfaiss.LinearTransform_is_orthonormal_set, doc=r""" - whether to use the bias term - check if matrix A is orthonormal (enables reverse_transform) - """) - A = property(_swigfaiss.LinearTransform_A_get, _swigfaiss.LinearTransform_A_set, doc=r"""Transformation matrix, size d_out * d_in""") - b = property(_swigfaiss.LinearTransform_b_get, _swigfaiss.LinearTransform_b_set, doc=r"""bias vector, size d_out""") - - def __init__(self, din=0, dout=0, have_bias_in=False): - r"""both d_in > d_out and d_out < d_in are supported""" - _swigfaiss.LinearTransform_swiginit(self, _swigfaiss.new_LinearTransform(din, dout, have_bias_in)) - - def apply_noalloc(self, n, x, xt): - r"""same as apply, but result is pre-allocated""" - return _swigfaiss.LinearTransform_apply_noalloc(self, n, x, xt) - - def transform_transpose(self, n, y, x): - r""" - compute x = A^T * (x - b) - is reverse transform if A has orthonormal lines - """ - return _swigfaiss.LinearTransform_transform_transpose(self, n, y, x) - - def reverse_transform(self, n, xt, x): - r"""works only if is_orthonormal""" - return _swigfaiss.LinearTransform_reverse_transform(self, n, xt, x) - - def set_is_orthonormal(self): - r"""compute A^T * A to set the is_orthonormal flag""" - return _swigfaiss.LinearTransform_set_is_orthonormal(self) - verbose = property(_swigfaiss.LinearTransform_verbose_get, _swigfaiss.LinearTransform_verbose_set) - - def print_if_verbose(self, name, mat, n, d): - return _swigfaiss.LinearTransform_print_if_verbose(self, name, mat, n, d) - - def check_identical(self, other): - return _swigfaiss.LinearTransform_check_identical(self, other) - __swig_destroy__ = _swigfaiss.delete_LinearTransform - -# Register LinearTransform in _swigfaiss: -_swigfaiss.LinearTransform_swigregister(LinearTransform) -class RandomRotationMatrix(LinearTransform): - r"""Randomly rotate a set of vectors""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def init(self, seed): - r"""must be called before the transform is used""" - return _swigfaiss.RandomRotationMatrix_init(self, seed) - - def train(self, n, x): - return _swigfaiss.RandomRotationMatrix_train(self, n, x) - - def __init__(self, *args): - r"""both d_in > d_out and d_out < d_in are supported""" - _swigfaiss.RandomRotationMatrix_swiginit(self, _swigfaiss.new_RandomRotationMatrix(*args)) - __swig_destroy__ = _swigfaiss.delete_RandomRotationMatrix - -# Register RandomRotationMatrix in _swigfaiss: -_swigfaiss.RandomRotationMatrix_swigregister(RandomRotationMatrix) -class HadamardRotation(VectorTransform): - r""" - Three rounds of random sign-flip + Fast Walsh-Hadamard Transform. - Produces a pseudo-random rotation in O(d log d) time. - d_out is the smallest power of 2 >= d_in (zero-padded as needed). - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - seed = property(_swigfaiss.HadamardRotation_seed_get, _swigfaiss.HadamardRotation_seed_set) - signs1 = property(_swigfaiss.HadamardRotation_signs1_get, _swigfaiss.HadamardRotation_signs1_set, doc=r"""Sign-flip vectors, each of size d_out, generated from seed.""") - signs2 = property(_swigfaiss.HadamardRotation_signs2_get, _swigfaiss.HadamardRotation_signs2_set) - signs3 = property(_swigfaiss.HadamardRotation_signs3_get, _swigfaiss.HadamardRotation_signs3_set) - - def init(self, seed_in): - return _swigfaiss.HadamardRotation_init(self, seed_in) - - def train(self, n, x): - return _swigfaiss.HadamardRotation_train(self, n, x) - - def apply_noalloc(self, n, x, xt): - return _swigfaiss.HadamardRotation_apply_noalloc(self, n, x, xt) - - def check_identical(self, other): - return _swigfaiss.HadamardRotation_check_identical(self, other) - - def __init__(self, *args): - _swigfaiss.HadamardRotation_swiginit(self, _swigfaiss.new_HadamardRotation(*args)) - __swig_destroy__ = _swigfaiss.delete_HadamardRotation - -# Register HadamardRotation in _swigfaiss: -_swigfaiss.HadamardRotation_swigregister(HadamardRotation) -class PCAMatrix(LinearTransform): - r""" - Applies a principal component analysis on a set of vectors, - with optionally whitening and random rotation. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - eigen_power = property(_swigfaiss.PCAMatrix_eigen_power_get, _swigfaiss.PCAMatrix_eigen_power_set, doc=r""" - after transformation the components are multiplied by - eigenvalues^eigen_power - - =0: no whitening - =-0.5: full whitening - """) - epsilon = property(_swigfaiss.PCAMatrix_epsilon_get, _swigfaiss.PCAMatrix_epsilon_set, doc=r"""value added to eigenvalues to avoid division by 0 when whitening""") - random_rotation = property(_swigfaiss.PCAMatrix_random_rotation_get, _swigfaiss.PCAMatrix_random_rotation_set, doc=r"""random rotation after PCA""") - max_points_per_d = property(_swigfaiss.PCAMatrix_max_points_per_d_get, _swigfaiss.PCAMatrix_max_points_per_d_set, doc=r"""ratio between # training vectors and dimension""") - balanced_bins = property(_swigfaiss.PCAMatrix_balanced_bins_get, _swigfaiss.PCAMatrix_balanced_bins_set, doc=r"""try to distribute output eigenvectors in this many bins""") - mean = property(_swigfaiss.PCAMatrix_mean_get, _swigfaiss.PCAMatrix_mean_set, doc=r"""Mean, size d_in""") - eigenvalues = property(_swigfaiss.PCAMatrix_eigenvalues_get, _swigfaiss.PCAMatrix_eigenvalues_set, doc=r"""eigenvalues of covariance matrix (= squared singular values)""") - PCAMat = property(_swigfaiss.PCAMatrix_PCAMat_get, _swigfaiss.PCAMatrix_PCAMat_set, doc=r"""PCA matrix, size d_in * d_in""") - - def __init__(self, din=0, dout=0, eigen_power_in=0, random_rotation_in=False): - _swigfaiss.PCAMatrix_swiginit(self, _swigfaiss.new_PCAMatrix(din, dout, eigen_power_in, random_rotation_in)) - - def train(self, n, x): - r""" - train on n vectors. If n < d_in then the eigenvector matrix - will be completed with 0s - """ - return _swigfaiss.PCAMatrix_train(self, n, x) - - def copy_from(self, other): - r"""copy pre-trained PCA matrix""" - return _swigfaiss.PCAMatrix_copy_from(self, other) - - def prepare_Ab(self): - r"""called after mean, PCAMat and eigenvalues are computed""" - return _swigfaiss.PCAMatrix_prepare_Ab(self) - __swig_destroy__ = _swigfaiss.delete_PCAMatrix - -# Register PCAMatrix in _swigfaiss: -_swigfaiss.PCAMatrix_swigregister(PCAMatrix) -class ITQMatrix(LinearTransform): - r""" - ITQ implementation from - - Iterative quantization: A procrustean approach to learning binary codes - for large-scale image retrieval, - - Yunchao Gong, Svetlana Lazebnik, Albert Gordo, Florent Perronnin, - PAMI'12. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - max_iter = property(_swigfaiss.ITQMatrix_max_iter_get, _swigfaiss.ITQMatrix_max_iter_set) - seed = property(_swigfaiss.ITQMatrix_seed_get, _swigfaiss.ITQMatrix_seed_set) - init_rotation = property(_swigfaiss.ITQMatrix_init_rotation_get, _swigfaiss.ITQMatrix_init_rotation_set) - - def __init__(self, d=0): - _swigfaiss.ITQMatrix_swiginit(self, _swigfaiss.new_ITQMatrix(d)) - - def train(self, n, x): - return _swigfaiss.ITQMatrix_train(self, n, x) - __swig_destroy__ = _swigfaiss.delete_ITQMatrix - -# Register ITQMatrix in _swigfaiss: -_swigfaiss.ITQMatrix_swigregister(ITQMatrix) -class ITQTransform(VectorTransform): - r"""The full ITQ transform, including normalizations and PCA transformation""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - mean = property(_swigfaiss.ITQTransform_mean_get, _swigfaiss.ITQTransform_mean_set) - do_pca = property(_swigfaiss.ITQTransform_do_pca_get, _swigfaiss.ITQTransform_do_pca_set) - itq = property(_swigfaiss.ITQTransform_itq_get, _swigfaiss.ITQTransform_itq_set) - max_train_per_dim = property(_swigfaiss.ITQTransform_max_train_per_dim_get, _swigfaiss.ITQTransform_max_train_per_dim_set, doc=r"""max training points per dimension""") - pca_then_itq = property(_swigfaiss.ITQTransform_pca_then_itq_get, _swigfaiss.ITQTransform_pca_then_itq_set) - - def __init__(self, din=0, dout=0, do_pca_in=False): - _swigfaiss.ITQTransform_swiginit(self, _swigfaiss.new_ITQTransform(din, dout, do_pca_in)) - - def train(self, n, x): - return _swigfaiss.ITQTransform_train(self, n, x) - - def apply_noalloc(self, n, x, xt): - return _swigfaiss.ITQTransform_apply_noalloc(self, n, x, xt) - - def check_identical(self, other): - return _swigfaiss.ITQTransform_check_identical(self, other) - __swig_destroy__ = _swigfaiss.delete_ITQTransform - -# Register ITQTransform in _swigfaiss: -_swigfaiss.ITQTransform_swigregister(ITQTransform) -class OPQMatrix(LinearTransform): - r""" - Applies a rotation to align the dimensions with a PQ to minimize - the reconstruction error. Can be used before an IndexPQ or an - IndexIVFPQ. The method is the non-parametric version described in: - - "Optimized Product Quantization for Approximate Nearest Neighbor Search" - Tiezheng Ge, Kaiming He, Qifa Ke, Jian Sun, CVPR'13 - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - M = property(_swigfaiss.OPQMatrix_M_get, _swigfaiss.OPQMatrix_M_set, doc=r"""nb of subquantizers""") - niter = property(_swigfaiss.OPQMatrix_niter_get, _swigfaiss.OPQMatrix_niter_set, doc=r"""Number of outer training iterations""") - niter_pq = property(_swigfaiss.OPQMatrix_niter_pq_get, _swigfaiss.OPQMatrix_niter_pq_set, doc=r"""Number of training iterations for the PQ""") - niter_pq_0 = property(_swigfaiss.OPQMatrix_niter_pq_0_get, _swigfaiss.OPQMatrix_niter_pq_0_set, doc=r"""same, for the first outer iteration""") - max_train_points = property(_swigfaiss.OPQMatrix_max_train_points_get, _swigfaiss.OPQMatrix_max_train_points_set, doc=r"""if there are too many training points, resample""") - verbose = property(_swigfaiss.OPQMatrix_verbose_get, _swigfaiss.OPQMatrix_verbose_set) - pq = property(_swigfaiss.OPQMatrix_pq_get, _swigfaiss.OPQMatrix_pq_set, doc=r""" - if non-NULL, use this product quantizer for training - should be constructed with (d_out, M, _) - """) - - def __init__(self, d=0, M_in=1, d2=-1): - r"""if d2 != -1, output vectors of this dimension""" - _swigfaiss.OPQMatrix_swiginit(self, _swigfaiss.new_OPQMatrix(d, M_in, d2)) - - def train(self, n, x): - return _swigfaiss.OPQMatrix_train(self, n, x) - __swig_destroy__ = _swigfaiss.delete_OPQMatrix - -# Register OPQMatrix in _swigfaiss: -_swigfaiss.OPQMatrix_swigregister(OPQMatrix) -class RemapDimensionsTransform(VectorTransform): - r""" - remap dimensions for input vectors, possibly inserting 0s - strictly speaking this is also a linear transform but we don't want - to compute it with matrix multiplies - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - map = property(_swigfaiss.RemapDimensionsTransform_map_get, _swigfaiss.RemapDimensionsTransform_map_set, doc=r""" - map from output dimension to input, size d_out - -1 -> set output to 0 - """) - - def apply_noalloc(self, n, x, xt): - return _swigfaiss.RemapDimensionsTransform_apply_noalloc(self, n, x, xt) - - def reverse_transform(self, n, xt, x): - r"""reverse transform correct only when the mapping is a permutation""" - return _swigfaiss.RemapDimensionsTransform_reverse_transform(self, n, xt, x) - - def __init__(self, *args): - r""" - *Overload 1:* - remap input to output, skipping or inserting dimensions as needed - if uniform: distribute dimensions uniformly - otherwise just take the d_out first ones. - - | - - *Overload 2:* - remap input to output, skipping or inserting dimensions as needed - if uniform: distribute dimensions uniformly - otherwise just take the d_out first ones. - """ - _swigfaiss.RemapDimensionsTransform_swiginit(self, _swigfaiss.new_RemapDimensionsTransform(*args)) - - def check_identical(self, other): - return _swigfaiss.RemapDimensionsTransform_check_identical(self, other) - __swig_destroy__ = _swigfaiss.delete_RemapDimensionsTransform - -# Register RemapDimensionsTransform in _swigfaiss: -_swigfaiss.RemapDimensionsTransform_swigregister(RemapDimensionsTransform) -class NormalizationTransform(VectorTransform): - r"""per-vector normalization""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - norm = property(_swigfaiss.NormalizationTransform_norm_get, _swigfaiss.NormalizationTransform_norm_set) - - def __init__(self, *args): - _swigfaiss.NormalizationTransform_swiginit(self, _swigfaiss.new_NormalizationTransform(*args)) - - def apply_noalloc(self, n, x, xt): - return _swigfaiss.NormalizationTransform_apply_noalloc(self, n, x, xt) - - def reverse_transform(self, n, xt, x): - r"""Identity transform since norm is not revertible""" - return _swigfaiss.NormalizationTransform_reverse_transform(self, n, xt, x) - - def check_identical(self, other): - return _swigfaiss.NormalizationTransform_check_identical(self, other) - __swig_destroy__ = _swigfaiss.delete_NormalizationTransform - -# Register NormalizationTransform in _swigfaiss: -_swigfaiss.NormalizationTransform_swigregister(NormalizationTransform) -class CenteringTransform(VectorTransform): - r"""Subtract the mean of each component from the vectors.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - mean = property(_swigfaiss.CenteringTransform_mean_get, _swigfaiss.CenteringTransform_mean_set, doc=r"""Mean, size d_in = d_out""") - - def __init__(self, d=0): - _swigfaiss.CenteringTransform_swiginit(self, _swigfaiss.new_CenteringTransform(d)) - - def train(self, n, x): - r"""train on n vectors.""" - return _swigfaiss.CenteringTransform_train(self, n, x) - - def apply_noalloc(self, n, x, xt): - r"""subtract the mean""" - return _swigfaiss.CenteringTransform_apply_noalloc(self, n, x, xt) - - def reverse_transform(self, n, xt, x): - r"""add the mean""" - return _swigfaiss.CenteringTransform_reverse_transform(self, n, xt, x) - - def check_identical(self, other): - return _swigfaiss.CenteringTransform_check_identical(self, other) - __swig_destroy__ = _swigfaiss.delete_CenteringTransform - -# Register CenteringTransform in _swigfaiss: -_swigfaiss.CenteringTransform_swigregister(CenteringTransform) -class SearchParametersPreTransform(SearchParameters): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - index_params = property(_swigfaiss.SearchParametersPreTransform_index_params_get, _swigfaiss.SearchParametersPreTransform_index_params_set) - - def __init__(self): - _swigfaiss.SearchParametersPreTransform_swiginit(self, _swigfaiss.new_SearchParametersPreTransform()) - __swig_destroy__ = _swigfaiss.delete_SearchParametersPreTransform - -# Register SearchParametersPreTransform in _swigfaiss: -_swigfaiss.SearchParametersPreTransform_swigregister(SearchParametersPreTransform) -class IndexPreTransform(Index): - r""" - Index that applies a LinearTransform transform on vectors before - handing them over to a sub-index - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - chain = property(_swigfaiss.IndexPreTransform_chain_get, _swigfaiss.IndexPreTransform_chain_set) - index = property(_swigfaiss.IndexPreTransform_index_get, _swigfaiss.IndexPreTransform_index_set, doc=r"""chain of transforms""") - own_fields = property(_swigfaiss.IndexPreTransform_own_fields_get, _swigfaiss.IndexPreTransform_own_fields_set, doc=r"""the sub-index""") - - def __init__(self, *args): - r""" - *Overload 1:* - whether pointers are deleted in destructor - - | - - *Overload 2:* - ltrans is the last transform before the index - """ - _swigfaiss.IndexPreTransform_swiginit(self, _swigfaiss.new_IndexPreTransform(*args)) - - def prepend_transform(self, ltrans): - return _swigfaiss.IndexPreTransform_prepend_transform(self, ltrans) - - def train(self, n, x): - return _swigfaiss.IndexPreTransform_train(self, n, x) - - def add(self, n, x): - return _swigfaiss.IndexPreTransform_add(self, n, x) - - def add_with_ids(self, n, x, xids): - return _swigfaiss.IndexPreTransform_add_with_ids(self, n, x, xids) - - def reset(self): - return _swigfaiss.IndexPreTransform_reset(self) - - def remove_ids(self, sel): - r"""removes IDs from the index. Not supported by all indexes.""" - return _swigfaiss.IndexPreTransform_remove_ids(self, sel) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexPreTransform_search(self, n, x, k, distances, labels, params) - - def search_subset(self, n, x, k_base, base_labels, k, distances, labels): - return _swigfaiss.IndexPreTransform_search_subset(self, n, x, k_base, base_labels, k, distances, labels) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexPreTransform_range_search(self, n, x, radius, result, params) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexPreTransform_reconstruct(self, key, recons) - - def reconstruct_n(self, i0, ni, recons): - return _swigfaiss.IndexPreTransform_reconstruct_n(self, i0, ni, recons) - - def search_and_reconstruct(self, n, x, k, distances, labels, recons, params=None): - return _swigfaiss.IndexPreTransform_search_and_reconstruct(self, n, x, k, distances, labels, recons, params) - - def apply_chain(self, n, x): - r""" - apply the transforms in the chain. The returned float * may be - equal to x, otherwise it should be deallocated. - """ - return _swigfaiss.IndexPreTransform_apply_chain(self, n, x) - - def reverse_chain(self, n, xt, x): - r""" - Reverse the transforms in the chain. May not be implemented for - all transforms in the chain or may return approximate results. - """ - return _swigfaiss.IndexPreTransform_reverse_chain(self, n, xt, x) - - def get_distance_computer(self): - return _swigfaiss.IndexPreTransform_get_distance_computer(self) - - def sa_code_size(self): - return _swigfaiss.IndexPreTransform_sa_code_size(self) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexPreTransform_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexPreTransform_sa_decode(self, n, bytes, x) - - def merge_from(self, otherIndex, add_id=0): - return _swigfaiss.IndexPreTransform_merge_from(self, otherIndex, add_id) - - def check_compatible_for_merge(self, otherIndex): - return _swigfaiss.IndexPreTransform_check_compatible_for_merge(self, otherIndex) - __swig_destroy__ = _swigfaiss.delete_IndexPreTransform - -# Register IndexPreTransform in _swigfaiss: -_swigfaiss.IndexPreTransform_swigregister(IndexPreTransform) -class IndexRefineSearchParameters(SearchParameters): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - k_factor = property(_swigfaiss.IndexRefineSearchParameters_k_factor_get, _swigfaiss.IndexRefineSearchParameters_k_factor_set) - base_index_params = property(_swigfaiss.IndexRefineSearchParameters_base_index_params_get, _swigfaiss.IndexRefineSearchParameters_base_index_params_set) - __swig_destroy__ = _swigfaiss.delete_IndexRefineSearchParameters - - def __init__(self): - _swigfaiss.IndexRefineSearchParameters_swiginit(self, _swigfaiss.new_IndexRefineSearchParameters()) - -# Register IndexRefineSearchParameters in _swigfaiss: -_swigfaiss.IndexRefineSearchParameters_swigregister(IndexRefineSearchParameters) -class IndexRefine(Index): - r""" - Index that queries in a base_index (a fast one) and refines the - results with an exact search, hopefully improving the results. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - base_index = property(_swigfaiss.IndexRefine_base_index_get, _swigfaiss.IndexRefine_base_index_set, doc=r"""faster index to pre-select the vectors that should be filtered""") - refine_index = property(_swigfaiss.IndexRefine_refine_index_get, _swigfaiss.IndexRefine_refine_index_set, doc=r"""refinement index""") - own_fields = property(_swigfaiss.IndexRefine_own_fields_get, _swigfaiss.IndexRefine_own_fields_set, doc=r"""should the base index be deallocated?""") - own_refine_index = property(_swigfaiss.IndexRefine_own_refine_index_get, _swigfaiss.IndexRefine_own_refine_index_set, doc=r"""same with the refinement index""") - k_factor = property(_swigfaiss.IndexRefine_k_factor_get, _swigfaiss.IndexRefine_k_factor_set, doc=r""" - factor between k requested in search and the k requested from - the base_index (should be >= 1) - """) - - def __init__(self, *args): - r"""initialize from empty index""" - _swigfaiss.IndexRefine_swiginit(self, _swigfaiss.new_IndexRefine(*args)) - - def train(self, n, x): - return _swigfaiss.IndexRefine_train(self, n, x) - - def add(self, n, x): - return _swigfaiss.IndexRefine_add(self, n, x) - - def reset(self): - return _swigfaiss.IndexRefine_reset(self) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexRefine_search(self, n, x, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexRefine_range_search(self, n, x, radius, result, params) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexRefine_reconstruct(self, key, recons) - - def sa_code_size(self): - return _swigfaiss.IndexRefine_sa_code_size(self) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexRefine_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - r""" - The sa_decode decodes from the index_refine, which is assumed to be more - accurate - """ - return _swigfaiss.IndexRefine_sa_decode(self, n, bytes, x) - __swig_destroy__ = _swigfaiss.delete_IndexRefine - -# Register IndexRefine in _swigfaiss: -_swigfaiss.IndexRefine_swigregister(IndexRefine) -class IndexRefineFlat(IndexRefine): - r""" - Version where the refinement index is an IndexFlat. It has one additional - constructor that takes a table of elements to add to the flat refinement - index - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexRefineFlat_swiginit(self, _swigfaiss.new_IndexRefineFlat(*args)) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexRefineFlat_search(self, n, x, k, distances, labels, params) - __swig_destroy__ = _swigfaiss.delete_IndexRefineFlat - -# Register IndexRefineFlat in _swigfaiss: -_swigfaiss.IndexRefineFlat_swigregister(IndexRefineFlat) -class IndexRefinePanorama(IndexRefine): - r""" - Version where the search calls search_subset, allowing for Panorama - refinement. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexRefinePanorama_swiginit(self, _swigfaiss.new_IndexRefinePanorama(*args)) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexRefinePanorama_search(self, n, x, k, distances, labels, params) - __swig_destroy__ = _swigfaiss.delete_IndexRefinePanorama - -# Register IndexRefinePanorama in _swigfaiss: -_swigfaiss.IndexRefinePanorama_swigregister(IndexRefinePanorama) -class IndexLSH(IndexFlatCodes): - r"""The sign of each vector component is put in a binary signature""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nbits = property(_swigfaiss.IndexLSH_nbits_get, _swigfaiss.IndexLSH_nbits_set, doc=r"""nb of bits per vector""") - rotate_data = property(_swigfaiss.IndexLSH_rotate_data_get, _swigfaiss.IndexLSH_rotate_data_set, doc=r"""whether to apply a random rotation to input""") - train_thresholds = property(_swigfaiss.IndexLSH_train_thresholds_get, _swigfaiss.IndexLSH_train_thresholds_set, doc=r"""whether we train thresholds or use 0""") - rrot = property(_swigfaiss.IndexLSH_rrot_get, _swigfaiss.IndexLSH_rrot_set, doc=r"""optional random rotation""") - thresholds = property(_swigfaiss.IndexLSH_thresholds_get, _swigfaiss.IndexLSH_thresholds_set, doc=r"""thresholds to compare with""") - - def apply_preprocess(self, n, x): - r""" - Preprocesses and resizes the input to the size required to - binarize the data - - :type x: float - :param x: input vectors, size n * d - :rtype: float - :return: output vectors, size n * bits. May be the same pointer - as x, otherwise it should be deleted by the caller - """ - return _swigfaiss.IndexLSH_apply_preprocess(self, n, x) - - def train(self, n, x): - return _swigfaiss.IndexLSH_train(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexLSH_search(self, n, x, k, distances, labels, params) - - def transfer_thresholds(self, vt): - r""" - transfer the thresholds to a pre-processing stage (and unset - train_thresholds) - """ - return _swigfaiss.IndexLSH_transfer_thresholds(self, vt) - __swig_destroy__ = _swigfaiss.delete_IndexLSH - - def __init__(self, *args): - _swigfaiss.IndexLSH_swiginit(self, _swigfaiss.new_IndexLSH(*args)) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexLSH_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexLSH_sa_decode(self, n, bytes, x) - -# Register IndexLSH in _swigfaiss: -_swigfaiss.IndexLSH_swigregister(IndexLSH) -class SimulatedAnnealingParameters(object): - r"""parameters used for the simulated annealing method""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - init_temperature = property(_swigfaiss.SimulatedAnnealingParameters_init_temperature_get, _swigfaiss.SimulatedAnnealingParameters_init_temperature_set) - temperature_decay = property(_swigfaiss.SimulatedAnnealingParameters_temperature_decay_get, _swigfaiss.SimulatedAnnealingParameters_temperature_decay_set) - n_iter = property(_swigfaiss.SimulatedAnnealingParameters_n_iter_get, _swigfaiss.SimulatedAnnealingParameters_n_iter_set) - n_redo = property(_swigfaiss.SimulatedAnnealingParameters_n_redo_get, _swigfaiss.SimulatedAnnealingParameters_n_redo_set) - seed = property(_swigfaiss.SimulatedAnnealingParameters_seed_get, _swigfaiss.SimulatedAnnealingParameters_seed_set) - verbose = property(_swigfaiss.SimulatedAnnealingParameters_verbose_get, _swigfaiss.SimulatedAnnealingParameters_verbose_set) - only_bit_flips = property(_swigfaiss.SimulatedAnnealingParameters_only_bit_flips_get, _swigfaiss.SimulatedAnnealingParameters_only_bit_flips_set) - init_random = property(_swigfaiss.SimulatedAnnealingParameters_init_random_get, _swigfaiss.SimulatedAnnealingParameters_init_random_set) - - def __init__(self): - _swigfaiss.SimulatedAnnealingParameters_swiginit(self, _swigfaiss.new_SimulatedAnnealingParameters()) - __swig_destroy__ = _swigfaiss.delete_SimulatedAnnealingParameters - -# Register SimulatedAnnealingParameters in _swigfaiss: -_swigfaiss.SimulatedAnnealingParameters_swigregister(SimulatedAnnealingParameters) -class PermutationObjective(object): - r"""abstract class for the loss function""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - n = property(_swigfaiss.PermutationObjective_n_get, _swigfaiss.PermutationObjective_n_set) - - def compute_cost(self, perm): - return _swigfaiss.PermutationObjective_compute_cost(self, perm) - - def cost_update(self, perm, iw, jw): - return _swigfaiss.PermutationObjective_cost_update(self, perm, iw, jw) - __swig_destroy__ = _swigfaiss.delete_PermutationObjective - -# Register PermutationObjective in _swigfaiss: -_swigfaiss.PermutationObjective_swigregister(PermutationObjective) -class ReproduceDistancesObjective(PermutationObjective): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - dis_weight_factor = property(_swigfaiss.ReproduceDistancesObjective_dis_weight_factor_get, _swigfaiss.ReproduceDistancesObjective_dis_weight_factor_set) - - @staticmethod - def sqr(x): - return _swigfaiss.ReproduceDistancesObjective_sqr(x) - - def dis_weight(self, x): - return _swigfaiss.ReproduceDistancesObjective_dis_weight(self, x) - source_dis = property(_swigfaiss.ReproduceDistancesObjective_source_dis_get, _swigfaiss.ReproduceDistancesObjective_source_dis_set, doc=r""""real" corrected distances (size n^2)""") - target_dis = property(_swigfaiss.ReproduceDistancesObjective_target_dis_get, _swigfaiss.ReproduceDistancesObjective_target_dis_set, doc=r"""wanted distances (size n^2)""") - weights = property(_swigfaiss.ReproduceDistancesObjective_weights_get, _swigfaiss.ReproduceDistancesObjective_weights_set, doc=r"""weights for each distance (size n^2)""") - - def get_source_dis(self, i, j): - return _swigfaiss.ReproduceDistancesObjective_get_source_dis(self, i, j) - - def compute_cost(self, perm): - return _swigfaiss.ReproduceDistancesObjective_compute_cost(self, perm) - - def cost_update(self, perm, iw, jw): - return _swigfaiss.ReproduceDistancesObjective_cost_update(self, perm, iw, jw) - - def __init__(self, n_in, source_dis_in, target_dis_in, dis_weight_factor_in): - _swigfaiss.ReproduceDistancesObjective_swiginit(self, _swigfaiss.new_ReproduceDistancesObjective(n_in, source_dis_in, target_dis_in, dis_weight_factor_in)) - - @staticmethod - def compute_mean_stdev(tab, n2, mean_out, stddev_out): - return _swigfaiss.ReproduceDistancesObjective_compute_mean_stdev(tab, n2, mean_out, stddev_out) - - def set_affine_target_dis(self, source_dis_in): - return _swigfaiss.ReproduceDistancesObjective_set_affine_target_dis(self, source_dis_in) - __swig_destroy__ = _swigfaiss.delete_ReproduceDistancesObjective - -# Register ReproduceDistancesObjective in _swigfaiss: -_swigfaiss.ReproduceDistancesObjective_swigregister(ReproduceDistancesObjective) -class SimulatedAnnealingOptimizer(SimulatedAnnealingParameters): - r"""Simulated annealing optimization algorithm for permutations.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - obj = property(_swigfaiss.SimulatedAnnealingOptimizer_obj_get, _swigfaiss.SimulatedAnnealingOptimizer_obj_set) - n = property(_swigfaiss.SimulatedAnnealingOptimizer_n_get, _swigfaiss.SimulatedAnnealingOptimizer_n_set, doc=r"""size of the permutation""") - logfile = property(_swigfaiss.SimulatedAnnealingOptimizer_logfile_get, _swigfaiss.SimulatedAnnealingOptimizer_logfile_set) - - def __init__(self, obj_in, p): - r"""logs values of the cost function""" - _swigfaiss.SimulatedAnnealingOptimizer_swiginit(self, _swigfaiss.new_SimulatedAnnealingOptimizer(obj_in, p)) - rnd = property(_swigfaiss.SimulatedAnnealingOptimizer_rnd_get, _swigfaiss.SimulatedAnnealingOptimizer_rnd_set) - init_cost = property(_swigfaiss.SimulatedAnnealingOptimizer_init_cost_get, _swigfaiss.SimulatedAnnealingOptimizer_init_cost_set, doc=r"""remember initial cost of optimization""") - - def optimize(self, perm): - return _swigfaiss.SimulatedAnnealingOptimizer_optimize(self, perm) - - def run_optimization(self, best_perm): - return _swigfaiss.SimulatedAnnealingOptimizer_run_optimization(self, best_perm) - __swig_destroy__ = _swigfaiss.delete_SimulatedAnnealingOptimizer - -# Register SimulatedAnnealingOptimizer in _swigfaiss: -_swigfaiss.SimulatedAnnealingOptimizer_swigregister(SimulatedAnnealingOptimizer) -class PolysemousTraining(SimulatedAnnealingParameters): - r"""optimizes the order of indices in a ProductQuantizer""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - OT_None = _swigfaiss.PolysemousTraining_OT_None - OT_ReproduceDistances_affine = _swigfaiss.PolysemousTraining_OT_ReproduceDistances_affine - r"""default""" - OT_Ranking_weighted_diff = _swigfaiss.PolysemousTraining_OT_Ranking_weighted_diff - r""" - same as _2, but use rank of y+ - rank of - y- - """ - optimization_type = property(_swigfaiss.PolysemousTraining_optimization_type_get, _swigfaiss.PolysemousTraining_optimization_type_set) - ntrain_permutation = property(_swigfaiss.PolysemousTraining_ntrain_permutation_get, _swigfaiss.PolysemousTraining_ntrain_permutation_set, doc=r""" - use 1/4 of the training points for the optimization, with - max. ntrain_permutation. If ntrain_permutation == 0: train on - centroids - """) - dis_weight_factor = property(_swigfaiss.PolysemousTraining_dis_weight_factor_get, _swigfaiss.PolysemousTraining_dis_weight_factor_set, doc=r"""decay of exp that weights distance loss""") - max_memory = property(_swigfaiss.PolysemousTraining_max_memory_get, _swigfaiss.PolysemousTraining_max_memory_set, doc=r"""refuse to train if it would require more than that amount of RAM""") - log_pattern = property(_swigfaiss.PolysemousTraining_log_pattern_get, _swigfaiss.PolysemousTraining_log_pattern_set) - - def __init__(self): - _swigfaiss.PolysemousTraining_swiginit(self, _swigfaiss.new_PolysemousTraining()) - - def optimize_pq_for_hamming(self, pq, n, x): - r""" - reorder the centroids so that the Hamming distance becomes a - good approximation of the SDC distance (called by train) - """ - return _swigfaiss.PolysemousTraining_optimize_pq_for_hamming(self, pq, n, x) - - def optimize_ranking(self, pq, n, x): - r"""called by optimize_pq_for_hamming""" - return _swigfaiss.PolysemousTraining_optimize_ranking(self, pq, n, x) - - def optimize_reproduce_distances(self, pq): - r"""called by optimize_pq_for_hamming""" - return _swigfaiss.PolysemousTraining_optimize_reproduce_distances(self, pq) - - def memory_usage_per_thread(self, pq): - r"""make sure we don't blow up the memory""" - return _swigfaiss.PolysemousTraining_memory_usage_per_thread(self, pq) - __swig_destroy__ = _swigfaiss.delete_PolysemousTraining - -# Register PolysemousTraining in _swigfaiss: -_swigfaiss.PolysemousTraining_swigregister(PolysemousTraining) -class IndexPQ(IndexFlatCodes): - r""" - Index based on a product quantizer. Stored vectors are - approximated by PQ codes. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - pq = property(_swigfaiss.IndexPQ_pq_get, _swigfaiss.IndexPQ_pq_set, doc=r"""The product quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexPQ_swiginit(self, _swigfaiss.new_IndexPQ(*args)) - - def train(self, n, x): - return _swigfaiss.IndexPQ_train(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexPQ_search(self, n, x, k, distances, labels, params) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexPQ_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexPQ_sa_decode(self, n, bytes, x) - - def get_FlatCodesDistanceComputer(self): - return _swigfaiss.IndexPQ_get_FlatCodesDistanceComputer(self) - do_polysemous_training = property(_swigfaiss.IndexPQ_do_polysemous_training_get, _swigfaiss.IndexPQ_do_polysemous_training_set, doc=r"""false = standard PQ""") - polysemous_training = property(_swigfaiss.IndexPQ_polysemous_training_get, _swigfaiss.IndexPQ_polysemous_training_set, doc=r"""parameters used for the polysemous training""") - ST_PQ = _swigfaiss.IndexPQ_ST_PQ - r"""asymmetric product quantizer (default)""" - ST_HE = _swigfaiss.IndexPQ_ST_HE - r"""Hamming distance on codes""" - ST_generalized_HE = _swigfaiss.IndexPQ_ST_generalized_HE - r"""nb of same codes""" - ST_SDC = _swigfaiss.IndexPQ_ST_SDC - r"""symmetric product quantizer (SDC)""" - ST_polysemous = _swigfaiss.IndexPQ_ST_polysemous - r"""HE filter (using ht) + PQ combination""" - ST_polysemous_generalize = _swigfaiss.IndexPQ_ST_polysemous_generalize - r"""Filter on generalized Hamming""" - search_type = property(_swigfaiss.IndexPQ_search_type_get, _swigfaiss.IndexPQ_search_type_set) - encode_signs = property(_swigfaiss.IndexPQ_encode_signs_get, _swigfaiss.IndexPQ_encode_signs_set) - polysemous_ht = property(_swigfaiss.IndexPQ_polysemous_ht_get, _swigfaiss.IndexPQ_polysemous_ht_set, doc=r"""Hamming threshold used for polysemy""") - - def search_core_polysemous(self, n, x, k, distances, labels, polysemous_ht, generalized_hamming): - return _swigfaiss.IndexPQ_search_core_polysemous(self, n, x, k, distances, labels, polysemous_ht, generalized_hamming) - - def hamming_distance_histogram(self, n, x, nb, xb, dist_histogram): - r""" - prepare query for a polysemous search, but instead of - computing the result, just get the histogram of Hamming - distances. May be computed on a provided dataset if xb != NULL - :type dist_histogram: int - :param dist_histogram: (M * nbits + 1) - """ - return _swigfaiss.IndexPQ_hamming_distance_histogram(self, n, x, nb, xb, dist_histogram) - - def hamming_distance_table(self, n, x, dis): - r""" - compute pairwise distances between queries and database - - :type n: int - :param n: nb of query vectors - :type x: float - :param x: query vector, size n * d - :type dis: int - :param dis: output distances, size n * ntotal - """ - return _swigfaiss.IndexPQ_hamming_distance_table(self, n, x, dis) - __swig_destroy__ = _swigfaiss.delete_IndexPQ - -# Register IndexPQ in _swigfaiss: -_swigfaiss.IndexPQ_swigregister(IndexPQ) -class SearchParametersPQ(SearchParameters): - r"""override search parameters from the class""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - search_type = property(_swigfaiss.SearchParametersPQ_search_type_get, _swigfaiss.SearchParametersPQ_search_type_set) - polysemous_ht = property(_swigfaiss.SearchParametersPQ_polysemous_ht_get, _swigfaiss.SearchParametersPQ_polysemous_ht_set) - - def __init__(self): - _swigfaiss.SearchParametersPQ_swiginit(self, _swigfaiss.new_SearchParametersPQ()) - __swig_destroy__ = _swigfaiss.delete_SearchParametersPQ - -# Register SearchParametersPQ in _swigfaiss: -_swigfaiss.SearchParametersPQ_swigregister(SearchParametersPQ) -class IndexPQStats(object): - r""" - statistics are robust to internal threading, but not if - IndexPQ::search is called by multiple threads - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nq = property(_swigfaiss.IndexPQStats_nq_get, _swigfaiss.IndexPQStats_nq_set) - ncode = property(_swigfaiss.IndexPQStats_ncode_get, _swigfaiss.IndexPQStats_ncode_set) - n_hamming_pass = property(_swigfaiss.IndexPQStats_n_hamming_pass_get, _swigfaiss.IndexPQStats_n_hamming_pass_set) - - def __init__(self): - _swigfaiss.IndexPQStats_swiginit(self, _swigfaiss.new_IndexPQStats()) - - def reset(self): - return _swigfaiss.IndexPQStats_reset(self) - __swig_destroy__ = _swigfaiss.delete_IndexPQStats - -# Register IndexPQStats in _swigfaiss: -_swigfaiss.IndexPQStats_swigregister(IndexPQStats) -class MultiIndexQuantizer(Index): - r""" - Quantizer where centroids are virtual: they are the Cartesian - product of sub-centroids. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - pq = property(_swigfaiss.MultiIndexQuantizer_pq_get, _swigfaiss.MultiIndexQuantizer_pq_set) - - def train(self, n, x): - return _swigfaiss.MultiIndexQuantizer_train(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.MultiIndexQuantizer_search(self, n, x, k, distances, labels, params) - - def add(self, n, x): - r"""add and reset will crash at runtime""" - return _swigfaiss.MultiIndexQuantizer_add(self, n, x) - - def reset(self): - return _swigfaiss.MultiIndexQuantizer_reset(self) - - def __init__(self, *args): - r""" - number of bit per subvector index - :type d: int - :param d: dimension of the input vectors - :type M: int - :param M: number of subquantizers - """ - _swigfaiss.MultiIndexQuantizer_swiginit(self, _swigfaiss.new_MultiIndexQuantizer(*args)) - - def reconstruct(self, key, recons): - return _swigfaiss.MultiIndexQuantizer_reconstruct(self, key, recons) - __swig_destroy__ = _swigfaiss.delete_MultiIndexQuantizer - -# Register MultiIndexQuantizer in _swigfaiss: -_swigfaiss.MultiIndexQuantizer_swigregister(MultiIndexQuantizer) -class MultiIndexQuantizer2(MultiIndexQuantizer): - r"""MultiIndexQuantizer where the PQ assignment is performed by sub-indexes""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - assign_indexes = property(_swigfaiss.MultiIndexQuantizer2_assign_indexes_get, _swigfaiss.MultiIndexQuantizer2_assign_indexes_set, doc=r"""M Indexes on d / M dimensions""") - own_fields = property(_swigfaiss.MultiIndexQuantizer2_own_fields_get, _swigfaiss.MultiIndexQuantizer2_own_fields_set) - - def __init__(self, *args): - _swigfaiss.MultiIndexQuantizer2_swiginit(self, _swigfaiss.new_MultiIndexQuantizer2(*args)) - - def train(self, n, x): - return _swigfaiss.MultiIndexQuantizer2_train(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.MultiIndexQuantizer2_search(self, n, x, k, distances, labels, params) - __swig_destroy__ = _swigfaiss.delete_MultiIndexQuantizer2 - -# Register MultiIndexQuantizer2 in _swigfaiss: -_swigfaiss.MultiIndexQuantizer2_swigregister(MultiIndexQuantizer2) -class IndexAdditiveQuantizer(IndexFlatCodes): - r"""Abstract class for additive quantizers. The search functions are in common.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - aq = property(_swigfaiss.IndexAdditiveQuantizer_aq_get, _swigfaiss.IndexAdditiveQuantizer_aq_set) - - def __init__(self, *args): - _swigfaiss.IndexAdditiveQuantizer_swiginit(self, _swigfaiss.new_IndexAdditiveQuantizer(*args)) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexAdditiveQuantizer_search(self, n, x, k, distances, labels, params) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexAdditiveQuantizer_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexAdditiveQuantizer_sa_decode(self, n, bytes, x) - - def get_FlatCodesDistanceComputer(self): - return _swigfaiss.IndexAdditiveQuantizer_get_FlatCodesDistanceComputer(self) - __swig_destroy__ = _swigfaiss.delete_IndexAdditiveQuantizer - -# Register IndexAdditiveQuantizer in _swigfaiss: -_swigfaiss.IndexAdditiveQuantizer_swigregister(IndexAdditiveQuantizer) -class IndexResidualQuantizer(IndexAdditiveQuantizer): - r""" - Index based on a residual quantizer. Stored vectors are - approximated by residual quantization codes. - Can also be used as a codec - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - rq = property(_swigfaiss.IndexResidualQuantizer_rq_get, _swigfaiss.IndexResidualQuantizer_rq_set, doc=r"""The residual quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexResidualQuantizer_swiginit(self, _swigfaiss.new_IndexResidualQuantizer(*args)) - - def train(self, n, x): - return _swigfaiss.IndexResidualQuantizer_train(self, n, x) - __swig_destroy__ = _swigfaiss.delete_IndexResidualQuantizer - -# Register IndexResidualQuantizer in _swigfaiss: -_swigfaiss.IndexResidualQuantizer_swigregister(IndexResidualQuantizer) -class IndexLocalSearchQuantizer(IndexAdditiveQuantizer): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - lsq = property(_swigfaiss.IndexLocalSearchQuantizer_lsq_get, _swigfaiss.IndexLocalSearchQuantizer_lsq_set) - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexLocalSearchQuantizer_swiginit(self, _swigfaiss.new_IndexLocalSearchQuantizer(*args)) - - def train(self, n, x): - return _swigfaiss.IndexLocalSearchQuantizer_train(self, n, x) - __swig_destroy__ = _swigfaiss.delete_IndexLocalSearchQuantizer - -# Register IndexLocalSearchQuantizer in _swigfaiss: -_swigfaiss.IndexLocalSearchQuantizer_swigregister(IndexLocalSearchQuantizer) -class IndexProductResidualQuantizer(IndexAdditiveQuantizer): - r"""Index based on a product residual quantizer.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - prq = property(_swigfaiss.IndexProductResidualQuantizer_prq_get, _swigfaiss.IndexProductResidualQuantizer_prq_set, doc=r"""The product residual quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of residual quantizers - :type Msub: int - :param Msub: number of subquantizers per RQ - :type nbits: int - :param nbits: number of bit per subvector index - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of residual quantizers - :type Msub: int - :param Msub: number of subquantizers per RQ - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexProductResidualQuantizer_swiginit(self, _swigfaiss.new_IndexProductResidualQuantizer(*args)) - - def train(self, n, x): - return _swigfaiss.IndexProductResidualQuantizer_train(self, n, x) - __swig_destroy__ = _swigfaiss.delete_IndexProductResidualQuantizer - -# Register IndexProductResidualQuantizer in _swigfaiss: -_swigfaiss.IndexProductResidualQuantizer_swigregister(IndexProductResidualQuantizer) -class IndexProductLocalSearchQuantizer(IndexAdditiveQuantizer): - r"""Index based on a product local search quantizer.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - plsq = property(_swigfaiss.IndexProductLocalSearchQuantizer_plsq_get, _swigfaiss.IndexProductLocalSearchQuantizer_plsq_set, doc=r"""The product local search quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of local search quantizers - :type Msub: int - :param Msub: number of subquantizers per LSQ - :type nbits: int - :param nbits: number of bit per subvector index - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of local search quantizers - :type Msub: int - :param Msub: number of subquantizers per LSQ - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexProductLocalSearchQuantizer_swiginit(self, _swigfaiss.new_IndexProductLocalSearchQuantizer(*args)) - - def train(self, n, x): - return _swigfaiss.IndexProductLocalSearchQuantizer_train(self, n, x) - __swig_destroy__ = _swigfaiss.delete_IndexProductLocalSearchQuantizer - -# Register IndexProductLocalSearchQuantizer in _swigfaiss: -_swigfaiss.IndexProductLocalSearchQuantizer_swigregister(IndexProductLocalSearchQuantizer) -class AdditiveCoarseQuantizer(Index): - r""" - A "virtual" index where the elements are the residual quantizer centroids. - - Intended for use as a coarse quantizer in an IndexIVF. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - aq = property(_swigfaiss.AdditiveCoarseQuantizer_aq_get, _swigfaiss.AdditiveCoarseQuantizer_aq_set) - - def __init__(self, *args): - _swigfaiss.AdditiveCoarseQuantizer_swiginit(self, _swigfaiss.new_AdditiveCoarseQuantizer(*args)) - centroid_norms = property(_swigfaiss.AdditiveCoarseQuantizer_centroid_norms_get, _swigfaiss.AdditiveCoarseQuantizer_centroid_norms_set, doc=r"""norms of centroids, useful for knn-search""") - - def add(self, n, x): - r"""N/A""" - return _swigfaiss.AdditiveCoarseQuantizer_add(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.AdditiveCoarseQuantizer_search(self, n, x, k, distances, labels, params) - - def reconstruct(self, key, recons): - return _swigfaiss.AdditiveCoarseQuantizer_reconstruct(self, key, recons) - - def train(self, n, x): - return _swigfaiss.AdditiveCoarseQuantizer_train(self, n, x) - - def reset(self): - r"""N/A""" - return _swigfaiss.AdditiveCoarseQuantizer_reset(self) - __swig_destroy__ = _swigfaiss.delete_AdditiveCoarseQuantizer - -# Register AdditiveCoarseQuantizer in _swigfaiss: -_swigfaiss.AdditiveCoarseQuantizer_swigregister(AdditiveCoarseQuantizer) -class SearchParametersResidualCoarseQuantizer(SearchParameters): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - beam_factor = property(_swigfaiss.SearchParametersResidualCoarseQuantizer_beam_factor_get, _swigfaiss.SearchParametersResidualCoarseQuantizer_beam_factor_set) - __swig_destroy__ = _swigfaiss.delete_SearchParametersResidualCoarseQuantizer - - def __init__(self): - _swigfaiss.SearchParametersResidualCoarseQuantizer_swiginit(self, _swigfaiss.new_SearchParametersResidualCoarseQuantizer()) - -# Register SearchParametersResidualCoarseQuantizer in _swigfaiss: -_swigfaiss.SearchParametersResidualCoarseQuantizer_swigregister(SearchParametersResidualCoarseQuantizer) -class ResidualCoarseQuantizer(AdditiveCoarseQuantizer): - r""" - The ResidualCoarseQuantizer is a bit specialized compared to the - default AdditiveCoarseQuantizer because it can use a beam search - at search time (slow but may be useful for very large vocabularies) - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - rq = property(_swigfaiss.ResidualCoarseQuantizer_rq_get, _swigfaiss.ResidualCoarseQuantizer_rq_set, doc=r"""The residual quantizer used to encode the vectors""") - beam_factor = property(_swigfaiss.ResidualCoarseQuantizer_beam_factor_get, _swigfaiss.ResidualCoarseQuantizer_beam_factor_set, doc=r""" - factor between the beam size and the search k - if negative, use exact search-to-centroid - """) - - def set_beam_factor(self, new_beam_factor): - r"""computes centroid norms if required""" - return _swigfaiss.ResidualCoarseQuantizer_set_beam_factor(self, new_beam_factor) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.ResidualCoarseQuantizer_search(self, n, x, k, distances, labels, params) - - def initialize_from(self, other): - r""" - Copy the M first codebook levels from other. Useful to crop a - ResidualQuantizer to its first M quantizers. - """ - return _swigfaiss.ResidualCoarseQuantizer_initialize_from(self, other) - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.ResidualCoarseQuantizer_swiginit(self, _swigfaiss.new_ResidualCoarseQuantizer(*args)) - __swig_destroy__ = _swigfaiss.delete_ResidualCoarseQuantizer - -# Register ResidualCoarseQuantizer in _swigfaiss: -_swigfaiss.ResidualCoarseQuantizer_swigregister(ResidualCoarseQuantizer) -class LocalSearchCoarseQuantizer(AdditiveCoarseQuantizer): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - lsq = property(_swigfaiss.LocalSearchCoarseQuantizer_lsq_get, _swigfaiss.LocalSearchCoarseQuantizer_lsq_set, doc=r"""The residual quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.LocalSearchCoarseQuantizer_swiginit(self, _swigfaiss.new_LocalSearchCoarseQuantizer(*args)) - __swig_destroy__ = _swigfaiss.delete_LocalSearchCoarseQuantizer - -# Register LocalSearchCoarseQuantizer in _swigfaiss: -_swigfaiss.LocalSearchCoarseQuantizer_swigregister(LocalSearchCoarseQuantizer) -class InvertedListsIterator(object): - r""" - Definition of inverted lists + a few common classes that implement - the interface. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - __swig_destroy__ = _swigfaiss.delete_InvertedListsIterator - - def is_available(self): - return _swigfaiss.InvertedListsIterator_is_available(self) - - def next(self): - return _swigfaiss.InvertedListsIterator_next(self) - - def get_id_and_codes(self): - return _swigfaiss.InvertedListsIterator_get_id_and_codes(self) - has_search_callbacks_ = property(_swigfaiss.InvertedListsIterator_has_search_callbacks__get, _swigfaiss.InvertedListsIterator_has_search_callbacks__set, doc=r""" - When true, iterate_codes will invoke on_distance_computed() and - on_heap_changed() via virtual dispatch. When false (the default), - iterate_codes skips the callbacks entirely — the guard branch is - perfectly predicted and costs ~0 cycles, so non-callback users - pay no overhead. Derived classes that override the callbacks - should set this to true in their constructor. - """) - - def on_distance_computed(self, arg2, arg3): - r""" - Called from iterate_codes after distance computation for the vector - returned by the most recent get_id_and_codes(). Default: no-op. - Only invoked when has_search_callbacks_ is true. - """ - return _swigfaiss.InvertedListsIterator_on_distance_computed(self, arg2, arg3) - - def on_heap_changed(self, arg2, arg3): - r""" - Called from iterate_codes when a vector replaces the current worst - in the top-K heap. evicted_id is the displaced entry. Default: no-op. - Only invoked when has_search_callbacks_ is true. - """ - return _swigfaiss.InvertedListsIterator_on_heap_changed(self, arg2, arg3) - -# Register InvertedListsIterator in _swigfaiss: -_swigfaiss.InvertedListsIterator_swigregister(InvertedListsIterator) -class InvertedLists(object): - r""" - Table of inverted lists - multithreading rules: - - concurrent read accesses are allowed - - concurrent update accesses are allowed - - for resize and add_entries, only concurrent access to different lists - are allowed - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - nlist = property(_swigfaiss.InvertedLists_nlist_get, _swigfaiss.InvertedLists_nlist_set, doc=r"""number of possible key values""") - code_size = property(_swigfaiss.InvertedLists_code_size_get, _swigfaiss.InvertedLists_code_size_set, doc=r"""code size per vector in bytes""") - use_iterator = property(_swigfaiss.InvertedLists_use_iterator_get, _swigfaiss.InvertedLists_use_iterator_set, doc=r"""request to use iterator rather than get_codes / get_ids""") - __swig_destroy__ = _swigfaiss.delete_InvertedLists - INVALID_CODE_SIZE = _swigfaiss.InvertedLists_INVALID_CODE_SIZE - r""" - used for BlockInvertedLists, where the codes are packed into groups - and the individual code size is meaningless - """ - - def list_size(self, list_no): - r"""get the size of a list""" - return _swigfaiss.InvertedLists_list_size(self, list_no) - - def get_codes(self, list_no): - r""" - get the codes for an inverted list - must be released by release_codes - - :rtype: uint8_t - :return: codes size list_size * code_size - """ - return _swigfaiss.InvertedLists_get_codes(self, list_no) - - def get_ids(self, list_no): - r""" - get the ids for an inverted list - must be released by release_ids - - :rtype: int - :return: ids size list_size - """ - return _swigfaiss.InvertedLists_get_ids(self, list_no) - - def release_codes(self, list_no, codes): - r"""release codes returned by get_codes (default implementation is nop""" - return _swigfaiss.InvertedLists_release_codes(self, list_no, codes) - - def release_ids(self, list_no, ids): - r"""release ids returned by get_ids""" - return _swigfaiss.InvertedLists_release_ids(self, list_no, ids) - - def get_single_id(self, list_no, offset): - r""" - :rtype: int - :return: a single id in an inverted list - """ - return _swigfaiss.InvertedLists_get_single_id(self, list_no, offset) - - def get_single_code(self, list_no, offset): - r""" - :rtype: uint8_t - :return: a single code in an inverted list - (should be deallocated with release_codes) - """ - return _swigfaiss.InvertedLists_get_single_code(self, list_no, offset) - - def prefetch_lists(self, list_nos, nlist_in): - r""" - prepare the following lists (default does nothing) - a list can be -1 hence the signed long - """ - return _swigfaiss.InvertedLists_prefetch_lists(self, list_nos, nlist_in) - - def is_empty(self, list_no, inverted_list_context=None): - r"""check if the list is empty""" - return _swigfaiss.InvertedLists_is_empty(self, list_no, inverted_list_context) - - def get_iterator(self, list_no, inverted_list_context=None): - r"""get iterable for lists that use_iterator""" - return _swigfaiss.InvertedLists_get_iterator(self, list_no, inverted_list_context) - - def add_entry(self, list_no, theid, code, inverted_list_context=None): - r"""add one entry to an inverted list""" - return _swigfaiss.InvertedLists_add_entry(self, list_no, theid, code, inverted_list_context) - - def add_entries(self, list_no, n_entry, ids, code): - return _swigfaiss.InvertedLists_add_entries(self, list_no, n_entry, ids, code) - - def update_entry(self, list_no, offset, id, code): - return _swigfaiss.InvertedLists_update_entry(self, list_no, offset, id, code) - - def update_entries(self, list_no, offset, n_entry, ids, code): - return _swigfaiss.InvertedLists_update_entries(self, list_no, offset, n_entry, ids, code) - - def resize(self, list_no, new_size): - return _swigfaiss.InvertedLists_resize(self, list_no, new_size) - - def reset(self): - return _swigfaiss.InvertedLists_reset(self) - - def merge_from(self, oivf, add_id): - r"""move all entries from oivf (empty on output)""" - return _swigfaiss.InvertedLists_merge_from(self, oivf, add_id) - SUBSET_TYPE_ID_RANGE = _swigfaiss.InvertedLists_SUBSET_TYPE_ID_RANGE - SUBSET_TYPE_ID_MOD = _swigfaiss.InvertedLists_SUBSET_TYPE_ID_MOD - SUBSET_TYPE_ELEMENT_RANGE = _swigfaiss.InvertedLists_SUBSET_TYPE_ELEMENT_RANGE - SUBSET_TYPE_INVLIST_FRACTION = _swigfaiss.InvertedLists_SUBSET_TYPE_INVLIST_FRACTION - SUBSET_TYPE_INVLIST = _swigfaiss.InvertedLists_SUBSET_TYPE_INVLIST - - def copy_subset_to(self, other, subset_type, a1, a2): - r""" - copy a subset of the entries index to the other index - :rtype: int - :return: number of entries copied - """ - return _swigfaiss.InvertedLists_copy_subset_to(self, other, subset_type, a1, a2) - - def imbalance_factor(self): - r"""1= perfectly balanced, >1: imbalanced""" - return _swigfaiss.InvertedLists_imbalance_factor(self) - - def print_stats(self): - r"""display some stats about the inverted lists""" - return _swigfaiss.InvertedLists_print_stats(self) - - def compute_ntotal(self): - r"""sum up list sizes""" - return _swigfaiss.InvertedLists_compute_ntotal(self) - -# Register InvertedLists in _swigfaiss: -_swigfaiss.InvertedLists_swigregister(InvertedLists) -class ArrayInvertedLists(InvertedLists): - r"""simple (default) implementation as an array of inverted lists""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - codes = property(_swigfaiss.ArrayInvertedLists_codes_get, _swigfaiss.ArrayInvertedLists_codes_set) - ids = property(_swigfaiss.ArrayInvertedLists_ids_get, _swigfaiss.ArrayInvertedLists_ids_set, doc=r"""Inverted lists for indexes""") - - def __init__(self, nlist_in, code_size_in): - _swigfaiss.ArrayInvertedLists_swiginit(self, _swigfaiss.new_ArrayInvertedLists(nlist_in, code_size_in)) - - def list_size(self, list_no): - return _swigfaiss.ArrayInvertedLists_list_size(self, list_no) - - def get_codes(self, list_no): - return _swigfaiss.ArrayInvertedLists_get_codes(self, list_no) - - def get_ids(self, list_no): - return _swigfaiss.ArrayInvertedLists_get_ids(self, list_no) - - def add_entries(self, list_no, n_entry, ids, code): - return _swigfaiss.ArrayInvertedLists_add_entries(self, list_no, n_entry, ids, code) - - def update_entries(self, list_no, offset, n_entry, ids, code): - return _swigfaiss.ArrayInvertedLists_update_entries(self, list_no, offset, n_entry, ids, code) - - def resize(self, list_no, new_size): - return _swigfaiss.ArrayInvertedLists_resize(self, list_no, new_size) - - def permute_invlists(self, map): - r"""permute the inverted lists, map maps new_id to old_id""" - return _swigfaiss.ArrayInvertedLists_permute_invlists(self, map) - - def is_empty(self, list_no, inverted_list_context=None): - return _swigfaiss.ArrayInvertedLists_is_empty(self, list_no, inverted_list_context) - __swig_destroy__ = _swigfaiss.delete_ArrayInvertedLists - -# Register ArrayInvertedLists in _swigfaiss: -_swigfaiss.ArrayInvertedLists_swigregister(ArrayInvertedLists) -class ArrayInvertedListsPanorama(ArrayInvertedLists): - r""" - Level-oriented storage as defined in the IVFFlat section of Panorama - (https://www.arxiv.org/pdf/2510.00566). - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - cum_sums = property(_swigfaiss.ArrayInvertedListsPanorama_cum_sums_get, _swigfaiss.ArrayInvertedListsPanorama_cum_sums_set) - n_levels = property(_swigfaiss.ArrayInvertedListsPanorama_n_levels_get) - level_width = property(_swigfaiss.ArrayInvertedListsPanorama_level_width_get) - pano = property(_swigfaiss.ArrayInvertedListsPanorama_pano_get, _swigfaiss.ArrayInvertedListsPanorama_pano_set) - - def __init__(self, *args): - _swigfaiss.ArrayInvertedListsPanorama_swiginit(self, _swigfaiss.new_ArrayInvertedListsPanorama(*args)) - - def get_cum_sums(self, list_no): - return _swigfaiss.ArrayInvertedListsPanorama_get_cum_sums(self, list_no) - - def add_entries(self, list_no, n_entry, ids, code): - return _swigfaiss.ArrayInvertedListsPanorama_add_entries(self, list_no, n_entry, ids, code) - - def update_entries(self, list_no, offset, n_entry, ids, code): - return _swigfaiss.ArrayInvertedListsPanorama_update_entries(self, list_no, offset, n_entry, ids, code) - - def resize(self, list_no, new_size): - return _swigfaiss.ArrayInvertedListsPanorama_resize(self, list_no, new_size) - - def get_iterator(self, list_no, inverted_list_context=None): - r""" - Panorama's layout make it impractical to support iterators as defined - by Faiss (i.e. `InvertedListsIterator` API). The iterator would require - to allocate and reassemble the vector at each call. - Hence, we override this method to throw an error, this effectively - disables the `iterate_codes` and `iterate_codes_range` methods. - """ - return _swigfaiss.ArrayInvertedListsPanorama_get_iterator(self, list_no, inverted_list_context) - - def get_single_code(self, list_no, offset): - r"""Reconstructs a single code from level-oriented storage to flat format.""" - return _swigfaiss.ArrayInvertedListsPanorama_get_single_code(self, list_no, offset) - - def release_codes(self, list_no, codes_in): - r"""Frees codes returned by `get_single_code`.""" - return _swigfaiss.ArrayInvertedListsPanorama_release_codes(self, list_no, codes_in) - __swig_destroy__ = _swigfaiss.delete_ArrayInvertedListsPanorama - -# Register ArrayInvertedListsPanorama in _swigfaiss: -_swigfaiss.ArrayInvertedListsPanorama_swigregister(ArrayInvertedListsPanorama) -class ReadOnlyInvertedLists(InvertedLists): - r"""invlists that fail for all write functions""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - - def add_entries(self, list_no, n_entry, ids, code): - return _swigfaiss.ReadOnlyInvertedLists_add_entries(self, list_no, n_entry, ids, code) - - def update_entries(self, list_no, offset, n_entry, ids, code): - return _swigfaiss.ReadOnlyInvertedLists_update_entries(self, list_no, offset, n_entry, ids, code) - - def resize(self, list_no, new_size): - return _swigfaiss.ReadOnlyInvertedLists_resize(self, list_no, new_size) - __swig_destroy__ = _swigfaiss.delete_ReadOnlyInvertedLists - -# Register ReadOnlyInvertedLists in _swigfaiss: -_swigfaiss.ReadOnlyInvertedLists_swigregister(ReadOnlyInvertedLists) -class HStackInvertedLists(ReadOnlyInvertedLists): - r"""Horizontal stack of inverted lists""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - ils = property(_swigfaiss.HStackInvertedLists_ils_get, _swigfaiss.HStackInvertedLists_ils_set) - - def __init__(self, n_il, ils): - r"""build InvertedLists by concatenating nil of them""" - _swigfaiss.HStackInvertedLists_swiginit(self, _swigfaiss.new_HStackInvertedLists(n_il, ils)) - - def list_size(self, list_no): - return _swigfaiss.HStackInvertedLists_list_size(self, list_no) - - def get_codes(self, list_no): - return _swigfaiss.HStackInvertedLists_get_codes(self, list_no) - - def get_ids(self, list_no): - return _swigfaiss.HStackInvertedLists_get_ids(self, list_no) - - def prefetch_lists(self, list_nos, nlist_in): - return _swigfaiss.HStackInvertedLists_prefetch_lists(self, list_nos, nlist_in) - - def release_codes(self, list_no, codes): - return _swigfaiss.HStackInvertedLists_release_codes(self, list_no, codes) - - def release_ids(self, list_no, ids): - return _swigfaiss.HStackInvertedLists_release_ids(self, list_no, ids) - - def get_single_id(self, list_no, offset): - return _swigfaiss.HStackInvertedLists_get_single_id(self, list_no, offset) - - def get_single_code(self, list_no, offset): - return _swigfaiss.HStackInvertedLists_get_single_code(self, list_no, offset) - __swig_destroy__ = _swigfaiss.delete_HStackInvertedLists - -# Register HStackInvertedLists in _swigfaiss: -_swigfaiss.HStackInvertedLists_swigregister(HStackInvertedLists) -class SliceInvertedLists(ReadOnlyInvertedLists): - r"""vertical slice of indexes in another InvertedLists""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - il = property(_swigfaiss.SliceInvertedLists_il_get, _swigfaiss.SliceInvertedLists_il_set) - i0 = property(_swigfaiss.SliceInvertedLists_i0_get, _swigfaiss.SliceInvertedLists_i0_set) - i1 = property(_swigfaiss.SliceInvertedLists_i1_get, _swigfaiss.SliceInvertedLists_i1_set) - - def __init__(self, il_, i0_, i1_): - _swigfaiss.SliceInvertedLists_swiginit(self, _swigfaiss.new_SliceInvertedLists(il_, i0_, i1_)) - - def list_size(self, list_no): - return _swigfaiss.SliceInvertedLists_list_size(self, list_no) - - def get_codes(self, list_no): - return _swigfaiss.SliceInvertedLists_get_codes(self, list_no) - - def get_ids(self, list_no): - return _swigfaiss.SliceInvertedLists_get_ids(self, list_no) - - def release_codes(self, list_no, codes): - return _swigfaiss.SliceInvertedLists_release_codes(self, list_no, codes) - - def release_ids(self, list_no, ids): - return _swigfaiss.SliceInvertedLists_release_ids(self, list_no, ids) - - def get_single_id(self, list_no, offset): - return _swigfaiss.SliceInvertedLists_get_single_id(self, list_no, offset) - - def get_single_code(self, list_no, offset): - return _swigfaiss.SliceInvertedLists_get_single_code(self, list_no, offset) - - def prefetch_lists(self, list_nos, nlist_in): - return _swigfaiss.SliceInvertedLists_prefetch_lists(self, list_nos, nlist_in) - __swig_destroy__ = _swigfaiss.delete_SliceInvertedLists - -# Register SliceInvertedLists in _swigfaiss: -_swigfaiss.SliceInvertedLists_swigregister(SliceInvertedLists) -class VStackInvertedLists(ReadOnlyInvertedLists): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - ils = property(_swigfaiss.VStackInvertedLists_ils_get, _swigfaiss.VStackInvertedLists_ils_set) - cumsz = property(_swigfaiss.VStackInvertedLists_cumsz_get, _swigfaiss.VStackInvertedLists_cumsz_set) - - def __init__(self, n_il, ils): - r"""build InvertedLists by concatenating nil of them""" - _swigfaiss.VStackInvertedLists_swiginit(self, _swigfaiss.new_VStackInvertedLists(n_il, ils)) - - def list_size(self, list_no): - return _swigfaiss.VStackInvertedLists_list_size(self, list_no) - - def get_codes(self, list_no): - return _swigfaiss.VStackInvertedLists_get_codes(self, list_no) - - def get_ids(self, list_no): - return _swigfaiss.VStackInvertedLists_get_ids(self, list_no) - - def release_codes(self, list_no, codes): - return _swigfaiss.VStackInvertedLists_release_codes(self, list_no, codes) - - def release_ids(self, list_no, ids): - return _swigfaiss.VStackInvertedLists_release_ids(self, list_no, ids) - - def get_single_id(self, list_no, offset): - return _swigfaiss.VStackInvertedLists_get_single_id(self, list_no, offset) - - def get_single_code(self, list_no, offset): - return _swigfaiss.VStackInvertedLists_get_single_code(self, list_no, offset) - - def prefetch_lists(self, list_nos, nlist_in): - return _swigfaiss.VStackInvertedLists_prefetch_lists(self, list_nos, nlist_in) - __swig_destroy__ = _swigfaiss.delete_VStackInvertedLists - -# Register VStackInvertedLists in _swigfaiss: -_swigfaiss.VStackInvertedLists_swigregister(VStackInvertedLists) -class MaskedInvertedLists(ReadOnlyInvertedLists): - r""" - use the first inverted lists if they are non-empty otherwise use the second - - This is useful if il1 has a few inverted lists that are too long, - and that il0 has replacement lists for those, with empty lists for - the others. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - il0 = property(_swigfaiss.MaskedInvertedLists_il0_get, _swigfaiss.MaskedInvertedLists_il0_set) - il1 = property(_swigfaiss.MaskedInvertedLists_il1_get, _swigfaiss.MaskedInvertedLists_il1_set) - - def __init__(self, il0_in, il1_in): - _swigfaiss.MaskedInvertedLists_swiginit(self, _swigfaiss.new_MaskedInvertedLists(il0_in, il1_in)) - - def list_size(self, list_no): - return _swigfaiss.MaskedInvertedLists_list_size(self, list_no) - - def get_codes(self, list_no): - return _swigfaiss.MaskedInvertedLists_get_codes(self, list_no) - - def get_ids(self, list_no): - return _swigfaiss.MaskedInvertedLists_get_ids(self, list_no) - - def release_codes(self, list_no, codes): - return _swigfaiss.MaskedInvertedLists_release_codes(self, list_no, codes) - - def release_ids(self, list_no, ids): - return _swigfaiss.MaskedInvertedLists_release_ids(self, list_no, ids) - - def get_single_id(self, list_no, offset): - return _swigfaiss.MaskedInvertedLists_get_single_id(self, list_no, offset) - - def get_single_code(self, list_no, offset): - return _swigfaiss.MaskedInvertedLists_get_single_code(self, list_no, offset) - - def prefetch_lists(self, list_nos, nlist_in): - return _swigfaiss.MaskedInvertedLists_prefetch_lists(self, list_nos, nlist_in) - __swig_destroy__ = _swigfaiss.delete_MaskedInvertedLists - -# Register MaskedInvertedLists in _swigfaiss: -_swigfaiss.MaskedInvertedLists_swigregister(MaskedInvertedLists) -class StopWordsInvertedLists(ReadOnlyInvertedLists): - r""" - if the inverted list in il is smaller than maxsize then return it, - otherwise return an empty invlist - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - il0 = property(_swigfaiss.StopWordsInvertedLists_il0_get, _swigfaiss.StopWordsInvertedLists_il0_set) - maxsize = property(_swigfaiss.StopWordsInvertedLists_maxsize_get, _swigfaiss.StopWordsInvertedLists_maxsize_set) - - def __init__(self, il0_in, maxsize_in): - _swigfaiss.StopWordsInvertedLists_swiginit(self, _swigfaiss.new_StopWordsInvertedLists(il0_in, maxsize_in)) - - def list_size(self, list_no): - return _swigfaiss.StopWordsInvertedLists_list_size(self, list_no) - - def get_codes(self, list_no): - return _swigfaiss.StopWordsInvertedLists_get_codes(self, list_no) - - def get_ids(self, list_no): - return _swigfaiss.StopWordsInvertedLists_get_ids(self, list_no) - - def release_codes(self, list_no, codes): - return _swigfaiss.StopWordsInvertedLists_release_codes(self, list_no, codes) - - def release_ids(self, list_no, ids): - return _swigfaiss.StopWordsInvertedLists_release_ids(self, list_no, ids) - - def get_single_id(self, list_no, offset): - return _swigfaiss.StopWordsInvertedLists_get_single_id(self, list_no, offset) - - def get_single_code(self, list_no, offset): - return _swigfaiss.StopWordsInvertedLists_get_single_code(self, list_no, offset) - - def prefetch_lists(self, list_nos, nlist_in): - return _swigfaiss.StopWordsInvertedLists_prefetch_lists(self, list_nos, nlist_in) - __swig_destroy__ = _swigfaiss.delete_StopWordsInvertedLists - -# Register StopWordsInvertedLists in _swigfaiss: -_swigfaiss.StopWordsInvertedLists_swigregister(StopWordsInvertedLists) -class CappedInvertedLists(InvertedLists): - r""" - Cap list sizes to maxsize for searching, while allowing writes. - Unlike StopWordsInvertedLists which skips large lists entirely, - this caps each list to maxsize entries (partial scan). - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - il0 = property(_swigfaiss.CappedInvertedLists_il0_get, _swigfaiss.CappedInvertedLists_il0_set) - maxsize = property(_swigfaiss.CappedInvertedLists_maxsize_get, _swigfaiss.CappedInvertedLists_maxsize_set) - - def __init__(self, il, maxsize): - _swigfaiss.CappedInvertedLists_swiginit(self, _swigfaiss.new_CappedInvertedLists(il, maxsize)) - - def list_size(self, list_no): - return _swigfaiss.CappedInvertedLists_list_size(self, list_no) - - def real_list_size(self, list_no): - return _swigfaiss.CappedInvertedLists_real_list_size(self, list_no) - - def get_codes(self, list_no): - return _swigfaiss.CappedInvertedLists_get_codes(self, list_no) - - def get_ids(self, list_no): - return _swigfaiss.CappedInvertedLists_get_ids(self, list_no) - - def release_codes(self, list_no, codes): - return _swigfaiss.CappedInvertedLists_release_codes(self, list_no, codes) - - def release_ids(self, list_no, ids): - return _swigfaiss.CappedInvertedLists_release_ids(self, list_no, ids) - - def get_single_id(self, list_no, offset): - return _swigfaiss.CappedInvertedLists_get_single_id(self, list_no, offset) - - def get_single_code(self, list_no, offset): - return _swigfaiss.CappedInvertedLists_get_single_code(self, list_no, offset) - - def prefetch_lists(self, list_nos, nlist): - return _swigfaiss.CappedInvertedLists_prefetch_lists(self, list_nos, nlist) - - def add_entries(self, list_no, n_entry, ids, code): - return _swigfaiss.CappedInvertedLists_add_entries(self, list_no, n_entry, ids, code) - - def update_entries(self, list_no, offset, n_entry, ids, code): - return _swigfaiss.CappedInvertedLists_update_entries(self, list_no, offset, n_entry, ids, code) - - def resize(self, list_no, new_size): - return _swigfaiss.CappedInvertedLists_resize(self, list_no, new_size) - __swig_destroy__ = _swigfaiss.delete_CappedInvertedLists - -# Register CappedInvertedLists in _swigfaiss: -_swigfaiss.CappedInvertedLists_swigregister(CappedInvertedLists) -class InvertedListsIOHook(object): - r""" - Callbacks to handle other types of InvertedList objects. - - The callbacks should be registered with add_callback before calling - read_index or read_InvertedLists. The callbacks for - OnDiskInvertedLists are registrered by default. The invlist type is - identified by: - - - the key (a fourcc) at read time - - the class name (as given by typeid.name) at write time - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - key = property(_swigfaiss.InvertedListsIOHook_key_get, doc=r"""string version of the fourcc""") - classname = property(_swigfaiss.InvertedListsIOHook_classname_get, doc=r"""typeid.name""") - - def write(self, ils, f): - r"""write the index to the IOWriter (including the fourcc)""" - return _swigfaiss.InvertedListsIOHook_write(self, ils, f) - - def read(self, f, io_flags): - r"""called when the fourcc matches this class's fourcc""" - return _swigfaiss.InvertedListsIOHook_read(self, f, io_flags) - - def read_ArrayInvertedLists(self, f, io_flags, nlist, code_size, sizes): - r""" - read from a ArrayInvertedLists into this invertedlist type. - For this to work, the callback has to be enabled and the io_flag has to - be set to IO_FLAG_SKIP_IVF_DATA | (16 upper bits of the fourcc) - - (default implementation fails) - """ - return _swigfaiss.InvertedListsIOHook_read_ArrayInvertedLists(self, f, io_flags, nlist, code_size, sizes) - __swig_destroy__ = _swigfaiss.delete_InvertedListsIOHook - - @staticmethod - def add_callback(arg1): - return _swigfaiss.InvertedListsIOHook_add_callback(arg1) - - @staticmethod - def print_callbacks(): - return _swigfaiss.InvertedListsIOHook_print_callbacks() - - @staticmethod - def lookup(h): - return _swigfaiss.InvertedListsIOHook_lookup(h) - - @staticmethod - def lookup_classname(classname): - return _swigfaiss.InvertedListsIOHook_lookup_classname(classname) - -# Register InvertedListsIOHook in _swigfaiss: -_swigfaiss.InvertedListsIOHook_swigregister(InvertedListsIOHook) -class BlockInvertedLists(InvertedLists): - r""" - Inverted Lists that are organized by blocks. - - Different from the regular inverted lists, the codes are organized by blocks - of size block_size bytes that represent a set of n_per_block. Therefore, code - allocations are always rounded up to block_size bytes. The codes are also - aligned on 32-byte boundaries for use with SIMD. - - To avoid misinterpretations, the code_size is set to (size_t)(-1), even if - arguably the amount of memory consumed by code is block_size / n_per_block. - - The writing functions add_entries and update_entries operate on block-aligned - data. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - n_per_block = property(_swigfaiss.BlockInvertedLists_n_per_block_get, _swigfaiss.BlockInvertedLists_n_per_block_set) - block_size = property(_swigfaiss.BlockInvertedLists_block_size_get, _swigfaiss.BlockInvertedLists_block_size_set) - packer = property(_swigfaiss.BlockInvertedLists_packer_get, _swigfaiss.BlockInvertedLists_packer_set) - codes = property(_swigfaiss.BlockInvertedLists_codes_get, _swigfaiss.BlockInvertedLists_codes_set) - ids = property(_swigfaiss.BlockInvertedLists_ids_get, _swigfaiss.BlockInvertedLists_ids_set) - - def __init__(self, *args): - _swigfaiss.BlockInvertedLists_swiginit(self, _swigfaiss.new_BlockInvertedLists(*args)) - - def list_size(self, list_no): - return _swigfaiss.BlockInvertedLists_list_size(self, list_no) - - def get_codes(self, list_no): - return _swigfaiss.BlockInvertedLists_get_codes(self, list_no) - - def get_ids(self, list_no): - return _swigfaiss.BlockInvertedLists_get_ids(self, list_no) - - def remove_ids(self, sel): - r"""remove ids from the InvertedLists""" - return _swigfaiss.BlockInvertedLists_remove_ids(self, sel) - - def add_entries(self, list_no, n_entry, ids, code): - return _swigfaiss.BlockInvertedLists_add_entries(self, list_no, n_entry, ids, code) - - def update_entries(self, list_no, offset, n_entry, ids, code): - r"""not implemented""" - return _swigfaiss.BlockInvertedLists_update_entries(self, list_no, offset, n_entry, ids, code) - - def resize(self, list_no, new_size): - return _swigfaiss.BlockInvertedLists_resize(self, list_no, new_size) - __swig_destroy__ = _swigfaiss.delete_BlockInvertedLists - -# Register BlockInvertedLists in _swigfaiss: -_swigfaiss.BlockInvertedLists_swigregister(BlockInvertedLists) - -def lo_build(list_id, offset): - return _swigfaiss.lo_build(list_id, offset) - -def lo_listno(lo): - return _swigfaiss.lo_listno(lo) - -def lo_offset(lo): - return _swigfaiss.lo_offset(lo) -class DirectMap(object): - r"""Direct map: a way to map back from ids to inverted lists""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - NoMap = _swigfaiss.DirectMap_NoMap - Array = _swigfaiss.DirectMap_Array - Hashtable = _swigfaiss.DirectMap_Hashtable - DMT_count = _swigfaiss.DirectMap_DMT_count - type = property(_swigfaiss.DirectMap_type_get, _swigfaiss.DirectMap_type_set) - array = property(_swigfaiss.DirectMap_array_get, _swigfaiss.DirectMap_array_set, doc=r"""map for direct access to the elements. Map ids to LO-encoded entries.""") - hashtable = property(_swigfaiss.DirectMap_hashtable_get, _swigfaiss.DirectMap_hashtable_set) - - def __init__(self): - _swigfaiss.DirectMap_swiginit(self, _swigfaiss.new_DirectMap()) - - def set_type(self, new_type, invlists, ntotal): - r"""set type and initialize""" - return _swigfaiss.DirectMap_set_type(self, new_type, invlists, ntotal) - - def get(self, id): - r"""get an entry""" - return _swigfaiss.DirectMap_get(self, id) - - def no(self): - r"""for quick checks""" - return _swigfaiss.DirectMap_no(self) - - def check_can_add(self, ids): - r""" - update the direct_map - - throw if Array and ids is not NULL - """ - return _swigfaiss.DirectMap_check_can_add(self, ids) - - def add_single_id(self, id, list_no, offset): - r"""non thread-safe version""" - return _swigfaiss.DirectMap_add_single_id(self, id, list_no, offset) - - def clear(self): - r"""remove all entries""" - return _swigfaiss.DirectMap_clear(self) - - def remove_ids(self, sel, invlists): - r""" - operations on inverted lists that require translation with a DirectMap - - remove ids from the InvertedLists, possibly using the direct map - """ - return _swigfaiss.DirectMap_remove_ids(self, sel, invlists) - - def update_codes(self, invlists, n, ids, list_nos, codes): - r"""update entries, using the direct map""" - return _swigfaiss.DirectMap_update_codes(self, invlists, n, ids, list_nos, codes) - __swig_destroy__ = _swigfaiss.delete_DirectMap - -# Register DirectMap in _swigfaiss: -_swigfaiss.DirectMap_swigregister(DirectMap) -class DirectMapAdd(object): - r"""Thread-safe way of updating the direct_map""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - direct_map = property(_swigfaiss.DirectMapAdd_direct_map_get, _swigfaiss.DirectMapAdd_direct_map_set) - type = property(_swigfaiss.DirectMapAdd_type_get, _swigfaiss.DirectMapAdd_type_set) - ntotal = property(_swigfaiss.DirectMapAdd_ntotal_get, _swigfaiss.DirectMapAdd_ntotal_set) - n = property(_swigfaiss.DirectMapAdd_n_get, _swigfaiss.DirectMapAdd_n_set) - xids = property(_swigfaiss.DirectMapAdd_xids_get, _swigfaiss.DirectMapAdd_xids_set) - all_ofs = property(_swigfaiss.DirectMapAdd_all_ofs_get, _swigfaiss.DirectMapAdd_all_ofs_set) - - def __init__(self, direct_map, n, xids): - _swigfaiss.DirectMapAdd_swiginit(self, _swigfaiss.new_DirectMapAdd(direct_map, n, xids)) - - def add(self, i, list_no, offset): - r"""add vector i (with id xids[i]) at list_no and offset""" - return _swigfaiss.DirectMapAdd_add(self, i, list_no, offset) - __swig_destroy__ = _swigfaiss.delete_DirectMapAdd - -# Register DirectMapAdd in _swigfaiss: -_swigfaiss.DirectMapAdd_swigregister(DirectMapAdd) -class InvertedListScannerStats(object): - r"""Per-list statistics returned by inverted-list scanners.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - scan_cnt = property(_swigfaiss.InvertedListScannerStats_scan_cnt_get, _swigfaiss.InvertedListScannerStats_scan_cnt_set, doc=r"""Number of distances computed after IDSelector filtering.""") - nheap_updates = property(_swigfaiss.InvertedListScannerStats_nheap_updates_get, _swigfaiss.InvertedListScannerStats_nheap_updates_set, doc=r"""Number of heap updates.""") - - def __init__(self): - _swigfaiss.InvertedListScannerStats_swiginit(self, _swigfaiss.new_InvertedListScannerStats()) - __swig_destroy__ = _swigfaiss.delete_InvertedListScannerStats - -# Register InvertedListScannerStats in _swigfaiss: -_swigfaiss.InvertedListScannerStats_swigregister(InvertedListScannerStats) -class Level1Quantizer(object): - r""" - Encapsulates a quantizer object for the IndexIVF - - The class isolates the fields that are independent of the storage - of the lists (especially training) - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - quantizer = property(_swigfaiss.Level1Quantizer_quantizer_get, _swigfaiss.Level1Quantizer_quantizer_set, doc=r"""quantizer that maps vectors to inverted lists""") - nlist = property(_swigfaiss.Level1Quantizer_nlist_get, _swigfaiss.Level1Quantizer_nlist_set, doc=r"""number of inverted lists""") - quantizer_trains_alone = property(_swigfaiss.Level1Quantizer_quantizer_trains_alone_get, _swigfaiss.Level1Quantizer_quantizer_trains_alone_set, doc=r""" - = 0: use the quantizer as index in a kmeans training - = 1: just pass on the training set to the train() of the quantizer - = 2: kmeans training on a flat index + add the centroids to the quantizer - """) - own_fields = property(_swigfaiss.Level1Quantizer_own_fields_get, _swigfaiss.Level1Quantizer_own_fields_set, doc=r"""whether object owns the quantizer""") - cp = property(_swigfaiss.Level1Quantizer_cp_get, _swigfaiss.Level1Quantizer_cp_set, doc=r"""to override default clustering params""") - clustering_index = property(_swigfaiss.Level1Quantizer_clustering_index_get, _swigfaiss.Level1Quantizer_clustering_index_set, doc=r"""to override index used during clustering""") - - def train_q1(self, n, x, verbose, metric_type): - r"""Trains the quantizer and calls train_residual to train sub-quantizers""" - return _swigfaiss.Level1Quantizer_train_q1(self, n, x, verbose, metric_type) - - def coarse_code_size(self): - r"""compute the number of bytes required to store list ids""" - return _swigfaiss.Level1Quantizer_coarse_code_size(self) - - def encode_listno(self, list_no, code): - return _swigfaiss.Level1Quantizer_encode_listno(self, list_no, code) - - def decode_listno(self, code): - return _swigfaiss.Level1Quantizer_decode_listno(self, code) - - def __init__(self, *args): - _swigfaiss.Level1Quantizer_swiginit(self, _swigfaiss.new_Level1Quantizer(*args)) - __swig_destroy__ = _swigfaiss.delete_Level1Quantizer - -# Register Level1Quantizer in _swigfaiss: -_swigfaiss.Level1Quantizer_swigregister(Level1Quantizer) -class SearchParametersIVF(SearchParameters): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nprobe = property(_swigfaiss.SearchParametersIVF_nprobe_get, _swigfaiss.SearchParametersIVF_nprobe_set, doc=r"""number of probes at query time""") - max_codes = property(_swigfaiss.SearchParametersIVF_max_codes_get, _swigfaiss.SearchParametersIVF_max_codes_set, doc=r"""max nb of codes to visit to do a query""") - max_lists_num = property(_swigfaiss.SearchParametersIVF_max_lists_num_get, _swigfaiss.SearchParametersIVF_max_lists_num_set, doc=r""" - FastScan k-NN only: maximum number of inverted lists to visit. - 0 means unlimited, i.e. bounded only by nprobe. When set together - with max_codes, either budget may stop the scan. With - ensure_topk_full, this limit is treated as at least k lists. - """) - ensure_topk_full = property(_swigfaiss.SearchParametersIVF_ensure_topk_full_get, _swigfaiss.SearchParametersIVF_ensure_topk_full_set, doc=r""" - For k-NN search, make small early-stop budgets less aggressive: - max_codes is treated as at least k post-IDSelector scans. Supported - by generic IVF in parallel_mode 0 and 3, and by FastScan k-NN - implementations 10 and 11. - """) - max_empty_result_buckets = property(_swigfaiss.SearchParametersIVF_max_empty_result_buckets_get, _swigfaiss.SearchParametersIVF_max_empty_result_buckets_set, doc=r""" - Range-search only: stop after this many consecutive probed lists add - no in-radius results. 0 disables the heuristic. This trades recall - for less work. Supported in parallel_mode 0; FastScan range search - uses implementation 10 for this option. - """) - quantizer_params = property(_swigfaiss.SearchParametersIVF_quantizer_params_get, _swigfaiss.SearchParametersIVF_quantizer_params_set) - inverted_list_context = property(_swigfaiss.SearchParametersIVF_inverted_list_context_get, _swigfaiss.SearchParametersIVF_inverted_list_context_set, doc=r"""context object to pass to InvertedLists""") - __swig_destroy__ = _swigfaiss.delete_SearchParametersIVF - - def __init__(self): - _swigfaiss.SearchParametersIVF_swiginit(self, _swigfaiss.new_SearchParametersIVF()) - -# Register SearchParametersIVF in _swigfaiss: -_swigfaiss.SearchParametersIVF_swigregister(SearchParametersIVF) -class IndexIVFInterface(Level1Quantizer): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - nprobe = property(_swigfaiss.IndexIVFInterface_nprobe_get, _swigfaiss.IndexIVFInterface_nprobe_set, doc=r"""number of probes at query time""") - max_codes = property(_swigfaiss.IndexIVFInterface_max_codes_get, _swigfaiss.IndexIVFInterface_max_codes_set, doc=r"""max nb of codes to visit to do a query""") - - def search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params=None, stats=None): - r""" - search a set of vectors, that are pre-quantized by the IVF - quantizer. Fill in the corresponding heaps with the query - results. The default implementation uses InvertedListScanners - to do the search. - - :type n: int - :param n: nb of vectors to query - :type x: float - :param x: query vectors, size nx * d - :type assign: int - :param assign: coarse quantization indices, size nx * nprobe - :type centroid_dis: float - :param centroid_dis: - distances to coarse centroids, size nx * nprobe - :param distance: - output distances, size n * k - :type labels: int - :param labels: output labels, size n * k - :type store_pairs: boolean - :param store_pairs: store inv list index + inv list offset - instead in upper/lower 32 bit of result, - instead of ids (used for reranking). - :type params: :py:class:`IVFSearchParameters`, optional - :param params: used to override the object's search parameters - :type stats: :py:class:`IndexIVFStats`, optional - :param stats: search stats to be updated (can be null) - """ - return _swigfaiss.IndexIVFInterface_search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params, stats) - - def range_search_preassigned(self, nx, x, radius, keys, coarse_dis, result, store_pairs=False, params=None, stats=None): - r""" - Range search a set of vectors, that are pre-quantized by the IVF - quantizer. Fill in the RangeSearchResults results. The default - implementation uses InvertedListScanners to do the search. - - :param n: nb of vectors to query - :type x: float - :param x: query vectors, size nx * d - :param assign: coarse quantization indices, size nx * nprobe - :param centroid_dis: - distances to coarse centroids, size nx * nprobe - :type result: :py:class:`RangeSearchResult` - :param result: Output results - :type store_pairs: boolean, optional - :param store_pairs: store inv list index + inv list offset - instead in upper/lower 32 bit of result, - instead of ids (used for reranking). - :type params: :py:class:`IVFSearchParameters`, optional - :param params: used to override the object's search parameters - :type stats: :py:class:`IndexIVFStats`, optional - :param stats: search stats to be updated (can be null) - """ - return _swigfaiss.IndexIVFInterface_range_search_preassigned(self, nx, x, radius, keys, coarse_dis, result, store_pairs, params, stats) - __swig_destroy__ = _swigfaiss.delete_IndexIVFInterface - -# Register IndexIVFInterface in _swigfaiss: -_swigfaiss.IndexIVFInterface_swigregister(IndexIVFInterface) -class IndexIVF(Index, IndexIVFInterface): - r""" - Index based on a inverted file (IVF) - - In the inverted file, the quantizer (an Index instance) provides a - quantization index for each vector to be added. The quantization - index maps to a list (aka inverted list or posting list), where the - id of the vector is stored. - - The inverted list object is required only after training. If none is - set externally, an ArrayInvertedLists is used automatically. - - At search time, the vector to be searched is also quantized, and - only the list corresponding to the quantization index is - searched. This speeds up the search by making it - non-exhaustive. This can be relaxed using multi-probe search: a few - (nprobe) quantization indices are selected and several inverted - lists are visited. - - Sub-classes implement a post-filtering of the index that refines - the distance estimation from the query to database vectors. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - invlists = property(_swigfaiss.IndexIVF_invlists_get, _swigfaiss.IndexIVF_invlists_set, doc=r"""Access to the actual data""") - own_invlists = property(_swigfaiss.IndexIVF_own_invlists_get, _swigfaiss.IndexIVF_own_invlists_set) - code_size = property(_swigfaiss.IndexIVF_code_size_get, _swigfaiss.IndexIVF_code_size_set, doc=r"""code size per vector in bytes""") - parallel_mode = property(_swigfaiss.IndexIVF_parallel_mode_get, _swigfaiss.IndexIVF_parallel_mode_set, doc=r""" - Parallel mode determines how queries are parallelized with OpenMP - - 0 (default): split over queries - 1: parallelize over inverted lists - 2: parallelize over both - 3: split over queries with a finer granularity - - PARALLEL_MODE_NO_HEAP_INIT: binary or with the previous to - prevent the heap to be initialized and finalized - """) - PARALLEL_MODE_NO_HEAP_INIT = property(_swigfaiss.IndexIVF_PARALLEL_MODE_NO_HEAP_INIT_get) - direct_map = property(_swigfaiss.IndexIVF_direct_map_get, _swigfaiss.IndexIVF_direct_map_set, doc=r""" - optional map that maps back ids to invlist entries. This - enables reconstruct() - """) - by_residual = property(_swigfaiss.IndexIVF_by_residual_get, _swigfaiss.IndexIVF_by_residual_set, doc=r""" - do the codes in the invlists encode the vectors relative to the - centroids? - """) - - def reset(self): - return _swigfaiss.IndexIVF_reset(self) - - def train(self, n, x): - r"""Trains the quantizer and calls train_encoder to train sub-quantizers""" - return _swigfaiss.IndexIVF_train(self, n, x) - - def add(self, n, x): - r"""Calls add_with_ids with NULL ids""" - return _swigfaiss.IndexIVF_add(self, n, x) - - def add_with_ids(self, n, x, xids): - r"""default implementation that calls encode_vectors""" - return _swigfaiss.IndexIVF_add_with_ids(self, n, x, xids) - - def add_core(self, n, x, xids, precomputed_idx, inverted_list_context=None): - r""" - Implementation of vector addition where the vector assignments are - predefined. The default implementation hands over the code extraction to - encode_vectors. - - :type precomputed_idx: int - :param precomputed_idx: quantization indices for the input vectors - (size n) - """ - return _swigfaiss.IndexIVF_add_core(self, n, x, xids, precomputed_idx, inverted_list_context) - - def encode_vectors(self, n, x, list_nos, codes, include_listno=False): - r""" - Encodes a set of vectors as they would appear in the inverted lists - - :type list_nos: int - :param list_nos: inverted list ids as returned by the - quantizer (size n). -1s are ignored. - :type codes: uint8_t - :param codes: output codes, size n * code_size - :type include_listno: boolean, optional - :param include_listno: - include the list ids in the code (in this case add - ceil(log8(nlist)) to the code size) - """ - return _swigfaiss.IndexIVF_encode_vectors(self, n, x, list_nos, codes, include_listno) - - def decode_vectors(self, n, codes, list_nos, x): - r""" - Decodes a set of vectors as they would appear in a given set of inverted - lists (inverse of encode_vectors) - - :type codes: uint8_t - :param codes: input codes, size n * code_size - :type x: float - :param x: output decoded vectors - :type list_nos: int - :param list_nos: input listnos, size n - """ - return _swigfaiss.IndexIVF_decode_vectors(self, n, codes, list_nos, x) - - def add_sa_codes(self, n, codes, xids): - r""" - Add vectors that are computed with the standalone codec - - :type codes: uint8_t - :param codes: codes to add size n * sa_code_size() - :type xids: int - :param xids: corresponding ids, size n - """ - return _swigfaiss.IndexIVF_add_sa_codes(self, n, codes, xids) - - def train_encoder(self, n, x, assign): - r""" - Train the encoder for the vectors. - - If by_residual then it is called with residuals and corresponding assign - array, otherwise x is the raw training vectors and assign=nullptr - """ - return _swigfaiss.IndexIVF_train_encoder(self, n, x, assign) - - def train_encoder_num_vectors(self): - r""" - can be redefined by subclasses to indicate how many training vectors - they need - """ - return _swigfaiss.IndexIVF_train_encoder_num_vectors(self) - - def search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params=None, stats=None): - return _swigfaiss.IndexIVF_search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params, stats) - - def range_search_preassigned(self, nx, x, radius, keys, coarse_dis, result, store_pairs=False, params=None, stats=None): - return _swigfaiss.IndexIVF_range_search_preassigned(self, nx, x, radius, keys, coarse_dis, result, store_pairs, params, stats) - - def search(self, n, x, k, distances, labels, params=None): - r"""assign the vectors, then call search_preassign""" - return _swigfaiss.IndexIVF_search(self, n, x, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexIVF_range_search(self, n, x, radius, result, params) - - def search1(self, x, handler, params=None): - r"""search one vector with a custom result handler""" - return _swigfaiss.IndexIVF_search1(self, x, handler, params) - - def get_InvertedListScanner(self, store_pairs=False, sel=None, params=None): - r""" - Get a scanner for this index (store_pairs means ignore labels) - - The default search implementation uses this to compute the distances. - Use sel instead of params->sel, because sel is initialized with - params->sel, but may get overridden by IndexIVF's internal logic. - """ - return _swigfaiss.IndexIVF_get_InvertedListScanner(self, store_pairs, sel, params) - - def reconstruct(self, key, recons): - r"""reconstruct a vector. Works only if maintain_direct_map is set to 1 or 2""" - return _swigfaiss.IndexIVF_reconstruct(self, key, recons) - - def update_vectors(self, nv, idx, v): - r""" - Update a subset of vectors. - - The index must have a direct_map - - :type nv: int - :param nv: nb of vectors to update - :type idx: int - :param idx: vector indices to update, size nv - :type v: float - :param v: vectors of new values, size nv*d - """ - return _swigfaiss.IndexIVF_update_vectors(self, nv, idx, v) - - def reconstruct_n(self, i0, ni, recons): - r""" - Reconstruct a subset of the indexed vectors. - - Overrides default implementation to bypass reconstruct() which requires - direct_map to be maintained. - - :type i0: int - :param i0: first vector to reconstruct - :type ni: int - :param ni: nb of vectors to reconstruct - :type recons: float - :param recons: output array of reconstructed vectors, size ni * d - """ - return _swigfaiss.IndexIVF_reconstruct_n(self, i0, ni, recons) - - def search_and_reconstruct(self, n, x, k, distances, labels, recons, params=None): - r""" - Similar to search, but also reconstructs the stored vectors (or an - approximation in the case of lossy coding) for the search results. - - Overrides default implementation to avoid having to maintain direct_map - and instead fetch the code offsets through the `store_pairs` flag in - search_preassigned(). - - :type recons: float - :param recons: reconstructed vectors size (n, k, d) - """ - return _swigfaiss.IndexIVF_search_and_reconstruct(self, n, x, k, distances, labels, recons, params) - - def search_and_return_codes(self, n, x, k, distances, labels, recons, include_listno=False, params=None): - r""" - Similar to search, but also returns the codes corresponding to the - stored vectors for the search results. - - :param codes: codes (n, k, code_size) - :type include_listno: boolean, optional - :param include_listno: - include the list ids in the code (in this case add - ceil(log8(nlist)) to the code size) - """ - return _swigfaiss.IndexIVF_search_and_return_codes(self, n, x, k, distances, labels, recons, include_listno, params) - - def reconstruct_from_offset(self, list_no, offset, recons): - r""" - Reconstruct a vector given the location in terms of (inv list index + - inv list offset) instead of the id. - - Useful for reconstructing when the direct_map is not maintained and - the inv list offset is computed by search_preassigned() with - `store_pairs` set. - """ - return _swigfaiss.IndexIVF_reconstruct_from_offset(self, list_no, offset, recons) - - def remove_ids(self, sel): - r"""Dataset manipulation functions""" - return _swigfaiss.IndexIVF_remove_ids(self, sel) - - def check_compatible_for_merge(self, otherIndex): - return _swigfaiss.IndexIVF_check_compatible_for_merge(self, otherIndex) - - def merge_from(self, otherIndex, add_id): - return _swigfaiss.IndexIVF_merge_from(self, otherIndex, add_id) - - def get_CodePacker(self): - return _swigfaiss.IndexIVF_get_CodePacker(self) - - def copy_subset_to(self, other, subset_type, a1, a2): - r""" - copy a subset of the entries index to the other index - see Invlists::copy_subset_to for the meaning of subset_type - """ - return _swigfaiss.IndexIVF_copy_subset_to(self, other, subset_type, a1, a2) - __swig_destroy__ = _swigfaiss.delete_IndexIVF - - def get_list_size(self, list_no): - return _swigfaiss.IndexIVF_get_list_size(self, list_no) - - def check_ids_sorted(self): - r"""are the ids sorted?""" - return _swigfaiss.IndexIVF_check_ids_sorted(self) - - def make_direct_map(self, new_maintain_direct_map=True): - r""" - initialize a direct map - - :type new_maintain_direct_map: boolean, optional - :param new_maintain_direct_map: if true, create a direct map, - else clear it - """ - return _swigfaiss.IndexIVF_make_direct_map(self, new_maintain_direct_map) - - def set_direct_map_type(self, type): - return _swigfaiss.IndexIVF_set_direct_map_type(self, type) - - def replace_invlists(self, il, own=False): - r"""replace the inverted lists, old one is deallocated if own_invlists""" - return _swigfaiss.IndexIVF_replace_invlists(self, il, own) - - def sa_code_size(self): - return _swigfaiss.IndexIVF_sa_code_size(self) - - def sa_encode(self, n, x, bytes): - r""" - encode a set of vectors - sa_encode will call encode_vectors with include_listno=true - :type n: int - :param n: nb of vectors to encode - :type x: float - :param x: the vectors to encode - :type bytes: uint8_t - :param bytes: output array for the codes - :rtype: void - :return: nb of bytes written to codes - """ - return _swigfaiss.IndexIVF_sa_encode(self, n, x, bytes) - -# Register IndexIVF in _swigfaiss: -_swigfaiss.IndexIVF_swigregister(IndexIVF) -class InvertedListScanner(object): - r""" - Object that handles a query. The inverted lists to scan are - provided externally. The object has a lot of state, but - distance_to_code and scan_codes can be called in multiple - threads - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - list_no = property(_swigfaiss.InvertedListScanner_list_no_get, _swigfaiss.InvertedListScanner_list_no_set, doc=r"""remember current list""") - keep_max = property(_swigfaiss.InvertedListScanner_keep_max_get, _swigfaiss.InvertedListScanner_keep_max_set, doc=r"""keep maximum instead of minimum""") - store_pairs = property(_swigfaiss.InvertedListScanner_store_pairs_get, _swigfaiss.InvertedListScanner_store_pairs_set, doc=r"""store positions in invlists rather than labels""") - sel = property(_swigfaiss.InvertedListScanner_sel_get, _swigfaiss.InvertedListScanner_sel_set, doc=r"""search in this subset of ids""") - code_size = property(_swigfaiss.InvertedListScanner_code_size_get, _swigfaiss.InvertedListScanner_code_size_set, doc=r"""used in default implementation of scan_codes""") - - def set_query(self, query_vector): - r"""from now on we handle this query.""" - return _swigfaiss.InvertedListScanner_set_query(self, query_vector) - - def set_list(self, list_no, coarse_dis): - r"""following codes come from this inverted list""" - return _swigfaiss.InvertedListScanner_set_list(self, list_no, coarse_dis) - - def distance_to_code(self, code): - r"""compute a single query-to-code distance""" - return _swigfaiss.InvertedListScanner_distance_to_code(self, code) - - def iterate_codes(self, iterator, distances, labels, k, list_size): - return _swigfaiss.InvertedListScanner_iterate_codes(self, iterator, distances, labels, k, list_size) - - def scan_codes_range(self, n, codes, ids, radius, result): - r""" - scan a set of codes, compute distances to current query and - update results if distances are below radius - - (default implementation fails) - """ - return _swigfaiss.InvertedListScanner_scan_codes_range(self, n, codes, ids, radius, result) - - def iterate_codes_range(self, iterator, radius, result, list_size): - return _swigfaiss.InvertedListScanner_iterate_codes_range(self, iterator, radius, result, list_size) - - def scan_codes(self, *args): - r""" - scan a set of codes, compute distances to current query, and - update heap of results if necessary. Default implementation - calls distance_to_code. - - :type n: int - :param n: number of codes to scan - :type codes: uint8_t - :param codes: codes to scan (n * code_size) - :type ids: int - :param ids: corresponding ids (ignored if store_pairs) - :type distances: float - :param distances: heap distances (size k) - :type labels: int - :param labels: heap labels (size k) - :type k: int - :param k: heap size - :rtype: int - :return: number of heap updates performed - """ - return _swigfaiss.InvertedListScanner_scan_codes(self, *args) - __swig_destroy__ = _swigfaiss.delete_InvertedListScanner - -# Register InvertedListScanner in _swigfaiss: -_swigfaiss.InvertedListScanner_swigregister(InvertedListScanner) -class IndexIVFStats(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nq = property(_swigfaiss.IndexIVFStats_nq_get, _swigfaiss.IndexIVFStats_nq_set) - nlist = property(_swigfaiss.IndexIVFStats_nlist_get, _swigfaiss.IndexIVFStats_nlist_set) - ndis = property(_swigfaiss.IndexIVFStats_ndis_get, _swigfaiss.IndexIVFStats_ndis_set) - nheap_updates = property(_swigfaiss.IndexIVFStats_nheap_updates_get, _swigfaiss.IndexIVFStats_nheap_updates_set) - quantization_time = property(_swigfaiss.IndexIVFStats_quantization_time_get, _swigfaiss.IndexIVFStats_quantization_time_set) - search_time = property(_swigfaiss.IndexIVFStats_search_time_get, _swigfaiss.IndexIVFStats_search_time_set) - - def __init__(self): - _swigfaiss.IndexIVFStats_swiginit(self, _swigfaiss.new_IndexIVFStats()) - - def reset(self): - return _swigfaiss.IndexIVFStats_reset(self) - - def add(self, other): - return _swigfaiss.IndexIVFStats_add(self, other) - __swig_destroy__ = _swigfaiss.delete_IndexIVFStats - -# Register IndexIVFStats in _swigfaiss: -_swigfaiss.IndexIVFStats_swigregister(IndexIVFStats) - -def check_compatible_for_merge(index1, index2): - r""" - check if two indexes have the same parameters and are trained in - the same way, otherwise throw. - """ - return _swigfaiss.check_compatible_for_merge(index1, index2) - -def extract_index_ivf(*args): - r""" - get an IndexIVF from an index. The index may be an IndexIVF or - some wrapper class that encloses an IndexIVF - - throws an exception if this is not the case. - """ - return _swigfaiss.extract_index_ivf(*args) - -def try_extract_index_ivf(*args): - r"""same as above but returns nullptr instead of throwing on failure""" - return _swigfaiss.try_extract_index_ivf(*args) - -def merge_into(index0, index1, shift_ids): - r""" - Merge index1 into index0. Works on IndexIVF's and IndexIVF's - embedded in a IndexPreTransform. On output, the index1 is empty. - - :type shift_ids: boolean - :param shift_ids:: translate the ids from index1 to index0->prev_ntotal - """ - return _swigfaiss.merge_into(index0, index1, shift_ids) - -def search_centroid(index, x, n, centroid_ids): - return _swigfaiss.search_centroid(index, x, n, centroid_ids) - -def search_and_return_centroids(index, n, xin, k, distances, labels, query_centroid_ids, result_centroid_ids): - return _swigfaiss.search_and_return_centroids(index, n, xin, k, distances, labels, query_centroid_ids, result_centroid_ids) -class SlidingIndexWindow(object): - r""" - A set of IndexIVFs concatenated together in a FIFO fashion. - at each "step", the oldest index slice is removed and a new index is added. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - index = property(_swigfaiss.SlidingIndexWindow_index_get, _swigfaiss.SlidingIndexWindow_index_set, doc=r"""common index that contains the sliding window""") - ils = property(_swigfaiss.SlidingIndexWindow_ils_get, _swigfaiss.SlidingIndexWindow_ils_set, doc=r"""InvertedLists of index""") - n_slice = property(_swigfaiss.SlidingIndexWindow_n_slice_get, _swigfaiss.SlidingIndexWindow_n_slice_set, doc=r"""number of slices currently in index""") - nlist = property(_swigfaiss.SlidingIndexWindow_nlist_get, _swigfaiss.SlidingIndexWindow_nlist_set, doc=r"""same as index->nlist""") - sizes = property(_swigfaiss.SlidingIndexWindow_sizes_get, _swigfaiss.SlidingIndexWindow_sizes_set, doc=r"""cumulative list sizes at each slice""") - - def __init__(self, index): - r"""index should be initially empty and trained""" - _swigfaiss.SlidingIndexWindow_swiginit(self, _swigfaiss.new_SlidingIndexWindow(index)) - - def step(self, sub_index, remove_oldest): - r""" - Add one index to the current index and remove the oldest one. - - :type sub_index: :py:class:`Index` - :param sub_index: slice to swap in (can be NULL) - :type remove_oldest: boolean - :param remove_oldest: if true, remove the oldest slices - """ - return _swigfaiss.SlidingIndexWindow_step(self, sub_index, remove_oldest) - __swig_destroy__ = _swigfaiss.delete_SlidingIndexWindow - -# Register SlidingIndexWindow in _swigfaiss: -_swigfaiss.SlidingIndexWindow_swigregister(SlidingIndexWindow) - -def get_invlist_range(index, i0, i1): - r"""Get a subset of inverted lists [i0, i1)""" - return _swigfaiss.get_invlist_range(index, i0, i1) - -def set_invlist_range(index, i0, i1, src): - r"""Set a subset of inverted lists""" - return _swigfaiss.set_invlist_range(index, i0, i1, src) - -def search_with_parameters(index, n, x, k, distances, labels, params, nb_dis=None, ms_per_stage=None): - r""" - search an IndexIVF, possibly embedded in an IndexPreTransform with - given parameters. This is a way to set the nprobe and get - statdistics in a thread-safe way. - - Optionally returns (if non-nullptr): - - nb_dis: number of distances computed - - ms_per_stage: [0]: preprocessing time - [1]: coarse quantization, - [2]: list scanning - """ - return _swigfaiss.search_with_parameters(index, n, x, k, distances, labels, params, nb_dis, ms_per_stage) - -def range_search_with_parameters(index, n, x, radius, result, params, nb_dis=None, ms_per_stage=None): - r"""same as search_with_parameters but for range search""" - return _swigfaiss.range_search_with_parameters(index, n, x, radius, result, params, nb_dis, ms_per_stage) - -def ivf_residual_from_quantizer(arg1, nlevel): - r""" - Build an IndexIVFResidualQuantizer from an ResidualQuantizer, using the - nlevel first components as coarse quantizer and the rest as codes in invlists - """ - return _swigfaiss.ivf_residual_from_quantizer(arg1, nlevel) - -def ivf_residual_add_from_flat_codes(ivfrq, ncode, codes, code_size=-1): - r""" - add from codes. NB that the norm component is not used, so the code_size can - be provided. - - :type ivfrq: :py:class:`IndexIVFResidualQuantizer` - :param ivfrq: index to populate with the codes - :type codes: uint8_t - :param codes: codes to add, size (ncode, code_size) - :type code_size: int, optional - :param code_size: override the ivfrq's code_size, useful if the norm encoding - is different - """ - return _swigfaiss.ivf_residual_add_from_flat_codes(ivfrq, ncode, codes, code_size) -class ShardingFunction(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - - def __call__(self, i, shard_count): - return _swigfaiss.ShardingFunction___call__(self, i, shard_count) - __swig_destroy__ = _swigfaiss.delete_ShardingFunction - -# Register ShardingFunction in _swigfaiss: -_swigfaiss.ShardingFunction_swigregister(ShardingFunction) -class DefaultShardingFunction(ShardingFunction): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __call__(self, i, shard_count): - return _swigfaiss.DefaultShardingFunction___call__(self, i, shard_count) - - def __init__(self): - _swigfaiss.DefaultShardingFunction_swiginit(self, _swigfaiss.new_DefaultShardingFunction()) - __swig_destroy__ = _swigfaiss.delete_DefaultShardingFunction - -# Register DefaultShardingFunction in _swigfaiss: -_swigfaiss.DefaultShardingFunction_swigregister(DefaultShardingFunction) - -def shard_ivf_index_centroids(*args): - r""" - Shards an IVF index centroids by the given sharding function, and writes - the index to the path given by filename_generator. The centroids must already - be added to the index quantizer. - - :type index: :py:class:`IndexIVF` - :param index: The IVF index containing centroids to shard. - :type shard_count: int, optional - :param shard_count: Number of shards. - :type filename_template: string, optional - :param filename_template: Template for shard filenames. - :type sharding_function: :py:class:`ShardingFunction`, optional - :param sharding_function: The function to shard by. The default is ith vector - mod shard_count. - :type generate_ids: boolean, optional - :param generate_ids: Generates ids using IndexIDMap2. If true, ids will - match the default ids in the unsharded index. - :rtype: void - :return: The number of shards written. - """ - return _swigfaiss.shard_ivf_index_centroids(*args) - -def shard_binary_ivf_index_centroids(*args): - return _swigfaiss.shard_binary_ivf_index_centroids(*args) -class ScalarQuantizer(Quantizer): - r""" - The uniform quantizer has a range [vmin, vmax]. The range can be - the same for all dimensions (uniform) or specific per dimension - (default). - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - QT_8bit = _swigfaiss.ScalarQuantizer_QT_8bit - r"""8 bits per component""" - QT_4bit = _swigfaiss.ScalarQuantizer_QT_4bit - r"""4 bits per component""" - QT_8bit_uniform = _swigfaiss.ScalarQuantizer_QT_8bit_uniform - r"""same, shared range for all dimensions""" - QT_4bit_uniform = _swigfaiss.ScalarQuantizer_QT_4bit_uniform - QT_fp16 = _swigfaiss.ScalarQuantizer_QT_fp16 - QT_8bit_direct = _swigfaiss.ScalarQuantizer_QT_8bit_direct - r"""fast indexing of uint8s""" - QT_6bit = _swigfaiss.ScalarQuantizer_QT_6bit - r"""6 bits per component""" - QT_bf16 = _swigfaiss.ScalarQuantizer_QT_bf16 - QT_8bit_direct_signed = _swigfaiss.ScalarQuantizer_QT_8bit_direct_signed - r""" - fast indexing of signed int8s ranging from - [-128 to 127] - """ - QT_0bit = _swigfaiss.ScalarQuantizer_QT_0bit - r"""0 bits per component, centroid-only distance (for IVF)""" - QT_1bit_tqmse = _swigfaiss.ScalarQuantizer_QT_1bit_tqmse - r"""TurboQuant MSE-optimized, 1 bit per component""" - QT_2bit_tqmse = _swigfaiss.ScalarQuantizer_QT_2bit_tqmse - r"""TurboQuant MSE-optimized, 2 bits per component""" - QT_3bit_tqmse = _swigfaiss.ScalarQuantizer_QT_3bit_tqmse - r"""TurboQuant MSE-optimized, 3 bits per component""" - QT_4bit_tqmse = _swigfaiss.ScalarQuantizer_QT_4bit_tqmse - r"""TurboQuant MSE-optimized, 4 bits per component""" - QT_8bit_tqmse = _swigfaiss.ScalarQuantizer_QT_8bit_tqmse - r"""TurboQuant MSE-optimized, 8 bits per component""" - QT_2bit_tq = _swigfaiss.ScalarQuantizer_QT_2bit_tq - r"""Full TurboQuant (1-bit MSE + 1-bit QJL + factors)""" - QT_3bit_tq = _swigfaiss.ScalarQuantizer_QT_3bit_tq - r"""Full TurboQuant (2-bit MSE + 1-bit QJL + factors)""" - QT_4bit_tq = _swigfaiss.ScalarQuantizer_QT_4bit_tq - r"""Full TurboQuant (3-bit MSE + 1-bit QJL + factors)""" - QT_5bit_tq = _swigfaiss.ScalarQuantizer_QT_5bit_tq - r"""Full TurboQuant (4-bit MSE + 1-bit QJL + factors)""" - QT_1bit_eden = _swigfaiss.ScalarQuantizer_QT_1bit_eden - r"""EDEN Lloyd-Max scalar code, 1 bit per component""" - QT_2bit_eden = _swigfaiss.ScalarQuantizer_QT_2bit_eden - r"""EDEN Lloyd-Max scalar code, 2 bits per component""" - QT_3bit_eden = _swigfaiss.ScalarQuantizer_QT_3bit_eden - r"""EDEN Lloyd-Max scalar code, 3 bits per component""" - QT_4bit_eden = _swigfaiss.ScalarQuantizer_QT_4bit_eden - r"""EDEN Lloyd-Max scalar code, 4 bits per component""" - QT_5bit_eden = _swigfaiss.ScalarQuantizer_QT_5bit_eden - r"""EDEN Lloyd-Max scalar code, 5 bits per component""" - QT_6bit_eden = _swigfaiss.ScalarQuantizer_QT_6bit_eden - r"""EDEN Lloyd-Max scalar code, 6 bits per component""" - QT_7bit_eden = _swigfaiss.ScalarQuantizer_QT_7bit_eden - r"""EDEN Lloyd-Max scalar code, 7 bits per component""" - QT_8bit_eden = _swigfaiss.ScalarQuantizer_QT_8bit_eden - r"""EDEN Lloyd-Max scalar code, 8 bits per component""" - QT_count = _swigfaiss.ScalarQuantizer_QT_count - qtype = property(_swigfaiss.ScalarQuantizer_qtype_get, _swigfaiss.ScalarQuantizer_qtype_set) - RS_minmax = _swigfaiss.ScalarQuantizer_RS_minmax - r"""[min - rs*(max-min), max + rs*(max-min)]""" - RS_meanstd = _swigfaiss.ScalarQuantizer_RS_meanstd - r"""[mean - std * rs, mean + std * rs]""" - RS_quantiles = _swigfaiss.ScalarQuantizer_RS_quantiles - r"""[Q(rs), Q(1-rs)]""" - RS_optim = _swigfaiss.ScalarQuantizer_RS_optim - r"""alternate optimization of reconstruction error""" - rangestat = property(_swigfaiss.ScalarQuantizer_rangestat_get, _swigfaiss.ScalarQuantizer_rangestat_set) - rangestat_arg = property(_swigfaiss.ScalarQuantizer_rangestat_arg_get, _swigfaiss.ScalarQuantizer_rangestat_arg_set) - bits = property(_swigfaiss.ScalarQuantizer_bits_get, _swigfaiss.ScalarQuantizer_bits_set, doc=r"""bits per scalar code""") - trained = property(_swigfaiss.ScalarQuantizer_trained_get, _swigfaiss.ScalarQuantizer_trained_set, doc=r"""trained values (including the range)""") - - def __init__(self, *args): - _swigfaiss.ScalarQuantizer_swiginit(self, _swigfaiss.new_ScalarQuantizer(*args)) - - def set_derived_sizes(self): - r"""updates internal values based on qtype and d""" - return _swigfaiss.ScalarQuantizer_set_derived_sizes(self) - - def train(self, n, x): - return _swigfaiss.ScalarQuantizer_train(self, n, x) - - def compute_codes(self, x, codes, n): - r""" - Encode a set of vectors - - :type x: float - :param x: vectors to encode, size n * d - :type codes: uint8_t - :param codes: output codes, size n * code_size - """ - return _swigfaiss.ScalarQuantizer_compute_codes(self, x, codes, n) - - def decode(self, code, x, n): - r""" - Decode a set of vectors - - :param codes: codes to decode, size n * code_size - :type x: float - :param x: output vectors, size n * d - """ - return _swigfaiss.ScalarQuantizer_decode(self, code, x, n) - - def select_quantizer(self): - return _swigfaiss.ScalarQuantizer_select_quantizer(self) - turboq_refine = property(_swigfaiss.ScalarQuantizer_turboq_refine_get, _swigfaiss.ScalarQuantizer_turboq_refine_set) - - def get_distance_computer(self, *args): - return _swigfaiss.ScalarQuantizer_get_distance_computer(self, *args) - - def select_InvertedListScanner(self, mt, quantizer, store_pairs, sel, by_residual=False): - return _swigfaiss.ScalarQuantizer_select_InvertedListScanner(self, mt, quantizer, store_pairs, sel, by_residual) - __swig_destroy__ = _swigfaiss.delete_ScalarQuantizer - -# Register ScalarQuantizer in _swigfaiss: -_swigfaiss.ScalarQuantizer_swigregister(ScalarQuantizer) -class IndexScalarQuantizer(IndexFlatCodes): - r"""Flat index built on a scalar quantizer.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - sq = property(_swigfaiss.IndexScalarQuantizer_sq_get, _swigfaiss.IndexScalarQuantizer_sq_set, doc=r"""Used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type qtype: int - :param qtype: type of scalar quantizer (e.g., QT_4bit) - :type metric: int, optional - :param metric: distance metric used for search (default: METRIC_L2) - """ - _swigfaiss.IndexScalarQuantizer_swiginit(self, _swigfaiss.new_IndexScalarQuantizer(*args)) - - def train(self, n, x): - return _swigfaiss.IndexScalarQuantizer_train(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexScalarQuantizer_search(self, n, x, k, distances, labels, params) - - def get_FlatCodesDistanceComputer(self): - return _swigfaiss.IndexScalarQuantizer_get_FlatCodesDistanceComputer(self) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexScalarQuantizer_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexScalarQuantizer_sa_decode(self, n, bytes, x) - __swig_destroy__ = _swigfaiss.delete_IndexScalarQuantizer - -# Register IndexScalarQuantizer in _swigfaiss: -_swigfaiss.IndexScalarQuantizer_swigregister(IndexScalarQuantizer) -class IVFSQTurboQSearchParameters(SearchParametersIVF): - r""" - An IVF implementation where the components of the residuals are - encoded with a scalar quantizer. All distance computations - are asymmetric, so the encoded vectors are decoded and approximate - distances are computed. - - Search parameters for TurboQuant full types (QT_*_tq). - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - qb = property(_swigfaiss.IVFSQTurboQSearchParameters_qb_get, _swigfaiss.IVFSQTurboQSearchParameters_qb_set, doc=r""" - Query quantization bits for integer MSE pre-screening. - 0 = float path (default), 1-8 = integer popcount path. - """) - int_qjl = property(_swigfaiss.IVFSQTurboQSearchParameters_int_qjl_get, _swigfaiss.IVFSQTurboQSearchParameters_int_qjl_set, doc=r"""Also use integer popcount for QJL stage (requires qb > 0).""") - - def __init__(self): - _swigfaiss.IVFSQTurboQSearchParameters_swiginit(self, _swigfaiss.new_IVFSQTurboQSearchParameters()) - __swig_destroy__ = _swigfaiss.delete_IVFSQTurboQSearchParameters - -# Register IVFSQTurboQSearchParameters in _swigfaiss: -_swigfaiss.IVFSQTurboQSearchParameters_swigregister(IVFSQTurboQSearchParameters) -class IndexIVFScalarQuantizer(IndexIVF): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - sq = property(_swigfaiss.IndexIVFScalarQuantizer_sq_get, _swigfaiss.IndexIVFScalarQuantizer_sq_set) - - def __init__(self, *args): - _swigfaiss.IndexIVFScalarQuantizer_swiginit(self, _swigfaiss.new_IndexIVFScalarQuantizer(*args)) - - def train_encoder(self, n, x, assign): - return _swigfaiss.IndexIVFScalarQuantizer_train_encoder(self, n, x, assign) - - def train_encoder_num_vectors(self): - return _swigfaiss.IndexIVFScalarQuantizer_train_encoder_num_vectors(self) - - def encode_vectors(self, n, x, list_nos, codes, include_listnos=False): - return _swigfaiss.IndexIVFScalarQuantizer_encode_vectors(self, n, x, list_nos, codes, include_listnos) - - def decode_vectors(self, n, codes, list_nos, x): - return _swigfaiss.IndexIVFScalarQuantizer_decode_vectors(self, n, codes, list_nos, x) - - def add_core(self, n, x, xids, precomputed_idx, inverted_list_context=None): - return _swigfaiss.IndexIVFScalarQuantizer_add_core(self, n, x, xids, precomputed_idx, inverted_list_context) - - def get_InvertedListScanner(self, store_pairs, sel, params): - return _swigfaiss.IndexIVFScalarQuantizer_get_InvertedListScanner(self, store_pairs, sel, params) - - def reconstruct_from_offset(self, list_no, offset, recons): - return _swigfaiss.IndexIVFScalarQuantizer_reconstruct_from_offset(self, list_no, offset, recons) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexIVFScalarQuantizer_sa_decode(self, n, bytes, x) - __swig_destroy__ = _swigfaiss.delete_IndexIVFScalarQuantizer - -# Register IndexIVFScalarQuantizer in _swigfaiss: -_swigfaiss.IndexIVFScalarQuantizer_swigregister(IndexIVFScalarQuantizer) -class IndexIVFSpectralHash(IndexIVF): - r""" - Inverted list that stores binary codes of size nbit. Before the - binary conversion, the dimension of the vectors is transformed from - dim d into dim nbit by vt (a random rotation by default). - - Each coordinate is subtracted from a value determined by - threshold_type, and split into intervals of size period. Half of - the interval is a 0 bit, the other half a 1. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - vt = property(_swigfaiss.IndexIVFSpectralHash_vt_get, _swigfaiss.IndexIVFSpectralHash_vt_set, doc=r"""transformation from d to nbit dim""") - own_fields = property(_swigfaiss.IndexIVFSpectralHash_own_fields_get, _swigfaiss.IndexIVFSpectralHash_own_fields_set, doc=r"""own the vt""") - nbit = property(_swigfaiss.IndexIVFSpectralHash_nbit_get, _swigfaiss.IndexIVFSpectralHash_nbit_set, doc=r"""nb of bits of the binary signature""") - period = property(_swigfaiss.IndexIVFSpectralHash_period_get, _swigfaiss.IndexIVFSpectralHash_period_set, doc=r"""interval size for 0s and 1s""") - Thresh_global = _swigfaiss.IndexIVFSpectralHash_Thresh_global - r"""global threshold at 0""" - Thresh_centroid = _swigfaiss.IndexIVFSpectralHash_Thresh_centroid - r"""compare to centroid""" - Thresh_centroid_half = _swigfaiss.IndexIVFSpectralHash_Thresh_centroid_half - r"""central interval around centroid""" - Thresh_median = _swigfaiss.IndexIVFSpectralHash_Thresh_median - r"""median of training set""" - threshold_type = property(_swigfaiss.IndexIVFSpectralHash_threshold_type_get, _swigfaiss.IndexIVFSpectralHash_threshold_type_set) - trained = property(_swigfaiss.IndexIVFSpectralHash_trained_get, _swigfaiss.IndexIVFSpectralHash_trained_set, doc=r""" - Trained threshold. - size nlist * nbit or 0 if Thresh_global - """) - - def __init__(self, *args): - _swigfaiss.IndexIVFSpectralHash_swiginit(self, _swigfaiss.new_IndexIVFSpectralHash(*args)) - - def train_encoder(self, n, x, assign): - return _swigfaiss.IndexIVFSpectralHash_train_encoder(self, n, x, assign) - - def encode_vectors(self, n, x, list_nos, codes, include_listnos=False): - return _swigfaiss.IndexIVFSpectralHash_encode_vectors(self, n, x, list_nos, codes, include_listnos) - - def get_InvertedListScanner(self, store_pairs, sel, params): - return _swigfaiss.IndexIVFSpectralHash_get_InvertedListScanner(self, store_pairs, sel, params) - - def replace_vt(self, *args): - r""" - *Overload 1:* - replace the vector transform for an empty (and possibly untrained) index - - | - - *Overload 2:* - convenience function to get the VT from an index constructed by an - index_factory (should end in "LSH") - - | - - *Overload 3:* - convenience function to get the VT from an index constructed by an - index_factory (should end in "LSH") - """ - return _swigfaiss.IndexIVFSpectralHash_replace_vt(self, *args) - __swig_destroy__ = _swigfaiss.delete_IndexIVFSpectralHash - -# Register IndexIVFSpectralHash in _swigfaiss: -_swigfaiss.IndexIVFSpectralHash_swigregister(IndexIVFSpectralHash) -class IndexIVFAdditiveQuantizer(IndexIVF): - r""" - Abstract class for IVF additive quantizers. - The search functions are in common. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - aq = property(_swigfaiss.IndexIVFAdditiveQuantizer_aq_get, _swigfaiss.IndexIVFAdditiveQuantizer_aq_set) - use_precomputed_table = property(_swigfaiss.IndexIVFAdditiveQuantizer_use_precomputed_table_get, _swigfaiss.IndexIVFAdditiveQuantizer_use_precomputed_table_set) - - def __init__(self, *args): - _swigfaiss.IndexIVFAdditiveQuantizer_swiginit(self, _swigfaiss.new_IndexIVFAdditiveQuantizer(*args)) - - def train_encoder(self, n, x, assign): - return _swigfaiss.IndexIVFAdditiveQuantizer_train_encoder(self, n, x, assign) - - def train_encoder_num_vectors(self): - return _swigfaiss.IndexIVFAdditiveQuantizer_train_encoder_num_vectors(self) - - def encode_vectors(self, n, x, list_nos, codes, include_listnos=False): - return _swigfaiss.IndexIVFAdditiveQuantizer_encode_vectors(self, n, x, list_nos, codes, include_listnos) - - def decode_vectors(self, n, codes, list_nos, x): - return _swigfaiss.IndexIVFAdditiveQuantizer_decode_vectors(self, n, codes, list_nos, x) - - def get_InvertedListScanner(self, store_pairs, sel, params): - return _swigfaiss.IndexIVFAdditiveQuantizer_get_InvertedListScanner(self, store_pairs, sel, params) - - def sa_decode(self, n, codes, x): - return _swigfaiss.IndexIVFAdditiveQuantizer_sa_decode(self, n, codes, x) - - def reconstruct_from_offset(self, list_no, offset, recons): - return _swigfaiss.IndexIVFAdditiveQuantizer_reconstruct_from_offset(self, list_no, offset, recons) - __swig_destroy__ = _swigfaiss.delete_IndexIVFAdditiveQuantizer - -# Register IndexIVFAdditiveQuantizer in _swigfaiss: -_swigfaiss.IndexIVFAdditiveQuantizer_swigregister(IndexIVFAdditiveQuantizer) -class IndexIVFResidualQuantizer(IndexIVFAdditiveQuantizer): - r""" - IndexIVF based on a residual quantizer. Stored vectors are - approximated by residual quantization codes. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - rq = property(_swigfaiss.IndexIVFResidualQuantizer_rq_get, _swigfaiss.IndexIVFResidualQuantizer_rq_set, doc=r"""The residual quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :param M: number of subquantizers - :type nbits: std::vector< size_t > - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexIVFResidualQuantizer_swiginit(self, _swigfaiss.new_IndexIVFResidualQuantizer(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFResidualQuantizer - -# Register IndexIVFResidualQuantizer in _swigfaiss: -_swigfaiss.IndexIVFResidualQuantizer_swigregister(IndexIVFResidualQuantizer) -class IndexIVFLocalSearchQuantizer(IndexIVFAdditiveQuantizer): - r""" - IndexIVF based on a residual quantizer. Stored vectors are - approximated by residual quantization codes. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - lsq = property(_swigfaiss.IndexIVFLocalSearchQuantizer_lsq_get, _swigfaiss.IndexIVFLocalSearchQuantizer_lsq_set, doc=r"""The LSQ quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexIVFLocalSearchQuantizer_swiginit(self, _swigfaiss.new_IndexIVFLocalSearchQuantizer(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFLocalSearchQuantizer - -# Register IndexIVFLocalSearchQuantizer in _swigfaiss: -_swigfaiss.IndexIVFLocalSearchQuantizer_swigregister(IndexIVFLocalSearchQuantizer) -class IndexIVFProductResidualQuantizer(IndexIVFAdditiveQuantizer): - r""" - IndexIVF based on a product residual quantizer. Stored vectors are - approximated by product residual quantization codes. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - prq = property(_swigfaiss.IndexIVFProductResidualQuantizer_prq_get, _swigfaiss.IndexIVFProductResidualQuantizer_prq_set, doc=r"""The product residual quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of residual quantizers - :type Msub: int - :param Msub: number of subquantizers per RQ - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexIVFProductResidualQuantizer_swiginit(self, _swigfaiss.new_IndexIVFProductResidualQuantizer(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFProductResidualQuantizer - -# Register IndexIVFProductResidualQuantizer in _swigfaiss: -_swigfaiss.IndexIVFProductResidualQuantizer_swigregister(IndexIVFProductResidualQuantizer) -class IndexIVFProductLocalSearchQuantizer(IndexIVFAdditiveQuantizer): - r""" - IndexIVF based on a product local search quantizer. Stored vectors are - approximated by product local search quantization codes. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - plsq = property(_swigfaiss.IndexIVFProductLocalSearchQuantizer_plsq_get, _swigfaiss.IndexIVFProductLocalSearchQuantizer_plsq_set, doc=r"""The product local search quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of local search quantizers - :type Msub: int - :param Msub: number of subquantizers per LSQ - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexIVFProductLocalSearchQuantizer_swiginit(self, _swigfaiss.new_IndexIVFProductLocalSearchQuantizer(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFProductLocalSearchQuantizer - -# Register IndexIVFProductLocalSearchQuantizer in _swigfaiss: -_swigfaiss.IndexIVFProductLocalSearchQuantizer_swigregister(IndexIVFProductLocalSearchQuantizer) -class SearchParametersHNSW(SearchParameters): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - efSearch = property(_swigfaiss.SearchParametersHNSW_efSearch_get, _swigfaiss.SearchParametersHNSW_efSearch_set) - check_relative_distance = property(_swigfaiss.SearchParametersHNSW_check_relative_distance_get, _swigfaiss.SearchParametersHNSW_check_relative_distance_set) - bounded_queue = property(_swigfaiss.SearchParametersHNSW_bounded_queue_get, _swigfaiss.SearchParametersHNSW_bounded_queue_set) - __swig_destroy__ = _swigfaiss.delete_SearchParametersHNSW - - def __init__(self): - _swigfaiss.SearchParametersHNSW_swiginit(self, _swigfaiss.new_SearchParametersHNSW()) - -# Register SearchParametersHNSW in _swigfaiss: -_swigfaiss.SearchParametersHNSW_swigregister(SearchParametersHNSW) -class HNSW(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - assign_probas = property(_swigfaiss.HNSW_assign_probas_get, _swigfaiss.HNSW_assign_probas_set, doc=r"""assignment probability to each layer (sum=1)""") - cum_nneighbor_per_level = property(_swigfaiss.HNSW_cum_nneighbor_per_level_get, _swigfaiss.HNSW_cum_nneighbor_per_level_set, doc=r""" - number of neighbors stored per layer (cumulative), should not - be changed after first add - """) - levels = property(_swigfaiss.HNSW_levels_get, _swigfaiss.HNSW_levels_set, doc=r"""level of each vector (base level = 1), size = ntotal""") - offsets = property(_swigfaiss.HNSW_offsets_get, _swigfaiss.HNSW_offsets_set, doc=r""" - offsets[i] is the offset in the neighbors array where vector i is stored - size ntotal + 1 - """) - neighbors = property(_swigfaiss.HNSW_neighbors_get, _swigfaiss.HNSW_neighbors_set, doc=r""" - neighbors[offsets[i]:offsets[i+1]] is the list of neighbors of vector i - for all levels. this is where all storage goes. - """) - entry_point = property(_swigfaiss.HNSW_entry_point_get, _swigfaiss.HNSW_entry_point_set, doc=r""" - entry point in the search structure (one of the points with maximum - level - """) - rng = property(_swigfaiss.HNSW_rng_get, _swigfaiss.HNSW_rng_set) - max_level = property(_swigfaiss.HNSW_max_level_get, _swigfaiss.HNSW_max_level_set, doc=r"""maximum level""") - efConstruction = property(_swigfaiss.HNSW_efConstruction_get, _swigfaiss.HNSW_efConstruction_set, doc=r"""expansion factor at construction time""") - efSearch = property(_swigfaiss.HNSW_efSearch_get, _swigfaiss.HNSW_efSearch_set, doc=r"""expansion factor at search time""") - prune_headroom = property(_swigfaiss.HNSW_prune_headroom_get, _swigfaiss.HNSW_prune_headroom_set, doc=r""" - when pruning, leave room for more neighbors to avoid O(n^2) - costs and lock contention on frequently-pruned nodes. - """) - check_relative_distance = property(_swigfaiss.HNSW_check_relative_distance_get, _swigfaiss.HNSW_check_relative_distance_set, doc=r""" - during search: do we check whether the next best distance is good - enough? - """) - search_bounded_queue = property(_swigfaiss.HNSW_search_bounded_queue_get, _swigfaiss.HNSW_search_bounded_queue_set, doc=r"""use bounded queue during exploration""") - is_panorama = property(_swigfaiss.HNSW_is_panorama_get, _swigfaiss.HNSW_is_panorama_set, doc=r"""use Panorama progressive pruning in search""") - is_similarity = property(_swigfaiss.HNSW_is_similarity_get, _swigfaiss.HNSW_is_similarity_set, doc=r""" - distance comparison semantics: when true, distances are treated as - similarity scores (larger is better). Default false matches the - historical L2/Hamming behavior (smaller is better). - Not serialized: must be re-set by the owning Index after loading. - """) - use_visited_hashset = property(_swigfaiss.HNSW_use_visited_hashset_get, _swigfaiss.HNSW_use_visited_hashset_set) - - def set_default_probas(self, M, levelMult): - r""" - initialize the assign_probas and cum_nneighbor_per_level to - have 2*M links on level 0 and M links on levels > 0 - """ - return _swigfaiss.HNSW_set_default_probas(self, M, levelMult) - - def set_nb_neighbors(self, level_no, n): - r"""set nb of neighbors for this level (before adding anything)""" - return _swigfaiss.HNSW_set_nb_neighbors(self, level_no, n) - - def nb_neighbors(self, layer_no): - r"""nb of neighbors for this level""" - return _swigfaiss.HNSW_nb_neighbors(self, layer_no) - - def cum_nb_neighbors(self, layer_no): - r"""cumulative nb up to (and excluding) this level""" - return _swigfaiss.HNSW_cum_nb_neighbors(self, layer_no) - - def neighbor_range(self, no, layer_no, begin, end): - r"""range of entries in the neighbors table of vertex no at layer_no""" - return _swigfaiss.HNSW_neighbor_range(self, no, layer_no, begin, end) - - def __init__(self, M=32): - r"""only mandatory parameter: nb of neighbors""" - _swigfaiss.HNSW_swiginit(self, _swigfaiss.new_HNSW(M)) - - def random_level(self): - r"""pick a random level for a new point""" - return _swigfaiss.HNSW_random_level(self) - - def fill_with_random_links(self, n): - r"""add n random levels to table (for debugging...)""" - return _swigfaiss.HNSW_fill_with_random_links(self, n) - - def add_links_starting_from(self, ptdis, pt_id, nearest, d_nearest, level, locks, vt, keep_max_size_level0=False): - return _swigfaiss.HNSW_add_links_starting_from(self, ptdis, pt_id, nearest, d_nearest, level, locks, vt, keep_max_size_level0) - - def add_with_locks(self, ptdis, pt_level, pt_id, locks, vt, keep_max_size_level0=False): - r""" - add point pt_id on all levels <= pt_level and build the link - structure for them. - """ - return _swigfaiss.HNSW_add_with_locks(self, ptdis, pt_level, pt_id, locks, vt, keep_max_size_level0) - - def search(self, qdis, index, res, vt, params=None): - r""" - Search interface for 1 point, single thread - - NOTE: We pass a reference to the index itself to allow for additional - state information to be passed (used for Panorama progressive pruning). - The alternative would be to override both HNSW::search and - HNSWIndex::search, which would be a nuisance of code duplication. - """ - return _swigfaiss.HNSW_search(self, qdis, index, res, vt, params) - - def search_level_0(self, qdis, res, nprobe, nearest_i, nearest_d, search_type, search_stats, vt, params=None): - r"""search only in level 0 from a given vertex""" - return _swigfaiss.HNSW_search_level_0(self, qdis, res, nprobe, nearest_i, nearest_d, search_type, search_stats, vt, params) - - def reset(self): - return _swigfaiss.HNSW_reset(self) - - def clear_neighbor_tables(self, level): - return _swigfaiss.HNSW_clear_neighbor_tables(self, level) - - def print_neighbor_stats(self, level): - return _swigfaiss.HNSW_print_neighbor_stats(self, level) - - def prepare_level_tab(self, n, preset_levels=False): - return _swigfaiss.HNSW_prepare_level_tab(self, n, preset_levels) - - def permute_entries(self, map): - return _swigfaiss.HNSW_permute_entries(self, map) - __swig_destroy__ = _swigfaiss.delete_HNSW - -# Register HNSW in _swigfaiss: -_swigfaiss.HNSW_swigregister(HNSW) -class HNSWStats(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - n1 = property(_swigfaiss.HNSWStats_n1_get, _swigfaiss.HNSWStats_n1_set) - n2 = property(_swigfaiss.HNSWStats_n2_get, _swigfaiss.HNSWStats_n2_set, doc=r"""number of vectors searched""") - ndis = property(_swigfaiss.HNSWStats_ndis_get, _swigfaiss.HNSWStats_ndis_set, doc=r"""number of queries for which the candidate list is exhausted""") - nhops = property(_swigfaiss.HNSWStats_nhops_get, _swigfaiss.HNSWStats_nhops_set, doc=r"""number of distances computed""") - - def reset(self): - r"""number of hops aka number of edges traversed""" - return _swigfaiss.HNSWStats_reset(self) - - def combine(self, other): - return _swigfaiss.HNSWStats_combine(self, other) - - def __init__(self): - _swigfaiss.HNSWStats_swiginit(self, _swigfaiss.new_HNSWStats()) - __swig_destroy__ = _swigfaiss.delete_HNSWStats - -# Register HNSWStats in _swigfaiss: -_swigfaiss.HNSWStats_swigregister(HNSWStats) - -def search_from_candidates(hnsw, qdis, res, candidates, vt, stats, level, nres_in=0, params=None): - r""" - Internal HNSW algorithm helpers. These are not part of the public API; they - are exposed here only so that unit tests (and a few cross-TU callers such as - the Panorama search variant) can reach them. - """ - return _swigfaiss.search_from_candidates(hnsw, qdis, res, candidates, vt, stats, level, nres_in, params) - -def search_from_candidates_panorama(hnsw, index, qdis, res, candidates, vt, stats, level, nres_in=0, params=None): - r""" - Equivalent to `search_from_candidates`, but applies pruning with progressive - refinement bounds. - This is used in `IndexHNSWFlatPanorama` to improve the search performance - for higher dimensional vectors. - """ - return _swigfaiss.search_from_candidates_panorama(hnsw, index, qdis, res, candidates, vt, stats, level, nres_in, params) - -def greedy_update_nearest(hnsw, qdis, level, nearest, d_nearest): - return _swigfaiss.greedy_update_nearest(hnsw, qdis, level, nearest, d_nearest) - -def search_from_candidate_unbounded(hnsw, node, qdis, ef, vt, stats): - return _swigfaiss.search_from_candidate_unbounded(hnsw, node, qdis, ef, vt, stats) - -def search_neighbors_to_add(hnsw, qdis, results, entry_point, d_entry_point, level, vt, reference_version=False): - return _swigfaiss.search_neighbors_to_add(hnsw, qdis, results, entry_point, d_entry_point, level, vt, reference_version) -class IndexHNSW(Index): - r""" - The HNSW index is a normal random-access index with a HNSW - link structure built on top - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - hnsw = property(_swigfaiss.IndexHNSW_hnsw_get, _swigfaiss.IndexHNSW_hnsw_set) - own_fields = property(_swigfaiss.IndexHNSW_own_fields_get, _swigfaiss.IndexHNSW_own_fields_set) - storage = property(_swigfaiss.IndexHNSW_storage_get, _swigfaiss.IndexHNSW_storage_set) - init_level0 = property(_swigfaiss.IndexHNSW_init_level0_get, _swigfaiss.IndexHNSW_init_level0_set) - keep_max_size_level0 = property(_swigfaiss.IndexHNSW_keep_max_size_level0_get, _swigfaiss.IndexHNSW_keep_max_size_level0_set) - use_visited_hashset = property(_swigfaiss.IndexHNSW_use_visited_hashset_get, _swigfaiss.IndexHNSW_use_visited_hashset_set) - retain_locks = property(_swigfaiss.IndexHNSW_retain_locks_get, _swigfaiss.IndexHNSW_retain_locks_set) - - def __init__(self, *args): - _swigfaiss.IndexHNSW_swiginit(self, _swigfaiss.new_IndexHNSW(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexHNSW - - def add(self, n, x): - r"""Adds vectors to the index. May not be called concurrently.""" - return _swigfaiss.IndexHNSW_add(self, n, x) - - def train(self, n, x): - r"""Trains the storage if needed""" - return _swigfaiss.IndexHNSW_train(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - r"""entry point for search""" - return _swigfaiss.IndexHNSW_search(self, n, x, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexHNSW_range_search(self, n, x, radius, result, params) - - def search1(self, x, handler, params=None): - r"""search one vector with a custom result handler""" - return _swigfaiss.IndexHNSW_search1(self, x, handler, params) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexHNSW_reconstruct(self, key, recons) - - def reset(self): - return _swigfaiss.IndexHNSW_reset(self) - - def shrink_level_0_neighbors(self, size): - return _swigfaiss.IndexHNSW_shrink_level_0_neighbors(self, size) - - def search_level_0(self, n, x, k, nearest, nearest_d, distances, labels, nprobe=1, search_type=1, params=None): - r""" - Perform search only on level 0, given the starting points for - each vertex. - - :type search_type: int, optional - :param search_type: 1:perform one search per nprobe, 2: enqueue - all entry points - """ - return _swigfaiss.IndexHNSW_search_level_0(self, n, x, k, nearest, nearest_d, distances, labels, nprobe, search_type, params) - - def init_level_0_from_knngraph(self, k, D, I): - r"""alternative graph building""" - return _swigfaiss.IndexHNSW_init_level_0_from_knngraph(self, k, D, I) - - def init_level_0_from_entry_points(self, npt, points, nearests): - r"""alternative graph building""" - return _swigfaiss.IndexHNSW_init_level_0_from_entry_points(self, npt, points, nearests) - - def reorder_links(self): - return _swigfaiss.IndexHNSW_reorder_links(self) - - def link_singletons(self): - return _swigfaiss.IndexHNSW_link_singletons(self) - - def permute_entries(self, perm): - return _swigfaiss.IndexHNSW_permute_entries(self, perm) - - def get_distance_computer(self): - return _swigfaiss.IndexHNSW_get_distance_computer(self) - -# Register IndexHNSW in _swigfaiss: -_swigfaiss.IndexHNSW_swigregister(IndexHNSW) -class IndexHNSWFlat(IndexHNSW): - r""" - Flat index topped with with a HNSW structure to access elements - more efficiently. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexHNSWFlat_swiginit(self, _swigfaiss.new_IndexHNSWFlat(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexHNSWFlat - -# Register IndexHNSWFlat in _swigfaiss: -_swigfaiss.IndexHNSWFlat_swigregister(IndexHNSWFlat) -class IndexHNSWFlatPanorama(IndexHNSWFlat): - r""" - Panorama implementation of IndexHNSWFlat following - https://www.arxiv.org/pdf/2510.00566. - - Unlike cluster-based Panorama, the vectors have to be higher dimensional - (i.e. typically d > 512) and/or be able to compress a lot of their energy in - the early dimensions to be effective. This is because HNSW accesses vectors - in a random order, which makes cache misses dominate the distance computation - time. - - The `num_panorama_levels` parameter controls the granularity of progressive - distance refinement, allowing candidates to be eliminated early using partial - distance computations rather than computing full distances. - - NOTE: This version of HNSW handles search slightly differently than the - vanilla HNSW, as it uses partial distance computations with progressive - refinement bounds. Instead of computing full distances immediately for all - candidates, Panorama maintains lower and upper bounds that are incrementally - tightened across refinement levels. Candidates are inserted into the search - beam using approximate distance estimates (LB+UB)/2 and are only fully - evaluated when they survive pruning and enter the result heap. This allows - the algorithm to prune unpromising candidates early using Cauchy-Schwarz - bounds on partial inner products. Hence, recall is not guaranteed to be the - same as vanilla HNSW due to the heterogeneous precision within the search - beam (exact vs. partial distance estimates affecting traversal order). - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexHNSWFlatPanorama_swiginit(self, _swigfaiss.new_IndexHNSWFlatPanorama(*args)) - - def add(self, n, x): - return _swigfaiss.IndexHNSWFlatPanorama_add(self, n, x) - - def reset(self): - return _swigfaiss.IndexHNSWFlatPanorama_reset(self) - - def permute_entries(self, perm): - return _swigfaiss.IndexHNSWFlatPanorama_permute_entries(self, perm) - - def get_cum_sum(self, i): - r"""Inline for performance - called frequently in search hot path.""" - return _swigfaiss.IndexHNSWFlatPanorama_get_cum_sum(self, i) - cum_sums = property(_swigfaiss.IndexHNSWFlatPanorama_cum_sums_get, _swigfaiss.IndexHNSWFlatPanorama_cum_sums_set) - pano = property(_swigfaiss.IndexHNSWFlatPanorama_pano_get, _swigfaiss.IndexHNSWFlatPanorama_pano_set) - num_panorama_levels = property(_swigfaiss.IndexHNSWFlatPanorama_num_panorama_levels_get) - __swig_destroy__ = _swigfaiss.delete_IndexHNSWFlatPanorama - -# Register IndexHNSWFlatPanorama in _swigfaiss: -_swigfaiss.IndexHNSWFlatPanorama_swigregister(IndexHNSWFlatPanorama) -class IndexHNSWPQ(IndexHNSW): - r""" - PQ index topped with with a HNSW structure to access elements - more efficiently. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexHNSWPQ_swiginit(self, _swigfaiss.new_IndexHNSWPQ(*args)) - - def train(self, n, x): - return _swigfaiss.IndexHNSWPQ_train(self, n, x) - __swig_destroy__ = _swigfaiss.delete_IndexHNSWPQ - -# Register IndexHNSWPQ in _swigfaiss: -_swigfaiss.IndexHNSWPQ_swigregister(IndexHNSWPQ) -class IndexHNSWSQ(IndexHNSW): - r""" - SQ index topped with a HNSW structure to access elements - more efficiently. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexHNSWSQ_swiginit(self, _swigfaiss.new_IndexHNSWSQ(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexHNSWSQ - -# Register IndexHNSWSQ in _swigfaiss: -_swigfaiss.IndexHNSWSQ_swigregister(IndexHNSWSQ) -class IndexHNSW2Level(IndexHNSW): - r"""2-level code structure with fast random access""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexHNSW2Level_swiginit(self, _swigfaiss.new_IndexHNSW2Level(*args)) - - def flip_to_ivf(self): - return _swigfaiss.IndexHNSW2Level_flip_to_ivf(self) - - def search(self, n, x, k, distances, labels, params=None): - r"""entry point for search""" - return _swigfaiss.IndexHNSW2Level_search(self, n, x, k, distances, labels, params) - __swig_destroy__ = _swigfaiss.delete_IndexHNSW2Level - -# Register IndexHNSW2Level in _swigfaiss: -_swigfaiss.IndexHNSW2Level_swigregister(IndexHNSW2Level) -class IndexHNSWCagra(IndexHNSW): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexHNSWCagra_swiginit(self, _swigfaiss.new_IndexHNSWCagra(*args)) - base_level_only = property(_swigfaiss.IndexHNSWCagra_base_level_only_get, _swigfaiss.IndexHNSWCagra_base_level_only_set, doc=r""" - When set to true, the index is immutable. - This option is used to copy the knn graph from GpuIndexCagra - to the base level of IndexHNSWCagra without adding upper levels. - Doing so enables to search the HNSW index, but removes the - ability to add vectors. - """) - num_base_level_search_entrypoints = property(_swigfaiss.IndexHNSWCagra_num_base_level_search_entrypoints_get, _swigfaiss.IndexHNSWCagra_num_base_level_search_entrypoints_set, doc=r""" - When `base_level_only` is set to `True`, the search function - searches only the base level knn graph of the HNSW index. - This parameter selects the entry point by randomly selecting - some points and using the best one. - """) - - def add(self, n, x): - return _swigfaiss.IndexHNSWCagra_add(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - r"""entry point for search""" - return _swigfaiss.IndexHNSWCagra_search(self, n, x, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexHNSWCagra_range_search(self, n, x, radius, result, params) - - def get_numeric_type(self): - return _swigfaiss.IndexHNSWCagra_get_numeric_type(self) - - def set_numeric_type(self, numeric_type): - return _swigfaiss.IndexHNSWCagra_set_numeric_type(self, numeric_type) - numeric_type_ = property(_swigfaiss.IndexHNSWCagra_numeric_type__get, _swigfaiss.IndexHNSWCagra_numeric_type__set) - __swig_destroy__ = _swigfaiss.delete_IndexHNSWCagra - -# Register IndexHNSWCagra in _swigfaiss: -_swigfaiss.IndexHNSWCagra_swigregister(IndexHNSWCagra) - -def smawk(nrows, ncols, x, argmins): - r""" - SMAWK algorithm. Find the row minima of a monotone matrix. - - Expose this for testing. - - :type nrows: int - :param nrows: number of rows - :type ncols: int - :param ncols: number of columns - :type x: float - :param x: input matrix, size (nrows, ncols) - :type argmins: int - :param argmins: argmin of each row - """ - return _swigfaiss.smawk(nrows, ncols, x, argmins) - -def kmeans1d(x, n, nclusters, centroids): - r""" - Exact 1D K-Means by dynamic programming - - From "Fast Exact k-Means, k-Medians and Bregman Divergence Clustering in 1D" - Allan Grønlund, Kasper Green Larsen, Alexander Mathiasen, Jesper Sindahl - Nielsen, Stefan Schneider, Mingzhou Song, ArXiV'17 - - Section 2.2 - - https://arxiv.org/abs/1701.07204 - - :type x: float - :param x: input 1D array - :type n: int - :param n: input array length - :type nclusters: int - :param nclusters: number of clusters - :type centroids: float - :param centroids: output centroids, size nclusters - :rtype: float - :return: imbalance factor - """ - return _swigfaiss.kmeans1d(x, n, nclusters, centroids) -class Neighbor(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - id = property(_swigfaiss.Neighbor_id_get, _swigfaiss.Neighbor_id_set) - distance = property(_swigfaiss.Neighbor_distance_get, _swigfaiss.Neighbor_distance_set) - flag = property(_swigfaiss.Neighbor_flag_get, _swigfaiss.Neighbor_flag_set) - - def __init__(self, *args): - _swigfaiss.Neighbor_swiginit(self, _swigfaiss.new_Neighbor(*args)) - - def __lt__(self, other): - return _swigfaiss.Neighbor___lt__(self, other) - __swig_destroy__ = _swigfaiss.delete_Neighbor - -# Register Neighbor in _swigfaiss: -_swigfaiss.Neighbor_swigregister(Neighbor) -class Nhood(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - pool = property(_swigfaiss.Nhood_pool_get, _swigfaiss.Nhood_pool_set) - M = property(_swigfaiss.Nhood_M_get, _swigfaiss.Nhood_M_set) - nn_old = property(_swigfaiss.Nhood_nn_old_get, _swigfaiss.Nhood_nn_old_set) - nn_new = property(_swigfaiss.Nhood_nn_new_get, _swigfaiss.Nhood_nn_new_set) - rnn_old = property(_swigfaiss.Nhood_rnn_old_get, _swigfaiss.Nhood_rnn_old_set) - rnn_new = property(_swigfaiss.Nhood_rnn_new_get, _swigfaiss.Nhood_rnn_new_set) - - def __init__(self, *args): - _swigfaiss.Nhood_swiginit(self, _swigfaiss.new_Nhood(*args)) - - def insert(self, id, dist): - return _swigfaiss.Nhood_insert(self, id, dist) - __swig_destroy__ = _swigfaiss.delete_Nhood - -# Register Nhood in _swigfaiss: -_swigfaiss.Nhood_swigregister(Nhood) -class NNDescent(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, d, K): - _swigfaiss.NNDescent_swiginit(self, _swigfaiss.new_NNDescent(d, K)) - __swig_destroy__ = _swigfaiss.delete_NNDescent - - def build(self, qdis, n, verbose): - return _swigfaiss.NNDescent_build(self, qdis, n, verbose) - - def search(self, qdis, topk, indices, dists, vt): - return _swigfaiss.NNDescent_search(self, qdis, topk, indices, dists, vt) - - def reset(self): - return _swigfaiss.NNDescent_reset(self) - - def init_graph(self, qdis): - r"""Initialize the KNN graph randomly""" - return _swigfaiss.NNDescent_init_graph(self, qdis) - - def nndescent(self, qdis, verbose): - r"""Perform NNDescent algorithm""" - return _swigfaiss.NNDescent_nndescent(self, qdis, verbose) - - def join(self, qdis): - r"""Perform local join on each node""" - return _swigfaiss.NNDescent_join(self, qdis) - - def update(self): - r"""Sample new neighbors for each node to perform local join later""" - return _swigfaiss.NNDescent_update(self) - - def generate_eval_set(self, qdis, c, v, N): - r"""Sample a small number of points to evaluate the quality of KNNG built""" - return _swigfaiss.NNDescent_generate_eval_set(self, qdis, c, v, N) - - def eval_recall(self, ctrl_points, acc_eval_set): - r"""Evaluate the quality of KNNG built""" - return _swigfaiss.NNDescent_eval_recall(self, ctrl_points, acc_eval_set) - has_built = property(_swigfaiss.NNDescent_has_built_get, _swigfaiss.NNDescent_has_built_set) - S = property(_swigfaiss.NNDescent_S_get, _swigfaiss.NNDescent_S_set) - R = property(_swigfaiss.NNDescent_R_get, _swigfaiss.NNDescent_R_set) - iter = property(_swigfaiss.NNDescent_iter_get, _swigfaiss.NNDescent_iter_set) - search_L = property(_swigfaiss.NNDescent_search_L_get, _swigfaiss.NNDescent_search_L_set) - random_seed = property(_swigfaiss.NNDescent_random_seed_get, _swigfaiss.NNDescent_random_seed_set) - K = property(_swigfaiss.NNDescent_K_get, _swigfaiss.NNDescent_K_set) - d = property(_swigfaiss.NNDescent_d_get, _swigfaiss.NNDescent_d_set) - L = property(_swigfaiss.NNDescent_L_get, _swigfaiss.NNDescent_L_set) - ntotal = property(_swigfaiss.NNDescent_ntotal_get, _swigfaiss.NNDescent_ntotal_set) - graph = property(_swigfaiss.NNDescent_graph_get, _swigfaiss.NNDescent_graph_set) - final_graph = property(_swigfaiss.NNDescent_final_graph_get, _swigfaiss.NNDescent_final_graph_set) - -# Register NNDescent in _swigfaiss: -_swigfaiss.NNDescent_swigregister(NNDescent) -class IndexNNDescent(Index): - r""" - The NNDescent index is a normal random-access index with an NNDescent - link structure built on top - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nndescent = property(_swigfaiss.IndexNNDescent_nndescent_get, _swigfaiss.IndexNNDescent_nndescent_set, doc=r"""Faiss results are 64-bit""") - own_fields = property(_swigfaiss.IndexNNDescent_own_fields_get, _swigfaiss.IndexNNDescent_own_fields_set) - storage = property(_swigfaiss.IndexNNDescent_storage_get, _swigfaiss.IndexNNDescent_storage_set) - - def __init__(self, *args): - _swigfaiss.IndexNNDescent_swiginit(self, _swigfaiss.new_IndexNNDescent(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexNNDescent - - def add(self, n, x): - return _swigfaiss.IndexNNDescent_add(self, n, x) - - def train(self, n, x): - r"""Trains the storage if needed""" - return _swigfaiss.IndexNNDescent_train(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - r"""entry point for search""" - return _swigfaiss.IndexNNDescent_search(self, n, x, k, distances, labels, params) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexNNDescent_reconstruct(self, key, recons) - - def reset(self): - return _swigfaiss.IndexNNDescent_reset(self) - -# Register IndexNNDescent in _swigfaiss: -_swigfaiss.IndexNNDescent_swigregister(IndexNNDescent) -class IndexNNDescentFlat(IndexNNDescent): - r""" - Flat index topped with with a NNDescent structure to access elements - more efficiently. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexNNDescentFlat_swiginit(self, _swigfaiss.new_IndexNNDescentFlat(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexNNDescentFlat - -# Register IndexNNDescentFlat in _swigfaiss: -_swigfaiss.IndexNNDescentFlat_swigregister(IndexNNDescentFlat) -class IndexIVFFlat(IndexIVF): - r""" - Inverted file with stored vectors. Here the inverted file - pre-selects the vectors to be searched, but they are not otherwise - encoded, the code array just contains the raw float entries. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def add_core(self, n, x, xids, precomputed_idx, inverted_list_context=None): - return _swigfaiss.IndexIVFFlat_add_core(self, n, x, xids, precomputed_idx, inverted_list_context) - - def encode_vectors(self, n, x, list_nos, codes, include_listnos=False): - return _swigfaiss.IndexIVFFlat_encode_vectors(self, n, x, list_nos, codes, include_listnos) - - def decode_vectors(self, n, codes, list_nos, x): - return _swigfaiss.IndexIVFFlat_decode_vectors(self, n, codes, list_nos, x) - - def get_InvertedListScanner(self, store_pairs, sel, params): - return _swigfaiss.IndexIVFFlat_get_InvertedListScanner(self, store_pairs, sel, params) - - def reconstruct_from_offset(self, list_no, offset, recons): - return _swigfaiss.IndexIVFFlat_reconstruct_from_offset(self, list_no, offset, recons) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexIVFFlat_sa_decode(self, n, bytes, x) - - def __init__(self, *args): - _swigfaiss.IndexIVFFlat_swiginit(self, _swigfaiss.new_IndexIVFFlat(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFFlat - -# Register IndexIVFFlat in _swigfaiss: -_swigfaiss.IndexIVFFlat_swigregister(IndexIVFFlat) -class IndexIVFFlatDedup(IndexIVFFlat): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def train(self, n, x): - r"""also dedups the training set""" - return _swigfaiss.IndexIVFFlatDedup_train(self, n, x) - - def add_with_ids(self, n, x, xids): - r"""implemented for all IndexIVF* classes""" - return _swigfaiss.IndexIVFFlatDedup_add_with_ids(self, n, x, xids) - - def search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params=None, stats=None): - return _swigfaiss.IndexIVFFlatDedup_search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params, stats) - - def remove_ids(self, sel): - return _swigfaiss.IndexIVFFlatDedup_remove_ids(self, sel) - - def range_search(self, n, x, radius, result, params=None): - r"""not implemented""" - return _swigfaiss.IndexIVFFlatDedup_range_search(self, n, x, radius, result, params) - - def update_vectors(self, nv, idx, v): - r"""not implemented""" - return _swigfaiss.IndexIVFFlatDedup_update_vectors(self, nv, idx, v) - - def reconstruct_from_offset(self, list_no, offset, recons): - r"""not implemented""" - return _swigfaiss.IndexIVFFlatDedup_reconstruct_from_offset(self, list_no, offset, recons) - - def __init__(self, *args): - _swigfaiss.IndexIVFFlatDedup_swiginit(self, _swigfaiss.new_IndexIVFFlatDedup(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFFlatDedup - -# Register IndexIVFFlatDedup in _swigfaiss: -_swigfaiss.IndexIVFFlatDedup_swigregister(IndexIVFFlatDedup) -class IndexIVFFlatPanorama(IndexIVFFlat): - r""" - Panorama adaptation of IndexIVFFlat following - https://www.arxiv.org/pdf/2510.00566. - - IDEA: - Panorama adapts the storage layout within each cluster and uses - pruning with bounds to improve the search performance. - Combined with orthogonal transforms upstream that concentrate the energy - in the early dimensions (like PCA, Cayley, etc.), Panorama can prune up - to 95% of the vectors in the cluster. - - OVERHEAD: - To be more efficient, we compute the residual energies at insertion time - and store them along the vectors, which comes with an additional storage - overhead of exactly (nlevels + 1) floats per vector. Add time is also - slightly higher due to the overhead of transposing the vectors. - - NOTE: - We inherit from IndexIVFFlat instead of IndexIVF so we can keep the same - insertion logic. The code responsible for level-oriented storage is in - `ArrayInvertedListsPanorama`, which is a struct member of `IndexIVF`. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - n_levels = property(_swigfaiss.IndexIVFFlatPanorama_n_levels_get, _swigfaiss.IndexIVFFlatPanorama_n_levels_set) - batch_size = property(_swigfaiss.IndexIVFFlatPanorama_batch_size_get, _swigfaiss.IndexIVFFlatPanorama_batch_size_set) - cum_sums = property(_swigfaiss.IndexIVFFlatPanorama_cum_sums_get, _swigfaiss.IndexIVFFlatPanorama_cum_sums_set) - - def get_InvertedListScanner(self, store_pairs, sel, params): - return _swigfaiss.IndexIVFFlatPanorama_get_InvertedListScanner(self, store_pairs, sel, params) - - def reconstruct_from_offset(self, list_no, offset, recons): - return _swigfaiss.IndexIVFFlatPanorama_reconstruct_from_offset(self, list_no, offset, recons) - - def __init__(self, *args): - _swigfaiss.IndexIVFFlatPanorama_swiginit(self, _swigfaiss.new_IndexIVFFlatPanorama(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFFlatPanorama - -# Register IndexIVFFlatPanorama in _swigfaiss: -_swigfaiss.IndexIVFFlatPanorama_swigregister(IndexIVFFlatPanorama) - -def storage_distance_computer(storage): - return _swigfaiss.storage_distance_computer(storage) -class NSG(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - ntotal = property(_swigfaiss.NSG_ntotal_get, _swigfaiss.NSG_ntotal_set, doc=r"""nb of nodes""") - R = property(_swigfaiss.NSG_R_get, _swigfaiss.NSG_R_set, doc=r"""nb of neighbors per node""") - L = property(_swigfaiss.NSG_L_get, _swigfaiss.NSG_L_set, doc=r"""length of the search path at construction time""") - C = property(_swigfaiss.NSG_C_get, _swigfaiss.NSG_C_set, doc=r"""candidate pool size at construction time""") - search_L = property(_swigfaiss.NSG_search_L_get, _swigfaiss.NSG_search_L_set, doc=r"""length of the search path""") - use_visited_hashset = property(_swigfaiss.NSG_use_visited_hashset_get, _swigfaiss.NSG_use_visited_hashset_set) - enterpoint = property(_swigfaiss.NSG_enterpoint_get, _swigfaiss.NSG_enterpoint_set, doc=r"""enterpoint""") - final_graph = property(_swigfaiss.NSG_final_graph_get, _swigfaiss.NSG_final_graph_set, doc=r"""NSG graph structure""") - is_built = property(_swigfaiss.NSG_is_built_get, _swigfaiss.NSG_is_built_set, doc=r"""NSG is built or not""") - rng = property(_swigfaiss.NSG_rng_get, _swigfaiss.NSG_rng_set, doc=r"""random generator""") - - def __init__(self, R=32): - _swigfaiss.NSG_swiginit(self, _swigfaiss.new_NSG(R)) - - def build(self, storage, n, knn_graph, verbose): - return _swigfaiss.NSG_build(self, storage, n, knn_graph, verbose) - - def reset(self): - return _swigfaiss.NSG_reset(self) - - def search(self, dis, k, I, D, vt): - return _swigfaiss.NSG_search(self, dis, k, I, D, vt) - - def init_graph(self, storage, knn_graph): - return _swigfaiss.NSG_init_graph(self, storage, knn_graph) - - def add_reverse_links(self, q, locks, dis, graph): - return _swigfaiss.NSG_add_reverse_links(self, q, locks, dis, graph) - - def sync_prune(self, q, pool, dis, vt, knn_graph, graph): - return _swigfaiss.NSG_sync_prune(self, q, pool, dis, vt, knn_graph, graph) - - def link(self, storage, knn_graph, graph, verbose): - return _swigfaiss.NSG_link(self, storage, knn_graph, graph, verbose) - - def tree_grow(self, storage, degrees): - return _swigfaiss.NSG_tree_grow(self, storage, degrees) - - def dfs(self, vt, root, cnt): - return _swigfaiss.NSG_dfs(self, vt, root, cnt) - - def attach_unlinked(self, storage, vt, vt2, degrees): - return _swigfaiss.NSG_attach_unlinked(self, storage, vt, vt2, degrees) - - def check_graph(self): - return _swigfaiss.NSG_check_graph(self) - - def get_final_graph(self): - return _swigfaiss.NSG_get_final_graph(self) - __swig_destroy__ = _swigfaiss.delete_NSG - -# Register NSG in _swigfaiss: -_swigfaiss.NSG_swigregister(NSG) -class NSG_Graph_int(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - data = property(_swigfaiss.NSG_Graph_int_data_get, _swigfaiss.NSG_Graph_int_data_set, doc=r"""the flattened adjacency matrix, size N-by-K""") - K = property(_swigfaiss.NSG_Graph_int_K_get, _swigfaiss.NSG_Graph_int_K_set, doc=r"""nb of neighbors per node""") - N = property(_swigfaiss.NSG_Graph_int_N_get, _swigfaiss.NSG_Graph_int_N_set, doc=r"""total nb of nodes""") - own_fields = property(_swigfaiss.NSG_Graph_int_own_fields_get, _swigfaiss.NSG_Graph_int_own_fields_set, doc=r"""the underlying data owned by itself or not""") - - def __init__(self, *args): - _swigfaiss.NSG_Graph_int_swiginit(self, _swigfaiss.new_NSG_Graph_int(*args)) - __swig_destroy__ = _swigfaiss.delete_NSG_Graph_int - - def at(self, *args): - return _swigfaiss.NSG_Graph_int_at(self, *args) - - def get_neighbors(self, i, neighbors): - return _swigfaiss.NSG_Graph_int_get_neighbors(self, i, neighbors) - -# Register NSG_Graph_int in _swigfaiss: -_swigfaiss.NSG_Graph_int_swigregister(NSG_Graph_int) -class IndexNSG(Index): - r""" - The NSG index is a normal random-access index with a NSG - link structure built on top - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nsg = property(_swigfaiss.IndexNSG_nsg_get, _swigfaiss.IndexNSG_nsg_set, doc=r"""the link structure""") - own_fields = property(_swigfaiss.IndexNSG_own_fields_get, _swigfaiss.IndexNSG_own_fields_set, doc=r"""the sequential storage""") - storage = property(_swigfaiss.IndexNSG_storage_get, _swigfaiss.IndexNSG_storage_set) - is_built = property(_swigfaiss.IndexNSG_is_built_get, _swigfaiss.IndexNSG_is_built_set, doc=r"""the index is built or not""") - GK = property(_swigfaiss.IndexNSG_GK_get, _swigfaiss.IndexNSG_GK_set, doc=r"""K of KNN graph for building""") - build_type = property(_swigfaiss.IndexNSG_build_type_get, _swigfaiss.IndexNSG_build_type_set, doc=r""" - indicate how to build a knn graph - - 0: build NSG with brute force search - - 1: build NSG with NNDescent - """) - nndescent_S = property(_swigfaiss.IndexNSG_nndescent_S_get, _swigfaiss.IndexNSG_nndescent_S_set, doc=r"""parameters for nndescent""") - nndescent_R = property(_swigfaiss.IndexNSG_nndescent_R_get, _swigfaiss.IndexNSG_nndescent_R_set) - nndescent_L = property(_swigfaiss.IndexNSG_nndescent_L_get, _swigfaiss.IndexNSG_nndescent_L_set) - nndescent_iter = property(_swigfaiss.IndexNSG_nndescent_iter_get, _swigfaiss.IndexNSG_nndescent_iter_set) - - def __init__(self, *args): - _swigfaiss.IndexNSG_swiginit(self, _swigfaiss.new_IndexNSG(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexNSG - - def build(self, n, x, knn_graph, GK): - return _swigfaiss.IndexNSG_build(self, n, x, knn_graph, GK) - - def add(self, n, x): - return _swigfaiss.IndexNSG_add(self, n, x) - - def train(self, n, x): - r"""Trains the storage if needed""" - return _swigfaiss.IndexNSG_train(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - r"""entry point for search""" - return _swigfaiss.IndexNSG_search(self, n, x, k, distances, labels, params) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexNSG_reconstruct(self, key, recons) - - def reset(self): - return _swigfaiss.IndexNSG_reset(self) - - def check_knn_graph(self, knn_graph, n, K): - return _swigfaiss.IndexNSG_check_knn_graph(self, knn_graph, n, K) - -# Register IndexNSG in _swigfaiss: -_swigfaiss.IndexNSG_swigregister(IndexNSG) -class IndexNSGFlat(IndexNSG): - r""" - Flat index topped with with a NSG structure to access elements - more efficiently. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexNSGFlat_swiginit(self, _swigfaiss.new_IndexNSGFlat(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexNSGFlat - -# Register IndexNSGFlat in _swigfaiss: -_swigfaiss.IndexNSGFlat_swigregister(IndexNSGFlat) -class IndexNSGPQ(IndexNSG): - r""" - PQ index topped with with a NSG structure to access elements - more efficiently. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexNSGPQ_swiginit(self, _swigfaiss.new_IndexNSGPQ(*args)) - - def train(self, n, x): - return _swigfaiss.IndexNSGPQ_train(self, n, x) - __swig_destroy__ = _swigfaiss.delete_IndexNSGPQ - -# Register IndexNSGPQ in _swigfaiss: -_swigfaiss.IndexNSGPQ_swigregister(IndexNSGPQ) -class IndexNSGSQ(IndexNSG): - r""" - SQ index topped with with a NSG structure to access elements - more efficiently. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexNSGSQ_swiginit(self, _swigfaiss.new_IndexNSGSQ(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexNSGSQ - -# Register IndexNSGSQ in _swigfaiss: -_swigfaiss.IndexNSGSQ_swigregister(IndexNSGSQ) -class ZnSphereSearch(object): - r""" - returns the nearest vertex in the sphere to a query. Returns only - the coordinates, not an id. - - Algorithm: all points are derived from a one atom vector up to a - permutation and sign changes. The search function finds the most - appropriate atom and transformation. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - dimS = property(_swigfaiss.ZnSphereSearch_dimS_get, _swigfaiss.ZnSphereSearch_dimS_set) - r2 = property(_swigfaiss.ZnSphereSearch_r2_get, _swigfaiss.ZnSphereSearch_r2_set) - natom = property(_swigfaiss.ZnSphereSearch_natom_get, _swigfaiss.ZnSphereSearch_natom_set) - voc = property(_swigfaiss.ZnSphereSearch_voc_get, _swigfaiss.ZnSphereSearch_voc_set, doc=r"""size dim * natom""") - - def __init__(self, dim, r2_in): - _swigfaiss.ZnSphereSearch_swiginit(self, _swigfaiss.new_ZnSphereSearch(dim, r2_in)) - - def search(self, *args): - r""" - *Overload 1:* - find nearest centroid. x does not need to be normalized - - | - - *Overload 2:* - full call. Requires externally-allocated temp space - - | - - *Overload 3:* - full call. Requires externally-allocated temp space - """ - return _swigfaiss.ZnSphereSearch_search(self, *args) - - def search_multi(self, n, x, c_out, dp_out): - return _swigfaiss.ZnSphereSearch_search_multi(self, n, x, c_out, dp_out) - __swig_destroy__ = _swigfaiss.delete_ZnSphereSearch - -# Register ZnSphereSearch in _swigfaiss: -_swigfaiss.ZnSphereSearch_swigregister(ZnSphereSearch) -class EnumeratedVectors(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - nv = property(_swigfaiss.EnumeratedVectors_nv_get, _swigfaiss.EnumeratedVectors_nv_set, doc=r"""size of the collection""") - dim = property(_swigfaiss.EnumeratedVectors_dim_get, _swigfaiss.EnumeratedVectors_dim_set) - - def encode(self, x): - r"""encode a vector from a collection""" - return _swigfaiss.EnumeratedVectors_encode(self, x) - - def decode(self, code, c): - r"""decode it""" - return _swigfaiss.EnumeratedVectors_decode(self, code, c) - - def encode_multi(self, nc, c, codes): - return _swigfaiss.EnumeratedVectors_encode_multi(self, nc, c, codes) - - def decode_multi(self, nc, codes, c): - return _swigfaiss.EnumeratedVectors_decode_multi(self, nc, codes, c) - - def find_nn(self, n, codes, nq, xq, idx, dis): - return _swigfaiss.EnumeratedVectors_find_nn(self, n, codes, nq, xq, idx, dis) - __swig_destroy__ = _swigfaiss.delete_EnumeratedVectors - -# Register EnumeratedVectors in _swigfaiss: -_swigfaiss.EnumeratedVectors_swigregister(EnumeratedVectors) -class Repeat(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - val = property(_swigfaiss.Repeat_val_get, _swigfaiss.Repeat_val_set) - n = property(_swigfaiss.Repeat_n_get, _swigfaiss.Repeat_n_set) - - def __init__(self): - _swigfaiss.Repeat_swiginit(self, _swigfaiss.new_Repeat()) - __swig_destroy__ = _swigfaiss.delete_Repeat - -# Register Repeat in _swigfaiss: -_swigfaiss.Repeat_swigregister(Repeat) -class Repeats(object): - r""" - Repeats: used to encode a vector that has n occurrences of - val. Encodes the signs and permutation of the vector. Useful for - atoms. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - dim = property(_swigfaiss.Repeats_dim_get, _swigfaiss.Repeats_dim_set) - repeats = property(_swigfaiss.Repeats_repeats_get, _swigfaiss.Repeats_repeats_set) - - def __init__(self, dim_in=0, c=None): - _swigfaiss.Repeats_swiginit(self, _swigfaiss.new_Repeats(dim_in, c)) - - def count(self): - return _swigfaiss.Repeats_count(self) - - def encode(self, c): - return _swigfaiss.Repeats_encode(self, c) - - def decode(self, code, c): - return _swigfaiss.Repeats_decode(self, code, c) - __swig_destroy__ = _swigfaiss.delete_Repeats - -# Register Repeats in _swigfaiss: -_swigfaiss.Repeats_swigregister(Repeats) -class ZnSphereCodec(ZnSphereSearch, EnumeratedVectors): - r""" - codec that can return ids for the encoded vectors - - uses the ZnSphereSearch to encode the vector by encoding the - permutation and signs. Depends on ZnSphereSearch because it uses - the atom numbers - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - code_segments = property(_swigfaiss.ZnSphereCodec_code_segments_get, _swigfaiss.ZnSphereCodec_code_segments_set) - nv = property(_swigfaiss.ZnSphereCodec_nv_get, _swigfaiss.ZnSphereCodec_nv_set) - code_size = property(_swigfaiss.ZnSphereCodec_code_size_get, _swigfaiss.ZnSphereCodec_code_size_set) - - def __init__(self, dim_in, r2_in): - _swigfaiss.ZnSphereCodec_swiginit(self, _swigfaiss.new_ZnSphereCodec(dim_in, r2_in)) - - def search_and_encode(self, x): - return _swigfaiss.ZnSphereCodec_search_and_encode(self, x) - - def decode(self, code, c): - return _swigfaiss.ZnSphereCodec_decode(self, code, c) - - def encode(self, x): - r"""takes vectors that do not need to be centroids""" - return _swigfaiss.ZnSphereCodec_encode(self, x) - __swig_destroy__ = _swigfaiss.delete_ZnSphereCodec - -# Register ZnSphereCodec in _swigfaiss: -_swigfaiss.ZnSphereCodec_swigregister(ZnSphereCodec) -class ZnSphereCodecRec(EnumeratedVectors): - r""" - recursive sphere codec - - Uses a recursive decomposition on the dimensions to encode - centroids found by the ZnSphereSearch. The codes are *not* - compatible with the ones of ZnSphereCodec - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - r2 = property(_swigfaiss.ZnSphereCodecRec_r2_get, _swigfaiss.ZnSphereCodecRec_r2_set) - log2_dim = property(_swigfaiss.ZnSphereCodecRec_log2_dim_get, _swigfaiss.ZnSphereCodecRec_log2_dim_set) - code_size = property(_swigfaiss.ZnSphereCodecRec_code_size_get, _swigfaiss.ZnSphereCodecRec_code_size_set) - - def __init__(self, dim_in, r2_in): - _swigfaiss.ZnSphereCodecRec_swiginit(self, _swigfaiss.new_ZnSphereCodecRec(dim_in, r2_in)) - - def encode_centroid(self, c): - return _swigfaiss.ZnSphereCodecRec_encode_centroid(self, c) - - def decode(self, code, c): - return _swigfaiss.ZnSphereCodecRec_decode(self, code, c) - - def encode(self, x): - r""" - vectors need to be centroids (does not work on arbitrary - vectors) - """ - return _swigfaiss.ZnSphereCodecRec_encode(self, x) - all_nv = property(_swigfaiss.ZnSphereCodecRec_all_nv_get, _swigfaiss.ZnSphereCodecRec_all_nv_set) - all_nv_cum = property(_swigfaiss.ZnSphereCodecRec_all_nv_cum_get, _swigfaiss.ZnSphereCodecRec_all_nv_cum_set) - decode_cache_ld = property(_swigfaiss.ZnSphereCodecRec_decode_cache_ld_get, _swigfaiss.ZnSphereCodecRec_decode_cache_ld_set) - decode_cache = property(_swigfaiss.ZnSphereCodecRec_decode_cache_get, _swigfaiss.ZnSphereCodecRec_decode_cache_set) - - def get_nv(self, ld, r2a): - return _swigfaiss.ZnSphereCodecRec_get_nv(self, ld, r2a) - - def get_nv_cum(self, ld, r2t, r2a): - return _swigfaiss.ZnSphereCodecRec_get_nv_cum(self, ld, r2t, r2a) - - def set_nv_cum(self, ld, r2t, r2a, v): - return _swigfaiss.ZnSphereCodecRec_set_nv_cum(self, ld, r2t, r2a, v) - __swig_destroy__ = _swigfaiss.delete_ZnSphereCodecRec - -# Register ZnSphereCodecRec in _swigfaiss: -_swigfaiss.ZnSphereCodecRec_swigregister(ZnSphereCodecRec) -class ZnSphereCodecAlt(ZnSphereCodec): - r""" - Codec that uses the recursive codec if dim is a power of 2 and - the regular one otherwise - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - use_rec = property(_swigfaiss.ZnSphereCodecAlt_use_rec_get, _swigfaiss.ZnSphereCodecAlt_use_rec_set) - znc_rec = property(_swigfaiss.ZnSphereCodecAlt_znc_rec_get, _swigfaiss.ZnSphereCodecAlt_znc_rec_set) - - def __init__(self, dim_in, r2_in): - _swigfaiss.ZnSphereCodecAlt_swiginit(self, _swigfaiss.new_ZnSphereCodecAlt(dim_in, r2_in)) - - def encode(self, x): - return _swigfaiss.ZnSphereCodecAlt_encode(self, x) - - def decode(self, code, c): - return _swigfaiss.ZnSphereCodecAlt_decode(self, code, c) - __swig_destroy__ = _swigfaiss.delete_ZnSphereCodecAlt - -# Register ZnSphereCodecAlt in _swigfaiss: -_swigfaiss.ZnSphereCodecAlt_swigregister(ZnSphereCodecAlt) -class IndexLattice(IndexFlatCodes): - r"""Index that encodes a vector with a series of Zn lattice quantizers""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nsq = property(_swigfaiss.IndexLattice_nsq_get, _swigfaiss.IndexLattice_nsq_set, doc=r"""number of sub-vectors""") - dsq = property(_swigfaiss.IndexLattice_dsq_get, _swigfaiss.IndexLattice_dsq_set, doc=r"""dimension of sub-vectors""") - zn_sphere_codec = property(_swigfaiss.IndexLattice_zn_sphere_codec_get, _swigfaiss.IndexLattice_zn_sphere_codec_set, doc=r"""the lattice quantizer""") - scale_nbit = property(_swigfaiss.IndexLattice_scale_nbit_get, _swigfaiss.IndexLattice_scale_nbit_set, doc=r"""nb bits used to encode the scale, per subvector""") - lattice_nbit = property(_swigfaiss.IndexLattice_lattice_nbit_get, _swigfaiss.IndexLattice_lattice_nbit_set) - trained = property(_swigfaiss.IndexLattice_trained_get, _swigfaiss.IndexLattice_trained_set, doc=r"""mins and maxes of the vector norms, per subquantizer""") - - def __init__(self, d, nsq, scale_nbit, r2): - _swigfaiss.IndexLattice_swiginit(self, _swigfaiss.new_IndexLattice(d, nsq, scale_nbit, r2)) - - def train(self, n, x): - return _swigfaiss.IndexLattice_train(self, n, x) - - def sa_code_size(self): - return _swigfaiss.IndexLattice_sa_code_size(self) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexLattice_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexLattice_sa_decode(self, n, bytes, x) - __swig_destroy__ = _swigfaiss.delete_IndexLattice - -# Register IndexLattice in _swigfaiss: -_swigfaiss.IndexLattice_swigregister(IndexLattice) -class IVFPQSearchParameters(SearchParametersIVF): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - scan_table_threshold = property(_swigfaiss.IVFPQSearchParameters_scan_table_threshold_get, _swigfaiss.IVFPQSearchParameters_scan_table_threshold_set, doc=r"""use table computation or on-the-fly?""") - polysemous_ht = property(_swigfaiss.IVFPQSearchParameters_polysemous_ht_get, _swigfaiss.IVFPQSearchParameters_polysemous_ht_set, doc=r"""Hamming thresh for polysemous filtering""") - - def __init__(self): - _swigfaiss.IVFPQSearchParameters_swiginit(self, _swigfaiss.new_IVFPQSearchParameters()) - __swig_destroy__ = _swigfaiss.delete_IVFPQSearchParameters - -# Register IVFPQSearchParameters in _swigfaiss: -_swigfaiss.IVFPQSearchParameters_swigregister(IVFPQSearchParameters) -class IndexIVFPQ(IndexIVF): - r""" - Inverted file with Product Quantizer encoding. Each residual - vector is encoded as a product quantizer code. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - pq = property(_swigfaiss.IndexIVFPQ_pq_get, _swigfaiss.IndexIVFPQ_pq_set, doc=r"""produces the codes""") - do_polysemous_training = property(_swigfaiss.IndexIVFPQ_do_polysemous_training_get, _swigfaiss.IndexIVFPQ_do_polysemous_training_set, doc=r"""reorder PQ centroids after training?""") - polysemous_training = property(_swigfaiss.IndexIVFPQ_polysemous_training_get, _swigfaiss.IndexIVFPQ_polysemous_training_set, doc=r"""if NULL, use default""") - scan_table_threshold = property(_swigfaiss.IndexIVFPQ_scan_table_threshold_get, _swigfaiss.IndexIVFPQ_scan_table_threshold_set, doc=r"""use table computation or on-the-fly?""") - polysemous_ht = property(_swigfaiss.IndexIVFPQ_polysemous_ht_get, _swigfaiss.IndexIVFPQ_polysemous_ht_set, doc=r"""Hamming thresh for polysemous filtering""") - use_precomputed_table = property(_swigfaiss.IndexIVFPQ_use_precomputed_table_get, _swigfaiss.IndexIVFPQ_use_precomputed_table_set, doc=r""" - Precompute table that speed up query preprocessing at some - memory cost (used only for by_residual with L2 metric) - """) - precomputed_table = property(_swigfaiss.IndexIVFPQ_precomputed_table_get, _swigfaiss.IndexIVFPQ_precomputed_table_set, doc=r""" - if use_precompute_table - size nlist * pq.M * pq.ksub - """) - - def encode_vectors(self, n, x, list_nos, codes, include_listnos=False): - return _swigfaiss.IndexIVFPQ_encode_vectors(self, n, x, list_nos, codes, include_listnos) - - def decode_vectors(self, n, codes, listnos, x): - return _swigfaiss.IndexIVFPQ_decode_vectors(self, n, codes, listnos, x) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexIVFPQ_sa_decode(self, n, bytes, x) - - def add_core(self, n, x, xids, precomputed_idx, inverted_list_context=None): - return _swigfaiss.IndexIVFPQ_add_core(self, n, x, xids, precomputed_idx, inverted_list_context) - - def add_core_o(self, n, x, xids, residuals_2, precomputed_idx=None, inverted_list_context=None): - r""" - same as add_core, also: - - output 2nd level residuals if residuals_2 != NULL - - accepts precomputed_idx = nullptr - """ - return _swigfaiss.IndexIVFPQ_add_core_o(self, n, x, xids, residuals_2, precomputed_idx, inverted_list_context) - - def train_encoder(self, n, x, assign): - r"""trains the product quantizer""" - return _swigfaiss.IndexIVFPQ_train_encoder(self, n, x, assign) - - def train_encoder_num_vectors(self): - return _swigfaiss.IndexIVFPQ_train_encoder_num_vectors(self) - - def reconstruct_from_offset(self, list_no, offset, recons): - return _swigfaiss.IndexIVFPQ_reconstruct_from_offset(self, list_no, offset, recons) - - def find_duplicates(self, ids, lims): - r""" - Find exact duplicates in the dataset. - - the duplicates are returned in pre-allocated arrays (see the - max sizes). - - :type lims: int - :param lims: limits between groups of duplicates - (max size ntotal / 2 + 1) - :type ids: int - :param ids: ids[lims[i]] : ids[lims[i+1]-1] is a group of - duplicates (max size ntotal) - :rtype: int - :return: n number of groups found - """ - return _swigfaiss.IndexIVFPQ_find_duplicates(self, ids, lims) - - def encode(self, key, x, code): - return _swigfaiss.IndexIVFPQ_encode(self, key, x, code) - - def encode_multiple(self, n, keys, x, codes, compute_keys=False): - r""" - Encode multiple vectors - - :type n: int - :param n: nb vectors to encode - :type keys: int - :param keys: posting list ids for those vectors (size n) - :type x: float - :param x: vectors (size n * d) - :type codes: uint8_t - :param codes: output codes (size n * code_size) - :type compute_keys: boolean, optional - :param compute_keys: if false, assume keys are precomputed, - otherwise compute them - """ - return _swigfaiss.IndexIVFPQ_encode_multiple(self, n, keys, x, codes, compute_keys) - - def decode_multiple(self, n, keys, xcodes, x): - r"""inverse of encode_multiple""" - return _swigfaiss.IndexIVFPQ_decode_multiple(self, n, keys, xcodes, x) - - def get_InvertedListScanner(self, store_pairs, sel, params): - return _swigfaiss.IndexIVFPQ_get_InvertedListScanner(self, store_pairs, sel, params) - - def precompute_table(self): - r"""build precomputed table""" - return _swigfaiss.IndexIVFPQ_precompute_table(self) - - def __init__(self, *args): - _swigfaiss.IndexIVFPQ_swiginit(self, _swigfaiss.new_IndexIVFPQ(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFPQ - -# Register IndexIVFPQ in _swigfaiss: -_swigfaiss.IndexIVFPQ_swigregister(IndexIVFPQ) - -def initialize_IVFPQ_precomputed_table(use_precomputed_table, quantizer, pq, precomputed_table, by_residual, verbose): - r""" - Pre-compute distance tables for IVFPQ with by-residual and METRIC_L2 - - :type use_precomputed_table: int - :param use_precomputed_table: (I/O) - =-1: force disable - =0: decide heuristically (default: use tables only if they are - < precomputed_tables_max_bytes), set use_precomputed_table on - output =1: tables that work for all quantizers (size 256 * nlist * M) =2: - specific version for MultiIndexQuantizer (much more compact) - :type precomputed_table: faiss::AlignedTable< float,32 > - :param precomputed_table: precomputed table to initialize - """ - return _swigfaiss.initialize_IVFPQ_precomputed_table(use_precomputed_table, quantizer, pq, precomputed_table, by_residual, verbose) -class IndexIVFPQStats(object): - r""" - statistics are robust to internal threading, but not if - IndexIVFPQ::search_preassigned is called by multiple threads - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nrefine = property(_swigfaiss.IndexIVFPQStats_nrefine_get, _swigfaiss.IndexIVFPQStats_nrefine_set, doc=r"""nb of refines (IVFPQR)""") - n_hamming_pass = property(_swigfaiss.IndexIVFPQStats_n_hamming_pass_get, _swigfaiss.IndexIVFPQStats_n_hamming_pass_set, doc=r"""nb of passed Hamming distance tests (for polysemous)""") - search_cycles = property(_swigfaiss.IndexIVFPQStats_search_cycles_get, _swigfaiss.IndexIVFPQStats_search_cycles_set) - refine_cycles = property(_swigfaiss.IndexIVFPQStats_refine_cycles_get, _swigfaiss.IndexIVFPQStats_refine_cycles_set, doc=r"""only for IVFPQR""") - - def __init__(self): - _swigfaiss.IndexIVFPQStats_swiginit(self, _swigfaiss.new_IndexIVFPQStats()) - - def reset(self): - return _swigfaiss.IndexIVFPQStats_reset(self) - __swig_destroy__ = _swigfaiss.delete_IndexIVFPQStats - -# Register IndexIVFPQStats in _swigfaiss: -_swigfaiss.IndexIVFPQStats_swigregister(IndexIVFPQStats) -class IndexIVFPQR(IndexIVFPQ): - r"""Index with an additional level of PQ refinement""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - refine_pq = property(_swigfaiss.IndexIVFPQR_refine_pq_get, _swigfaiss.IndexIVFPQR_refine_pq_set, doc=r"""3rd level quantizer""") - refine_codes = property(_swigfaiss.IndexIVFPQR_refine_codes_get, _swigfaiss.IndexIVFPQR_refine_codes_set, doc=r"""corresponding codes""") - k_factor = property(_swigfaiss.IndexIVFPQR_k_factor_get, _swigfaiss.IndexIVFPQR_k_factor_set, doc=r"""factor between k requested in search and the k requested from the IVFPQ""") - - def reset(self): - return _swigfaiss.IndexIVFPQR_reset(self) - - def remove_ids(self, sel): - return _swigfaiss.IndexIVFPQR_remove_ids(self, sel) - - def train_encoder(self, n, x, assign): - r"""trains the two product quantizers""" - return _swigfaiss.IndexIVFPQR_train_encoder(self, n, x, assign) - - def train_encoder_num_vectors(self): - return _swigfaiss.IndexIVFPQR_train_encoder_num_vectors(self) - - def add_with_ids(self, n, x, xids): - return _swigfaiss.IndexIVFPQR_add_with_ids(self, n, x, xids) - - def add_core(self, n, x, xids, precomputed_idx, inverted_list_context=None): - r"""same as add_with_ids, but optionally use the precomputed list ids""" - return _swigfaiss.IndexIVFPQR_add_core(self, n, x, xids, precomputed_idx, inverted_list_context) - - def reconstruct_from_offset(self, list_no, offset, recons): - return _swigfaiss.IndexIVFPQR_reconstruct_from_offset(self, list_no, offset, recons) - - def merge_from(self, otherIndex, add_id): - return _swigfaiss.IndexIVFPQR_merge_from(self, otherIndex, add_id) - - def search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params=None, stats=None): - return _swigfaiss.IndexIVFPQR_search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params, stats) - - def __init__(self, *args): - _swigfaiss.IndexIVFPQR_swiginit(self, _swigfaiss.new_IndexIVFPQR(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFPQR - -# Register IndexIVFPQR in _swigfaiss: -_swigfaiss.IndexIVFPQR_swigregister(IndexIVFPQR) -class Index2Layer(IndexFlatCodes): - r""" - Same as an IndexIVFPQ without the inverted lists: codes are stored - sequentially - - The class is mainly intended to store encoded vectors that can be - accessed randomly, the search function is not implemented. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - q1 = property(_swigfaiss.Index2Layer_q1_get, _swigfaiss.Index2Layer_q1_set, doc=r"""first level quantizer""") - pq = property(_swigfaiss.Index2Layer_pq_get, _swigfaiss.Index2Layer_pq_set, doc=r"""second level quantizer is always a PQ""") - code_size_1 = property(_swigfaiss.Index2Layer_code_size_1_get, _swigfaiss.Index2Layer_code_size_1_set, doc=r"""size of the code for the first level (ceil(log8(q1.nlist)))""") - code_size_2 = property(_swigfaiss.Index2Layer_code_size_2_get, _swigfaiss.Index2Layer_code_size_2_set, doc=r"""size of the code for the second level""") - - def __init__(self, *args): - _swigfaiss.Index2Layer_swiginit(self, _swigfaiss.new_Index2Layer(*args)) - __swig_destroy__ = _swigfaiss.delete_Index2Layer - - def train(self, n, x): - return _swigfaiss.Index2Layer_train(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - r"""not implemented""" - return _swigfaiss.Index2Layer_search(self, n, x, k, distances, labels, params) - - def get_distance_computer(self): - return _swigfaiss.Index2Layer_get_distance_computer(self) - - def transfer_to_IVFPQ(self, other): - r"""transfer the flat codes to an IVFPQ index""" - return _swigfaiss.Index2Layer_transfer_to_IVFPQ(self, other) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.Index2Layer_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.Index2Layer_sa_decode(self, n, bytes, x) - -# Register Index2Layer in _swigfaiss: -_swigfaiss.Index2Layer_swigregister(Index2Layer) -class FastScanDistancePostProcessing(object): - r"""Simple context object that holds processors for FastScan operations.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - pq2x4_scale = property(_swigfaiss.FastScanDistancePostProcessing_pq2x4_scale_get, _swigfaiss.FastScanDistancePostProcessing_pq2x4_scale_set, doc=r""" - Norm scaling processor for Additive Quantizers. - The scale is encoded in a 2x4 bit PQ table, then scaled by this int. - Set to 0 if unused. - """) - query_factors = property(_swigfaiss.FastScanDistancePostProcessing_query_factors_get, _swigfaiss.FastScanDistancePostProcessing_query_factors_set, doc=r""" - Query factors data pointer for RaBitQ (nullptr if not needed) - This pointer should point to the beginning of the relevant - QueryFactorsData subset for this context. - """) - nprobe = property(_swigfaiss.FastScanDistancePostProcessing_nprobe_get, _swigfaiss.FastScanDistancePostProcessing_nprobe_set, doc=r""" - The nprobe value used when allocating query_factors storage. - This is needed because the allocation size (n * nprobe) may use a - different nprobe than index->nprobe if search params override it. - Set to 0 to use index->nprobe as fallback. - """) - qb = property(_swigfaiss.FastScanDistancePostProcessing_qb_get, _swigfaiss.FastScanDistancePostProcessing_qb_set, doc=r""" - RaBitQ query quantization bits override. - Set to 0 to use the index default (index->qb). - """) - centered = property(_swigfaiss.FastScanDistancePostProcessing_centered_get, _swigfaiss.FastScanDistancePostProcessing_centered_set, doc=r""" - RaBitQ centered scalar quantizer override. - Only used when qb > 0 (i.e., when params are overridden). - """) - - def __init__(self): - r"""Default constructor - no processing""" - _swigfaiss.FastScanDistancePostProcessing_swiginit(self, _swigfaiss.new_FastScanDistancePostProcessing()) - - def has_norm_scaling(self): - r"""Check if norm scaling is enabled""" - return _swigfaiss.FastScanDistancePostProcessing_has_norm_scaling(self) - - def has_query_processing(self): - r"""Check if query factors processing is enabled""" - return _swigfaiss.FastScanDistancePostProcessing_has_query_processing(self) - __swig_destroy__ = _swigfaiss.delete_FastScanDistancePostProcessing - -# Register FastScanDistancePostProcessing in _swigfaiss: -_swigfaiss.FastScanDistancePostProcessing_swigregister(FastScanDistancePostProcessing) -class IndexFastScan(Index): - r""" - Fast scan version of IndexPQ and IndexAQ. Works for 4-bit PQ and AQ for now. - - The codes are not stored sequentially but grouped in blocks of size bbs. - This makes it possible to compute distances quickly with SIMD instructions. - The trailing codes (padding codes that are added to complete the last code) - are garbage. - - Implementations: - 12: blocked loop with internal loop on Q with qbs - 13: same with reservoir accumulator to store results - 14: no qbs with heap accumulator - 15: no qbs with reservoir accumulator - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - implem = property(_swigfaiss.IndexFastScan_implem_get, _swigfaiss.IndexFastScan_implem_set) - skip = property(_swigfaiss.IndexFastScan_skip_get, _swigfaiss.IndexFastScan_skip_set) - bbs = property(_swigfaiss.IndexFastScan_bbs_get, _swigfaiss.IndexFastScan_bbs_set) - qbs = property(_swigfaiss.IndexFastScan_qbs_get, _swigfaiss.IndexFastScan_qbs_set) - M = property(_swigfaiss.IndexFastScan_M_get, _swigfaiss.IndexFastScan_M_set) - nbits = property(_swigfaiss.IndexFastScan_nbits_get, _swigfaiss.IndexFastScan_nbits_set) - ksub = property(_swigfaiss.IndexFastScan_ksub_get, _swigfaiss.IndexFastScan_ksub_set) - code_size = property(_swigfaiss.IndexFastScan_code_size_get, _swigfaiss.IndexFastScan_code_size_set) - ntotal2 = property(_swigfaiss.IndexFastScan_ntotal2_get, _swigfaiss.IndexFastScan_ntotal2_set) - M2 = property(_swigfaiss.IndexFastScan_M2_get, _swigfaiss.IndexFastScan_M2_set) - codes = property(_swigfaiss.IndexFastScan_codes_get, _swigfaiss.IndexFastScan_codes_set) - orig_codes = property(_swigfaiss.IndexFastScan_orig_codes_get, _swigfaiss.IndexFastScan_orig_codes_set) - - def init_fastscan(self, d, M, nbits, metric, bbs): - r""" - Initialize the fast scan index - - :type d: int - :param d: dimensionality of vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bits per subquantizer - :type metric: int - :param metric: distance metric to use - :type bbs: int - :param bbs: block size for SIMD processing - """ - return _swigfaiss.IndexFastScan_init_fastscan(self, d, M, nbits, metric, bbs) - - def reset(self): - return _swigfaiss.IndexFastScan_reset(self) - - def search(self, n, x, k, distances, labels, params=None): - r""" - Search for k nearest neighbors - - :type n: int - :param n: number of query vectors - :type x: float - :param x: query vectors (n * d) - :type k: int - :param k: number of nearest neighbors to find - :type distances: float - :param distances: output distances (n * k) - :type labels: int - :param labels: output labels/indices (n * k) - :type params: :py:class:`SearchParameters`, optional - :param params: optional search parameters - """ - return _swigfaiss.IndexFastScan_search(self, n, x, k, distances, labels, params) - - def add(self, n, x): - r""" - Add vectors to the index - - :type n: int - :param n: number of vectors to add - :type x: float - :param x: vectors to add (n * d) - """ - return _swigfaiss.IndexFastScan_add(self, n, x) - - def compute_codes(self, codes, n, x): - r""" - Compute codes for vectors - - :type codes: uint8_t - :param codes: output codes - :type n: int - :param n: number of vectors to encode - :type x: float - :param x: vectors to encode (n * d) - """ - return _swigfaiss.IndexFastScan_compute_codes(self, codes, n, x) - - def compute_float_LUT(self, lut, n, x, context): - r""" - Compute floating-point lookup table for distance computation - - :type lut: float - :param lut: output lookup table - :type n: int - :param n: number of query vectors - :type x: float - :param x: query vectors (n * d) - :type context: :py:class:`FastScanDistancePostProcessing` - :param context: processing context containing all processors - """ - return _swigfaiss.IndexFastScan_compute_float_LUT(self, lut, n, x, context) - - def compute_quantized_LUT(self, n, x, lut, normalizers, context): - return _swigfaiss.IndexFastScan_compute_quantized_LUT(self, n, x, lut, normalizers, context) - - def reconstruct(self, key, recons): - r""" - Reconstruct a vector from its code - - :type key: int - :param key: index of vector to reconstruct - :type recons: float - :param recons: output reconstructed vector - """ - return _swigfaiss.IndexFastScan_reconstruct(self, key, recons) - - def remove_ids(self, sel): - r""" - Remove vectors by ID selector - - :type sel: :py:class:`IDSelector` - :param sel: selector defining which vectors to remove - :rtype: int - :return: number of vectors removed - """ - return _swigfaiss.IndexFastScan_remove_ids(self, sel) - - def get_CodePacker(self): - r""" - Get the code packer for this index - - :rtype: :py:class:`CodePacker` - :return: pointer to the code packer - """ - return _swigfaiss.IndexFastScan_get_CodePacker(self) - - def get_block_stride(self): - r""" - Get stride in bytes between consecutive SIMD blocks. - - Derived from get_CodePacker()->block_size so that there is a - single source of truth for the block layout. - - :rtype: int - :return: stride in bytes - """ - return _swigfaiss.IndexFastScan_get_block_stride(self) - - def merge_from(self, otherIndex, add_id=0): - r""" - Merge another index into this one - - :type otherIndex: :py:class:`Index` - :param otherIndex: index to merge from - :type add_id: int, optional - :param add_id: ID offset to add to merged vectors - """ - return _swigfaiss.IndexFastScan_merge_from(self, otherIndex, add_id) - - def check_compatible_for_merge(self, otherIndex): - r""" - Check if another index is compatible for merging - - :type otherIndex: :py:class:`Index` - :param otherIndex: index to check compatibility with - """ - return _swigfaiss.IndexFastScan_check_compatible_for_merge(self, otherIndex) - - def sa_code_size(self): - r"""standalone codes interface (but the codes are flattened)""" - return _swigfaiss.IndexFastScan_sa_code_size(self) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexFastScan_sa_encode(self, n, x, bytes) - - def fast_scan_code_size(self): - r""" - Get the size of the code portion packed by pq4_pack_codes. - - Returns the number of bytes per vector that are interleaved into - SIMD blocks by pq4_pack_codes, excluding any embedded metadata - (e.g., RaBitQ factors). The meaning of these bytes depends on the - quantizer: for PQ/AQ they are 4-bit sub-quantizer nibbles, for - RaBitQ they are 1-bit-per-dimension sign bits packed into nibbles. - - Must be implemented by all derived classes. - """ - return _swigfaiss.IndexFastScan_fast_scan_code_size(self) - __swig_destroy__ = _swigfaiss.delete_IndexFastScan - -# Register IndexFastScan in _swigfaiss: -_swigfaiss.IndexFastScan_swigregister(IndexFastScan) -class FastScanStats(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - t0 = property(_swigfaiss.FastScanStats_t0_get, _swigfaiss.FastScanStats_t0_set) - t1 = property(_swigfaiss.FastScanStats_t1_get, _swigfaiss.FastScanStats_t1_set) - t2 = property(_swigfaiss.FastScanStats_t2_get, _swigfaiss.FastScanStats_t2_set) - t3 = property(_swigfaiss.FastScanStats_t3_get, _swigfaiss.FastScanStats_t3_set) - - def __init__(self): - _swigfaiss.FastScanStats_swiginit(self, _swigfaiss.new_FastScanStats()) - - def reset(self): - return _swigfaiss.FastScanStats_reset(self) - __swig_destroy__ = _swigfaiss.delete_FastScanStats - -# Register FastScanStats in _swigfaiss: -_swigfaiss.FastScanStats_swigregister(FastScanStats) -class IndexAdditiveQuantizerFastScan(IndexFastScan): - r""" - Fast scan version of IndexAQ. Works for 4-bit AQ for now. - - The codes are not stored sequentially but grouped in blocks of size bbs. - This makes it possible to compute distances quickly with SIMD instructions. - - Implementations: - 12: blocked loop with internal loop on Q with qbs - 13: same with reservoir accumulator to store results - 14: no qbs with heap accumulator - 15: no qbs with reservoir accumulator - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - aq = property(_swigfaiss.IndexAdditiveQuantizerFastScan_aq_get, _swigfaiss.IndexAdditiveQuantizerFastScan_aq_set) - rescale_norm = property(_swigfaiss.IndexAdditiveQuantizerFastScan_rescale_norm_get, _swigfaiss.IndexAdditiveQuantizerFastScan_rescale_norm_set) - norm_scale = property(_swigfaiss.IndexAdditiveQuantizerFastScan_norm_scale_get, _swigfaiss.IndexAdditiveQuantizerFastScan_norm_scale_set) - max_train_points = property(_swigfaiss.IndexAdditiveQuantizerFastScan_max_train_points_get, _swigfaiss.IndexAdditiveQuantizerFastScan_max_train_points_set) - - def init(self, *args): - return _swigfaiss.IndexAdditiveQuantizerFastScan_init(self, *args) - __swig_destroy__ = _swigfaiss.delete_IndexAdditiveQuantizerFastScan - - def __init__(self, *args): - r""" - *Overload 1:* - build from an existing IndexAQ - - | - - *Overload 2:* - build from an existing IndexAQ - """ - _swigfaiss.IndexAdditiveQuantizerFastScan_swiginit(self, _swigfaiss.new_IndexAdditiveQuantizerFastScan(*args)) - - def train(self, n, x): - return _swigfaiss.IndexAdditiveQuantizerFastScan_train(self, n, x) - - def estimate_norm_scale(self, n, x): - return _swigfaiss.IndexAdditiveQuantizerFastScan_estimate_norm_scale(self, n, x) - - def compute_codes(self, codes, n, x): - return _swigfaiss.IndexAdditiveQuantizerFastScan_compute_codes(self, codes, n, x) - - def compute_float_LUT(self, lut, n, x, context): - return _swigfaiss.IndexAdditiveQuantizerFastScan_compute_float_LUT(self, lut, n, x, context) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexAdditiveQuantizerFastScan_search(self, n, x, k, distances, labels, params) - - def sa_decode(self, n, bytes, x): - r""" - Decode a set of vectors. - - NOTE: The codes in the IndexAdditiveQuantizerFastScan object are non- - contiguous. But this method requires a contiguous representation. - - :type n: int - :param n: number of vectors - :type bytes: uint8_t - :param bytes: input encoded vectors, size n * code_size - :type x: float - :param x: output vectors, size n * d - """ - return _swigfaiss.IndexAdditiveQuantizerFastScan_sa_decode(self, n, bytes, x) - - def fast_scan_code_size(self): - r"""Packed code size: M2 / 2 bytes (4-bit AQ sub-quantizer nibbles)""" - return _swigfaiss.IndexAdditiveQuantizerFastScan_fast_scan_code_size(self) - -# Register IndexAdditiveQuantizerFastScan in _swigfaiss: -_swigfaiss.IndexAdditiveQuantizerFastScan_swigregister(IndexAdditiveQuantizerFastScan) -class IndexResidualQuantizerFastScan(IndexAdditiveQuantizerFastScan): - r""" - Index based on a residual quantizer. Stored vectors are - approximated by residual quantization codes. - Can also be used as a codec - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - rq = property(_swigfaiss.IndexResidualQuantizerFastScan_rq_get, _swigfaiss.IndexResidualQuantizerFastScan_rq_set, doc=r"""The residual quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - :type metric: int, optional - :param metric: metric type - :type search_type: int, optional - :param search_type: AQ search type - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexResidualQuantizerFastScan_swiginit(self, _swigfaiss.new_IndexResidualQuantizerFastScan(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexResidualQuantizerFastScan - -# Register IndexResidualQuantizerFastScan in _swigfaiss: -_swigfaiss.IndexResidualQuantizerFastScan_swigregister(IndexResidualQuantizerFastScan) -class IndexLocalSearchQuantizerFastScan(IndexAdditiveQuantizerFastScan): - r""" - Index based on a local search quantizer. Stored vectors are - approximated by local search quantization codes. - Can also be used as a codec - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - lsq = property(_swigfaiss.IndexLocalSearchQuantizerFastScan_lsq_get, _swigfaiss.IndexLocalSearchQuantizerFastScan_lsq_set) - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - :type metric: int, optional - :param metric: metric type - :type search_type: int, optional - :param search_type: AQ search type - - :type d: int - :param d: dimensionality of the input vectors - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexLocalSearchQuantizerFastScan_swiginit(self, _swigfaiss.new_IndexLocalSearchQuantizerFastScan(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexLocalSearchQuantizerFastScan - -# Register IndexLocalSearchQuantizerFastScan in _swigfaiss: -_swigfaiss.IndexLocalSearchQuantizerFastScan_swigregister(IndexLocalSearchQuantizerFastScan) -class IndexProductResidualQuantizerFastScan(IndexAdditiveQuantizerFastScan): - r""" - Index based on a product residual quantizer. Stored vectors are - approximated by product residual quantization codes. - Can also be used as a codec - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - prq = property(_swigfaiss.IndexProductResidualQuantizerFastScan_prq_get, _swigfaiss.IndexProductResidualQuantizerFastScan_prq_set, doc=r"""The product residual quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of residual quantizers - :type Msub: int - :param Msub: number of subquantizers per RQ - :type nbits: int - :param nbits: number of bit per subvector index - :type metric: int, optional - :param metric: metric type - :type search_type: int, optional - :param search_type: AQ search type - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of residual quantizers - :type Msub: int - :param Msub: number of subquantizers per RQ - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexProductResidualQuantizerFastScan_swiginit(self, _swigfaiss.new_IndexProductResidualQuantizerFastScan(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexProductResidualQuantizerFastScan - -# Register IndexProductResidualQuantizerFastScan in _swigfaiss: -_swigfaiss.IndexProductResidualQuantizerFastScan_swigregister(IndexProductResidualQuantizerFastScan) -class IndexProductLocalSearchQuantizerFastScan(IndexAdditiveQuantizerFastScan): - r""" - Index based on a product local search quantizer. Stored vectors are - approximated by product local search quantization codes. - Can also be used as a codec - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - plsq = property(_swigfaiss.IndexProductLocalSearchQuantizerFastScan_plsq_get, _swigfaiss.IndexProductLocalSearchQuantizerFastScan_plsq_set, doc=r"""The product local search quantizer used to encode the vectors""") - - def __init__(self, *args): - r""" - Constructor. - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of local search quantizers - :type Msub: int - :param Msub: number of subquantizers per LSQ - :type nbits: int - :param nbits: number of bit per subvector index - :type metric: int, optional - :param metric: metric type - :type search_type: int, optional - :param search_type: AQ search type - - :type d: int - :param d: dimensionality of the input vectors - :type nsplits: int - :param nsplits: number of local search quantizers - :type Msub: int - :param Msub: number of subquantizers per LSQ - :type nbits: int - :param nbits: number of bit per subvector index - """ - _swigfaiss.IndexProductLocalSearchQuantizerFastScan_swiginit(self, _swigfaiss.new_IndexProductLocalSearchQuantizerFastScan(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexProductLocalSearchQuantizerFastScan - -# Register IndexProductLocalSearchQuantizerFastScan in _swigfaiss: -_swigfaiss.IndexProductLocalSearchQuantizerFastScan_swigregister(IndexProductLocalSearchQuantizerFastScan) -class IndexPQFastScan(IndexFastScan): - r""" - Fast scan version of IndexPQ. Works for 4-bit PQ for now. - - The codes are not stored sequentially but grouped in blocks of size bbs. - This makes it possible to compute distances quickly with SIMD instructions. - - Implementations: - 12: blocked loop with internal loop on Q with qbs - 13: same with reservoir accumulator to store results - 14: no qbs with heap accumulator - 15: no qbs with reservoir accumulator - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - pq = property(_swigfaiss.IndexPQFastScan_pq_get, _swigfaiss.IndexPQFastScan_pq_set) - - def __init__(self, *args): - r""" - *Overload 1:* - build from an existing IndexPQ - - | - - *Overload 2:* - build from an existing IndexPQ - """ - _swigfaiss.IndexPQFastScan_swiginit(self, _swigfaiss.new_IndexPQFastScan(*args)) - - def train(self, n, x): - return _swigfaiss.IndexPQFastScan_train(self, n, x) - - def compute_codes(self, codes, n, x): - return _swigfaiss.IndexPQFastScan_compute_codes(self, codes, n, x) - - def compute_float_LUT(self, lut, n, x, context): - return _swigfaiss.IndexPQFastScan_compute_float_LUT(self, lut, n, x, context) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexPQFastScan_sa_decode(self, n, bytes, x) - - def fast_scan_code_size(self): - r"""Packed code size: M2 / 2 bytes (4-bit PQ sub-quantizer nibbles)""" - return _swigfaiss.IndexPQFastScan_fast_scan_code_size(self) - __swig_destroy__ = _swigfaiss.delete_IndexPQFastScan - -# Register IndexPQFastScan in _swigfaiss: -_swigfaiss.IndexPQFastScan_swigregister(IndexPQFastScan) -class IndexIVFFastScan(IndexIVF): - r""" - Fast scan version of IVFPQ and IVFAQ. Works for 4-bit PQ/AQ for now. - - The codes in the inverted lists are not stored sequentially but - grouped in blocks of size bbs. This makes it possible to very quickly - compute distances with SIMD instructions. - - Implementations (implem): - 0: auto-select implementation (default) - 1: orig's search, re-implemented - 2: orig's search, re-ordered by invlist - 10: optimizer int16 search, collect results in heap, no qbs - 11: idem, collect results in reservoir - 12: optimizer int16 search, collect results in heap, uses qbs - 13: idem, collect results in reservoir - 14: internally multithreaded implem over nq * nprobe - 15: same with reservoir - - For range search, only 10 and 12 are supported. - add 100 to the implem to force single-thread scanning (the coarse quantizer - may still use multiple threads). - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - bbs = property(_swigfaiss.IndexIVFFastScan_bbs_get, _swigfaiss.IndexIVFFastScan_bbs_set) - M = property(_swigfaiss.IndexIVFFastScan_M_get, _swigfaiss.IndexIVFFastScan_M_set) - nbits = property(_swigfaiss.IndexIVFFastScan_nbits_get, _swigfaiss.IndexIVFFastScan_nbits_set) - ksub = property(_swigfaiss.IndexIVFFastScan_ksub_get, _swigfaiss.IndexIVFFastScan_ksub_set) - M2 = property(_swigfaiss.IndexIVFFastScan_M2_get, _swigfaiss.IndexIVFFastScan_M2_set) - implem = property(_swigfaiss.IndexIVFFastScan_implem_get, _swigfaiss.IndexIVFFastScan_implem_set) - skip = property(_swigfaiss.IndexIVFFastScan_skip_get, _swigfaiss.IndexIVFFastScan_skip_set) - qbs = property(_swigfaiss.IndexIVFFastScan_qbs_get, _swigfaiss.IndexIVFFastScan_qbs_set) - qbs2 = property(_swigfaiss.IndexIVFFastScan_qbs2_get, _swigfaiss.IndexIVFFastScan_qbs2_set) - fine_quantizer = property(_swigfaiss.IndexIVFFastScan_fine_quantizer_get, _swigfaiss.IndexIVFFastScan_fine_quantizer_set) - - def init_fastscan(self, fine_quantizer, M, nbits, nlist, metric, bbs, own_invlists): - r""" - Initialize the fast scan functionality (called by implementations) - - :type fine_quantizer: :py:class:`Quantizer` - :param fine_quantizer: fine quantizer for encoding - :type M: int - :param M: number of subquantizers - :type nbits: int - :param nbits: number of bits per subquantizer - :type nlist: int - :param nlist: number of inverted lists - :type metric: int - :param metric: distance metric to use - :type bbs: int - :param bbs: block size for SIMD processing - :type own_invlists: boolean - :param own_invlists: whether to own the inverted lists - """ - return _swigfaiss.IndexIVFFastScan_init_fastscan(self, fine_quantizer, M, nbits, nlist, metric, bbs, own_invlists) - - def init_code_packer(self): - return _swigfaiss.IndexIVFFastScan_init_code_packer(self) - __swig_destroy__ = _swigfaiss.delete_IndexIVFFastScan - orig_invlists = property(_swigfaiss.IndexIVFFastScan_orig_invlists_get, _swigfaiss.IndexIVFFastScan_orig_invlists_set, doc=r"""orig's inverted lists (for debugging)""") - - def add_with_ids(self, n, x, xids): - r""" - Add vectors with specific IDs to the index - - :type n: int - :param n: number of vectors to add - :type x: float - :param x: vectors to add (n * d) - :type xids: int - :param xids: IDs for the vectors (n) - """ - return _swigfaiss.IndexIVFFastScan_add_with_ids(self, n, x, xids) - - def lookup_table_is_3d(self): - return _swigfaiss.IndexIVFFastScan_lookup_table_is_3d(self) - - def compute_LUT(self, n, x, cq, dis_tables, biases, context): - return _swigfaiss.IndexIVFFastScan_compute_LUT(self, n, x, cq, dis_tables, biases, context) - - def compute_LUT_uint8(self, n, x, cq, dis_tables, biases, normalizers, context): - r""" - Compute quantized lookup tables for distance computation - - :type n: int - :param n: number of query vectors - :type x: float - :param x: query vectors (n * d) - :type cq: faiss::IndexIVFFastScan::CoarseQuantized - :param cq: coarse quantization results - :type dis_tables: faiss::AlignedTable< uint8_t > - :param dis_tables: output quantized distance tables - :type biases: faiss::AlignedTable< uint16_t > - :param biases: output quantized bias values - :type normalizers: float - :param normalizers: output normalization factors - :type context: :py:class:`FastScanDistancePostProcessing` - :param context: processing context containing query factors - processor - """ - return _swigfaiss.IndexIVFFastScan_compute_LUT_uint8(self, n, x, cq, dis_tables, biases, normalizers, context) - - def search(self, n, x, k, distances, labels, params=None): - r""" - Search for k nearest neighbors - - :type n: int - :param n: number of query vectors - :type x: float - :param x: query vectors (n * d) - :type k: int - :param k: number of nearest neighbors to find - :type distances: float - :param distances: output distances (n * k) - :type labels: int - :param labels: output labels/indices (n * k) - :type params: :py:class:`SearchParameters`, optional - :param params: optional search parameters - """ - return _swigfaiss.IndexIVFFastScan_search(self, n, x, k, distances, labels, params) - - def search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params=None, stats=None): - r""" - Search with pre-assigned coarse quantization - - :type n: int - :param n: number of query vectors - :type x: float - :param x: query vectors (n * d) - :type k: int - :param k: number of nearest neighbors to find - :type assign: int - :param assign: coarse cluster assignments (n * nprobe) - :type centroid_dis: float - :param centroid_dis: distances to centroids (n * nprobe) - :type distances: float - :param distances: output distances (n * k) - :type labels: int - :param labels: output labels/indices (n * k) - :type store_pairs: boolean - :param store_pairs: whether to store cluster-relative pairs - :type params: :py:class:`IVFSearchParameters`, optional - :param params: optional IVF search parameters - :type stats: :py:class:`IndexIVFStats`, optional - :param stats: optional search statistics - """ - return _swigfaiss.IndexIVFFastScan_search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params, stats) - - def range_search(self, n, x, radius, result, params=None): - r""" - Range search for all neighbors within radius - - :type n: int - :param n: number of query vectors - :type x: float - :param x: query vectors (n * d) - :type radius: float - :param radius: search radius - :type result: :py:class:`RangeSearchResult` - :param result: output range search results - :type params: :py:class:`SearchParameters`, optional - :param params: optional search parameters - """ - return _swigfaiss.IndexIVFFastScan_range_search(self, n, x, radius, result, params) - - def search_dispatch_implem(self, n, x, k, distances, labels, cq, context, params=None): - return _swigfaiss.IndexIVFFastScan_search_dispatch_implem(self, n, x, k, distances, labels, cq, context, params) - - def range_search_dispatch_implem(self, n, x, radius, rres, cq_in, context, params=None): - return _swigfaiss.IndexIVFFastScan_range_search_dispatch_implem(self, n, x, radius, rres, cq_in, context, params) - - def search_implem_14(self, n, x, k, distances, labels, cq, impl, context, params=None): - return _swigfaiss.IndexIVFFastScan_search_implem_14(self, n, x, k, distances, labels, cq, impl, context, params) - - def reconstruct_from_offset(self, list_no, offset, recons): - return _swigfaiss.IndexIVFFastScan_reconstruct_from_offset(self, list_no, offset, recons) - - def get_CodePacker(self): - return _swigfaiss.IndexIVFFastScan_get_CodePacker(self) - - def reconstruct_orig_invlists(self): - return _swigfaiss.IndexIVFFastScan_reconstruct_orig_invlists(self) - - def sa_decode(self, n, bytes, x): - r""" - Decode a set of vectors - - NOTE: The codes in the IndexFastScan object are non-contiguous. - But this method requires a contiguous representation. - - :type n: int - :param n: number of vectors - :type bytes: uint8_t - :param bytes: input encoded vectors, size n * code_size - :type x: float - :param x: output vectors, size n * d - """ - return _swigfaiss.IndexIVFFastScan_sa_decode(self, n, bytes, x) - - def fast_scan_code_size(self): - r""" - Get the size of the code portion packed by pq4_pack_codes. - - Returns the number of bytes per vector that are interleaved into - SIMD blocks by pq4_pack_codes, excluding any embedded metadata - (e.g., RaBitQ factors). The meaning of these bytes depends on the - quantizer: for PQ/AQ they are 4-bit sub-quantizer nibbles, for - RaBitQ they are 1-bit-per-dimension sign bits packed into nibbles. - - Must be implemented by all derived classes. - """ - return _swigfaiss.IndexIVFFastScan_fast_scan_code_size(self) - - def get_block_stride(self): - r""" - Get stride in bytes between consecutive SIMD blocks. - - Derived from get_CodePacker()->block_size so that there is a - single source of truth for the block layout. - - :rtype: int - :return: stride in bytes - """ - return _swigfaiss.IndexIVFFastScan_get_block_stride(self) - - def postprocess_packed_codes(self, list_no, list_offset, n_added, flat_codes): - r""" - Post-process packed codes after pq4_pack_codes_range. - - Called during add_with_ids after codes have been packed into - SIMD-friendly blocks. - - :type list_no: int - :param list_no: inverted list number - :type list_offset: int - :param list_offset: starting offset within the list (pre-existing size) - :type n_added: int - :param n_added: number of vectors added in this batch - :type flat_codes: uint8_t - :param flat_codes: encoded vectors for this batch (n_added * code_size) - """ - return _swigfaiss.IndexIVFFastScan_postprocess_packed_codes(self, list_no, list_offset, n_added, flat_codes) - -# Register IndexIVFFastScan in _swigfaiss: -_swigfaiss.IndexIVFFastScan_swigregister(IndexIVFFastScan) -class IVFFastScanStats(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - times = property(_swigfaiss.IVFFastScanStats_times_get, _swigfaiss.IVFFastScanStats_times_set) - t_compute_distance_tables = property(_swigfaiss.IVFFastScanStats_t_compute_distance_tables_get, _swigfaiss.IVFFastScanStats_t_compute_distance_tables_set) - t_round = property(_swigfaiss.IVFFastScanStats_t_round_get, _swigfaiss.IVFFastScanStats_t_round_set) - t_copy_pack = property(_swigfaiss.IVFFastScanStats_t_copy_pack_get, _swigfaiss.IVFFastScanStats_t_copy_pack_set) - t_scan = property(_swigfaiss.IVFFastScanStats_t_scan_get, _swigfaiss.IVFFastScanStats_t_scan_set) - t_to_flat = property(_swigfaiss.IVFFastScanStats_t_to_flat_get, _swigfaiss.IVFFastScanStats_t_to_flat_set) - reservoir_times = property(_swigfaiss.IVFFastScanStats_reservoir_times_get, _swigfaiss.IVFFastScanStats_reservoir_times_set) - t_aq_encode = property(_swigfaiss.IVFFastScanStats_t_aq_encode_get, _swigfaiss.IVFFastScanStats_t_aq_encode_set) - t_aq_norm_encode = property(_swigfaiss.IVFFastScanStats_t_aq_norm_encode_get, _swigfaiss.IVFFastScanStats_t_aq_norm_encode_set) - - def Mcy_at(self, i): - return _swigfaiss.IVFFastScanStats_Mcy_at(self, i) - - def Mcy_reservoir_at(self, i): - return _swigfaiss.IVFFastScanStats_Mcy_reservoir_at(self, i) - - def __init__(self): - _swigfaiss.IVFFastScanStats_swiginit(self, _swigfaiss.new_IVFFastScanStats()) - - def reset(self): - return _swigfaiss.IVFFastScanStats_reset(self) - __swig_destroy__ = _swigfaiss.delete_IVFFastScanStats - -# Register IVFFastScanStats in _swigfaiss: -_swigfaiss.IVFFastScanStats_swigregister(IVFFastScanStats) -class IndexIVFAdditiveQuantizerFastScan(IndexIVFFastScan): - r""" - Fast scan version of IVFAQ. Works for 4-bit AQ for now. - - The codes in the inverted lists are not stored sequentially but - grouped in blocks of size bbs. This makes it possible to very quickly - compute distances with SIMD instructions. - - Implementations (implem): - 0: auto-select implementation (default) - 1: orig's search, re-implemented - 2: orig's search, re-ordered by invlist - 10: optimizer int16 search, collect results in heap, no qbs - 11: idem, collect results in reservoir - 12: optimizer int16 search, collect results in heap, uses qbs - 13: idem, collect results in reservoir - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - aq = property(_swigfaiss.IndexIVFAdditiveQuantizerFastScan_aq_get, _swigfaiss.IndexIVFAdditiveQuantizerFastScan_aq_set) - rescale_norm = property(_swigfaiss.IndexIVFAdditiveQuantizerFastScan_rescale_norm_get, _swigfaiss.IndexIVFAdditiveQuantizerFastScan_rescale_norm_set) - norm_scale = property(_swigfaiss.IndexIVFAdditiveQuantizerFastScan_norm_scale_get, _swigfaiss.IndexIVFAdditiveQuantizerFastScan_norm_scale_set) - max_train_points = property(_swigfaiss.IndexIVFAdditiveQuantizerFastScan_max_train_points_get, _swigfaiss.IndexIVFAdditiveQuantizerFastScan_max_train_points_set) - - def init(self, aq, nlist, metric, bbs, own_invlists): - return _swigfaiss.IndexIVFAdditiveQuantizerFastScan_init(self, aq, nlist, metric, bbs, own_invlists) - __swig_destroy__ = _swigfaiss.delete_IndexIVFAdditiveQuantizerFastScan - - def __init__(self, *args): - _swigfaiss.IndexIVFAdditiveQuantizerFastScan_swiginit(self, _swigfaiss.new_IndexIVFAdditiveQuantizerFastScan(*args)) - - def fast_scan_code_size(self): - r"""Packed code size: M2 / 2 bytes (4-bit AQ sub-quantizer nibbles)""" - return _swigfaiss.IndexIVFAdditiveQuantizerFastScan_fast_scan_code_size(self) - - def train_encoder(self, n, x, assign): - return _swigfaiss.IndexIVFAdditiveQuantizerFastScan_train_encoder(self, n, x, assign) - - def train_encoder_num_vectors(self): - return _swigfaiss.IndexIVFAdditiveQuantizerFastScan_train_encoder_num_vectors(self) - - def estimate_norm_scale(self, n, x): - return _swigfaiss.IndexIVFAdditiveQuantizerFastScan_estimate_norm_scale(self, n, x) - - def encode_vectors(self, n, x, list_nos, codes, include_listno=False): - r""" - same as the regular IVFAQ encoder. The codes are not reorganized by - blocks a that point - """ - return _swigfaiss.IndexIVFAdditiveQuantizerFastScan_encode_vectors(self, n, x, list_nos, codes, include_listno) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexIVFAdditiveQuantizerFastScan_search(self, n, x, k, distances, labels, params) - - def lookup_table_is_3d(self): - return _swigfaiss.IndexIVFAdditiveQuantizerFastScan_lookup_table_is_3d(self) - - def compute_LUT(self, n, x, cq, dis_tables, biases, context): - return _swigfaiss.IndexIVFAdditiveQuantizerFastScan_compute_LUT(self, n, x, cq, dis_tables, biases, context) - -# Register IndexIVFAdditiveQuantizerFastScan in _swigfaiss: -_swigfaiss.IndexIVFAdditiveQuantizerFastScan_swigregister(IndexIVFAdditiveQuantizerFastScan) -class IndexIVFLocalSearchQuantizerFastScan(IndexIVFAdditiveQuantizerFastScan): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - lsq = property(_swigfaiss.IndexIVFLocalSearchQuantizerFastScan_lsq_get, _swigfaiss.IndexIVFLocalSearchQuantizerFastScan_lsq_set) - - def __init__(self, *args): - _swigfaiss.IndexIVFLocalSearchQuantizerFastScan_swiginit(self, _swigfaiss.new_IndexIVFLocalSearchQuantizerFastScan(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFLocalSearchQuantizerFastScan - -# Register IndexIVFLocalSearchQuantizerFastScan in _swigfaiss: -_swigfaiss.IndexIVFLocalSearchQuantizerFastScan_swigregister(IndexIVFLocalSearchQuantizerFastScan) -class IndexIVFResidualQuantizerFastScan(IndexIVFAdditiveQuantizerFastScan): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - rq = property(_swigfaiss.IndexIVFResidualQuantizerFastScan_rq_get, _swigfaiss.IndexIVFResidualQuantizerFastScan_rq_set) - - def __init__(self, *args): - _swigfaiss.IndexIVFResidualQuantizerFastScan_swiginit(self, _swigfaiss.new_IndexIVFResidualQuantizerFastScan(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFResidualQuantizerFastScan - -# Register IndexIVFResidualQuantizerFastScan in _swigfaiss: -_swigfaiss.IndexIVFResidualQuantizerFastScan_swigregister(IndexIVFResidualQuantizerFastScan) -class IndexIVFProductLocalSearchQuantizerFastScan(IndexIVFAdditiveQuantizerFastScan): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - plsq = property(_swigfaiss.IndexIVFProductLocalSearchQuantizerFastScan_plsq_get, _swigfaiss.IndexIVFProductLocalSearchQuantizerFastScan_plsq_set) - - def __init__(self, *args): - _swigfaiss.IndexIVFProductLocalSearchQuantizerFastScan_swiginit(self, _swigfaiss.new_IndexIVFProductLocalSearchQuantizerFastScan(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFProductLocalSearchQuantizerFastScan - -# Register IndexIVFProductLocalSearchQuantizerFastScan in _swigfaiss: -_swigfaiss.IndexIVFProductLocalSearchQuantizerFastScan_swigregister(IndexIVFProductLocalSearchQuantizerFastScan) -class IndexIVFProductResidualQuantizerFastScan(IndexIVFAdditiveQuantizerFastScan): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - prq = property(_swigfaiss.IndexIVFProductResidualQuantizerFastScan_prq_get, _swigfaiss.IndexIVFProductResidualQuantizerFastScan_prq_set) - - def __init__(self, *args): - _swigfaiss.IndexIVFProductResidualQuantizerFastScan_swiginit(self, _swigfaiss.new_IndexIVFProductResidualQuantizerFastScan(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexIVFProductResidualQuantizerFastScan - -# Register IndexIVFProductResidualQuantizerFastScan in _swigfaiss: -_swigfaiss.IndexIVFProductResidualQuantizerFastScan_swigregister(IndexIVFProductResidualQuantizerFastScan) -class IndexIVFIndependentQuantizer(Index): - r""" - An IVF index with a quantizer that has a different input dimension from the - payload size. The vectors to encode are obtained from the input vectors by a - VectorTransform. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - quantizer = property(_swigfaiss.IndexIVFIndependentQuantizer_quantizer_get, _swigfaiss.IndexIVFIndependentQuantizer_quantizer_set, doc=r"""quantizer is fed directly with the input vectors""") - vt = property(_swigfaiss.IndexIVFIndependentQuantizer_vt_get, _swigfaiss.IndexIVFIndependentQuantizer_vt_set, doc=r"""transform before the IVF vectors are applied""") - index_ivf = property(_swigfaiss.IndexIVFIndependentQuantizer_index_ivf_get, _swigfaiss.IndexIVFIndependentQuantizer_index_ivf_set, doc=r"""the IVF index, controls nlist and nprobe""") - own_fields = property(_swigfaiss.IndexIVFIndependentQuantizer_own_fields_get, _swigfaiss.IndexIVFIndependentQuantizer_own_fields_set, doc=r"""whether *this owns the 3 fields""") - - def __init__(self, *args): - _swigfaiss.IndexIVFIndependentQuantizer_swiginit(self, _swigfaiss.new_IndexIVFIndependentQuantizer(*args)) - - def train(self, n, x): - return _swigfaiss.IndexIVFIndependentQuantizer_train(self, n, x) - - def add(self, n, x): - return _swigfaiss.IndexIVFIndependentQuantizer_add(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexIVFIndependentQuantizer_search(self, n, x, k, distances, labels, params) - - def reset(self): - return _swigfaiss.IndexIVFIndependentQuantizer_reset(self) - __swig_destroy__ = _swigfaiss.delete_IndexIVFIndependentQuantizer - -# Register IndexIVFIndependentQuantizer in _swigfaiss: -_swigfaiss.IndexIVFIndependentQuantizer_swigregister(IndexIVFIndependentQuantizer) -class IndexIVFPQFastScan(IndexIVFFastScan): - r""" - Fast scan version of IVFPQ. Works for 4-bit PQ for now. - - The codes in the inverted lists are not stored sequentially but - grouped in blocks of size bbs. This makes it possible to very quickly - compute distances with SIMD instructions. - - Implementations (implem): - 0: auto-select implementation (default) - 1: orig's search, re-implemented - 2: orig's search, re-ordered by invlist - 10: optimizer int16 search, collect results in heap, no qbs - 11: idem, collect results in reservoir - 12: optimizer int16 search, collect results in heap, uses qbs - 13: idem, collect results in reservoir - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - pq = property(_swigfaiss.IndexIVFPQFastScan_pq_get, _swigfaiss.IndexIVFPQFastScan_pq_set, doc=r"""produces the codes""") - use_precomputed_table = property(_swigfaiss.IndexIVFPQFastScan_use_precomputed_table_get, _swigfaiss.IndexIVFPQFastScan_use_precomputed_table_set, doc=r"""precomputed tables management""") - precomputed_table = property(_swigfaiss.IndexIVFPQFastScan_precomputed_table_get, _swigfaiss.IndexIVFPQFastScan_precomputed_table_set, doc=r"""if use_precompute_table size (nlist, pq.M, pq.ksub)""") - - def __init__(self, *args): - _swigfaiss.IndexIVFPQFastScan_swiginit(self, _swigfaiss.new_IndexIVFPQFastScan(*args)) - - def fast_scan_code_size(self): - r"""Packed code size: M2 / 2 bytes (4-bit PQ sub-quantizer nibbles)""" - return _swigfaiss.IndexIVFPQFastScan_fast_scan_code_size(self) - - def train_encoder(self, n, x, assign): - return _swigfaiss.IndexIVFPQFastScan_train_encoder(self, n, x, assign) - - def train_encoder_num_vectors(self): - return _swigfaiss.IndexIVFPQFastScan_train_encoder_num_vectors(self) - - def precompute_table(self): - r"""build precomputed table, possibly updating use_precomputed_table""" - return _swigfaiss.IndexIVFPQFastScan_precompute_table(self) - - def encode_vectors(self, n, x, list_nos, codes, include_listno=False): - r""" - same as the regular IVFPQ encoder. The codes are not reorganized by - blocks a that point - """ - return _swigfaiss.IndexIVFPQFastScan_encode_vectors(self, n, x, list_nos, codes, include_listno) - - def lookup_table_is_3d(self): - return _swigfaiss.IndexIVFPQFastScan_lookup_table_is_3d(self) - - def compute_LUT(self, n, x, cq, dis_tables, biases, context): - return _swigfaiss.IndexIVFPQFastScan_compute_LUT(self, n, x, cq, dis_tables, biases, context) - - def get_InvertedListScanner(self, store_pairs, sel, arg4): - return _swigfaiss.IndexIVFPQFastScan_get_InvertedListScanner(self, store_pairs, sel, arg4) - __swig_destroy__ = _swigfaiss.delete_IndexIVFPQFastScan - -# Register IndexIVFPQFastScan in _swigfaiss: -_swigfaiss.IndexIVFPQFastScan_swigregister(IndexIVFPQFastScan) - -def round_uint8_per_column(tab, n, d, a_out=None, b_out=None): - r""" - Functions to quantize PQ floating-point Look Up Tables (LUT) to uint8, and - biases to uint16. The accumulation is supposed to take place in uint16. - The quantization coefficients are float (a, b) such that - - original_value = quantized_value * a / b - - The hardest part of the quantization is with multiple LUTs that need to be - added up together. In that case, coefficient a has to be chosen so that - the sum fits in a uint16 accumulator. - """ - return _swigfaiss.round_uint8_per_column(tab, n, d, a_out, b_out) - -def round_uint8_per_column_multi(tab, m, n, d, a_out=None, b_out=None): - return _swigfaiss.round_uint8_per_column_multi(tab, m, n, d, a_out, b_out) - -def quantize_LUT_and_bias(nprobe, M, ksub, lut_is_3d, LUT, bias, LUTq, M2, biasq, a_out=None, b_out=None): - r""" - LUT quantization to uint8 and bias to uint16. - - (nprobe, M, ksub, lut_is_3d) determine the size of the LUT - - LUT input: - - 2D size (M, ksub): single matrix per probe (lut_is_3d=false) - - 3D size (nprobe, M, ksub): separate LUT per probe (lut_is_3d=true) - bias input: - - nullptr: bias is 0 - - size (nprobe): one bias per probe - Output: - - LUTq uint8 version of the LUT (M size is rounded up to M2) - - biasq (or nullptr): uint16 version of the LUT - - a, b: scalars to approximate the true distance - """ - return _swigfaiss.quantize_LUT_and_bias(nprobe, M, ksub, lut_is_3d, LUT, bias, LUTq, M2, biasq, a_out, b_out) - -def aq_quantize_LUT_and_bias(nprobe, M, ksub, LUT, bias, M_norm, norm_scale, LUTq, M2, biasq, a_out, b_out): - return _swigfaiss.aq_quantize_LUT_and_bias(nprobe, M, ksub, LUT, bias, M_norm, norm_scale, LUTq, M2, biasq, a_out, b_out) - -def aq_estimate_norm_scale(M, ksub, M_norm, LUT): - return _swigfaiss.aq_estimate_norm_scale(M, ksub, M_norm, LUT) -class IndexBinary(object): - r""" - Abstract structure for a binary index. - - Supports adding vertices and searching them. - - All queries are symmetric because there is no distinction between codes and - vectors. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - d = property(_swigfaiss.IndexBinary_d_get, _swigfaiss.IndexBinary_d_set, doc=r"""vector dimension""") - code_size = property(_swigfaiss.IndexBinary_code_size_get, _swigfaiss.IndexBinary_code_size_set, doc=r"""number of bytes per vector ( = d / 8 )""") - ntotal = property(_swigfaiss.IndexBinary_ntotal_get, _swigfaiss.IndexBinary_ntotal_set, doc=r"""total nb of indexed vectors""") - verbose = property(_swigfaiss.IndexBinary_verbose_get, _swigfaiss.IndexBinary_verbose_set, doc=r"""verbosity level""") - is_trained = property(_swigfaiss.IndexBinary_is_trained_get, _swigfaiss.IndexBinary_is_trained_set, doc=r""" - set if the Index does not require training, or if training is done - already - """) - metric_type = property(_swigfaiss.IndexBinary_metric_type_get, _swigfaiss.IndexBinary_metric_type_set, doc=r"""type of metric this index uses for search""") - __swig_destroy__ = _swigfaiss.delete_IndexBinary - - def train(self, n, x): - r""" - Perform training on a representative set of vectors. - - :type n: int - :param n: nb of training vectors - :type x: uint8_t - :param x: training vectors, size n * d / 8 - """ - return _swigfaiss.IndexBinary_train(self, n, x) - - def train_ex(self, n, x, numeric_type): - return _swigfaiss.IndexBinary_train_ex(self, n, x, numeric_type) - - def add(self, n, x): - r""" - Add n vectors of dimension d to the index. - - Vectors are implicitly assigned labels ntotal .. ntotal + n - 1 - :type x: uint8_t - :param x: input matrix, size n * d / 8 - """ - return _swigfaiss.IndexBinary_add(self, n, x) - - def add_ex(self, n, x, numeric_type): - return _swigfaiss.IndexBinary_add_ex(self, n, x, numeric_type) - - def add_with_ids(self, n, x, xids): - r""" - Same as add, but stores xids instead of sequential ids. - - The default implementation fails with an assertion, as it is - not supported by all indexes. - - :type xids: int - :param xids: if non-null, ids to store for the vectors (size n) - """ - return _swigfaiss.IndexBinary_add_with_ids(self, n, x, xids) - - def add_with_ids_ex(self, n, x, numeric_type, xids): - return _swigfaiss.IndexBinary_add_with_ids_ex(self, n, x, numeric_type, xids) - - def search(self, n, x, k, distances, labels, params=None): - r""" - Query n vectors of dimension d to the index. - - return at most k vectors. If there are not enough results for a - query, the result array is padded with -1s. - - :type x: uint8_t - :param x: input vectors to search, size n * d / 8 - :type labels: int - :param labels: output labels of the NNs, size n*k - :type distances: int - :param distances: output pairwise distances, size n*k - """ - return _swigfaiss.IndexBinary_search(self, n, x, k, distances, labels, params) - - def search_ex(self, n, x, numeric_type, k, distances, labels, params=None): - return _swigfaiss.IndexBinary_search_ex(self, n, x, numeric_type, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - r""" - Query n vectors of dimension d to the index. - - return all vectors with distance < radius. Note that many indexes - do not implement the range_search (only the k-NN search is - mandatory). The distances are converted to float to reuse the - RangeSearchResult structure, but they are integer. By convention, - only distances < radius (strict comparison) are returned, - ie. radius = 0 does not return any result and 1 returns only - exact same vectors. - - :type x: uint8_t - :param x: input vectors to search, size n * d / 8 - :type radius: int - :param radius: search radius - :type result: :py:class:`RangeSearchResult` - :param result: result table - """ - return _swigfaiss.IndexBinary_range_search(self, n, x, radius, result, params) - - def assign(self, n, x, labels, k=1): - r""" - Return the indexes of the k vectors closest to the query x. - - This function is identical to search but only returns labels of - neighbors. - :type x: uint8_t - :param x: input vectors to search, size n * d / 8 - :type labels: int - :param labels: output labels of the NNs, size n*k - """ - return _swigfaiss.IndexBinary_assign(self, n, x, labels, k) - - def reset(self): - r"""Removes all elements from the database.""" - return _swigfaiss.IndexBinary_reset(self) - - def remove_ids(self, sel): - r"""Removes IDs from the index. Not supported by all indexes.""" - return _swigfaiss.IndexBinary_remove_ids(self, sel) - - def reconstruct(self, key, recons): - r""" - Reconstruct a stored vector. - - This function may not be defined for some indexes. - :type key: int - :param key: id of the vector to reconstruct - :type recons: uint8_t - :param recons: reconstructed vector (size d / 8) - """ - return _swigfaiss.IndexBinary_reconstruct(self, key, recons) - - def reconstruct_n(self, i0, ni, recons): - r""" - Reconstruct vectors i0 to i0 + ni - 1. - - This function may not be defined for some indexes. - :type recons: uint8_t - :param recons: reconstructed vectors (size ni * d / 8) - """ - return _swigfaiss.IndexBinary_reconstruct_n(self, i0, ni, recons) - - def search_and_reconstruct(self, n, x, k, distances, labels, recons, params=None): - r""" - Similar to search, but also reconstructs the stored vectors (or an - approximation in the case of lossy coding) for the search results. - - If there are not enough results for a query, the resulting array - is padded with -1s. - - :type recons: uint8_t - :param recons: reconstructed vectors size (n, k, d) - """ - return _swigfaiss.IndexBinary_search_and_reconstruct(self, n, x, k, distances, labels, recons, params) - - def display(self): - r"""Display the actual class name and some more info.""" - return _swigfaiss.IndexBinary_display(self) - - def merge_from(self, otherIndex, add_id=0): - r""" - moves the entries from another dataset to self. - On output, other is empty. - add_id is added to all moved ids - (for sequential ids, this would be this->ntotal) - """ - return _swigfaiss.IndexBinary_merge_from(self, otherIndex, add_id) - - def check_compatible_for_merge(self, otherIndex): - r""" - check that the two indexes are compatible (ie, they are - trained in the same way and have the same - parameters). Otherwise throw. - """ - return _swigfaiss.IndexBinary_check_compatible_for_merge(self, otherIndex) - - def sa_code_size(self): - r"""size of the produced codes in bytes""" - return _swigfaiss.IndexBinary_sa_code_size(self) - - def add_sa_codes(self, n, codes, xids): - r"""Same as add_with_ids for IndexBinary.""" - return _swigfaiss.IndexBinary_add_sa_codes(self, n, codes, xids) - -# Register IndexBinary in _swigfaiss: -_swigfaiss.IndexBinary_swigregister(IndexBinary) -class IndexBinaryFlat(IndexBinary): - r"""Index that stores the full vectors and performs exhaustive search.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - xb = property(_swigfaiss.IndexBinaryFlat_xb_get, _swigfaiss.IndexBinaryFlat_xb_set, doc=r"""database vectors, size ntotal * d / 8""") - use_heap = property(_swigfaiss.IndexBinaryFlat_use_heap_get, _swigfaiss.IndexBinaryFlat_use_heap_set, doc=r""" - Select between using a heap or counting to select the k smallest values - when scanning inverted lists. - """) - query_batch_size = property(_swigfaiss.IndexBinaryFlat_query_batch_size_get, _swigfaiss.IndexBinaryFlat_query_batch_size_set) - approx_topk_mode = property(_swigfaiss.IndexBinaryFlat_approx_topk_mode_get, _swigfaiss.IndexBinaryFlat_approx_topk_mode_set) - - def add(self, n, x): - return _swigfaiss.IndexBinaryFlat_add(self, n, x) - - def reset(self): - return _swigfaiss.IndexBinaryFlat_reset(self) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexBinaryFlat_search(self, n, x, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexBinaryFlat_range_search(self, n, x, radius, result, params) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexBinaryFlat_reconstruct(self, key, recons) - - def remove_ids(self, sel): - r""" - Remove some ids. Note that because of the indexing structure, - the semantics of this operation are different from the usual ones: - the new ids are shifted. - """ - return _swigfaiss.IndexBinaryFlat_remove_ids(self, sel) - - def __init__(self, *args): - _swigfaiss.IndexBinaryFlat_swiginit(self, _swigfaiss.new_IndexBinaryFlat(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryFlat - -# Register IndexBinaryFlat in _swigfaiss: -_swigfaiss.IndexBinaryFlat_swigregister(IndexBinaryFlat) -class IndexBinaryIVF(IndexBinary): - r""" - Index based on a inverted file (IVF) - - In the inverted file, the quantizer (an IndexBinary instance) provides a - quantization index for each vector to be added. The quantization - index maps to a list (aka inverted list or posting list), where the - id of the vector is stored. - - Otherwise the object is similar to the IndexIVF - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - invlists = property(_swigfaiss.IndexBinaryIVF_invlists_get, _swigfaiss.IndexBinaryIVF_invlists_set, doc=r"""Access to the actual data""") - own_invlists = property(_swigfaiss.IndexBinaryIVF_own_invlists_get, _swigfaiss.IndexBinaryIVF_own_invlists_set) - nprobe = property(_swigfaiss.IndexBinaryIVF_nprobe_get, _swigfaiss.IndexBinaryIVF_nprobe_set, doc=r"""number of probes at query time""") - max_codes = property(_swigfaiss.IndexBinaryIVF_max_codes_get, _swigfaiss.IndexBinaryIVF_max_codes_set, doc=r"""max nb of codes to visit to do a query""") - use_heap = property(_swigfaiss.IndexBinaryIVF_use_heap_get, _swigfaiss.IndexBinaryIVF_use_heap_set, doc=r""" - Select between using a heap or counting to select the k smallest values - when scanning inverted lists. - """) - per_invlist_search = property(_swigfaiss.IndexBinaryIVF_per_invlist_search_get, _swigfaiss.IndexBinaryIVF_per_invlist_search_set, doc=r"""collect computations per batch""") - direct_map = property(_swigfaiss.IndexBinaryIVF_direct_map_get, _swigfaiss.IndexBinaryIVF_direct_map_set, doc=r"""map for direct access to the elements. Enables reconstruct().""") - quantizer = property(_swigfaiss.IndexBinaryIVF_quantizer_get, _swigfaiss.IndexBinaryIVF_quantizer_set, doc=r"""quantizer that maps vectors to inverted lists""") - nlist = property(_swigfaiss.IndexBinaryIVF_nlist_get, _swigfaiss.IndexBinaryIVF_nlist_set, doc=r"""number of possible key values""") - own_fields = property(_swigfaiss.IndexBinaryIVF_own_fields_get, _swigfaiss.IndexBinaryIVF_own_fields_set, doc=r"""whether object owns the quantizer""") - cp = property(_swigfaiss.IndexBinaryIVF_cp_get, _swigfaiss.IndexBinaryIVF_cp_set, doc=r"""to override default clustering params""") - clustering_index = property(_swigfaiss.IndexBinaryIVF_clustering_index_get, _swigfaiss.IndexBinaryIVF_clustering_index_set, doc=r"""to override index used during clustering""") - - def __init__(self, *args): - r""" - The Inverted file takes a quantizer (an IndexBinary) on input, - which implements the function mapping a vector to a list - identifier. The pointer is borrowed: the quantizer should not - be deleted while the IndexBinaryIVF is in use. - """ - _swigfaiss.IndexBinaryIVF_swiginit(self, _swigfaiss.new_IndexBinaryIVF(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryIVF - - def reset(self): - return _swigfaiss.IndexBinaryIVF_reset(self) - - def train(self, n, x): - r"""Trains the quantizer""" - return _swigfaiss.IndexBinaryIVF_train(self, n, x) - - def add(self, n, x): - return _swigfaiss.IndexBinaryIVF_add(self, n, x) - - def add_with_ids(self, n, x, xids): - return _swigfaiss.IndexBinaryIVF_add_with_ids(self, n, x, xids) - - def add_core(self, n, x, xids, precomputed_idx): - r""" - Implementation of vector addition where the vector assignments are - predefined. - - :type precomputed_idx: int - :param precomputed_idx: quantization indices for the input vectors - (size n) - """ - return _swigfaiss.IndexBinaryIVF_add_core(self, n, x, xids, precomputed_idx) - - def search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params=None): - r""" - Search a set of vectors, that are pre-quantized by the IVF - quantizer. Fill in the corresponding heaps with the query - results. search() calls this. - - :type n: int - :param n: nb of vectors to query - :type x: uint8_t - :param x: query vectors, size nx * d - :type assign: int - :param assign: coarse quantization indices, size nx * nprobe - :type centroid_dis: int - :param centroid_dis: - distances to coarse centroids, size nx * nprobe - :param distance: - output distances, size n * k - :type labels: int - :param labels: output labels, size n * k - :type store_pairs: boolean - :param store_pairs: store inv list index + inv list offset - instead in upper/lower 32 bit of result, - instead of ids (used for reranking). - :type params: :py:class:`IVFSearchParameters`, optional - :param params: used to override the object's search parameters - """ - return _swigfaiss.IndexBinaryIVF_search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params) - - def get_InvertedListScanner(self, store_pairs=False): - return _swigfaiss.IndexBinaryIVF_get_InvertedListScanner(self, store_pairs) - - def search(self, n, x, k, distances, labels, params=None): - r"""assign the vectors, then call search_preassign""" - return _swigfaiss.IndexBinaryIVF_search(self, n, x, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexBinaryIVF_range_search(self, n, x, radius, result, params) - - def range_search_preassigned(self, n, x, radius, assign, centroid_dis, result): - return _swigfaiss.IndexBinaryIVF_range_search_preassigned(self, n, x, radius, assign, centroid_dis, result) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexBinaryIVF_reconstruct(self, key, recons) - - def reconstruct_n(self, i0, ni, recons): - r""" - Reconstruct a subset of the indexed vectors. - - Overrides default implementation to bypass reconstruct() which requires - direct_map to be maintained. - - :type i0: int - :param i0: first vector to reconstruct - :type ni: int - :param ni: nb of vectors to reconstruct - :type recons: uint8_t - :param recons: output array of reconstructed vectors, size ni * d / 8 - """ - return _swigfaiss.IndexBinaryIVF_reconstruct_n(self, i0, ni, recons) - - def search_and_reconstruct(self, n, x, k, distances, labels, recons, params=None): - r""" - Similar to search, but also reconstructs the stored vectors (or an - approximation in the case of lossy coding) for the search results. - - Overrides default implementation to avoid having to maintain direct_map - and instead fetch the code offsets through the `store_pairs` flag in - search_preassigned(). - - :type recons: uint8_t - :param recons: reconstructed vectors size (n, k, d / 8) - """ - return _swigfaiss.IndexBinaryIVF_search_and_reconstruct(self, n, x, k, distances, labels, recons, params) - - def reconstruct_from_offset(self, list_no, offset, recons): - r""" - Reconstruct a vector given the location in terms of (inv list index + - inv list offset) instead of the id. - - Useful for reconstructing when the direct_map is not maintained and - the inv list offset is computed by search_preassigned() with - `store_pairs` set. - """ - return _swigfaiss.IndexBinaryIVF_reconstruct_from_offset(self, list_no, offset, recons) - - def remove_ids(self, sel): - r"""Dataset manipulation functions""" - return _swigfaiss.IndexBinaryIVF_remove_ids(self, sel) - - def merge_from(self, other, add_id): - return _swigfaiss.IndexBinaryIVF_merge_from(self, other, add_id) - - def check_compatible_for_merge(self, otherIndex): - return _swigfaiss.IndexBinaryIVF_check_compatible_for_merge(self, otherIndex) - - def get_list_size(self, list_no): - return _swigfaiss.IndexBinaryIVF_get_list_size(self, list_no) - - def make_direct_map(self, new_maintain_direct_map=True): - r""" - initialize a direct map - - :type new_maintain_direct_map: boolean, optional - :param new_maintain_direct_map: if true, create a direct map, - else clear it - """ - return _swigfaiss.IndexBinaryIVF_make_direct_map(self, new_maintain_direct_map) - - def set_direct_map_type(self, type): - return _swigfaiss.IndexBinaryIVF_set_direct_map_type(self, type) - - def replace_invlists(self, il, own=False): - return _swigfaiss.IndexBinaryIVF_replace_invlists(self, il, own) - -# Register IndexBinaryIVF in _swigfaiss: -_swigfaiss.IndexBinaryIVF_swigregister(IndexBinaryIVF) -class BinaryInvertedListScanner(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - - def set_query(self, query_vector): - r"""from now on we handle this query.""" - return _swigfaiss.BinaryInvertedListScanner_set_query(self, query_vector) - - def set_list(self, list_no, coarse_dis): - r"""following codes come from this inverted list""" - return _swigfaiss.BinaryInvertedListScanner_set_list(self, list_no, coarse_dis) - - def distance_to_code(self, code): - r"""compute a single query-to-code distance""" - return _swigfaiss.BinaryInvertedListScanner_distance_to_code(self, code) - - def scan_codes(self, n, codes, ids, distances, labels, k): - r""" - compute the distances to codes. (distances, labels) should be - organized as a min- or max-heap - - :type n: int - :param n: number of codes to scan - :type codes: uint8_t - :param codes: codes to scan (n * code_size) - :type ids: int - :param ids: corresponding ids (ignored if store_pairs) - :type distances: int - :param distances: heap distances (size k) - :type labels: int - :param labels: heap labels (size k) - :type k: int - :param k: heap size - """ - return _swigfaiss.BinaryInvertedListScanner_scan_codes(self, n, codes, ids, distances, labels, k) - - def scan_codes_range(self, n, codes, ids, radius, result): - return _swigfaiss.BinaryInvertedListScanner_scan_codes_range(self, n, codes, ids, radius, result) - __swig_destroy__ = _swigfaiss.delete_BinaryInvertedListScanner - -# Register BinaryInvertedListScanner in _swigfaiss: -_swigfaiss.BinaryInvertedListScanner_swigregister(BinaryInvertedListScanner) -class IndexBinaryFromFloat(IndexBinary): - r""" - IndexBinary backed by a float Index. - - Supports adding vertices and searching them. - - All queries are symmetric because there is no distinction between codes and - vectors. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - index = property(_swigfaiss.IndexBinaryFromFloat_index_get, _swigfaiss.IndexBinaryFromFloat_index_set) - own_fields = property(_swigfaiss.IndexBinaryFromFloat_own_fields_get, _swigfaiss.IndexBinaryFromFloat_own_fields_set, doc=r"""Whether object owns the index pointer.""") - - def __init__(self, *args): - _swigfaiss.IndexBinaryFromFloat_swiginit(self, _swigfaiss.new_IndexBinaryFromFloat(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryFromFloat - - def add(self, n, x): - return _swigfaiss.IndexBinaryFromFloat_add(self, n, x) - - def reset(self): - return _swigfaiss.IndexBinaryFromFloat_reset(self) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexBinaryFromFloat_search(self, n, x, k, distances, labels, params) - - def train(self, n, x): - return _swigfaiss.IndexBinaryFromFloat_train(self, n, x) - -# Register IndexBinaryFromFloat in _swigfaiss: -_swigfaiss.IndexBinaryFromFloat_swigregister(IndexBinaryFromFloat) -class IndexBinaryHNSW(IndexBinary): - r""" - The HNSW index is a normal random-access index with a HNSW - link structure built on top - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - hnsw = property(_swigfaiss.IndexBinaryHNSW_hnsw_get, _swigfaiss.IndexBinaryHNSW_hnsw_set) - own_fields = property(_swigfaiss.IndexBinaryHNSW_own_fields_get, _swigfaiss.IndexBinaryHNSW_own_fields_set) - storage = property(_swigfaiss.IndexBinaryHNSW_storage_get, _swigfaiss.IndexBinaryHNSW_storage_set) - init_level0 = property(_swigfaiss.IndexBinaryHNSW_init_level0_get, _swigfaiss.IndexBinaryHNSW_init_level0_set) - keep_max_size_level0 = property(_swigfaiss.IndexBinaryHNSW_keep_max_size_level0_get, _swigfaiss.IndexBinaryHNSW_keep_max_size_level0_set) - retain_locks = property(_swigfaiss.IndexBinaryHNSW_retain_locks_get, _swigfaiss.IndexBinaryHNSW_retain_locks_set) - - def __init__(self, *args): - _swigfaiss.IndexBinaryHNSW_swiginit(self, _swigfaiss.new_IndexBinaryHNSW(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryHNSW - - def get_distance_computer(self): - return _swigfaiss.IndexBinaryHNSW_get_distance_computer(self) - - def add(self, n, x): - return _swigfaiss.IndexBinaryHNSW_add(self, n, x) - - def train(self, n, x): - r"""Trains the storage if needed""" - return _swigfaiss.IndexBinaryHNSW_train(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - r"""entry point for search""" - return _swigfaiss.IndexBinaryHNSW_search(self, n, x, k, distances, labels, params) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexBinaryHNSW_reconstruct(self, key, recons) - - def reset(self): - return _swigfaiss.IndexBinaryHNSW_reset(self) - -# Register IndexBinaryHNSW in _swigfaiss: -_swigfaiss.IndexBinaryHNSW_swigregister(IndexBinaryHNSW) -class IndexBinaryHNSWCagra(IndexBinaryHNSW): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexBinaryHNSWCagra_swiginit(self, _swigfaiss.new_IndexBinaryHNSWCagra(*args)) - base_level_only = property(_swigfaiss.IndexBinaryHNSWCagra_base_level_only_get, _swigfaiss.IndexBinaryHNSWCagra_base_level_only_set, doc=r""" - When set to true, the index is immutable. - This option is used to copy the knn graph from GpuIndexBinaryCagra - to the base level of IndexBinaryHNSWCagra without adding upper levels. - Doing so enables to search the HNSW index, but removes the - ability to add vectors. - """) - num_base_level_search_entrypoints = property(_swigfaiss.IndexBinaryHNSWCagra_num_base_level_search_entrypoints_get, _swigfaiss.IndexBinaryHNSWCagra_num_base_level_search_entrypoints_set, doc=r""" - When `base_level_only` is set to `True`, the search function - searches only the base level knn graph of the HNSW index. - This parameter selects the entry point by randomly selecting - some points and using the best one. - """) - - def add(self, n, x): - return _swigfaiss.IndexBinaryHNSWCagra_add(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - r"""entry point for search""" - return _swigfaiss.IndexBinaryHNSWCagra_search(self, n, x, k, distances, labels, params) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryHNSWCagra - -# Register IndexBinaryHNSWCagra in _swigfaiss: -_swigfaiss.IndexBinaryHNSWCagra_swigregister(IndexBinaryHNSWCagra) -class IndexBinaryHash(IndexBinary): - r"""just uses the b first bits as a hash value""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - invlists = property(_swigfaiss.IndexBinaryHash_invlists_get, _swigfaiss.IndexBinaryHash_invlists_set) - b = property(_swigfaiss.IndexBinaryHash_b_get, _swigfaiss.IndexBinaryHash_b_set) - nflip = property(_swigfaiss.IndexBinaryHash_nflip_get, _swigfaiss.IndexBinaryHash_nflip_set) - - def __init__(self, *args): - _swigfaiss.IndexBinaryHash_swiginit(self, _swigfaiss.new_IndexBinaryHash(*args)) - - def reset(self): - return _swigfaiss.IndexBinaryHash_reset(self) - - def add(self, n, x): - return _swigfaiss.IndexBinaryHash_add(self, n, x) - - def add_with_ids(self, n, x, xids): - return _swigfaiss.IndexBinaryHash_add_with_ids(self, n, x, xids) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexBinaryHash_range_search(self, n, x, radius, result, params) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexBinaryHash_search(self, n, x, k, distances, labels, params) - - def display(self): - return _swigfaiss.IndexBinaryHash_display(self) - - def hashtable_size(self): - return _swigfaiss.IndexBinaryHash_hashtable_size(self) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryHash - -# Register IndexBinaryHash in _swigfaiss: -_swigfaiss.IndexBinaryHash_swigregister(IndexBinaryHash) -class IndexBinaryHashStats(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nq = property(_swigfaiss.IndexBinaryHashStats_nq_get, _swigfaiss.IndexBinaryHashStats_nq_set) - n0 = property(_swigfaiss.IndexBinaryHashStats_n0_get, _swigfaiss.IndexBinaryHashStats_n0_set) - nlist = property(_swigfaiss.IndexBinaryHashStats_nlist_get, _swigfaiss.IndexBinaryHashStats_nlist_set) - ndis = property(_swigfaiss.IndexBinaryHashStats_ndis_get, _swigfaiss.IndexBinaryHashStats_ndis_set) - - def __init__(self): - _swigfaiss.IndexBinaryHashStats_swiginit(self, _swigfaiss.new_IndexBinaryHashStats()) - - def reset(self): - return _swigfaiss.IndexBinaryHashStats_reset(self) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryHashStats - -# Register IndexBinaryHashStats in _swigfaiss: -_swigfaiss.IndexBinaryHashStats_swigregister(IndexBinaryHashStats) -class IndexBinaryMultiHash(IndexBinary): - r"""just uses the b first bits as a hash value""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - storage = property(_swigfaiss.IndexBinaryMultiHash_storage_get, _swigfaiss.IndexBinaryMultiHash_storage_set) - own_fields = property(_swigfaiss.IndexBinaryMultiHash_own_fields_get, _swigfaiss.IndexBinaryMultiHash_own_fields_set) - maps = property(_swigfaiss.IndexBinaryMultiHash_maps_get, _swigfaiss.IndexBinaryMultiHash_maps_set) - nhash = property(_swigfaiss.IndexBinaryMultiHash_nhash_get, _swigfaiss.IndexBinaryMultiHash_nhash_set, doc=r"""nb of hash maps""") - b = property(_swigfaiss.IndexBinaryMultiHash_b_get, _swigfaiss.IndexBinaryMultiHash_b_set, doc=r"""nb bits per hash map""") - nflip = property(_swigfaiss.IndexBinaryMultiHash_nflip_get, _swigfaiss.IndexBinaryMultiHash_nflip_set, doc=r"""nb bit flips to use at search time""") - - def __init__(self, *args): - _swigfaiss.IndexBinaryMultiHash_swiginit(self, _swigfaiss.new_IndexBinaryMultiHash(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryMultiHash - - def reset(self): - return _swigfaiss.IndexBinaryMultiHash_reset(self) - - def add(self, n, x): - return _swigfaiss.IndexBinaryMultiHash_add(self, n, x) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexBinaryMultiHash_range_search(self, n, x, radius, result, params) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexBinaryMultiHash_search(self, n, x, k, distances, labels, params) - - def hashtable_size(self): - return _swigfaiss.IndexBinaryMultiHash_hashtable_size(self) - -# Register IndexBinaryMultiHash in _swigfaiss: -_swigfaiss.IndexBinaryMultiHash_swigregister(IndexBinaryMultiHash) -class ThreadedIndexBase(Index): - r""" - A holder of indices in a collection of threads - The interface to this class itself is not thread safe - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - __swig_destroy__ = _swigfaiss.delete_ThreadedIndexBase - - def addIndex(self, index): - r""" - override an index that is managed by ourselves. - WARNING: once an index is added, it becomes unsafe to touch it from any - other thread than that on which is managing it, until we are shut - down. Use runOnIndex to perform work on it instead. - """ - return _swigfaiss.ThreadedIndexBase_addIndex(self, index) - - def removeIndex(self, index): - r""" - Remove an index that is managed by ourselves. - This will flush all pending work on that index, and then shut - down its managing thread, and will remove the index. - """ - return _swigfaiss.ThreadedIndexBase_removeIndex(self, index) - - def runOnIndex(self, *args): - r""" - Run a function on all indices, in the thread that the index is - managed in. - Function arguments are (index in collection, index pointer) - """ - return _swigfaiss.ThreadedIndexBase_runOnIndex(self, *args) - - def reset(self): - r""" - faiss::Index API - All indices receive the same call - """ - return _swigfaiss.ThreadedIndexBase_reset(self) - - def count(self): - r"""Returns the number of sub-indices""" - return _swigfaiss.ThreadedIndexBase_count(self) - - def at(self, *args): - r""" - *Overload 1:* - Returns the i-th sub-index - - | - - *Overload 2:* - Returns the i-th sub-index (const version) - """ - return _swigfaiss.ThreadedIndexBase_at(self, *args) - own_indices = property(_swigfaiss.ThreadedIndexBase_own_indices_get, _swigfaiss.ThreadedIndexBase_own_indices_set, doc=r"""Whether or not we are responsible for deleting our contained indices""") - -# Register ThreadedIndexBase in _swigfaiss: -_swigfaiss.ThreadedIndexBase_swigregister(ThreadedIndexBase) -class ThreadedIndexBaseBinary(IndexBinary): - r""" - A holder of indices in a collection of threads - The interface to this class itself is not thread safe - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - __swig_destroy__ = _swigfaiss.delete_ThreadedIndexBaseBinary - - def addIndex(self, index): - r""" - override an index that is managed by ourselves. - WARNING: once an index is added, it becomes unsafe to touch it from any - other thread than that on which is managing it, until we are shut - down. Use runOnIndex to perform work on it instead. - """ - return _swigfaiss.ThreadedIndexBaseBinary_addIndex(self, index) - - def removeIndex(self, index): - r""" - Remove an index that is managed by ourselves. - This will flush all pending work on that index, and then shut - down its managing thread, and will remove the index. - """ - return _swigfaiss.ThreadedIndexBaseBinary_removeIndex(self, index) - - def runOnIndex(self, *args): - r""" - Run a function on all indices, in the thread that the index is - managed in. - Function arguments are (index in collection, index pointer) - """ - return _swigfaiss.ThreadedIndexBaseBinary_runOnIndex(self, *args) - - def reset(self): - r""" - faiss::Index API - All indices receive the same call - """ - return _swigfaiss.ThreadedIndexBaseBinary_reset(self) - - def count(self): - r"""Returns the number of sub-indices""" - return _swigfaiss.ThreadedIndexBaseBinary_count(self) - - def at(self, *args): - r""" - *Overload 1:* - Returns the i-th sub-index - - | - - *Overload 2:* - Returns the i-th sub-index (const version) - """ - return _swigfaiss.ThreadedIndexBaseBinary_at(self, *args) - own_indices = property(_swigfaiss.ThreadedIndexBaseBinary_own_indices_get, _swigfaiss.ThreadedIndexBaseBinary_own_indices_set, doc=r"""Whether or not we are responsible for deleting our contained indices""") - -# Register ThreadedIndexBaseBinary in _swigfaiss: -_swigfaiss.ThreadedIndexBaseBinary_swigregister(ThreadedIndexBaseBinary) -class IndexShards(ThreadedIndexBase): - r"""Index that concatenates the results from several sub-indexes""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - r""" - *Overload 1:* - - The dimension that all sub-indices must share will be the dimension of - the first sub-index added - - :type threaded: boolean, optional - :param threaded: do we use one thread per sub_index or do - queries sequentially? - :type successive_ids: boolean, optional - :param successive_ids: should we shift the returned ids by - the size of each sub-index or return them - as they are? - - | - - *Overload 2:* - - :type threaded: boolean, optional - :param threaded: do we use one thread per sub_index or do - queries sequentially? - :type successive_ids: boolean, optional - :param successive_ids: should we shift the returned ids by - the size of each sub-index or return them - as they are? - - | - - *Overload 3:* - - :type threaded: boolean, optional - :param threaded: do we use one thread per sub_index or do - queries sequentially? - :param successive_ids: should we shift the returned ids by - the size of each sub-index or return them - as they are? - - | - - *Overload 4:* - - :param threaded: do we use one thread per sub_index or do - queries sequentially? - :param successive_ids: should we shift the returned ids by - the size of each sub-index or return them - as they are? - - | - - *Overload 5:* - int version due to the implicit bool conversion ambiguity of int as - dimension - - | - - *Overload 6:* - int version due to the implicit bool conversion ambiguity of int as - dimension - - | - - *Overload 7:* - int version due to the implicit bool conversion ambiguity of int as - dimension - """ - _swigfaiss.IndexShards_swiginit(self, _swigfaiss.new_IndexShards(*args)) - - def add_shard(self, index): - r"""Alias for addIndex()""" - return _swigfaiss.IndexShards_add_shard(self, index) - - def remove_shard(self, index): - r"""Alias for removeIndex()""" - return _swigfaiss.IndexShards_remove_shard(self, index) - - def add(self, n, x): - r"""supported only for sub-indices that implement add_with_ids""" - return _swigfaiss.IndexShards_add(self, n, x) - - def add_with_ids(self, n, x, xids): - r""" - Cases (successive_ids, xids): - - true, non-NULL ERROR: it makes no sense to pass in ids and - request them to be shifted - - true, NULL OK: but should be called only once (calls add() - on sub-indexes). - - false, non-NULL OK: will call add_with_ids with passed in xids - distributed evenly over shards - - false, NULL OK: will call add_with_ids on each sub-index, - starting at ntotal - """ - return _swigfaiss.IndexShards_add_with_ids(self, n, x, xids) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexShards_search(self, n, x, k, distances, labels, params) - - def train(self, n, x): - return _swigfaiss.IndexShards_train(self, n, x) - successive_ids = property(_swigfaiss.IndexShards_successive_ids_get, _swigfaiss.IndexShards_successive_ids_set) - - def syncWithSubIndexes(self): - r""" - Synchronize the top-level index (IndexShards) with data in the - sub-indices - """ - return _swigfaiss.IndexShards_syncWithSubIndexes(self) - __swig_destroy__ = _swigfaiss.delete_IndexShards - -# Register IndexShards in _swigfaiss: -_swigfaiss.IndexShards_swigregister(IndexShards) -class IndexBinaryShards(ThreadedIndexBaseBinary): - r"""Index that concatenates the results from several sub-indexes""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - r""" - *Overload 1:* - - The dimension that all sub-indices must share will be the dimension of - the first sub-index added - - :type threaded: boolean, optional - :param threaded: do we use one thread per sub_index or do - queries sequentially? - :type successive_ids: boolean, optional - :param successive_ids: should we shift the returned ids by - the size of each sub-index or return them - as they are? - - | - - *Overload 2:* - - :type threaded: boolean, optional - :param threaded: do we use one thread per sub_index or do - queries sequentially? - :type successive_ids: boolean, optional - :param successive_ids: should we shift the returned ids by - the size of each sub-index or return them - as they are? - - | - - *Overload 3:* - - :type threaded: boolean, optional - :param threaded: do we use one thread per sub_index or do - queries sequentially? - :param successive_ids: should we shift the returned ids by - the size of each sub-index or return them - as they are? - - | - - *Overload 4:* - - :param threaded: do we use one thread per sub_index or do - queries sequentially? - :param successive_ids: should we shift the returned ids by - the size of each sub-index or return them - as they are? - - | - - *Overload 5:* - int version due to the implicit bool conversion ambiguity of int as - dimension - - | - - *Overload 6:* - int version due to the implicit bool conversion ambiguity of int as - dimension - - | - - *Overload 7:* - int version due to the implicit bool conversion ambiguity of int as - dimension - """ - _swigfaiss.IndexBinaryShards_swiginit(self, _swigfaiss.new_IndexBinaryShards(*args)) - - def add_shard(self, index): - r"""Alias for addIndex()""" - return _swigfaiss.IndexBinaryShards_add_shard(self, index) - - def remove_shard(self, index): - r"""Alias for removeIndex()""" - return _swigfaiss.IndexBinaryShards_remove_shard(self, index) - - def add(self, n, x): - r"""supported only for sub-indices that implement add_with_ids""" - return _swigfaiss.IndexBinaryShards_add(self, n, x) - - def add_with_ids(self, n, x, xids): - r""" - Cases (successive_ids, xids): - - true, non-NULL ERROR: it makes no sense to pass in ids and - request them to be shifted - - true, NULL OK: but should be called only once (calls add() - on sub-indexes). - - false, non-NULL OK: will call add_with_ids with passed in xids - distributed evenly over shards - - false, NULL OK: will call add_with_ids on each sub-index, - starting at ntotal - """ - return _swigfaiss.IndexBinaryShards_add_with_ids(self, n, x, xids) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexBinaryShards_search(self, n, x, k, distances, labels, params) - - def train(self, n, x): - return _swigfaiss.IndexBinaryShards_train(self, n, x) - successive_ids = property(_swigfaiss.IndexBinaryShards_successive_ids_get, _swigfaiss.IndexBinaryShards_successive_ids_set) - - def syncWithSubIndexes(self): - r""" - Synchronize the top-level index (IndexShards) with data in the - sub-indices - """ - return _swigfaiss.IndexBinaryShards_syncWithSubIndexes(self) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryShards - -# Register IndexBinaryShards in _swigfaiss: -_swigfaiss.IndexBinaryShards_swigregister(IndexBinaryShards) -class IndexShardsIVF(IndexShards, Level1Quantizer): - r""" - IndexShards with a common coarse quantizer. All the indexes added should be - IndexIVFInterface indexes so that the search_precomputed can be called. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, quantizer, nlist, threaded=False, successive_ids=True): - _swigfaiss.IndexShardsIVF_swiginit(self, _swigfaiss.new_IndexShardsIVF(quantizer, nlist, threaded, successive_ids)) - - def addIndex(self, index): - return _swigfaiss.IndexShardsIVF_addIndex(self, index) - - def add_with_ids(self, n, x, xids): - return _swigfaiss.IndexShardsIVF_add_with_ids(self, n, x, xids) - - def train(self, n, x): - return _swigfaiss.IndexShardsIVF_train(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexShardsIVF_search(self, n, x, k, distances, labels, params) - __swig_destroy__ = _swigfaiss.delete_IndexShardsIVF - -# Register IndexShardsIVF in _swigfaiss: -_swigfaiss.IndexShardsIVF_swigregister(IndexShardsIVF) -class IndexReplicas(ThreadedIndexBase): - r""" - Takes individual faiss::Index instances, and splits queries for - sending to each Index instance, and joins the results together - when done. - Each index is managed by a separate CPU thread. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - r""" - *Overload 1:* - The dimension that all sub-indices must share will be the dimension of - the first sub-index added - :type threaded: boolean, optional - :param threaded: do we use one thread per sub-index or do queries - sequentially? - - | - - *Overload 2:* - :type d: int - :param d: the dimension that all sub-indices must share - :type threaded: boolean, optional - :param threaded: do we use one thread per sub index or do queries - sequentially? - - | - - *Overload 3:* - :type d: int - :param d: the dimension that all sub-indices must share - :param threaded: do we use one thread per sub index or do queries - sequentially? - - | - - *Overload 4:* - int version due to the implicit bool conversion ambiguity of int as - dimension - - | - - *Overload 5:* - int version due to the implicit bool conversion ambiguity of int as - dimension - """ - _swigfaiss.IndexReplicas_swiginit(self, _swigfaiss.new_IndexReplicas(*args)) - - def add_replica(self, index): - r"""Alias for addIndex()""" - return _swigfaiss.IndexReplicas_add_replica(self, index) - - def remove_replica(self, index): - r"""Alias for removeIndex()""" - return _swigfaiss.IndexReplicas_remove_replica(self, index) - - def train(self, n, x): - r""" - faiss::Index API - All indices receive the same call - """ - return _swigfaiss.IndexReplicas_train(self, n, x) - - def add(self, n, x): - r""" - faiss::Index API - All indices receive the same call - """ - return _swigfaiss.IndexReplicas_add(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - r""" - faiss::Index API - Query is partitioned into a slice for each sub-index - split by ceil(n / #indices) for our sub-indices - """ - return _swigfaiss.IndexReplicas_search(self, n, x, k, distances, labels, params) - - def reconstruct(self, arg2, v): - r"""reconstructs from the first index""" - return _swigfaiss.IndexReplicas_reconstruct(self, arg2, v) - - def syncWithSubIndexes(self): - r""" - Synchronize the top-level index (IndexShards) with data in the - sub-indices - """ - return _swigfaiss.IndexReplicas_syncWithSubIndexes(self) - __swig_destroy__ = _swigfaiss.delete_IndexReplicas - -# Register IndexReplicas in _swigfaiss: -_swigfaiss.IndexReplicas_swigregister(IndexReplicas) -class IndexBinaryReplicas(ThreadedIndexBaseBinary): - r""" - Takes individual faiss::Index instances, and splits queries for - sending to each Index instance, and joins the results together - when done. - Each index is managed by a separate CPU thread. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - r""" - *Overload 1:* - The dimension that all sub-indices must share will be the dimension of - the first sub-index added - :type threaded: boolean, optional - :param threaded: do we use one thread per sub-index or do queries - sequentially? - - | - - *Overload 2:* - :type d: int - :param d: the dimension that all sub-indices must share - :type threaded: boolean, optional - :param threaded: do we use one thread per sub index or do queries - sequentially? - - | - - *Overload 3:* - :type d: int - :param d: the dimension that all sub-indices must share - :param threaded: do we use one thread per sub index or do queries - sequentially? - - | - - *Overload 4:* - int version due to the implicit bool conversion ambiguity of int as - dimension - - | - - *Overload 5:* - int version due to the implicit bool conversion ambiguity of int as - dimension - """ - _swigfaiss.IndexBinaryReplicas_swiginit(self, _swigfaiss.new_IndexBinaryReplicas(*args)) - - def add_replica(self, index): - r"""Alias for addIndex()""" - return _swigfaiss.IndexBinaryReplicas_add_replica(self, index) - - def remove_replica(self, index): - r"""Alias for removeIndex()""" - return _swigfaiss.IndexBinaryReplicas_remove_replica(self, index) - - def train(self, n, x): - r""" - faiss::Index API - All indices receive the same call - """ - return _swigfaiss.IndexBinaryReplicas_train(self, n, x) - - def add(self, n, x): - r""" - faiss::Index API - All indices receive the same call - """ - return _swigfaiss.IndexBinaryReplicas_add(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - r""" - faiss::Index API - Query is partitioned into a slice for each sub-index - split by ceil(n / #indices) for our sub-indices - """ - return _swigfaiss.IndexBinaryReplicas_search(self, n, x, k, distances, labels, params) - - def reconstruct(self, arg2, v): - r"""reconstructs from the first index""" - return _swigfaiss.IndexBinaryReplicas_reconstruct(self, arg2, v) - - def syncWithSubIndexes(self): - r""" - Synchronize the top-level index (IndexShards) with data in the - sub-indices - """ - return _swigfaiss.IndexBinaryReplicas_syncWithSubIndexes(self) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryReplicas - -# Register IndexBinaryReplicas in _swigfaiss: -_swigfaiss.IndexBinaryReplicas_swigregister(IndexBinaryReplicas) -class IndexSplitVectors(Index): - r""" - splits input vectors in segments and assigns each segment to a sub-index - used to distribute a MultiIndexQuantizer - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - own_fields = property(_swigfaiss.IndexSplitVectors_own_fields_get, _swigfaiss.IndexSplitVectors_own_fields_set) - threaded = property(_swigfaiss.IndexSplitVectors_threaded_get, _swigfaiss.IndexSplitVectors_threaded_set) - sub_indexes = property(_swigfaiss.IndexSplitVectors_sub_indexes_get, _swigfaiss.IndexSplitVectors_sub_indexes_set) - sum_d = property(_swigfaiss.IndexSplitVectors_sum_d_get, _swigfaiss.IndexSplitVectors_sum_d_set) - - def __init__(self, d, threaded=False): - r"""sum of dimensions seen so far""" - _swigfaiss.IndexSplitVectors_swiginit(self, _swigfaiss.new_IndexSplitVectors(d, threaded)) - - def add_sub_index(self, arg2): - return _swigfaiss.IndexSplitVectors_add_sub_index(self, arg2) - - def sync_with_sub_indexes(self): - return _swigfaiss.IndexSplitVectors_sync_with_sub_indexes(self) - - def add(self, n, x): - return _swigfaiss.IndexSplitVectors_add(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexSplitVectors_search(self, n, x, k, distances, labels, params) - - def train(self, n, x): - return _swigfaiss.IndexSplitVectors_train(self, n, x) - - def reset(self): - return _swigfaiss.IndexSplitVectors_reset(self) - __swig_destroy__ = _swigfaiss.delete_IndexSplitVectors - -# Register IndexSplitVectors in _swigfaiss: -_swigfaiss.IndexSplitVectors_swigregister(IndexSplitVectors) -class IndexRandom(Index): - r""" - index that returns random results. - used mainly for time benchmarks - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - seed = property(_swigfaiss.IndexRandom_seed_get, _swigfaiss.IndexRandom_seed_set) - - def __init__(self, *args): - _swigfaiss.IndexRandom_swiginit(self, _swigfaiss.new_IndexRandom(*args)) - - def add(self, n, x): - return _swigfaiss.IndexRandom_add(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexRandom_search(self, n, x, k, distances, labels, params) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexRandom_reconstruct(self, key, recons) - - def reset(self): - return _swigfaiss.IndexRandom_reset(self) - __swig_destroy__ = _swigfaiss.delete_IndexRandom - -# Register IndexRandom in _swigfaiss: -_swigfaiss.IndexRandom_swigregister(IndexRandom) -class IndexRowwiseMinMaxBase(Index): - r""" - Index wrapper that performs rowwise normalization to [0,1], preserving - the coefficients. This is a vector codec index only. - - Basically, this index performs a rowwise scaling to [0,1] of every row - in an input dataset before calling subindex::train() and - subindex::sa_encode(). sa_encode() call stores the scaling coefficients - (scaler and minv) in the very beginning of every output code. The format: - [scaler][minv][subindex::sa_encode() output] - The de-scaling in sa_decode() is done using: - output_rescaled = scaler * output + minv - - An additional ::train_inplace() function is provided in order to do - an inplace scaling before calling subindex::train() and, thus, avoiding - the cloning of the input dataset, but modifying the input dataset because - of the scaling and the scaling back. It is up to user to call - this function instead of ::train() - - Derived classes provide different data types for scaling coefficients. - Currently, versions with fp16 and fp32 scaling coefficients are available. - fp16 version adds 4 extra bytes per encoded vector - fp32 version adds 8 extra bytes per encoded vector - Provides base functions for rowwise normalizing indices. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - index = property(_swigfaiss.IndexRowwiseMinMaxBase_index_get, _swigfaiss.IndexRowwiseMinMaxBase_index_set, doc=r"""sub-index""") - own_fields = property(_swigfaiss.IndexRowwiseMinMaxBase_own_fields_get, _swigfaiss.IndexRowwiseMinMaxBase_own_fields_set, doc=r"""whether the subindex needs to be freed in the destructor.""") - __swig_destroy__ = _swigfaiss.delete_IndexRowwiseMinMaxBase - - def add(self, n, x): - return _swigfaiss.IndexRowwiseMinMaxBase_add(self, n, x) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexRowwiseMinMaxBase_search(self, n, x, k, distances, labels, params) - - def reset(self): - return _swigfaiss.IndexRowwiseMinMaxBase_reset(self) - - def train_inplace(self, n, x): - return _swigfaiss.IndexRowwiseMinMaxBase_train_inplace(self, n, x) - -# Register IndexRowwiseMinMaxBase in _swigfaiss: -_swigfaiss.IndexRowwiseMinMaxBase_swigregister(IndexRowwiseMinMaxBase) -class IndexRowwiseMinMaxFP16(IndexRowwiseMinMaxBase): - r"""Stores scaling coefficients as fp16 values.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexRowwiseMinMaxFP16_swiginit(self, _swigfaiss.new_IndexRowwiseMinMaxFP16(*args)) - - def train(self, n, x): - return _swigfaiss.IndexRowwiseMinMaxFP16_train(self, n, x) - - def train_inplace(self, n, x): - return _swigfaiss.IndexRowwiseMinMaxFP16_train_inplace(self, n, x) - - def sa_code_size(self): - return _swigfaiss.IndexRowwiseMinMaxFP16_sa_code_size(self) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexRowwiseMinMaxFP16_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexRowwiseMinMaxFP16_sa_decode(self, n, bytes, x) - __swig_destroy__ = _swigfaiss.delete_IndexRowwiseMinMaxFP16 - -# Register IndexRowwiseMinMaxFP16 in _swigfaiss: -_swigfaiss.IndexRowwiseMinMaxFP16_swigregister(IndexRowwiseMinMaxFP16) -class IndexRowwiseMinMax(IndexRowwiseMinMaxBase): - r"""Stores scaling coefficients as fp32 values.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, *args): - _swigfaiss.IndexRowwiseMinMax_swiginit(self, _swigfaiss.new_IndexRowwiseMinMax(*args)) - - def train(self, n, x): - return _swigfaiss.IndexRowwiseMinMax_train(self, n, x) - - def train_inplace(self, n, x): - return _swigfaiss.IndexRowwiseMinMax_train_inplace(self, n, x) - - def sa_code_size(self): - return _swigfaiss.IndexRowwiseMinMax_sa_code_size(self) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexRowwiseMinMax_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexRowwiseMinMax_sa_decode(self, n, bytes, x) - __swig_destroy__ = _swigfaiss.delete_IndexRowwiseMinMax - -# Register IndexRowwiseMinMax in _swigfaiss: -_swigfaiss.IndexRowwiseMinMax_swigregister(IndexRowwiseMinMax) -class Linear(object): - r"""minimal translation of nn.Linear""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - in_features = property(_swigfaiss.Linear_in_features_get, _swigfaiss.Linear_in_features_set) - out_features = property(_swigfaiss.Linear_out_features_get, _swigfaiss.Linear_out_features_set) - weight = property(_swigfaiss.Linear_weight_get, _swigfaiss.Linear_weight_set) - bias = property(_swigfaiss.Linear_bias_get, _swigfaiss.Linear_bias_set) - - def __init__(self, in_features, out_features, bias=True): - _swigfaiss.Linear_swiginit(self, _swigfaiss.new_Linear(in_features, out_features, bias)) - - def __call__(self, x): - return _swigfaiss.Linear___call__(self, x) - __swig_destroy__ = _swigfaiss.delete_Linear - -# Register Linear in _swigfaiss: -_swigfaiss.Linear_swigregister(Linear) -class Embedding(object): - r"""minimal translation of nn.Embedding""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - num_embeddings = property(_swigfaiss.Embedding_num_embeddings_get, _swigfaiss.Embedding_num_embeddings_set) - embedding_dim = property(_swigfaiss.Embedding_embedding_dim_get, _swigfaiss.Embedding_embedding_dim_set) - weight = property(_swigfaiss.Embedding_weight_get, _swigfaiss.Embedding_weight_set) - - def __init__(self, num_embeddings, embedding_dim): - _swigfaiss.Embedding_swiginit(self, _swigfaiss.new_Embedding(num_embeddings, embedding_dim)) - - def __call__(self, arg2): - return _swigfaiss.Embedding___call__(self, arg2) - - def data(self, *args): - return _swigfaiss.Embedding_data(self, *args) - __swig_destroy__ = _swigfaiss.delete_Embedding - -# Register Embedding in _swigfaiss: -_swigfaiss.Embedding_swigregister(Embedding) -class FFN(object): - r""" - Feed forward layer that expands to a hidden dimension, applies a ReLU non - linearity and maps back to the original dimension - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - linear1 = property(_swigfaiss.FFN_linear1_get, _swigfaiss.FFN_linear1_set) - linear2 = property(_swigfaiss.FFN_linear2_get, _swigfaiss.FFN_linear2_set) - - def __init__(self, d, h): - _swigfaiss.FFN_swiginit(self, _swigfaiss.new_FFN(d, h)) - - def __call__(self, x): - return _swigfaiss.FFN___call__(self, x) - __swig_destroy__ = _swigfaiss.delete_FFN - -# Register FFN in _swigfaiss: -_swigfaiss.FFN_swigregister(FFN) -class QINCoStep(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - d = property(_swigfaiss.QINCoStep_d_get, _swigfaiss.QINCoStep_d_set, doc=r"""d: input dim, K: codebook size, L: # of residual blocks, h: hidden dim""") - K = property(_swigfaiss.QINCoStep_K_get, _swigfaiss.QINCoStep_K_set) - L = property(_swigfaiss.QINCoStep_L_get, _swigfaiss.QINCoStep_L_set) - h = property(_swigfaiss.QINCoStep_h_get, _swigfaiss.QINCoStep_h_set) - - def __init__(self, d, K, L, h): - _swigfaiss.QINCoStep_swiginit(self, _swigfaiss.new_QINCoStep(d, K, L, h)) - codebook = property(_swigfaiss.QINCoStep_codebook_get, _swigfaiss.QINCoStep_codebook_set) - MLPconcat = property(_swigfaiss.QINCoStep_MLPconcat_get, _swigfaiss.QINCoStep_MLPconcat_set) - residual_blocks = property(_swigfaiss.QINCoStep_residual_blocks_get, _swigfaiss.QINCoStep_residual_blocks_set) - - def get_residual_block(self, i): - return _swigfaiss.QINCoStep_get_residual_block(self, i) - - def encode(self, xhat, x, residuals=None): - r""" - encode a set of vectors x with initial estimate xhat. Optionally return - the delta to be added to xhat to form the new xhat - """ - return _swigfaiss.QINCoStep_encode(self, xhat, x, residuals) - - def decode(self, xhat, codes): - return _swigfaiss.QINCoStep_decode(self, xhat, codes) - __swig_destroy__ = _swigfaiss.delete_QINCoStep - -# Register QINCoStep in _swigfaiss: -_swigfaiss.QINCoStep_swigregister(QINCoStep) -class NeuralNetCodec(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - d = property(_swigfaiss.NeuralNetCodec_d_get, _swigfaiss.NeuralNetCodec_d_set) - M = property(_swigfaiss.NeuralNetCodec_M_get, _swigfaiss.NeuralNetCodec_M_set) - - def decode(self, codes): - return _swigfaiss.NeuralNetCodec_decode(self, codes) - - def encode(self, x): - return _swigfaiss.NeuralNetCodec_encode(self, x) - __swig_destroy__ = _swigfaiss.delete_NeuralNetCodec - -# Register NeuralNetCodec in _swigfaiss: -_swigfaiss.NeuralNetCodec_swigregister(NeuralNetCodec) -class QINCo(NeuralNetCodec): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - K = property(_swigfaiss.QINCo_K_get, _swigfaiss.QINCo_K_set) - L = property(_swigfaiss.QINCo_L_get, _swigfaiss.QINCo_L_set) - h = property(_swigfaiss.QINCo_h_get, _swigfaiss.QINCo_h_set) - codebook0 = property(_swigfaiss.QINCo_codebook0_get, _swigfaiss.QINCo_codebook0_set) - steps = property(_swigfaiss.QINCo_steps_get, _swigfaiss.QINCo_steps_set) - - def __init__(self, d, K, L, M, h): - _swigfaiss.QINCo_swiginit(self, _swigfaiss.new_QINCo(d, K, L, M, h)) - - def get_step(self, i): - return _swigfaiss.QINCo_get_step(self, i) - - def decode(self, codes): - return _swigfaiss.QINCo_decode(self, codes) - - def encode(self, x): - return _swigfaiss.QINCo_encode(self, x) - __swig_destroy__ = _swigfaiss.delete_QINCo - -# Register QINCo in _swigfaiss: -_swigfaiss.QINCo_swigregister(QINCo) -class Tensor2D(object): - r"""Implements a few neural net layers, mainly to support QINCo""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - shape = property(_swigfaiss.Tensor2D_shape_get, _swigfaiss.Tensor2D_shape_set) - v = property(_swigfaiss.Tensor2D_v_get, _swigfaiss.Tensor2D_v_set) - - def __init__(self, n0, n1, data=None): - _swigfaiss.Tensor2D_swiginit(self, _swigfaiss.new_Tensor2D(n0, n1, data)) - - def __iadd__(self, arg2): - return _swigfaiss.Tensor2D___iadd__(self, arg2) - - def column(self, j): - r"""get column #j as a 1-column Tensor2D""" - return _swigfaiss.Tensor2D_column(self, j) - - def numel(self): - return _swigfaiss.Tensor2D_numel(self) - - def data(self, *args): - return _swigfaiss.Tensor2D_data(self, *args) - __swig_destroy__ = _swigfaiss.delete_Tensor2D - -# Register Tensor2D in _swigfaiss: -_swigfaiss.Tensor2D_swigregister(Tensor2D) -class Int32Tensor2D(object): - r"""Implements a few neural net layers, mainly to support QINCo""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - shape = property(_swigfaiss.Int32Tensor2D_shape_get, _swigfaiss.Int32Tensor2D_shape_set) - v = property(_swigfaiss.Int32Tensor2D_v_get, _swigfaiss.Int32Tensor2D_v_set) - - def __init__(self, n0, n1, data=None): - _swigfaiss.Int32Tensor2D_swiginit(self, _swigfaiss.new_Int32Tensor2D(n0, n1, data)) - - def __iadd__(self, arg2): - return _swigfaiss.Int32Tensor2D___iadd__(self, arg2) - - def column(self, j): - r"""get column #j as a 1-column Tensor2D""" - return _swigfaiss.Int32Tensor2D_column(self, j) - - def numel(self): - return _swigfaiss.Int32Tensor2D_numel(self) - - def data(self, *args): - return _swigfaiss.Int32Tensor2D_data(self, *args) - __swig_destroy__ = _swigfaiss.delete_Int32Tensor2D - -# Register Int32Tensor2D in _swigfaiss: -_swigfaiss.Int32Tensor2D_swigregister(Int32Tensor2D) -class IndexNeuralNetCodec(IndexFlatCodes): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - net = property(_swigfaiss.IndexNeuralNetCodec_net_get, _swigfaiss.IndexNeuralNetCodec_net_set) - M = property(_swigfaiss.IndexNeuralNetCodec_M_get, _swigfaiss.IndexNeuralNetCodec_M_set) - nbits = property(_swigfaiss.IndexNeuralNetCodec_nbits_get, _swigfaiss.IndexNeuralNetCodec_nbits_set) - - def __init__(self, *args): - _swigfaiss.IndexNeuralNetCodec_swiginit(self, _swigfaiss.new_IndexNeuralNetCodec(*args)) - - def train(self, n, x): - return _swigfaiss.IndexNeuralNetCodec_train(self, n, x) - - def sa_encode(self, n, x, codes): - return _swigfaiss.IndexNeuralNetCodec_sa_encode(self, n, x, codes) - - def sa_decode(self, n, codes, x): - return _swigfaiss.IndexNeuralNetCodec_sa_decode(self, n, codes, x) - __swig_destroy__ = _swigfaiss.delete_IndexNeuralNetCodec - -# Register IndexNeuralNetCodec in _swigfaiss: -_swigfaiss.IndexNeuralNetCodec_swigregister(IndexNeuralNetCodec) -class IndexQINCo(IndexNeuralNetCodec): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - qinco = property(_swigfaiss.IndexQINCo_qinco_get, _swigfaiss.IndexQINCo_qinco_set) - - def __init__(self, *args): - _swigfaiss.IndexQINCo_swiginit(self, _swigfaiss.new_IndexQINCo(*args)) - __swig_destroy__ = _swigfaiss.delete_IndexQINCo - -# Register IndexQINCo in _swigfaiss: -_swigfaiss.IndexQINCo_swigregister(IndexQINCo) -class RaBitQuantizer(Quantizer): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - centroid = property(_swigfaiss.RaBitQuantizer_centroid_get, _swigfaiss.RaBitQuantizer_centroid_set) - metric_type = property(_swigfaiss.RaBitQuantizer_metric_type_get, _swigfaiss.RaBitQuantizer_metric_type_set) - nb_bits = property(_swigfaiss.RaBitQuantizer_nb_bits_get, _swigfaiss.RaBitQuantizer_nb_bits_set) - - def __init__(self, *args): - _swigfaiss.RaBitQuantizer_swiginit(self, _swigfaiss.new_RaBitQuantizer(*args)) - - def compute_code_size(self, d, num_bits): - return _swigfaiss.RaBitQuantizer_compute_code_size(self, d, num_bits) - - def train(self, n, x): - return _swigfaiss.RaBitQuantizer_train(self, n, x) - - def compute_codes(self, x, codes, n): - return _swigfaiss.RaBitQuantizer_compute_codes(self, x, codes, n) - - def compute_codes_core(self, x, codes, n, centroid_in): - return _swigfaiss.RaBitQuantizer_compute_codes_core(self, x, codes, n, centroid_in) - - def decode(self, codes, x, n): - return _swigfaiss.RaBitQuantizer_decode(self, codes, x, n) - - def decode_core(self, codes, x, n, centroid_in): - return _swigfaiss.RaBitQuantizer_decode_core(self, codes, x, n, centroid_in) - - def get_distance_computer(self, qb=0, centroid=None, centered=False): - return _swigfaiss.RaBitQuantizer_get_distance_computer(self, qb, centroid, centered) - __swig_destroy__ = _swigfaiss.delete_RaBitQuantizer - -# Register RaBitQuantizer in _swigfaiss: -_swigfaiss.RaBitQuantizer_swigregister(RaBitQuantizer) -class RaBitQDistanceComputer(FlatCodesDistanceComputer): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - d = property(_swigfaiss.RaBitQDistanceComputer_d_get, _swigfaiss.RaBitQDistanceComputer_d_set) - centroid = property(_swigfaiss.RaBitQDistanceComputer_centroid_get, _swigfaiss.RaBitQDistanceComputer_centroid_set) - metric_type = property(_swigfaiss.RaBitQDistanceComputer_metric_type_get, _swigfaiss.RaBitQDistanceComputer_metric_type_set) - nb_bits = property(_swigfaiss.RaBitQDistanceComputer_nb_bits_get, _swigfaiss.RaBitQDistanceComputer_nb_bits_set) - g_error = property(_swigfaiss.RaBitQDistanceComputer_g_error_get, _swigfaiss.RaBitQDistanceComputer_g_error_set) - - def symmetric_dis(self, arg2, arg3): - return _swigfaiss.RaBitQDistanceComputer_symmetric_dis(self, arg2, arg3) - - def distance_to_code_1bit(self, code): - return _swigfaiss.RaBitQDistanceComputer_distance_to_code_1bit(self, code) - - def distance_to_code_full(self, code): - return _swigfaiss.RaBitQDistanceComputer_distance_to_code_full(self, code) - - def set_centroid(self, centroid_in): - return _swigfaiss.RaBitQDistanceComputer_set_centroid(self, centroid_in) - - def scan_codes_multibit(self, list_size, codes, ids, code_size, list_no, store_pairs, sel, keep_max, handler): - return _swigfaiss.RaBitQDistanceComputer_scan_codes_multibit(self, list_size, codes, ids, code_size, list_no, store_pairs, sel, keep_max, handler) - - def distance_to_code(self, code): - return _swigfaiss.RaBitQDistanceComputer_distance_to_code(self, code) - __swig_destroy__ = _swigfaiss.delete_RaBitQDistanceComputer - -# Register RaBitQDistanceComputer in _swigfaiss: -_swigfaiss.RaBitQDistanceComputer_swigregister(RaBitQDistanceComputer) -class SignBitFactors(object): - r""" - Base factors computed per database vector for RaBitQ distance computation. - Used by both 1-bit and multi-bit RaBitQ variants. - These can be stored either embedded in codes (IndexRaBitQ) or separately - (IndexRaBitQFastScan). - - For 1-bit mode only - contains the minimal factors needed for distance - estimation using just sign bits. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - or_minus_c_l2sqr = property(_swigfaiss.SignBitFactors_or_minus_c_l2sqr_get, _swigfaiss.SignBitFactors_or_minus_c_l2sqr_set) - dp_multiplier = property(_swigfaiss.SignBitFactors_dp_multiplier_get, _swigfaiss.SignBitFactors_dp_multiplier_set) - - def __init__(self): - _swigfaiss.SignBitFactors_swiginit(self, _swigfaiss.new_SignBitFactors()) - __swig_destroy__ = _swigfaiss.delete_SignBitFactors - -# Register SignBitFactors in _swigfaiss: -_swigfaiss.SignBitFactors_swigregister(SignBitFactors) -class SignBitFactorsWithError(SignBitFactors): - r""" - Extended factors for multi-bit RaBitQ (nb_bits > 1). - Includes error bound for lower bound computation in two-stage search. - Inherits base factors to maintain layout compatibility. - - Used in multi-bit mode - the error bound enables quick filtering of - unlikely candidates in the first stage of two-stage search. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - f_error = property(_swigfaiss.SignBitFactorsWithError_f_error_get, _swigfaiss.SignBitFactorsWithError_f_error_set) - - def __init__(self): - _swigfaiss.SignBitFactorsWithError_swiginit(self, _swigfaiss.new_SignBitFactorsWithError()) - __swig_destroy__ = _swigfaiss.delete_SignBitFactorsWithError - -# Register SignBitFactorsWithError in _swigfaiss: -_swigfaiss.SignBitFactorsWithError_swigregister(SignBitFactorsWithError) -class ExtraBitsFactors(object): - r""" - Additional factors for multi-bit RaBitQ (nb_bits > 1). - Used to store normalization and scaling factors for the refinement bits - that encode additional precision beyond the sign bit. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - f_add_ex = property(_swigfaiss.ExtraBitsFactors_f_add_ex_get, _swigfaiss.ExtraBitsFactors_f_add_ex_set) - f_rescale_ex = property(_swigfaiss.ExtraBitsFactors_f_rescale_ex_get, _swigfaiss.ExtraBitsFactors_f_rescale_ex_set) - - def __init__(self): - _swigfaiss.ExtraBitsFactors_swiginit(self, _swigfaiss.new_ExtraBitsFactors()) - __swig_destroy__ = _swigfaiss.delete_ExtraBitsFactors - -# Register ExtraBitsFactors in _swigfaiss: -_swigfaiss.ExtraBitsFactors_swigregister(ExtraBitsFactors) -class QueryFactorsData(object): - r""" - Query-specific factors computed during search for RaBitQ distance - computation. Used by both IndexRaBitQ and IndexRaBitQFastScan - implementations. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - c1 = property(_swigfaiss.QueryFactorsData_c1_get, _swigfaiss.QueryFactorsData_c1_set) - c2 = property(_swigfaiss.QueryFactorsData_c2_get, _swigfaiss.QueryFactorsData_c2_set) - c34 = property(_swigfaiss.QueryFactorsData_c34_get, _swigfaiss.QueryFactorsData_c34_set) - qr_to_c_L2sqr = property(_swigfaiss.QueryFactorsData_qr_to_c_L2sqr_get, _swigfaiss.QueryFactorsData_qr_to_c_L2sqr_set) - qr_norm_L2sqr = property(_swigfaiss.QueryFactorsData_qr_norm_L2sqr_get, _swigfaiss.QueryFactorsData_qr_norm_L2sqr_set) - q_dot_c = property(_swigfaiss.QueryFactorsData_q_dot_c_get, _swigfaiss.QueryFactorsData_q_dot_c_set) - int_dot_scale = property(_swigfaiss.QueryFactorsData_int_dot_scale_get, _swigfaiss.QueryFactorsData_int_dot_scale_set) - g_error = property(_swigfaiss.QueryFactorsData_g_error_get, _swigfaiss.QueryFactorsData_g_error_set) - rotated_q = property(_swigfaiss.QueryFactorsData_rotated_q_get, _swigfaiss.QueryFactorsData_rotated_q_set) - - def __init__(self): - _swigfaiss.QueryFactorsData_swiginit(self, _swigfaiss.new_QueryFactorsData()) - __swig_destroy__ = _swigfaiss.delete_QueryFactorsData - -# Register QueryFactorsData in _swigfaiss: -_swigfaiss.QueryFactorsData_swigregister(QueryFactorsData) - -def round_nonnegative_to_uint8(x): - r""" - Fast half-away rounding for non-negative RaBitQ quantization values. - - Contract: x must be finite, non-NaN, and in [0, 255.5). This is not a - bit-exact replacement for roundf within a few ulps below k + 0.5, where the - addition may round across the tie. It is intended only for RaBitQ LUT - quantization paths where a +/- 1 code difference at those boundaries is - acceptable. - """ - return _swigfaiss.round_nonnegative_to_uint8(x) - -def round_nonnegative_to_uint16(x): - r"""Same as round_nonnegative_to_uint8 for uint16 RaBitQ bias values.""" - return _swigfaiss.round_nonnegative_to_uint16(x) - -def round_clamped_to_uint8(x, max_code): - r""" - Fast clamped rounding for query-byte quantization. - - Contract: x must be non-NaN. Values outside [0, max_code] are clamped before - applying the same non-bit-exact rounding used by - round_nonnegative_to_uint8(). - """ - return _swigfaiss.round_clamped_to_uint8(x, max_code) - -def compute_vector_factors(x, d, centroid, metric_type, compute_error=True): - r""" - Compute factors for a single database vector using RaBitQ algorithm. - This function consolidates the mathematical logic that was duplicated - between IndexRaBitQ and IndexRaBitQFastScan. - - :type x: float - :param x: input vector (d dimensions) - :type d: int - :param d: dimensionality - :type centroid: float - :param centroid: database centroid (nullptr if not used) - :type metric_type: int - :param metric_type: distance metric (L2 or Inner Product) - :type compute_error: boolean, optional - :param compute_error: whether to compute f_error (false for 1-bit mode) - :rtype: :py:class:`SignBitFactorsWithError` - :return: computed factors for distance computation - """ - return _swigfaiss.compute_vector_factors(x, d, centroid, metric_type, compute_error) - -def compute_vector_intermediate_values(x, d, centroid, norm_L2sqr, or_L2sqr, dp_oO): - r""" - Compute intermediate values needed for vector factor computation. - Separated out to allow different bit packing strategies while sharing - the core mathematical computation. - - :type x: float - :param x: input vector (d dimensions) - :type d: int - :param d: dimensionality - :type centroid: float - :param centroid: database centroid (nullptr if not used) - :type norm_L2sqr: float - :param norm_L2sqr: output: ||or - c||^2 - :type or_L2sqr: float - :param or_L2sqr: output: ||or||^2 - :type dp_oO: float - :param dp_oO: output: sum of |or_i - c_i| (absolute deviations) - """ - return _swigfaiss.compute_vector_intermediate_values(x, d, centroid, norm_L2sqr, or_L2sqr, dp_oO) - -def compute_factors_from_intermediates(norm_L2sqr, or_L2sqr, dp_oO, d, metric_type, compute_error=True): - r""" - Compute final factors from intermediate values. - :type norm_L2sqr: float - :param norm_L2sqr: ||or - c||^2 - :type or_L2sqr: float - :param or_L2sqr: ||or||^2 - :type dp_oO: float - :param dp_oO: sum of |or_i - c_i| - :type d: int - :param d: dimensionality - :type metric_type: int - :param metric_type: distance metric - :type compute_error: boolean, optional - :param compute_error: whether to compute f_error (false for 1-bit mode) - :rtype: :py:class:`SignBitFactorsWithError` - :return: computed factors - """ - return _swigfaiss.compute_factors_from_intermediates(norm_L2sqr, or_L2sqr, dp_oO, d, metric_type, compute_error) - -def compute_query_factors(query, d, centroid, qb, centered, metric_type, rotated_q, rotated_qq): - r""" - Compute query factors for RaBitQ distance computation. - This consolidates the query processing logic shared between implementations. - - :type query: float - :param query: query vector (d dimensions) - :type d: int - :param d: dimensionality - :type centroid: float - :param centroid: database centroid (nullptr if not used) - :type qb: uint8_t - :param qb: number of quantization bits (1-8) - :type centered: boolean - :param centered: whether to use centered quantization - :type metric_type: int - :param metric_type: distance metric - :type rotated_q: std::vector< float > - :param rotated_q: output: query - centroid - :type rotated_qq: std::vector< uint8_t > - :param rotated_qq: output: quantized query values - :rtype: :py:class:`QueryFactorsData` - :return: computed query factors - """ - return _swigfaiss.compute_query_factors(query, d, centroid, qb, centered, metric_type, rotated_q, rotated_qq) - -def extract_bit_standard(code, bit_index): - r""" - Extract bit value from RaBitQ code in standard format. - Used by IndexRaBitQ which stores bits sequentially. - - :type code: uint8_t - :param code: RaBitQ code data - :type bit_index: int - :param bit_index: which bit to extract (0 to d-1) - :rtype: boolean - :return: bit value (true/false) - """ - return _swigfaiss.extract_bit_standard(code, bit_index) - -def extract_bit_fastscan(code, bit_index): - r""" - Extract bit value from FastScan code format. - Used by IndexRaBitQFastScan which packs bits into 4-bit sub-quantizers. - - :type code: uint8_t - :param code: FastScan code data - :type bit_index: int - :param bit_index: which bit to extract (0 to d-1) - :rtype: boolean - :return: bit value (true/false) - """ - return _swigfaiss.extract_bit_fastscan(code, bit_index) - -def set_bit_standard(code, bit_index): - r""" - Set bit value in standard RaBitQ code format. - :type code: uint8_t - :param code: RaBitQ code data to modify - :type bit_index: int - :param bit_index: which bit to set (0 to d-1) - """ - return _swigfaiss.set_bit_standard(code, bit_index) - -def set_bit_fastscan(code, bit_index): - r""" - Set bit value in FastScan code format. - :type code: uint8_t - :param code: FastScan code data to modify - :type bit_index: int - :param bit_index: which bit to set (0 to d-1) - """ - return _swigfaiss.set_bit_fastscan(code, bit_index) - -def compute_1bit_adjusted_distance(normalized_distance, db_factors, query_factors, centered, qb, d): - r""" - Compute adjusted 1-bit distance from normalized LUT distance. - This is the core distance formula shared by all RaBitQ handlers. - - :type normalized_distance: float - :param normalized_distance: Distance from SIMD LUT lookup (after - normalization) - :type db_factors: :py:class:`SignBitFactors` - :param db_factors: Database vector factors (SignBitFactors or - SignBitFactorsWithError) - :type query_factors: :py:class:`QueryFactorsData` - :param query_factors: Query factors computed during search - :type centered: boolean - :param centered: Whether centered quantization is used - :type qb: int - :param qb: Number of quantization bits - :type d: int - :param d: Dimensionality - :rtype: float - :return: Adjusted distance value - """ - return _swigfaiss.compute_1bit_adjusted_distance(normalized_distance, db_factors, query_factors, centered, qb, d) - -def should_refine_candidate(est_distance, f_error, g_error, threshold, is_similarity): - r""" - Determine whether a candidate should be refined in two-stage search. - Consolidates the filtering logic for both L2 and IP metrics. - - For L2 (min-heap): uses lower_bound = est_distance - error_adjustment - - Skip if lower_bound >= threshold (can't beat current worst) - For IP (max-heap): uses upper_bound = est_distance + error_adjustment - - Skip if upper_bound <= threshold (can't beat current best) - - :type est_distance: float - :param est_distance: Estimated 1-bit distance - :type f_error: float - :param f_error: Database vector error factor - :type g_error: float - :param g_error: Query vector error factor - :type threshold: float - :param threshold: Current heap threshold (worst result in heap) - :type is_similarity: boolean - :param is_similarity: True for IP metric (max-heap), false for L2 - (min-heap) - :rtype: boolean - :return: True if candidate should be refined with full - multi-bit distance - """ - return _swigfaiss.should_refine_candidate(est_distance, f_error, g_error, threshold, is_similarity) - -def extract_code_inline(ex_code, index, ex_bits): - r""" - Extract multi-bit code on-the-fly from packed ex-bit codes. - This inline function extracts a single code value without unpacking the - entire array, enabling efficient on-the-fly decoding during distance - computation. - - :type ex_code: uint8_t - :param ex_code: packed ex-bit codes - :type index: int - :param index: which code to extract (0 to d-1) - :type ex_bits: int - :param ex_bits: number of bits per code (1-8) - :rtype: int - :return: extracted code value in range [0, 2^ex_bits - 1] - """ - return _swigfaiss.extract_code_inline(ex_code, index, ex_bits) - -def compute_full_multibit_distance(sign_bits, ex_code, ex_fac, rotated_q, qr_base, d, ex_bits, metric_type): - r""" - Compute full multi-bit distance from sign bits and ex-bit codes. - This is the core distance computation shared by RaBitQFastScan handlers. - - The multi-bit distance combines the sign bit (1-bit) with additional - magnitude bits (ex_bits) to compute a more accurate distance estimate. - Uses SIMD-optimized bit-plane decomposition (AVX2+BMI2) for ex_bits 1-7, - with scalar fallback for non-x86 or non-BMI2 platforms. - - :type sign_bits: uint8_t - :param sign_bits: unpacked sign bits (1-bit codes in standard format) - :type ex_code: uint8_t - :param ex_code: packed ex-bit codes - :type ex_fac: :py:class:`ExtraBitsFactors` - :param ex_fac: ex-bit factors (f_add_ex, f_rescale_ex) - :type rotated_q: float - :param rotated_q: rotated query vector - :type qr_base: float - :param qr_base: precomputed base term: ||q-c||^2 for L2, for IP - :type d: int - :param d: dimensionality - :type ex_bits: int - :param ex_bits: number of extra bits (nb_bits - 1) - :type metric_type: int - :param metric_type: distance metric (L2 or Inner Product) - :rtype: float - :return: computed full multi-bit distance - """ - return _swigfaiss.compute_full_multibit_distance(sign_bits, ex_code, ex_fac, rotated_q, qr_base, d, ex_bits, metric_type) - -def unpack_sign_bits_from_packed(block, bbs, nsq, offset, block_stride, sign_bits_out): - r""" - Extract sign bits from PQ4-interleaved block into flat byte packing. - Like CodePackerRaBitQ::unpack_1 but sign-bits-only and with the - vector's in-block address hoisted out of the per-SQ loop. - """ - return _swigfaiss.unpack_sign_bits_from_packed(block, bbs, nsq, offset, block_stride, sign_bits_out) - -def compute_per_vector_storage_size(nb_bits, d): - r""" - Compute per-vector auxiliary storage size. - - :type nb_bits: int - :param nb_bits: number of quantization bits (1 = sign-bit only) - :type d: int - :param d: dimensionality - :rtype: int - :return: storage size in bytes - """ - return _swigfaiss.compute_per_vector_storage_size(nb_bits, d) - -def populate_block_aux_from_flat_storage(flat_storage, codes, num_vectors, bbs, M2, old_block_stride, new_block_stride, storage_size, id_map=None): - r""" - [LEGACY FORMAT SUPPORT] Migrate block data from old I/O format to new - format. - - This function is used only when reading indexes saved with the legacy format - (fourcc "Irfs"/"Iwrf") to convert them to the new embedded auxiliary data - format. Not needed for indexes saved with the new format ("Irfn"/"Iwrn"). - - Re-layouts blocks in-place and copies aux data from flat_storage. - - :type flat_storage: std::vector< uint8_t > - :param flat_storage: legacy per-vector aux data indexed by global ID - :type codes: faiss::AlignedTable< uint8_t > - :param codes: block data (will be resized and re-laid out) - :type num_vectors: int - :param num_vectors: number of vectors in this segment - :type bbs: int - :param bbs: block batch size (vectors per block) - :type M2: int - :param M2: rounded sub-quantizer count - :type old_block_stride: int - :param old_block_stride: old block size (packed codes only, or current) - :type new_block_stride: int - :param new_block_stride: new block size (packed codes + aux region) - :type storage_size: int - :param storage_size: per-vector aux storage size in bytes - :type id_map: int, optional - :param id_map: maps local offset to global ID; null = sequential - """ - return _swigfaiss.populate_block_aux_from_flat_storage(flat_storage, codes, num_vectors, bbs, M2, old_block_stride, new_block_stride, storage_size, id_map) -EDENScaleType_UNBIASED = _swigfaiss.EDENScaleType_UNBIASED -EDENScaleType_BIASED = _swigfaiss.EDENScaleType_BIASED -class EDENCodeFactors(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - l2_norm_term = property(_swigfaiss.EDENCodeFactors_l2_norm_term_get, _swigfaiss.EDENCodeFactors_l2_norm_term_set) - scale = property(_swigfaiss.EDENCodeFactors_scale_get, _swigfaiss.EDENCodeFactors_scale_set) - - def __init__(self): - _swigfaiss.EDENCodeFactors_swiginit(self, _swigfaiss.new_EDENCodeFactors()) - __swig_destroy__ = _swigfaiss.delete_EDENCodeFactors - -# Register EDENCodeFactors in _swigfaiss: -_swigfaiss.EDENCodeFactors_swigregister(EDENCodeFactors) -Z_MAX_BY_QB = cvar.Z_MAX_BY_QB - -class EDENFlatCodesDistanceComputer(FlatCodesDistanceComputer): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - - def consecutive_distances_batch_8(self, first, distances): - return _swigfaiss.EDENFlatCodesDistanceComputer_consecutive_distances_batch_8(self, first, distances) - - def consecutive_distances_batch_16(self, first, distances): - return _swigfaiss.EDENFlatCodesDistanceComputer_consecutive_distances_batch_16(self, first, distances) - __swig_destroy__ = _swigfaiss.delete_EDENFlatCodesDistanceComputer - -# Register EDENFlatCodesDistanceComputer in _swigfaiss: -_swigfaiss.EDENFlatCodesDistanceComputer_swigregister(EDENFlatCodesDistanceComputer) - -def quantizer_type_for_bits(nb_bits): - return _swigfaiss.quantizer_type_for_bits(nb_bits) - -def is_eden_quantizer_type(qtype): - return _swigfaiss.is_eden_quantizer_type(qtype) - -def nb_bits_for_qtype(qtype): - return _swigfaiss.nb_bits_for_qtype(qtype) - -def packed_code_size(d, nb_bits): - return _swigfaiss.packed_code_size(d, nb_bits) - -def code_size(d, nb_bits): - return _swigfaiss.code_size(d, nb_bits) - -def extract_code(codes, index, nb_bits): - return _swigfaiss.extract_code(codes, index, nb_bits) - -def compute_codes(sq, metric_type, scale_type, x, codes, n, centroid=None): - return _swigfaiss.compute_codes(sq, metric_type, scale_type, x, codes, n, centroid) - -def decode(sq, codes, x, n, centroid=None): - return _swigfaiss.decode(sq, codes, x, n, centroid) - -def get_distance_computer(sq, metric_type, centroid=None): - return _swigfaiss.get_distance_computer(sq, metric_type, centroid) -class IndexEDEN(IndexFlatCodes): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - sq = property(_swigfaiss.IndexEDEN_sq_get, _swigfaiss.IndexEDEN_sq_set) - scale_type = property(_swigfaiss.IndexEDEN_scale_type_get, _swigfaiss.IndexEDEN_scale_type_set) - center = property(_swigfaiss.IndexEDEN_center_get, _swigfaiss.IndexEDEN_center_set) - - def __init__(self, *args): - _swigfaiss.IndexEDEN_swiginit(self, _swigfaiss.new_IndexEDEN(*args)) - - def train(self, n, x): - return _swigfaiss.IndexEDEN_train(self, n, x) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexEDEN_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexEDEN_sa_decode(self, n, bytes, x) - - def get_FlatCodesDistanceComputer(self): - return _swigfaiss.IndexEDEN_get_FlatCodesDistanceComputer(self) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexEDEN_search(self, n, x, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexEDEN_range_search(self, n, x, radius, result, params) - __swig_destroy__ = _swigfaiss.delete_IndexEDEN - -# Register IndexEDEN in _swigfaiss: -_swigfaiss.IndexEDEN_swigregister(IndexEDEN) -class IndexIVFEDEN(IndexIVF): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - sq = property(_swigfaiss.IndexIVFEDEN_sq_get, _swigfaiss.IndexIVFEDEN_sq_set) - scale_type = property(_swigfaiss.IndexIVFEDEN_scale_type_get, _swigfaiss.IndexIVFEDEN_scale_type_set) - - def __init__(self, *args): - _swigfaiss.IndexIVFEDEN_swiginit(self, _swigfaiss.new_IndexIVFEDEN(*args)) - - def train_encoder(self, n, x, assign): - return _swigfaiss.IndexIVFEDEN_train_encoder(self, n, x, assign) - - def encode_vectors(self, n, x, list_nos, codes, include_listnos=False): - return _swigfaiss.IndexIVFEDEN_encode_vectors(self, n, x, list_nos, codes, include_listnos) - - def decode_vectors(self, n, codes, list_nos, x): - return _swigfaiss.IndexIVFEDEN_decode_vectors(self, n, codes, list_nos, x) - - def add_core(self, n, x, xids, precomputed_idx, inverted_list_context=None): - return _swigfaiss.IndexIVFEDEN_add_core(self, n, x, xids, precomputed_idx, inverted_list_context) - - def get_InvertedListScanner(self, store_pairs, sel, params): - return _swigfaiss.IndexIVFEDEN_get_InvertedListScanner(self, store_pairs, sel, params) - - def reconstruct_from_offset(self, list_no, offset, recons): - return _swigfaiss.IndexIVFEDEN_reconstruct_from_offset(self, list_no, offset, recons) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexIVFEDEN_sa_decode(self, n, bytes, x) - - def get_distance_computer(self): - return _swigfaiss.IndexIVFEDEN_get_distance_computer(self) - __swig_destroy__ = _swigfaiss.delete_IndexIVFEDEN - -# Register IndexIVFEDEN in _swigfaiss: -_swigfaiss.IndexIVFEDEN_swigregister(IndexIVFEDEN) -class RaBitQSearchParameters(SearchParameters): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - qb = property(_swigfaiss.RaBitQSearchParameters_qb_get, _swigfaiss.RaBitQSearchParameters_qb_set) - centered = property(_swigfaiss.RaBitQSearchParameters_centered_get, _swigfaiss.RaBitQSearchParameters_centered_set) - - def __init__(self): - _swigfaiss.RaBitQSearchParameters_swiginit(self, _swigfaiss.new_RaBitQSearchParameters()) - __swig_destroy__ = _swigfaiss.delete_RaBitQSearchParameters - -# Register RaBitQSearchParameters in _swigfaiss: -_swigfaiss.RaBitQSearchParameters_swigregister(RaBitQSearchParameters) -class IndexRaBitQ(IndexFlatCodes): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - rabitq = property(_swigfaiss.IndexRaBitQ_rabitq_get, _swigfaiss.IndexRaBitQ_rabitq_set) - center = property(_swigfaiss.IndexRaBitQ_center_get, _swigfaiss.IndexRaBitQ_center_set) - qb = property(_swigfaiss.IndexRaBitQ_qb_get, _swigfaiss.IndexRaBitQ_qb_set) - centered = property(_swigfaiss.IndexRaBitQ_centered_get, _swigfaiss.IndexRaBitQ_centered_set) - - def __init__(self, *args): - _swigfaiss.IndexRaBitQ_swiginit(self, _swigfaiss.new_IndexRaBitQ(*args)) - - def train(self, n, x): - return _swigfaiss.IndexRaBitQ_train(self, n, x) - - def sa_encode(self, n, x, bytes): - return _swigfaiss.IndexRaBitQ_sa_encode(self, n, x, bytes) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexRaBitQ_sa_decode(self, n, bytes, x) - - def get_FlatCodesDistanceComputer(self): - return _swigfaiss.IndexRaBitQ_get_FlatCodesDistanceComputer(self) - - def get_quantized_distance_computer(self, qb_in, centered): - return _swigfaiss.IndexRaBitQ_get_quantized_distance_computer(self, qb_in, centered) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexRaBitQ_search(self, n, x, k, distances, labels, params) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexRaBitQ_range_search(self, n, x, radius, result, params) - __swig_destroy__ = _swigfaiss.delete_IndexRaBitQ - -# Register IndexRaBitQ in _swigfaiss: -_swigfaiss.IndexRaBitQ_swigregister(IndexRaBitQ) -class IndexRaBitQFastScan(IndexFastScan): - r""" - Fast-scan version of RaBitQ index that processes 32 database vectors at a - time using SIMD operations. Similar to IndexPQFastScan but adapted for - RaBitQ's bit-level quantization with factors. - - The key differences from IndexRaBitQ: - - Processes vectors in batches of 32 - - Uses 4-bit groupings for SIMD optimization (4 dimensions per 4-bit unit) - - Separates factors from quantized bits for efficient processing - - Leverages existing PQ4 FastScan infrastructure where possible - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - rabitq = property(_swigfaiss.IndexRaBitQFastScan_rabitq_get, _swigfaiss.IndexRaBitQFastScan_rabitq_set, doc=r"""RaBitQ quantizer for encoding/decoding""") - center = property(_swigfaiss.IndexRaBitQFastScan_center_get, _swigfaiss.IndexRaBitQFastScan_center_set, doc=r"""Center of all points (same as IndexRaBitQ)""") - qb = property(_swigfaiss.IndexRaBitQFastScan_qb_get, _swigfaiss.IndexRaBitQFastScan_qb_set, doc=r"""Default number of bits to quantize a query with""") - centered = property(_swigfaiss.IndexRaBitQFastScan_centered_get, _swigfaiss.IndexRaBitQFastScan_centered_set) - - def __init__(self, *args): - r""" - *Overload 1:* - build from an existing IndexRaBitQ - - | - - *Overload 2:* - build from an existing IndexRaBitQ - """ - _swigfaiss.IndexRaBitQFastScan_swiginit(self, _swigfaiss.new_IndexRaBitQFastScan(*args)) - - def train(self, n, x): - return _swigfaiss.IndexRaBitQFastScan_train(self, n, x) - - def add(self, n, x): - return _swigfaiss.IndexRaBitQFastScan_add(self, n, x) - - def compute_codes(self, codes, n, x): - return _swigfaiss.IndexRaBitQFastScan_compute_codes(self, codes, n, x) - - def compute_per_vector_storage_size(self): - r"""Compute per-vector auxiliary data size in block aux region""" - return _swigfaiss.IndexRaBitQFastScan_compute_per_vector_storage_size(self) - - def compute_float_LUT(self, lut, n, x, context): - return _swigfaiss.IndexRaBitQFastScan_compute_float_LUT(self, lut, n, x, context) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexRaBitQFastScan_sa_decode(self, n, bytes, x) - - def fast_scan_code_size(self): - r""" - Packed code size: (d + 7) / 8 bytes (1-bit-per-dimension sign bits, - excluding factors) - """ - return _swigfaiss.IndexRaBitQFastScan_fast_scan_code_size(self) - - def get_CodePacker(self): - r"""Return CodePackerRaBitQ with enlarged block size""" - return _swigfaiss.IndexRaBitQFastScan_get_CodePacker(self) - - def remove_ids(self, sel): - r"""Remove vectors and compact both PQ4 codes and auxiliary data""" - return _swigfaiss.IndexRaBitQFastScan_remove_ids(self, sel) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexRaBitQFastScan_search(self, n, x, k, distances, labels, params) - __swig_destroy__ = _swigfaiss.delete_IndexRaBitQFastScan - -# Register IndexRaBitQFastScan in _swigfaiss: -_swigfaiss.IndexRaBitQFastScan_swigregister(IndexRaBitQFastScan) -class IVFRaBitQSearchParameters(SearchParametersIVF): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - qb = property(_swigfaiss.IVFRaBitQSearchParameters_qb_get, _swigfaiss.IVFRaBitQSearchParameters_qb_set) - centered = property(_swigfaiss.IVFRaBitQSearchParameters_centered_get, _swigfaiss.IVFRaBitQSearchParameters_centered_set) - - def __init__(self): - _swigfaiss.IVFRaBitQSearchParameters_swiginit(self, _swigfaiss.new_IVFRaBitQSearchParameters()) - __swig_destroy__ = _swigfaiss.delete_IVFRaBitQSearchParameters - -# Register IVFRaBitQSearchParameters in _swigfaiss: -_swigfaiss.IVFRaBitQSearchParameters_swigregister(IVFRaBitQSearchParameters) -class IndexIVFRaBitQ(IndexIVF): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - rabitq = property(_swigfaiss.IndexIVFRaBitQ_rabitq_get, _swigfaiss.IndexIVFRaBitQ_rabitq_set) - qb = property(_swigfaiss.IndexIVFRaBitQ_qb_get, _swigfaiss.IndexIVFRaBitQ_qb_set) - - def __init__(self, *args): - _swigfaiss.IndexIVFRaBitQ_swiginit(self, _swigfaiss.new_IndexIVFRaBitQ(*args)) - - def train_encoder(self, n, x, assign): - return _swigfaiss.IndexIVFRaBitQ_train_encoder(self, n, x, assign) - - def encode_vectors(self, n, x, list_nos, codes, include_listnos=False): - return _swigfaiss.IndexIVFRaBitQ_encode_vectors(self, n, x, list_nos, codes, include_listnos) - - def decode_vectors(self, n, codes, list_nos, x): - return _swigfaiss.IndexIVFRaBitQ_decode_vectors(self, n, codes, list_nos, x) - - def add_core(self, n, x, xids, precomputed_idx, inverted_list_context=None): - return _swigfaiss.IndexIVFRaBitQ_add_core(self, n, x, xids, precomputed_idx, inverted_list_context) - - def get_InvertedListScanner(self, store_pairs, sel, params): - return _swigfaiss.IndexIVFRaBitQ_get_InvertedListScanner(self, store_pairs, sel, params) - - def reconstruct_from_offset(self, list_no, offset, recons): - return _swigfaiss.IndexIVFRaBitQ_reconstruct_from_offset(self, list_no, offset, recons) - - def sa_decode(self, n, bytes, x): - return _swigfaiss.IndexIVFRaBitQ_sa_decode(self, n, bytes, x) - - def get_distance_computer(self): - return _swigfaiss.IndexIVFRaBitQ_get_distance_computer(self) - __swig_destroy__ = _swigfaiss.delete_IndexIVFRaBitQ - -# Register IndexIVFRaBitQ in _swigfaiss: -_swigfaiss.IndexIVFRaBitQ_swigregister(IndexIVFRaBitQ) -class IndexIVFRaBitQFastScan(IndexIVFFastScan): - r""" - Fast-scan version of IndexIVFRaBitQ that processes vectors in batches - using SIMD operations. Combines the inverted file structure of IVF - with RaBitQ's bit-level quantization and FastScan's batch processing. - - Key features: - - Inherits from IndexIVFFastScan for IVF structure and search algorithms - - Processes 32 database vectors at a time using SIMD - - Separates factors from quantized bits for efficient processing - - Supports both L2 and inner product metrics - - Maintains compatibility with existing IVF search parameters - - Implementation details: - - Batch size (bbs) is typically 32 for optimal SIMD performance - - Factors are stored separately from packed codes for cache efficiency - - Query factors are computed once per search and reused across lists - - Uses specialized result handlers for RaBitQ distance corrections - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - rabitq = property(_swigfaiss.IndexIVFRaBitQFastScan_rabitq_get, _swigfaiss.IndexIVFRaBitQFastScan_rabitq_set) - qb = property(_swigfaiss.IndexIVFRaBitQFastScan_qb_get, _swigfaiss.IndexIVFRaBitQFastScan_qb_set, doc=r"""Default number of bits to quantize a query with""") - centered = property(_swigfaiss.IndexIVFRaBitQFastScan_centered_get, _swigfaiss.IndexIVFRaBitQFastScan_centered_set, doc=r"""Use zero-centered scalar quantizer for queries""") - - def __init__(self, *args): - r""" - *Overload 1:* - Build from an existing IndexIVFRaBitQ - - | - - *Overload 2:* - Build from an existing IndexIVFRaBitQ - """ - _swigfaiss.IndexIVFRaBitQFastScan_swiginit(self, _swigfaiss.new_IndexIVFRaBitQFastScan(*args)) - - def train_encoder(self, n, x, assign): - return _swigfaiss.IndexIVFRaBitQFastScan_train_encoder(self, n, x, assign) - - def encode_vectors(self, n, x, list_nos, codes, include_listnos=False): - return _swigfaiss.IndexIVFRaBitQFastScan_encode_vectors(self, n, x, list_nos, codes, include_listnos) - - def fast_scan_code_size(self): - r""" - Packed code size: (d + 7) / 8 bytes (1-bit-per-dimension sign bits, - excluding factors) - """ - return _swigfaiss.IndexIVFRaBitQFastScan_fast_scan_code_size(self) - - def get_CodePacker(self): - r"""Return CodePackerRaBitQ with enlarged block size""" - return _swigfaiss.IndexIVFRaBitQFastScan_get_CodePacker(self) - - def postprocess_packed_codes(self, list_no, list_offset, n_added, flat_codes): - r"""Write per-vector auxiliary data into block auxiliary region""" - return _swigfaiss.IndexIVFRaBitQFastScan_postprocess_packed_codes(self, list_no, list_offset, n_added, flat_codes) - - def reconstruct_from_offset(self, list_no, offset, recons): - r"""Reconstruct a single vector from an inverted list""" - return _swigfaiss.IndexIVFRaBitQFastScan_reconstruct_from_offset(self, list_no, offset, recons) - - def sa_decode(self, n, bytes, x): - r"""Override sa_decode to handle RaBitQ reconstruction""" - return _swigfaiss.IndexIVFRaBitQFastScan_sa_decode(self, n, bytes, x) - - def compute_per_vector_storage_size(self): - r"""Compute per-vector auxiliary storage size based on nb_bits""" - return _swigfaiss.IndexIVFRaBitQFastScan_compute_per_vector_storage_size(self) - - def compute_LUT_uint8(self, n, x, cq, dis_tables, biases, normalizers, context): - r""" - Override: compute and quantize LUT per-query to avoid O(n*nprobe*M*16) - float table allocation. - """ - return _swigfaiss.IndexIVFRaBitQFastScan_compute_LUT_uint8(self, n, x, cq, dis_tables, biases, normalizers, context) - - def compute_residual_LUT(self, query, centroid_id, query_factors, lut_out, qb_param, centered_param, rotated_q, centroid_buf): - r"""Compute residual, query factors, and float LUT in two passes over d.""" - return _swigfaiss.IndexIVFRaBitQFastScan_compute_residual_LUT(self, query, centroid_id, query_factors, lut_out, qb_param, centered_param, rotated_q, centroid_buf) - - def lookup_table_is_3d(self): - r"""Implementation methods for IVFRaBitQFastScan specialization""" - return _swigfaiss.IndexIVFRaBitQFastScan_lookup_table_is_3d(self) - - def compute_LUT(self, n, x, cq, dis_tables, biases, context): - return _swigfaiss.IndexIVFRaBitQFastScan_compute_LUT(self, n, x, cq, dis_tables, biases, context) - - def search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params=None, stats=None): - return _swigfaiss.IndexIVFRaBitQFastScan_search_preassigned(self, n, x, k, assign, centroid_dis, distances, labels, store_pairs, params, stats) - - def get_InvertedListScanner(self, store_pairs=False, sel=None, params=None): - r""" - Get an InvertedListScanner for single-query scanning. - This provides compatibility with the standard IVF search interface - """ - return _swigfaiss.IndexIVFRaBitQFastScan_get_InvertedListScanner(self, store_pairs, sel, params) - __swig_destroy__ = _swigfaiss.delete_IndexIVFRaBitQFastScan - -# Register IndexIVFRaBitQFastScan in _swigfaiss: -_swigfaiss.IndexIVFRaBitQFastScan_swigregister(IndexIVFRaBitQFastScan) -class RangeSearchResult(object): - r""" - The objective is to have a simple result structure while - minimizing the number of mem copies in the result. The method - do_allocation can be overloaded to allocate the result tables in - the matrix type of a scripting language like Lua or Python. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nq = property(_swigfaiss.RangeSearchResult_nq_get, _swigfaiss.RangeSearchResult_nq_set, doc=r"""nb of queries""") - lims = property(_swigfaiss.RangeSearchResult_lims_get, _swigfaiss.RangeSearchResult_lims_set, doc=r"""size (nq + 1)""") - labels = property(_swigfaiss.RangeSearchResult_labels_get, _swigfaiss.RangeSearchResult_labels_set, doc=r"""result for query i is labels[lims[i]:lims[i+1]]""") - distances = property(_swigfaiss.RangeSearchResult_distances_get, _swigfaiss.RangeSearchResult_distances_set, doc=r"""corresponding distances (not sorted)""") - buffer_size = property(_swigfaiss.RangeSearchResult_buffer_size_get, _swigfaiss.RangeSearchResult_buffer_size_set, doc=r"""size of the result buffers used""") - - def __init__(self, nq, alloc_lims=True): - r"""lims must be allocated on input to range_search.""" - _swigfaiss.RangeSearchResult_swiginit(self, _swigfaiss.new_RangeSearchResult(nq, alloc_lims)) - - def do_allocation(self): - r""" - called when lims contains the nb of elements result entries - for each query - """ - return _swigfaiss.RangeSearchResult_do_allocation(self) - __swig_destroy__ = _swigfaiss.delete_RangeSearchResult - -# Register RangeSearchResult in _swigfaiss: -_swigfaiss.RangeSearchResult_swigregister(RangeSearchResult) -class BufferList(object): - r""" - List of temporary buffers used to store results before they are - copied to the RangeSearchResult object. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - buffer_size = property(_swigfaiss.BufferList_buffer_size_get, _swigfaiss.BufferList_buffer_size_set) - buffers = property(_swigfaiss.BufferList_buffers_get, _swigfaiss.BufferList_buffers_set) - wp = property(_swigfaiss.BufferList_wp_get, _swigfaiss.BufferList_wp_set, doc=r"""write pointer in the last buffer.""") - - def __init__(self, buffer_size): - _swigfaiss.BufferList_swiginit(self, _swigfaiss.new_BufferList(buffer_size)) - __swig_destroy__ = _swigfaiss.delete_BufferList - - def append_buffer(self): - r"""create a new buffer""" - return _swigfaiss.BufferList_append_buffer(self) - - def add(self, id, dis): - r"""add one result, possibly appending a new buffer if needed""" - return _swigfaiss.BufferList_add(self, id, dis) - - def copy_range(self, ofs, n, dest_ids, dest_dis): - r""" - copy elements ofs:ofs+n-1 seen as linear data in the buffers to - tables dest_ids, dest_dis - """ - return _swigfaiss.BufferList_copy_range(self, ofs, n, dest_ids, dest_dis) - -# Register BufferList in _swigfaiss: -_swigfaiss.BufferList_swigregister(BufferList) -class RangeQueryResult(object): - r"""result structure for a single query""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - qno = property(_swigfaiss.RangeQueryResult_qno_get, _swigfaiss.RangeQueryResult_qno_set) - nres = property(_swigfaiss.RangeQueryResult_nres_get, _swigfaiss.RangeQueryResult_nres_set) - pres = property(_swigfaiss.RangeQueryResult_pres_get, _swigfaiss.RangeQueryResult_pres_set) - stats = property(_swigfaiss.RangeQueryResult_stats_get, _swigfaiss.RangeQueryResult_stats_set) - - def add(self, dis, id): - r"""called by search function to report a new result""" - return _swigfaiss.RangeQueryResult_add(self, dis, id) - - def __init__(self): - _swigfaiss.RangeQueryResult_swiginit(self, _swigfaiss.new_RangeQueryResult()) - __swig_destroy__ = _swigfaiss.delete_RangeQueryResult - -# Register RangeQueryResult in _swigfaiss: -_swigfaiss.RangeQueryResult_swigregister(RangeQueryResult) -class RangeSearchPartialResult(BufferList): - r"""the entries in the buffers are split per query""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - res = property(_swigfaiss.RangeSearchPartialResult_res_get, _swigfaiss.RangeSearchPartialResult_res_set) - - def __init__(self, res_in): - r"""eventually the result will be stored in res_in""" - _swigfaiss.RangeSearchPartialResult_swiginit(self, _swigfaiss.new_RangeSearchPartialResult(res_in)) - queries = property(_swigfaiss.RangeSearchPartialResult_queries_get, _swigfaiss.RangeSearchPartialResult_queries_set, doc=r"""query ids + nb of results per query.""") - - def new_result(self, qno): - r"""begin a new result""" - return _swigfaiss.RangeSearchPartialResult_new_result(self, qno) - - def finalize(self): - return _swigfaiss.RangeSearchPartialResult_finalize(self) - - def set_lims(self): - r"""called by range_search before do_allocation""" - return _swigfaiss.RangeSearchPartialResult_set_lims(self) - - def copy_result(self, incremental=False): - r"""called by range_search after do_allocation""" - return _swigfaiss.RangeSearchPartialResult_copy_result(self, incremental) - - @staticmethod - def merge(partial_results, do_delete=True): - r""" - merge a set of PartialResult's into one RangeSearchResult - on output the partialresults are empty! - """ - return _swigfaiss.RangeSearchPartialResult_merge(partial_results, do_delete) - __swig_destroy__ = _swigfaiss.delete_RangeSearchPartialResult - -# Register RangeSearchPartialResult in _swigfaiss: -_swigfaiss.RangeSearchPartialResult_swigregister(RangeSearchPartialResult) -class InterruptCallback(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - - def want_interrupt(self): - return _swigfaiss.InterruptCallback_want_interrupt(self) - __swig_destroy__ = _swigfaiss.delete_InterruptCallback - - @staticmethod - def clear_instance(): - return _swigfaiss.InterruptCallback_clear_instance() - - @staticmethod - def check(): - r""" - check if: - - an interrupt callback is set - - the callback returns true - if this is the case, then throw an exception. Should not be called - from multiple threads. - """ - return _swigfaiss.InterruptCallback_check() - - @staticmethod - def is_interrupted(): - r""" - same as check() but return true if is interrupted instead of - throwing. Can be called from multiple threads. - """ - return _swigfaiss.InterruptCallback_is_interrupted() - - @staticmethod - def get_period_hint(flops): - r""" - assuming each iteration takes a certain number of flops, what - is a reasonable interval to check for interrupts? - """ - return _swigfaiss.InterruptCallback_get_period_hint(flops) - -# Register InterruptCallback in _swigfaiss: -_swigfaiss.InterruptCallback_swigregister(InterruptCallback) -class TimeoutCallback(InterruptCallback): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - start = property(_swigfaiss.TimeoutCallback_start_get, _swigfaiss.TimeoutCallback_start_set) - timeout = property(_swigfaiss.TimeoutCallback_timeout_get, _swigfaiss.TimeoutCallback_timeout_set) - - def want_interrupt(self): - return _swigfaiss.TimeoutCallback_want_interrupt(self) - - def set_timeout(self, timeout_in_seconds): - return _swigfaiss.TimeoutCallback_set_timeout(self, timeout_in_seconds) - - @staticmethod - def reset(timeout_in_seconds): - return _swigfaiss.TimeoutCallback_reset(timeout_in_seconds) - - def __init__(self): - _swigfaiss.TimeoutCallback_swiginit(self, _swigfaiss.new_TimeoutCallback()) - __swig_destroy__ = _swigfaiss.delete_TimeoutCallback - -# Register TimeoutCallback in _swigfaiss: -_swigfaiss.TimeoutCallback_swigregister(TimeoutCallback) -class VisitedTable(object): - r""" - Abstract base class for a fast, reusable Visited Set for graph search - algorithms. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - __swig_destroy__ = _swigfaiss.delete_VisitedTable - - def set(self, no): - r"""set flag #no to true, return whether this changed it.""" - return _swigfaiss.VisitedTable_set(self, no) - - def get(self, no): - r"""get flag #no""" - return _swigfaiss.VisitedTable_get(self, no) - - def prefetch(self, no): - r"""prefetch flag #no""" - return _swigfaiss.VisitedTable_prefetch(self, no) - - def reserve(self, arg2): - r"""pre-allocate bucket space to avoid rehashing during repeated set() calls""" - return _swigfaiss.VisitedTable_reserve(self, arg2) - - def advance(self): - r"""reset all flags to false""" - return _swigfaiss.VisitedTable_advance(self) - - @staticmethod - def get_reusable(*args): - r""" - Returns a thread-local, reusable table sized for at least `size` and - reset to a clean state. Unlike create(), it does not allocate on each - call: the O(size) versioned array is allocated once per thread and - reused across searches, avoiding a per-search alloc+zero of the whole - array when a static index is searched repeatedly. - - The returned reference is owned by thread-local storage: do not delete - it and do not use it beyond the current search on the calling thread. - """ - return _swigfaiss.VisitedTable_get_reusable(*args) - -# Register VisitedTable in _swigfaiss: -_swigfaiss.VisitedTable_swigregister(VisitedTable) -class VisitedTableSet(VisitedTable): - r""" - Set-based implementation using unordered_set. - O(1) to construct and O(visits) to advance. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - visited_set = property(_swigfaiss.VisitedTableSet_visited_set_get, _swigfaiss.VisitedTableSet_visited_set_set) - - def __init__(self): - _swigfaiss.VisitedTableSet_swiginit(self, _swigfaiss.new_VisitedTableSet()) - - def set(self, no): - return _swigfaiss.VisitedTableSet_set(self, no) - - def get(self, no): - return _swigfaiss.VisitedTableSet_get(self, no) - - def prefetch(self, arg2): - return _swigfaiss.VisitedTableSet_prefetch(self, arg2) - - def reserve(self, n): - return _swigfaiss.VisitedTableSet_reserve(self, n) - - def advance(self): - return _swigfaiss.VisitedTableSet_advance(self) - __swig_destroy__ = _swigfaiss.delete_VisitedTableSet - -# Register VisitedTableSet in _swigfaiss: -_swigfaiss.VisitedTableSet_swigregister(VisitedTableSet) -class VisitedTableVector(VisitedTable): - r""" - Vector-based implementation using a versioned byte array. - Faster for get()/set(), but O(size) to initialize. - advance() is O(1) except every 250 calls, which are O(size). - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - visited = property(_swigfaiss.VisitedTableVector_visited_get, _swigfaiss.VisitedTableVector_visited_set) - visno = property(_swigfaiss.VisitedTableVector_visno_get, _swigfaiss.VisitedTableVector_visno_set) - - def __init__(self, size): - _swigfaiss.VisitedTableVector_swiginit(self, _swigfaiss.new_VisitedTableVector(size)) - - def ensure_size(self, size): - r""" - Grow so indices in [0, size) are valid; new slots read as unvisited. - Never shrinks, so capacity is retained when the table is reused. - """ - return _swigfaiss.VisitedTableVector_ensure_size(self, size) - - def set(self, no): - return _swigfaiss.VisitedTableVector_set(self, no) - - def get(self, no): - return _swigfaiss.VisitedTableVector_get(self, no) - - def prefetch(self, no): - return _swigfaiss.VisitedTableVector_prefetch(self, no) - - def advance(self): - return _swigfaiss.VisitedTableVector_advance(self) - __swig_destroy__ = _swigfaiss.delete_VisitedTableVector - -# Register VisitedTableVector in _swigfaiss: -_swigfaiss.VisitedTableVector_swigregister(VisitedTableVector) -class IDSelector(object): - r"""Encapsulates a set of ids to handle.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - - def is_member(self, id): - return _swigfaiss.IDSelector_is_member(self, id) - __swig_destroy__ = _swigfaiss.delete_IDSelector - -# Register IDSelector in _swigfaiss: -_swigfaiss.IDSelector_swigregister(IDSelector) -class IDSelectorRange(IDSelector): - r"""ids between [imin, imax)""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - imin = property(_swigfaiss.IDSelectorRange_imin_get, _swigfaiss.IDSelectorRange_imin_set) - imax = property(_swigfaiss.IDSelectorRange_imax_get, _swigfaiss.IDSelectorRange_imax_set) - assume_sorted = property(_swigfaiss.IDSelectorRange_assume_sorted_get, _swigfaiss.IDSelectorRange_assume_sorted_set, doc=r""" - Assume that the ids to handle are sorted. In some cases this can speed - up processing - """) - - def __init__(self, imin, imax, assume_sorted=False): - _swigfaiss.IDSelectorRange_swiginit(self, _swigfaiss.new_IDSelectorRange(imin, imax, assume_sorted)) - - def is_member(self, id): - return _swigfaiss.IDSelectorRange_is_member(self, id) - - def find_sorted_ids_bounds(self, list_size, ids, jmin, jmax): - r""" - for sorted ids, find the range of list indices where the valid ids are - stored - """ - return _swigfaiss.IDSelectorRange_find_sorted_ids_bounds(self, list_size, ids, jmin, jmax) - __swig_destroy__ = _swigfaiss.delete_IDSelectorRange - -# Register IDSelectorRange in _swigfaiss: -_swigfaiss.IDSelectorRange_swigregister(IDSelectorRange) -class IDSelectorArray(IDSelector): - r""" - Simple array of elements - - is_member calls are very inefficient, but some operations can use the ids - directly. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - n = property(_swigfaiss.IDSelectorArray_n_get, _swigfaiss.IDSelectorArray_n_set) - ids = property(_swigfaiss.IDSelectorArray_ids_get, _swigfaiss.IDSelectorArray_ids_set) - - def __init__(self, n, ids): - r""" - Construct with an array of ids to process - - :type n: int - :param n: number of ids to store - :type ids: int - :param ids: elements to store. The pointer should remain valid during - IDSelectorArray's lifetime - """ - _swigfaiss.IDSelectorArray_swiginit(self, _swigfaiss.new_IDSelectorArray(n, ids)) - - def is_member(self, id): - return _swigfaiss.IDSelectorArray_is_member(self, id) - __swig_destroy__ = _swigfaiss.delete_IDSelectorArray - -# Register IDSelectorArray in _swigfaiss: -_swigfaiss.IDSelectorArray_swigregister(IDSelectorArray) -class IDSelectorBatch(IDSelector): - r""" - Ids from a set. - - Repetitions of ids in the indices set passed to the constructor does not hurt - performance. - - The hash function used for the bloom filter and GCC's implementation of - unordered_set are just the least significant bits of the id. This works fine - for random ids or ids in sequences but will produce many hash collisions if - lsb's are always the same - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nbits = property(_swigfaiss.IDSelectorBatch_nbits_get, _swigfaiss.IDSelectorBatch_nbits_set) - mask = property(_swigfaiss.IDSelectorBatch_mask_get, _swigfaiss.IDSelectorBatch_mask_set) - - def __init__(self, n, indices): - r""" - Construct with an array of ids to process - - :type n: int - :param n: number of ids to store - :param ids: elements to store. The pointer can be released after - construction - """ - _swigfaiss.IDSelectorBatch_swiginit(self, _swigfaiss.new_IDSelectorBatch(n, indices)) - - def is_member(self, id): - return _swigfaiss.IDSelectorBatch_is_member(self, id) - __swig_destroy__ = _swigfaiss.delete_IDSelectorBatch - -# Register IDSelectorBatch in _swigfaiss: -_swigfaiss.IDSelectorBatch_swigregister(IDSelectorBatch) -class IDSelectorBitmap(IDSelector): - r"""One bit per element. Constructed with a bitmap, size ceil(n / 8).""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - n = property(_swigfaiss.IDSelectorBitmap_n_get, _swigfaiss.IDSelectorBitmap_n_set) - bitmap = property(_swigfaiss.IDSelectorBitmap_bitmap_get, _swigfaiss.IDSelectorBitmap_bitmap_set) - - def __init__(self, n, bitmap): - r""" - Construct with a binary mask - - :type n: int - :param n: size of the bitmap array - :type bitmap: uint8_t - :param bitmap: id will be selected iff id / 8 < n and bit number - (i%8) of bitmap[floor(i / 8)] is 1. - """ - _swigfaiss.IDSelectorBitmap_swiginit(self, _swigfaiss.new_IDSelectorBitmap(n, bitmap)) - - def is_member(self, id): - return _swigfaiss.IDSelectorBitmap_is_member(self, id) - __swig_destroy__ = _swigfaiss.delete_IDSelectorBitmap - -# Register IDSelectorBitmap in _swigfaiss: -_swigfaiss.IDSelectorBitmap_swigregister(IDSelectorBitmap) -class IDSelectorNot(IDSelector): - r"""reverts the membership test of another selector""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - sel = property(_swigfaiss.IDSelectorNot_sel_get, _swigfaiss.IDSelectorNot_sel_set) - - def __init__(self, sel_): - _swigfaiss.IDSelectorNot_swiginit(self, _swigfaiss.new_IDSelectorNot(sel_)) - - def is_member(self, id): - return _swigfaiss.IDSelectorNot_is_member(self, id) - __swig_destroy__ = _swigfaiss.delete_IDSelectorNot - -# Register IDSelectorNot in _swigfaiss: -_swigfaiss.IDSelectorNot_swigregister(IDSelectorNot) -class IDSelectorAll(IDSelector): - r"""selects all entries (useful for benchmarking)""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def is_member(self, arg2): - return _swigfaiss.IDSelectorAll_is_member(self, arg2) - __swig_destroy__ = _swigfaiss.delete_IDSelectorAll - - def __init__(self): - _swigfaiss.IDSelectorAll_swiginit(self, _swigfaiss.new_IDSelectorAll()) - -# Register IDSelectorAll in _swigfaiss: -_swigfaiss.IDSelectorAll_swigregister(IDSelectorAll) -class IDSelectorAnd(IDSelector): - r""" - does an AND operation on the two given IDSelector's is_membership - results. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - lhs = property(_swigfaiss.IDSelectorAnd_lhs_get, _swigfaiss.IDSelectorAnd_lhs_set) - rhs = property(_swigfaiss.IDSelectorAnd_rhs_get, _swigfaiss.IDSelectorAnd_rhs_set) - - def __init__(self, lhs_, rhs_): - _swigfaiss.IDSelectorAnd_swiginit(self, _swigfaiss.new_IDSelectorAnd(lhs_, rhs_)) - - def is_member(self, id): - return _swigfaiss.IDSelectorAnd_is_member(self, id) - __swig_destroy__ = _swigfaiss.delete_IDSelectorAnd - -# Register IDSelectorAnd in _swigfaiss: -_swigfaiss.IDSelectorAnd_swigregister(IDSelectorAnd) -class IDSelectorOr(IDSelector): - r""" - does an OR operation on the two given IDSelector's is_membership - results. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - lhs = property(_swigfaiss.IDSelectorOr_lhs_get, _swigfaiss.IDSelectorOr_lhs_set) - rhs = property(_swigfaiss.IDSelectorOr_rhs_get, _swigfaiss.IDSelectorOr_rhs_set) - - def __init__(self, lhs_, rhs_): - _swigfaiss.IDSelectorOr_swiginit(self, _swigfaiss.new_IDSelectorOr(lhs_, rhs_)) - - def is_member(self, id): - return _swigfaiss.IDSelectorOr_is_member(self, id) - __swig_destroy__ = _swigfaiss.delete_IDSelectorOr - -# Register IDSelectorOr in _swigfaiss: -_swigfaiss.IDSelectorOr_swigregister(IDSelectorOr) -class IDSelectorXOr(IDSelector): - r""" - does an XOR operation on the two given IDSelector's is_membership - results. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - lhs = property(_swigfaiss.IDSelectorXOr_lhs_get, _swigfaiss.IDSelectorXOr_lhs_set) - rhs = property(_swigfaiss.IDSelectorXOr_rhs_get, _swigfaiss.IDSelectorXOr_rhs_set) - - def __init__(self, lhs_, rhs_): - _swigfaiss.IDSelectorXOr_swiginit(self, _swigfaiss.new_IDSelectorXOr(lhs_, rhs_)) - - def is_member(self, id): - return _swigfaiss.IDSelectorXOr_is_member(self, id) - __swig_destroy__ = _swigfaiss.delete_IDSelectorXOr - -# Register IDSelectorXOr in _swigfaiss: -_swigfaiss.IDSelectorXOr_swigregister(IDSelectorXOr) -class IDSelectorTranslated(IDSelector): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - id_map = property(_swigfaiss.IDSelectorTranslated_id_map_get) - sel = property(_swigfaiss.IDSelectorTranslated_sel_get, _swigfaiss.IDSelectorTranslated_sel_set) - - def __init__(self, *args): - _swigfaiss.IDSelectorTranslated_swiginit(self, _swigfaiss.new_IDSelectorTranslated(*args)) - - def is_member(self, id): - return _swigfaiss.IDSelectorTranslated_is_member(self, id) - __swig_destroy__ = _swigfaiss.delete_IDSelectorTranslated - -# Register IDSelectorTranslated in _swigfaiss: -_swigfaiss.IDSelectorTranslated_swigregister(IDSelectorTranslated) -class IndexIDMap(Index): - r"""Index that translates search results to ids""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - index = property(_swigfaiss.IndexIDMap_index_get, _swigfaiss.IndexIDMap_index_set) - own_fields = property(_swigfaiss.IndexIDMap_own_fields_get, _swigfaiss.IndexIDMap_own_fields_set, doc=r"""the sub-index""") - id_map = property(_swigfaiss.IndexIDMap_id_map_get, _swigfaiss.IndexIDMap_id_map_set, doc=r"""whether pointers are deleted in destructor""") - - def add_with_ids(self, n, x, xids): - r""" - :type xids: int - :param xids: if non-null, ids to store for the vectors (size n) - """ - return _swigfaiss.IndexIDMap_add_with_ids(self, n, x, xids) - - def add_with_ids_ex(self, n, x, numeric_type, xids): - return _swigfaiss.IndexIDMap_add_with_ids_ex(self, n, x, numeric_type, xids) - - def add(self, n, x): - r"""this will fail. Use add_with_ids""" - return _swigfaiss.IndexIDMap_add(self, n, x) - - def add_ex(self, n, x, numeric_type): - return _swigfaiss.IndexIDMap_add_ex(self, n, x, numeric_type) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexIDMap_search(self, n, x, k, distances, labels, params) - - def search_ex(self, n, x, numeric_type, k, distances, labels, params=None): - return _swigfaiss.IndexIDMap_search_ex(self, n, x, numeric_type, k, distances, labels, params) - - def train(self, n, x): - return _swigfaiss.IndexIDMap_train(self, n, x) - - def train_ex(self, n, x, numeric_type): - return _swigfaiss.IndexIDMap_train_ex(self, n, x, numeric_type) - - def reset(self): - return _swigfaiss.IndexIDMap_reset(self) - - def remove_ids(self, sel): - r"""remove ids adapted to IndexFlat""" - return _swigfaiss.IndexIDMap_remove_ids(self, sel) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexIDMap_range_search(self, n, x, radius, result, params) - - def merge_from(self, otherIndex, add_id=0): - return _swigfaiss.IndexIDMap_merge_from(self, otherIndex, add_id) - - def check_compatible_for_merge(self, otherIndex): - return _swigfaiss.IndexIDMap_check_compatible_for_merge(self, otherIndex) - - def sa_code_size(self): - return _swigfaiss.IndexIDMap_sa_code_size(self) - - def add_sa_codes(self, n, x, xids): - return _swigfaiss.IndexIDMap_add_sa_codes(self, n, x, xids) - __swig_destroy__ = _swigfaiss.delete_IndexIDMap - - def __init__(self, *args): - _swigfaiss.IndexIDMap_swiginit(self, _swigfaiss.new_IndexIDMap(*args)) - -# Register IndexIDMap in _swigfaiss: -_swigfaiss.IndexIDMap_swigregister(IndexIDMap) -class IndexBinaryIDMap(IndexBinary): - r"""Index that translates search results to ids""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - index = property(_swigfaiss.IndexBinaryIDMap_index_get, _swigfaiss.IndexBinaryIDMap_index_set) - own_fields = property(_swigfaiss.IndexBinaryIDMap_own_fields_get, _swigfaiss.IndexBinaryIDMap_own_fields_set, doc=r"""the sub-index""") - id_map = property(_swigfaiss.IndexBinaryIDMap_id_map_get, _swigfaiss.IndexBinaryIDMap_id_map_set, doc=r"""whether pointers are deleted in destructor""") - - def add_with_ids(self, n, x, xids): - r""" - :type xids: int - :param xids: if non-null, ids to store for the vectors (size n) - """ - return _swigfaiss.IndexBinaryIDMap_add_with_ids(self, n, x, xids) - - def add_with_ids_ex(self, n, x, numeric_type, xids): - return _swigfaiss.IndexBinaryIDMap_add_with_ids_ex(self, n, x, numeric_type, xids) - - def add(self, n, x): - r"""this will fail. Use add_with_ids""" - return _swigfaiss.IndexBinaryIDMap_add(self, n, x) - - def add_ex(self, n, x, numeric_type): - return _swigfaiss.IndexBinaryIDMap_add_ex(self, n, x, numeric_type) - - def search(self, n, x, k, distances, labels, params=None): - return _swigfaiss.IndexBinaryIDMap_search(self, n, x, k, distances, labels, params) - - def search_ex(self, n, x, numeric_type, k, distances, labels, params=None): - return _swigfaiss.IndexBinaryIDMap_search_ex(self, n, x, numeric_type, k, distances, labels, params) - - def train(self, n, x): - return _swigfaiss.IndexBinaryIDMap_train(self, n, x) - - def train_ex(self, n, x, numeric_type): - return _swigfaiss.IndexBinaryIDMap_train_ex(self, n, x, numeric_type) - - def reset(self): - return _swigfaiss.IndexBinaryIDMap_reset(self) - - def remove_ids(self, sel): - r"""remove ids adapted to IndexFlat""" - return _swigfaiss.IndexBinaryIDMap_remove_ids(self, sel) - - def range_search(self, n, x, radius, result, params=None): - return _swigfaiss.IndexBinaryIDMap_range_search(self, n, x, radius, result, params) - - def merge_from(self, otherIndex, add_id=0): - return _swigfaiss.IndexBinaryIDMap_merge_from(self, otherIndex, add_id) - - def check_compatible_for_merge(self, otherIndex): - return _swigfaiss.IndexBinaryIDMap_check_compatible_for_merge(self, otherIndex) - - def sa_code_size(self): - return _swigfaiss.IndexBinaryIDMap_sa_code_size(self) - - def add_sa_codes(self, n, x, xids): - return _swigfaiss.IndexBinaryIDMap_add_sa_codes(self, n, x, xids) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryIDMap - - def __init__(self, *args): - _swigfaiss.IndexBinaryIDMap_swiginit(self, _swigfaiss.new_IndexBinaryIDMap(*args)) - -# Register IndexBinaryIDMap in _swigfaiss: -_swigfaiss.IndexBinaryIDMap_swigregister(IndexBinaryIDMap) -class IndexIDMap2(IndexIDMap): - r""" - same as IndexIDMap but also provides an efficient reconstruction - implementation via a 2-way index - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - rev_map = property(_swigfaiss.IndexIDMap2_rev_map_get, _swigfaiss.IndexIDMap2_rev_map_set) - - def construct_rev_map(self): - r"""make the rev_map from scratch""" - return _swigfaiss.IndexIDMap2_construct_rev_map(self) - - def add_with_ids(self, n, x, xids): - return _swigfaiss.IndexIDMap2_add_with_ids(self, n, x, xids) - - def add_with_ids_ex(self, n, x, numeric_type, xids): - return _swigfaiss.IndexIDMap2_add_with_ids_ex(self, n, x, numeric_type, xids) - - def add_sa_codes(self, n, x, xids): - return _swigfaiss.IndexIDMap2_add_sa_codes(self, n, x, xids) - - def remove_ids(self, sel): - return _swigfaiss.IndexIDMap2_remove_ids(self, sel) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexIDMap2_reconstruct(self, key, recons) - - def check_consistency(self): - r"""check that the rev_map and the id_map are in sync""" - return _swigfaiss.IndexIDMap2_check_consistency(self) - - def merge_from(self, otherIndex, add_id=0): - return _swigfaiss.IndexIDMap2_merge_from(self, otherIndex, add_id) - __swig_destroy__ = _swigfaiss.delete_IndexIDMap2 - - def __init__(self, *args): - _swigfaiss.IndexIDMap2_swiginit(self, _swigfaiss.new_IndexIDMap2(*args)) - -# Register IndexIDMap2 in _swigfaiss: -_swigfaiss.IndexIDMap2_swigregister(IndexIDMap2) -class IndexBinaryIDMap2(IndexBinaryIDMap): - r""" - same as IndexIDMap but also provides an efficient reconstruction - implementation via a 2-way index - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - rev_map = property(_swigfaiss.IndexBinaryIDMap2_rev_map_get, _swigfaiss.IndexBinaryIDMap2_rev_map_set) - - def construct_rev_map(self): - r"""make the rev_map from scratch""" - return _swigfaiss.IndexBinaryIDMap2_construct_rev_map(self) - - def add_with_ids(self, n, x, xids): - return _swigfaiss.IndexBinaryIDMap2_add_with_ids(self, n, x, xids) - - def add_with_ids_ex(self, n, x, numeric_type, xids): - return _swigfaiss.IndexBinaryIDMap2_add_with_ids_ex(self, n, x, numeric_type, xids) - - def add_sa_codes(self, n, x, xids): - return _swigfaiss.IndexBinaryIDMap2_add_sa_codes(self, n, x, xids) - - def remove_ids(self, sel): - return _swigfaiss.IndexBinaryIDMap2_remove_ids(self, sel) - - def reconstruct(self, key, recons): - return _swigfaiss.IndexBinaryIDMap2_reconstruct(self, key, recons) - - def check_consistency(self): - r"""check that the rev_map and the id_map are in sync""" - return _swigfaiss.IndexBinaryIDMap2_check_consistency(self) - - def merge_from(self, otherIndex, add_id=0): - return _swigfaiss.IndexBinaryIDMap2_merge_from(self, otherIndex, add_id) - __swig_destroy__ = _swigfaiss.delete_IndexBinaryIDMap2 - - def __init__(self, *args): - _swigfaiss.IndexBinaryIDMap2_swiginit(self, _swigfaiss.new_IndexBinaryIDMap2(*args)) - -# Register IndexBinaryIDMap2 in _swigfaiss: -_swigfaiss.IndexBinaryIDMap2_swigregister(IndexBinaryIDMap2) -EXACT_TOPK = _swigfaiss.EXACT_TOPK -APPROX_TOPK_BUCKETS_B32_D2 = _swigfaiss.APPROX_TOPK_BUCKETS_B32_D2 -APPROX_TOPK_BUCKETS_B8_D3 = _swigfaiss.APPROX_TOPK_BUCKETS_B8_D3 -APPROX_TOPK_BUCKETS_B16_D2 = _swigfaiss.APPROX_TOPK_BUCKETS_B16_D2 -APPROX_TOPK_BUCKETS_B8_D2 = _swigfaiss.APPROX_TOPK_BUCKETS_B8_D2 - -def approx_topk_by_mode(mode, beam_size, n_per_beam, distances, k, bh_val, bh_ids): - return _swigfaiss.approx_topk_by_mode(mode, beam_size, n_per_beam, distances, k, bh_val, bh_ids) - -def downcast_index(index): - return _swigfaiss.downcast_index(index) - -def downcast_VectorTransform(vt): - return _swigfaiss.downcast_VectorTransform(vt) - -def downcast_IndexBinary(index): - return _swigfaiss.downcast_IndexBinary(index) - -def downcast_InvertedLists(il): - return _swigfaiss.downcast_InvertedLists(il) - -def downcast_AdditiveQuantizer(aq): - return _swigfaiss.downcast_AdditiveQuantizer(aq) - -def downcast_Quantizer(aq): - return _swigfaiss.downcast_Quantizer(aq) - -def write_index(*args): - return _swigfaiss.write_index(*args) - -def write_index_binary(*args): - return _swigfaiss.write_index_binary(*args) - -def read_index(*args): - return _swigfaiss.read_index(*args) - -def read_index_binary(*args): - return _swigfaiss.read_index_binary(*args) - -def write_VectorTransform(*args): - return _swigfaiss.write_VectorTransform(*args) - -def read_VectorTransform(*args): - return _swigfaiss.read_VectorTransform(*args) - -def read_ProductQuantizer(*args): - return _swigfaiss.read_ProductQuantizer(*args) - -def write_ProductQuantizer(*args): - return _swigfaiss.write_ProductQuantizer(*args) - -def write_InvertedLists(ils, f): - return _swigfaiss.write_InvertedLists(ils, f) - -def read_InvertedLists(reader, io_flags=0): - return _swigfaiss.read_InvertedLists(reader, io_flags) - -def get_deserialization_loop_limit(): - return _swigfaiss.get_deserialization_loop_limit() - -def set_deserialization_loop_limit(value): - return _swigfaiss.set_deserialization_loop_limit(value) - -def get_deserialization_vector_byte_limit(): - return _swigfaiss.get_deserialization_vector_byte_limit() - -def set_deserialization_vector_byte_limit(value): - return _swigfaiss.set_deserialization_vector_byte_limit(value) - -def get_deserialization_lattice_r2_limit(): - return _swigfaiss.get_deserialization_lattice_r2_limit() - -def set_deserialization_lattice_r2_limit(value): - return _swigfaiss.set_deserialization_lattice_r2_limit(value) - -def clone_index(arg1): - return _swigfaiss.clone_index(arg1) -class Cloner(object): - r""" - Cloner class, useful to override classes with other cloning - functions. The cloning function above just calls - Cloner::clone_Index. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def clone_VectorTransform(self, arg2): - return _swigfaiss.Cloner_clone_VectorTransform(self, arg2) - - def clone_Index(self, arg2): - return _swigfaiss.Cloner_clone_Index(self, arg2) - - def clone_IndexIVF(self, arg2): - return _swigfaiss.Cloner_clone_IndexIVF(self, arg2) - __swig_destroy__ = _swigfaiss.delete_Cloner - - def __init__(self): - _swigfaiss.Cloner_swiginit(self, _swigfaiss.new_Cloner()) - -# Register Cloner in _swigfaiss: -_swigfaiss.Cloner_swigregister(Cloner) -IO_FLAG_SKIP_STORAGE = cvar.IO_FLAG_SKIP_STORAGE -IO_FLAG_READ_ONLY = cvar.IO_FLAG_READ_ONLY -IO_FLAG_ONDISK_SAME_DIR = cvar.IO_FLAG_ONDISK_SAME_DIR -IO_FLAG_SKIP_IVF_DATA = cvar.IO_FLAG_SKIP_IVF_DATA -IO_FLAG_SKIP_PRECOMPUTE_TABLE = cvar.IO_FLAG_SKIP_PRECOMPUTE_TABLE -IO_FLAG_PQ_SKIP_SDC_TABLE = cvar.IO_FLAG_PQ_SKIP_SDC_TABLE -IO_FLAG_MMAP = cvar.IO_FLAG_MMAP -IO_FLAG_MMAP_IFC = cvar.IO_FLAG_MMAP_IFC - - -def clone_Quantizer(quant): - return _swigfaiss.clone_Quantizer(quant) - -def clone_binary_index(index): - return _swigfaiss.clone_binary_index(index) -class AutoTuneCriterion(object): - r""" - Evaluation criterion. Returns a performance measure in [0,1], - higher is better. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - - def __init__(self, *args, **kwargs): - raise AttributeError("No constructor defined - class is abstract") - __repr__ = _swig_repr - nq = property(_swigfaiss.AutoTuneCriterion_nq_get, _swigfaiss.AutoTuneCriterion_nq_set, doc=r"""nb of queries this criterion is evaluated on""") - nnn = property(_swigfaiss.AutoTuneCriterion_nnn_get, _swigfaiss.AutoTuneCriterion_nnn_set, doc=r"""nb of NNs that the query should request""") - gt_nnn = property(_swigfaiss.AutoTuneCriterion_gt_nnn_get, _swigfaiss.AutoTuneCriterion_gt_nnn_set, doc=r"""nb of GT NNs required to evaluate criterion""") - gt_D = property(_swigfaiss.AutoTuneCriterion_gt_D_get, _swigfaiss.AutoTuneCriterion_gt_D_set, doc=r"""Ground-truth distances (size nq * gt_nnn)""") - gt_I = property(_swigfaiss.AutoTuneCriterion_gt_I_get, _swigfaiss.AutoTuneCriterion_gt_I_set, doc=r"""Ground-truth indexes (size nq * gt_nnn)""") - - def set_groundtruth(self, gt_nnn, gt_D_in, gt_I_in): - r""" - Initializes the gt_D and gt_I vectors. Must be called before evaluating - - :type gt_D_in: float - :param gt_D_in: size nq * gt_nnn - :type gt_I_in: int - :param gt_I_in: size nq * gt_nnn - """ - return _swigfaiss.AutoTuneCriterion_set_groundtruth(self, gt_nnn, gt_D_in, gt_I_in) - - def evaluate(self, D, I): - r""" - Evaluate the criterion. - - :type D: float - :param D: size nq * nnn - :type I: int - :param I: size nq * nnn - :rtype: float - :return: the criterion, between 0 and 1. Larger is better. - """ - return _swigfaiss.AutoTuneCriterion_evaluate(self, D, I) - __swig_destroy__ = _swigfaiss.delete_AutoTuneCriterion - -# Register AutoTuneCriterion in _swigfaiss: -_swigfaiss.AutoTuneCriterion_swigregister(AutoTuneCriterion) -class OneRecallAtRCriterion(AutoTuneCriterion): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - R = property(_swigfaiss.OneRecallAtRCriterion_R_get, _swigfaiss.OneRecallAtRCriterion_R_set) - - def __init__(self, nq, R): - _swigfaiss.OneRecallAtRCriterion_swiginit(self, _swigfaiss.new_OneRecallAtRCriterion(nq, R)) - - def evaluate(self, D, I): - return _swigfaiss.OneRecallAtRCriterion_evaluate(self, D, I) - __swig_destroy__ = _swigfaiss.delete_OneRecallAtRCriterion - -# Register OneRecallAtRCriterion in _swigfaiss: -_swigfaiss.OneRecallAtRCriterion_swigregister(OneRecallAtRCriterion) -class IntersectionCriterion(AutoTuneCriterion): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - R = property(_swigfaiss.IntersectionCriterion_R_get, _swigfaiss.IntersectionCriterion_R_set) - - def __init__(self, nq, R): - _swigfaiss.IntersectionCriterion_swiginit(self, _swigfaiss.new_IntersectionCriterion(nq, R)) - - def evaluate(self, D, I): - return _swigfaiss.IntersectionCriterion_evaluate(self, D, I) - __swig_destroy__ = _swigfaiss.delete_IntersectionCriterion - -# Register IntersectionCriterion in _swigfaiss: -_swigfaiss.IntersectionCriterion_swigregister(IntersectionCriterion) -class OperatingPoint(object): - r""" - Maintains a list of experimental results. Each operating point is a - (perf, t, key) triplet, where higher perf and lower t is - better. The key field is an arbitrary identifier for the operating point. - - Includes primitives to extract the Pareto-optimal operating points in the - (perf, t) space. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - perf = property(_swigfaiss.OperatingPoint_perf_get, _swigfaiss.OperatingPoint_perf_set, doc=r"""performance measure (output of a Criterion)""") - t = property(_swigfaiss.OperatingPoint_t_get, _swigfaiss.OperatingPoint_t_set, doc=r"""corresponding execution time (ms)""") - key = property(_swigfaiss.OperatingPoint_key_get, _swigfaiss.OperatingPoint_key_set, doc=r"""key that identifies this op pt""") - cno = property(_swigfaiss.OperatingPoint_cno_get, _swigfaiss.OperatingPoint_cno_set, doc=r"""integer identifier""") - - def __init__(self): - _swigfaiss.OperatingPoint_swiginit(self, _swigfaiss.new_OperatingPoint()) - __swig_destroy__ = _swigfaiss.delete_OperatingPoint - -# Register OperatingPoint in _swigfaiss: -_swigfaiss.OperatingPoint_swigregister(OperatingPoint) -class OperatingPoints(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - all_pts = property(_swigfaiss.OperatingPoints_all_pts_get, _swigfaiss.OperatingPoints_all_pts_set, doc=r"""all operating points""") - optimal_pts = property(_swigfaiss.OperatingPoints_optimal_pts_get, _swigfaiss.OperatingPoints_optimal_pts_set, doc=r"""optimal operating points, sorted by perf""") - - def __init__(self): - _swigfaiss.OperatingPoints_swiginit(self, _swigfaiss.new_OperatingPoints()) - - def merge_with(self, *args): - r"""add operating points from other to this, with a prefix to the keys""" - return _swigfaiss.OperatingPoints_merge_with(self, *args) - - def clear(self): - return _swigfaiss.OperatingPoints_clear(self) - - def add(self, perf, t, key, cno=0): - r"""add a performance measure. Return whether it is an optimal point""" - return _swigfaiss.OperatingPoints_add(self, perf, t, key, cno) - - def t_for_perf(self, perf): - r"""get time required to obtain a given performance measure""" - return _swigfaiss.OperatingPoints_t_for_perf(self, perf) - - def display(self, only_optimal=True): - r"""easy-to-read output""" - return _swigfaiss.OperatingPoints_display(self, only_optimal) - - def all_to_gnuplot(self, fname): - r"""output to a format easy to digest by gnuplot""" - return _swigfaiss.OperatingPoints_all_to_gnuplot(self, fname) - - def optimal_to_gnuplot(self, fname): - return _swigfaiss.OperatingPoints_optimal_to_gnuplot(self, fname) - __swig_destroy__ = _swigfaiss.delete_OperatingPoints - -# Register OperatingPoints in _swigfaiss: -_swigfaiss.OperatingPoints_swigregister(OperatingPoints) -class ParameterRange(object): - r"""possible values of a parameter, sorted from least to most expensive/accurate""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - name = property(_swigfaiss.ParameterRange_name_get, _swigfaiss.ParameterRange_name_set) - values = property(_swigfaiss.ParameterRange_values_get, _swigfaiss.ParameterRange_values_set) - - def __init__(self): - _swigfaiss.ParameterRange_swiginit(self, _swigfaiss.new_ParameterRange()) - __swig_destroy__ = _swigfaiss.delete_ParameterRange - -# Register ParameterRange in _swigfaiss: -_swigfaiss.ParameterRange_swigregister(ParameterRange) -class ParameterSpace(object): - r"""Uses a-priori knowledge on the Faiss indexes to extract tunable parameters.""" - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - parameter_ranges = property(_swigfaiss.ParameterSpace_parameter_ranges_get, _swigfaiss.ParameterSpace_parameter_ranges_set, doc=r"""all tunable parameters""") - verbose = property(_swigfaiss.ParameterSpace_verbose_get, _swigfaiss.ParameterSpace_verbose_set, doc=r"""verbosity during exploration""") - n_experiments = property(_swigfaiss.ParameterSpace_n_experiments_get, _swigfaiss.ParameterSpace_n_experiments_set, doc=r"""nb of experiments during optimization (0 = try all combinations)""") - batchsize = property(_swigfaiss.ParameterSpace_batchsize_get, _swigfaiss.ParameterSpace_batchsize_set, doc=r"""maximum number of queries to submit at a time.""") - thread_over_batches = property(_swigfaiss.ParameterSpace_thread_over_batches_get, _swigfaiss.ParameterSpace_thread_over_batches_set, doc=r""" - use multithreading over batches (useful to benchmark - independent single-searches) - """) - min_test_duration = property(_swigfaiss.ParameterSpace_min_test_duration_get, _swigfaiss.ParameterSpace_min_test_duration_set, doc=r""" - run tests several times until they reach at least this - duration (to avoid jittering in MT mode) - """) - - def __init__(self): - _swigfaiss.ParameterSpace_swiginit(self, _swigfaiss.new_ParameterSpace()) - - def n_combinations(self): - r"""nb of combinations, = product of values sizes""" - return _swigfaiss.ParameterSpace_n_combinations(self) - - def combination_ge(self, c1, c2): - r"""returns whether combinations c1 >= c2 in the tuple sense""" - return _swigfaiss.ParameterSpace_combination_ge(self, c1, c2) - - def combination_name(self, cno): - r"""get string representation of the combination""" - return _swigfaiss.ParameterSpace_combination_name(self, cno) - - def display(self): - r"""print a description on stdout""" - return _swigfaiss.ParameterSpace_display(self) - - def add_range(self, name): - r"""add a new parameter (or return it if it exists)""" - return _swigfaiss.ParameterSpace_add_range(self, name) - - def initialize(self, index): - r"""initialize with reasonable parameters for the index""" - return _swigfaiss.ParameterSpace_initialize(self, index) - - def set_index_parameters(self, *args): - r""" - *Overload 1:* - set a combination of parameters on an index - - | - - *Overload 2:* - set a combination of parameters described by a string - - | - - *Overload 3:* - set a combination of parameters on a binary index - - | - - *Overload 4:* - set a combination of parameters described by a string on a binary index - """ - return _swigfaiss.ParameterSpace_set_index_parameters(self, *args) - - def set_index_parameter(self, *args): - r""" - *Overload 1:* - set one of the parameters - - | - - *Overload 2:* - set one of the parameters on a binary index - """ - return _swigfaiss.ParameterSpace_set_index_parameter(self, *args) - - def update_bounds(self, cno, op, upper_bound_perf, lower_bound_t): - r""" - find an upper bound on the performance and a lower bound on t - for configuration cno given another operating point op - """ - return _swigfaiss.ParameterSpace_update_bounds(self, cno, op, upper_bound_perf, lower_bound_t) - - def explore(self, index, nq, xq, crit, ops): - r""" - explore operating points - :type index: :py:class:`Index` - :param index: index to run on - :type xq: float - :param xq: query vectors (size nq * index.d) - :type crit: :py:class:`AutoTuneCriterion` - :param crit: selection criterion - :type ops: :py:class:`OperatingPoints` - :param ops: resulting operating points - """ - return _swigfaiss.ParameterSpace_explore(self, index, nq, xq, crit, ops) - __swig_destroy__ = _swigfaiss.delete_ParameterSpace - -# Register ParameterSpace in _swigfaiss: -_swigfaiss.ParameterSpace_swigregister(ParameterSpace) - -def index_factory(*args): - r""" - Build an index with the sequence of processing steps described in - the string. - """ - return _swigfaiss.index_factory(*args) - -def index_binary_factory(d, description, own_invlists=True): - return _swigfaiss.index_binary_factory(d, description, own_invlists) -class MatrixStats(object): - r""" - Reports some statistics on a dataset and comments on them. - - It is a class rather than a function so that all stats can also be - accessed from code - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def __init__(self, n, d, x): - _swigfaiss.MatrixStats_swiginit(self, _swigfaiss.new_MatrixStats(n, d, x)) - comments = property(_swigfaiss.MatrixStats_comments_get, _swigfaiss.MatrixStats_comments_set) - n = property(_swigfaiss.MatrixStats_n_get, _swigfaiss.MatrixStats_n_set) - d = property(_swigfaiss.MatrixStats_d_get, _swigfaiss.MatrixStats_d_set) - n_collision = property(_swigfaiss.MatrixStats_n_collision_get, _swigfaiss.MatrixStats_n_collision_set) - n_valid = property(_swigfaiss.MatrixStats_n_valid_get, _swigfaiss.MatrixStats_n_valid_set) - n0 = property(_swigfaiss.MatrixStats_n0_get, _swigfaiss.MatrixStats_n0_set) - min_norm2 = property(_swigfaiss.MatrixStats_min_norm2_get, _swigfaiss.MatrixStats_min_norm2_set) - max_norm2 = property(_swigfaiss.MatrixStats_max_norm2_get, _swigfaiss.MatrixStats_max_norm2_set) - hash_value = property(_swigfaiss.MatrixStats_hash_value_get, _swigfaiss.MatrixStats_hash_value_set) - per_dim_stats = property(_swigfaiss.MatrixStats_per_dim_stats_get, _swigfaiss.MatrixStats_per_dim_stats_set) - occurrences = property(_swigfaiss.MatrixStats_occurrences_get, _swigfaiss.MatrixStats_occurrences_set) - buf = property(_swigfaiss.MatrixStats_buf_get, _swigfaiss.MatrixStats_buf_set) - nbuf = property(_swigfaiss.MatrixStats_nbuf_get, _swigfaiss.MatrixStats_nbuf_set) - - def do_comment(self, fmt): - return _swigfaiss.MatrixStats_do_comment(self, fmt) - __swig_destroy__ = _swigfaiss.delete_MatrixStats - -# Register MatrixStats in _swigfaiss: -_swigfaiss.MatrixStats_swigregister(MatrixStats) -class PyCallbackIOWriter(IOWriter): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - callback = property(_swigfaiss.PyCallbackIOWriter_callback_get, _swigfaiss.PyCallbackIOWriter_callback_set) - bs = property(_swigfaiss.PyCallbackIOWriter_bs_get, _swigfaiss.PyCallbackIOWriter_bs_set) - - def __init__(self, *args): - r""" - Callback: Python function that takes a bytes object and - returns the number of bytes successfully written. - """ - _swigfaiss.PyCallbackIOWriter_swiginit(self, _swigfaiss.new_PyCallbackIOWriter(*args)) - - def __call__(self, ptrv, size, nitems): - return _swigfaiss.PyCallbackIOWriter___call__(self, ptrv, size, nitems) - __swig_destroy__ = _swigfaiss.delete_PyCallbackIOWriter - -# Register PyCallbackIOWriter in _swigfaiss: -_swigfaiss.PyCallbackIOWriter_swigregister(PyCallbackIOWriter) -class PyCallbackIOReader(IOReader): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - callback = property(_swigfaiss.PyCallbackIOReader_callback_get, _swigfaiss.PyCallbackIOReader_callback_set) - bs = property(_swigfaiss.PyCallbackIOReader_bs_get, _swigfaiss.PyCallbackIOReader_bs_set) - - def __init__(self, *args): - r""" - Callback: Python function that takes a size and returns a - bytes object with the resulting read - """ - _swigfaiss.PyCallbackIOReader_swiginit(self, _swigfaiss.new_PyCallbackIOReader(*args)) - - def __call__(self, ptrv, size, nitems): - return _swigfaiss.PyCallbackIOReader___call__(self, ptrv, size, nitems) - __swig_destroy__ = _swigfaiss.delete_PyCallbackIOReader - -# Register PyCallbackIOReader in _swigfaiss: -_swigfaiss.PyCallbackIOReader_swigregister(PyCallbackIOReader) -class PyCallbackIDSelector(IDSelector): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - callback = property(_swigfaiss.PyCallbackIDSelector_callback_get, _swigfaiss.PyCallbackIDSelector_callback_set) - - def __init__(self, callback): - _swigfaiss.PyCallbackIDSelector_swiginit(self, _swigfaiss.new_PyCallbackIDSelector(callback)) - - def is_member(self, id): - return _swigfaiss.PyCallbackIDSelector_is_member(self, id) - __swig_destroy__ = _swigfaiss.delete_PyCallbackIDSelector - -# Register PyCallbackIDSelector in _swigfaiss: -_swigfaiss.PyCallbackIDSelector_swigregister(PyCallbackIDSelector) -class PyCallbackShardingFunction(ShardingFunction): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - callback = property(_swigfaiss.PyCallbackShardingFunction_callback_get, _swigfaiss.PyCallbackShardingFunction_callback_set) - - def __call__(self, i, shard_count): - return _swigfaiss.PyCallbackShardingFunction___call__(self, i, shard_count) - __swig_destroy__ = _swigfaiss.delete_PyCallbackShardingFunction - - def __init__(self, *args): - _swigfaiss.PyCallbackShardingFunction_swiginit(self, _swigfaiss.new_PyCallbackShardingFunction(*args)) - -# Register PyCallbackShardingFunction in _swigfaiss: -_swigfaiss.PyCallbackShardingFunction_swigregister(PyCallbackShardingFunction) -class float_minheap_array_t(object): - r""" - a template structure for a set of [min|max]-heaps it is tailored - so that the actual data of the heaps can just live in compact - arrays. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nh = property(_swigfaiss.float_minheap_array_t_nh_get, _swigfaiss.float_minheap_array_t_nh_set, doc=r"""number of heaps""") - k = property(_swigfaiss.float_minheap_array_t_k_get, _swigfaiss.float_minheap_array_t_k_set, doc=r"""allocated size per heap""") - ids = property(_swigfaiss.float_minheap_array_t_ids_get, _swigfaiss.float_minheap_array_t_ids_set, doc=r"""identifiers (size nh * k)""") - val = property(_swigfaiss.float_minheap_array_t_val_get, _swigfaiss.float_minheap_array_t_val_set, doc=r"""values (distances or similarities), size nh * k""") - - def get_val(self, key): - r"""Return the list of values for a heap""" - return _swigfaiss.float_minheap_array_t_get_val(self, key) - - def get_ids(self, key): - r"""Corresponding identifiers""" - return _swigfaiss.float_minheap_array_t_get_ids(self, key) - - def heapify(self): - r"""prepare all the heaps before adding""" - return _swigfaiss.float_minheap_array_t_heapify(self) - - def addn(self, nj, vin, j0=0, i0=0, ni=-1): - r""" - add nj elements to heaps i0:i0+ni, with sequential ids - - :type nj: int - :param nj: nb of elements to add to each heap - :type vin: float - :param vin: elements to add, size ni * nj - :type j0: int, optional - :param j0: add this to the ids that are added - :type i0: int, optional - :param i0: first heap to update - :type ni: int, optional - :param ni: nb of elements to update (-1 = use nh) - """ - return _swigfaiss.float_minheap_array_t_addn(self, nj, vin, j0, i0, ni) - - def addn_with_ids(self, nj, vin, id_in=None, id_stride=0, i0=0, ni=-1): - r""" - same as addn - - :type id_in: int, optional - :param id_in: ids of the elements to add, size ni * nj - :type id_stride: int, optional - :param id_stride: stride for id_in - """ - return _swigfaiss.float_minheap_array_t_addn_with_ids(self, nj, vin, id_in, id_stride, i0, ni) - - def addn_query_subset_with_ids(self, nsubset, subset, nj, vin, id_in=None, id_stride=0): - r""" - same as addn_with_ids, but for just a subset of queries - - :type nsubset: int - :param nsubset: number of query entries to update - :type subset: int - :param subset: indexes of queries to update, in 0..nh-1, size nsubset - """ - return _swigfaiss.float_minheap_array_t_addn_query_subset_with_ids(self, nsubset, subset, nj, vin, id_in, id_stride) - - def reorder(self): - r"""reorder all the heaps""" - return _swigfaiss.float_minheap_array_t_reorder(self) - - def per_line_extrema(self, vals_out, idx_out): - r""" - this is not really a heap function. It just finds the per-line - extrema of each line of array D - :type vals_out: float - :param vals_out: extreme value of each line (size nh, or NULL) - :type idx_out: int - :param idx_out: index of extreme value (size nh or NULL) - """ - return _swigfaiss.float_minheap_array_t_per_line_extrema(self, vals_out, idx_out) - - def __init__(self): - _swigfaiss.float_minheap_array_t_swiginit(self, _swigfaiss.new_float_minheap_array_t()) - __swig_destroy__ = _swigfaiss.delete_float_minheap_array_t - -# Register float_minheap_array_t in _swigfaiss: -_swigfaiss.float_minheap_array_t_swigregister(float_minheap_array_t) -class int_minheap_array_t(object): - r""" - a template structure for a set of [min|max]-heaps it is tailored - so that the actual data of the heaps can just live in compact - arrays. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nh = property(_swigfaiss.int_minheap_array_t_nh_get, _swigfaiss.int_minheap_array_t_nh_set, doc=r"""number of heaps""") - k = property(_swigfaiss.int_minheap_array_t_k_get, _swigfaiss.int_minheap_array_t_k_set, doc=r"""allocated size per heap""") - ids = property(_swigfaiss.int_minheap_array_t_ids_get, _swigfaiss.int_minheap_array_t_ids_set, doc=r"""identifiers (size nh * k)""") - val = property(_swigfaiss.int_minheap_array_t_val_get, _swigfaiss.int_minheap_array_t_val_set, doc=r"""values (distances or similarities), size nh * k""") - - def get_val(self, key): - r"""Return the list of values for a heap""" - return _swigfaiss.int_minheap_array_t_get_val(self, key) - - def get_ids(self, key): - r"""Corresponding identifiers""" - return _swigfaiss.int_minheap_array_t_get_ids(self, key) - - def heapify(self): - r"""prepare all the heaps before adding""" - return _swigfaiss.int_minheap_array_t_heapify(self) - - def addn(self, nj, vin, j0=0, i0=0, ni=-1): - r""" - add nj elements to heaps i0:i0+ni, with sequential ids - - :type nj: int - :param nj: nb of elements to add to each heap - :type vin: int - :param vin: elements to add, size ni * nj - :type j0: int, optional - :param j0: add this to the ids that are added - :type i0: int, optional - :param i0: first heap to update - :type ni: int, optional - :param ni: nb of elements to update (-1 = use nh) - """ - return _swigfaiss.int_minheap_array_t_addn(self, nj, vin, j0, i0, ni) - - def addn_with_ids(self, nj, vin, id_in=None, id_stride=0, i0=0, ni=-1): - r""" - same as addn - - :type id_in: int, optional - :param id_in: ids of the elements to add, size ni * nj - :type id_stride: int, optional - :param id_stride: stride for id_in - """ - return _swigfaiss.int_minheap_array_t_addn_with_ids(self, nj, vin, id_in, id_stride, i0, ni) - - def addn_query_subset_with_ids(self, nsubset, subset, nj, vin, id_in=None, id_stride=0): - r""" - same as addn_with_ids, but for just a subset of queries - - :type nsubset: int - :param nsubset: number of query entries to update - :type subset: int - :param subset: indexes of queries to update, in 0..nh-1, size nsubset - """ - return _swigfaiss.int_minheap_array_t_addn_query_subset_with_ids(self, nsubset, subset, nj, vin, id_in, id_stride) - - def reorder(self): - r"""reorder all the heaps""" - return _swigfaiss.int_minheap_array_t_reorder(self) - - def per_line_extrema(self, vals_out, idx_out): - r""" - this is not really a heap function. It just finds the per-line - extrema of each line of array D - :type vals_out: int - :param vals_out: extreme value of each line (size nh, or NULL) - :type idx_out: int - :param idx_out: index of extreme value (size nh or NULL) - """ - return _swigfaiss.int_minheap_array_t_per_line_extrema(self, vals_out, idx_out) - - def __init__(self): - _swigfaiss.int_minheap_array_t_swiginit(self, _swigfaiss.new_int_minheap_array_t()) - __swig_destroy__ = _swigfaiss.delete_int_minheap_array_t - -# Register int_minheap_array_t in _swigfaiss: -_swigfaiss.int_minheap_array_t_swigregister(int_minheap_array_t) -class float_maxheap_array_t(object): - r""" - a template structure for a set of [min|max]-heaps it is tailored - so that the actual data of the heaps can just live in compact - arrays. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nh = property(_swigfaiss.float_maxheap_array_t_nh_get, _swigfaiss.float_maxheap_array_t_nh_set, doc=r"""number of heaps""") - k = property(_swigfaiss.float_maxheap_array_t_k_get, _swigfaiss.float_maxheap_array_t_k_set, doc=r"""allocated size per heap""") - ids = property(_swigfaiss.float_maxheap_array_t_ids_get, _swigfaiss.float_maxheap_array_t_ids_set, doc=r"""identifiers (size nh * k)""") - val = property(_swigfaiss.float_maxheap_array_t_val_get, _swigfaiss.float_maxheap_array_t_val_set, doc=r"""values (distances or similarities), size nh * k""") - - def get_val(self, key): - r"""Return the list of values for a heap""" - return _swigfaiss.float_maxheap_array_t_get_val(self, key) - - def get_ids(self, key): - r"""Corresponding identifiers""" - return _swigfaiss.float_maxheap_array_t_get_ids(self, key) - - def heapify(self): - r"""prepare all the heaps before adding""" - return _swigfaiss.float_maxheap_array_t_heapify(self) - - def addn(self, nj, vin, j0=0, i0=0, ni=-1): - r""" - add nj elements to heaps i0:i0+ni, with sequential ids - - :type nj: int - :param nj: nb of elements to add to each heap - :type vin: float - :param vin: elements to add, size ni * nj - :type j0: int, optional - :param j0: add this to the ids that are added - :type i0: int, optional - :param i0: first heap to update - :type ni: int, optional - :param ni: nb of elements to update (-1 = use nh) - """ - return _swigfaiss.float_maxheap_array_t_addn(self, nj, vin, j0, i0, ni) - - def addn_with_ids(self, nj, vin, id_in=None, id_stride=0, i0=0, ni=-1): - r""" - same as addn - - :type id_in: int, optional - :param id_in: ids of the elements to add, size ni * nj - :type id_stride: int, optional - :param id_stride: stride for id_in - """ - return _swigfaiss.float_maxheap_array_t_addn_with_ids(self, nj, vin, id_in, id_stride, i0, ni) - - def addn_query_subset_with_ids(self, nsubset, subset, nj, vin, id_in=None, id_stride=0): - r""" - same as addn_with_ids, but for just a subset of queries - - :type nsubset: int - :param nsubset: number of query entries to update - :type subset: int - :param subset: indexes of queries to update, in 0..nh-1, size nsubset - """ - return _swigfaiss.float_maxheap_array_t_addn_query_subset_with_ids(self, nsubset, subset, nj, vin, id_in, id_stride) - - def reorder(self): - r"""reorder all the heaps""" - return _swigfaiss.float_maxheap_array_t_reorder(self) - - def per_line_extrema(self, vals_out, idx_out): - r""" - this is not really a heap function. It just finds the per-line - extrema of each line of array D - :type vals_out: float - :param vals_out: extreme value of each line (size nh, or NULL) - :type idx_out: int - :param idx_out: index of extreme value (size nh or NULL) - """ - return _swigfaiss.float_maxheap_array_t_per_line_extrema(self, vals_out, idx_out) - - def __init__(self): - _swigfaiss.float_maxheap_array_t_swiginit(self, _swigfaiss.new_float_maxheap_array_t()) - __swig_destroy__ = _swigfaiss.delete_float_maxheap_array_t - -# Register float_maxheap_array_t in _swigfaiss: -_swigfaiss.float_maxheap_array_t_swigregister(float_maxheap_array_t) -class int_maxheap_array_t(object): - r""" - a template structure for a set of [min|max]-heaps it is tailored - so that the actual data of the heaps can just live in compact - arrays. - """ - - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - nh = property(_swigfaiss.int_maxheap_array_t_nh_get, _swigfaiss.int_maxheap_array_t_nh_set, doc=r"""number of heaps""") - k = property(_swigfaiss.int_maxheap_array_t_k_get, _swigfaiss.int_maxheap_array_t_k_set, doc=r"""allocated size per heap""") - ids = property(_swigfaiss.int_maxheap_array_t_ids_get, _swigfaiss.int_maxheap_array_t_ids_set, doc=r"""identifiers (size nh * k)""") - val = property(_swigfaiss.int_maxheap_array_t_val_get, _swigfaiss.int_maxheap_array_t_val_set, doc=r"""values (distances or similarities), size nh * k""") - - def get_val(self, key): - r"""Return the list of values for a heap""" - return _swigfaiss.int_maxheap_array_t_get_val(self, key) - - def get_ids(self, key): - r"""Corresponding identifiers""" - return _swigfaiss.int_maxheap_array_t_get_ids(self, key) - - def heapify(self): - r"""prepare all the heaps before adding""" - return _swigfaiss.int_maxheap_array_t_heapify(self) - - def addn(self, nj, vin, j0=0, i0=0, ni=-1): - r""" - add nj elements to heaps i0:i0+ni, with sequential ids - - :type nj: int - :param nj: nb of elements to add to each heap - :type vin: int - :param vin: elements to add, size ni * nj - :type j0: int, optional - :param j0: add this to the ids that are added - :type i0: int, optional - :param i0: first heap to update - :type ni: int, optional - :param ni: nb of elements to update (-1 = use nh) - """ - return _swigfaiss.int_maxheap_array_t_addn(self, nj, vin, j0, i0, ni) - - def addn_with_ids(self, nj, vin, id_in=None, id_stride=0, i0=0, ni=-1): - r""" - same as addn - - :type id_in: int, optional - :param id_in: ids of the elements to add, size ni * nj - :type id_stride: int, optional - :param id_stride: stride for id_in - """ - return _swigfaiss.int_maxheap_array_t_addn_with_ids(self, nj, vin, id_in, id_stride, i0, ni) - - def addn_query_subset_with_ids(self, nsubset, subset, nj, vin, id_in=None, id_stride=0): - r""" - same as addn_with_ids, but for just a subset of queries - - :type nsubset: int - :param nsubset: number of query entries to update - :type subset: int - :param subset: indexes of queries to update, in 0..nh-1, size nsubset - """ - return _swigfaiss.int_maxheap_array_t_addn_query_subset_with_ids(self, nsubset, subset, nj, vin, id_in, id_stride) - - def reorder(self): - r"""reorder all the heaps""" - return _swigfaiss.int_maxheap_array_t_reorder(self) - - def per_line_extrema(self, vals_out, idx_out): - r""" - this is not really a heap function. It just finds the per-line - extrema of each line of array D - :type vals_out: int - :param vals_out: extreme value of each line (size nh, or NULL) - :type idx_out: int - :param idx_out: index of extreme value (size nh or NULL) - """ - return _swigfaiss.int_maxheap_array_t_per_line_extrema(self, vals_out, idx_out) - - def __init__(self): - _swigfaiss.int_maxheap_array_t_swiginit(self, _swigfaiss.new_int_maxheap_array_t()) - __swig_destroy__ = _swigfaiss.delete_int_maxheap_array_t - -# Register int_maxheap_array_t in _swigfaiss: -_swigfaiss.int_maxheap_array_t_swigregister(int_maxheap_array_t) - -def CMin_float_partition_fuzzy(vals, ids, n, q_min, q_max, q_out): - r""" - partitions the table into 0:q and q:n where all elements above q are >= all - elements below q (for C = CMax, for CMin comparisons are reversed) - - Returns the partition threshold. The elements q:n are destroyed on output. - """ - return _swigfaiss.CMin_float_partition_fuzzy(vals, ids, n, q_min, q_max, q_out) - -def CMax_float_partition_fuzzy(vals, ids, n, q_min, q_max, q_out): - r""" - partitions the table into 0:q and q:n where all elements above q are >= all - elements below q (for C = CMax, for CMin comparisons are reversed) - - Returns the partition threshold. The elements q:n are destroyed on output. - """ - return _swigfaiss.CMax_float_partition_fuzzy(vals, ids, n, q_min, q_max, q_out) -class AlignedTableUint8(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - tab = property(_swigfaiss.AlignedTableUint8_tab_get, _swigfaiss.AlignedTableUint8_tab_set) - numel = property(_swigfaiss.AlignedTableUint8_numel_get, _swigfaiss.AlignedTableUint8_numel_set) - - @staticmethod - def round_capacity(n): - return _swigfaiss.AlignedTableUint8_round_capacity(n) - - def __init__(self, *args): - _swigfaiss.AlignedTableUint8_swiginit(self, _swigfaiss.new_AlignedTableUint8(*args)) - - def itemsize(self): - return _swigfaiss.AlignedTableUint8_itemsize(self) - - def resize(self, n): - return _swigfaiss.AlignedTableUint8_resize(self, n) - - def clear(self): - return _swigfaiss.AlignedTableUint8_clear(self) - - def size(self): - return _swigfaiss.AlignedTableUint8_size(self) - - def nbytes(self): - return _swigfaiss.AlignedTableUint8_nbytes(self) - - def get(self, *args): - return _swigfaiss.AlignedTableUint8_get(self, *args) - - def data(self, *args): - return _swigfaiss.AlignedTableUint8_data(self, *args) - __swig_destroy__ = _swigfaiss.delete_AlignedTableUint8 - -# Register AlignedTableUint8 in _swigfaiss: -_swigfaiss.AlignedTableUint8_swigregister(AlignedTableUint8) -class AlignedTableUint16(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - tab = property(_swigfaiss.AlignedTableUint16_tab_get, _swigfaiss.AlignedTableUint16_tab_set) - numel = property(_swigfaiss.AlignedTableUint16_numel_get, _swigfaiss.AlignedTableUint16_numel_set) - - @staticmethod - def round_capacity(n): - return _swigfaiss.AlignedTableUint16_round_capacity(n) - - def __init__(self, *args): - _swigfaiss.AlignedTableUint16_swiginit(self, _swigfaiss.new_AlignedTableUint16(*args)) - - def itemsize(self): - return _swigfaiss.AlignedTableUint16_itemsize(self) - - def resize(self, n): - return _swigfaiss.AlignedTableUint16_resize(self, n) - - def clear(self): - return _swigfaiss.AlignedTableUint16_clear(self) - - def size(self): - return _swigfaiss.AlignedTableUint16_size(self) - - def nbytes(self): - return _swigfaiss.AlignedTableUint16_nbytes(self) - - def get(self, *args): - return _swigfaiss.AlignedTableUint16_get(self, *args) - - def data(self, *args): - return _swigfaiss.AlignedTableUint16_data(self, *args) - __swig_destroy__ = _swigfaiss.delete_AlignedTableUint16 - -# Register AlignedTableUint16 in _swigfaiss: -_swigfaiss.AlignedTableUint16_swigregister(AlignedTableUint16) -class AlignedTableFloat32(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - tab = property(_swigfaiss.AlignedTableFloat32_tab_get, _swigfaiss.AlignedTableFloat32_tab_set) - numel = property(_swigfaiss.AlignedTableFloat32_numel_get, _swigfaiss.AlignedTableFloat32_numel_set) - - @staticmethod - def round_capacity(n): - return _swigfaiss.AlignedTableFloat32_round_capacity(n) - - def __init__(self, *args): - _swigfaiss.AlignedTableFloat32_swiginit(self, _swigfaiss.new_AlignedTableFloat32(*args)) - - def itemsize(self): - return _swigfaiss.AlignedTableFloat32_itemsize(self) - - def resize(self, n): - return _swigfaiss.AlignedTableFloat32_resize(self, n) - - def clear(self): - return _swigfaiss.AlignedTableFloat32_clear(self) - - def size(self): - return _swigfaiss.AlignedTableFloat32_size(self) - - def nbytes(self): - return _swigfaiss.AlignedTableFloat32_nbytes(self) - - def get(self, *args): - return _swigfaiss.AlignedTableFloat32_get(self, *args) - - def data(self, *args): - return _swigfaiss.AlignedTableFloat32_data(self, *args) - __swig_destroy__ = _swigfaiss.delete_AlignedTableFloat32 - -# Register AlignedTableFloat32 in _swigfaiss: -_swigfaiss.AlignedTableFloat32_swigregister(AlignedTableFloat32) -class MaybeOwnedVectorUInt8(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - is_owned = property(_swigfaiss.MaybeOwnedVectorUInt8_is_owned_get, _swigfaiss.MaybeOwnedVectorUInt8_is_owned_set) - owned_data = property(_swigfaiss.MaybeOwnedVectorUInt8_owned_data_get, _swigfaiss.MaybeOwnedVectorUInt8_owned_data_set) - view_data = property(_swigfaiss.MaybeOwnedVectorUInt8_view_data_get, _swigfaiss.MaybeOwnedVectorUInt8_view_data_set) - view_size = property(_swigfaiss.MaybeOwnedVectorUInt8_view_size_get, _swigfaiss.MaybeOwnedVectorUInt8_view_size_set) - owner = property(_swigfaiss.MaybeOwnedVectorUInt8_owner_get, _swigfaiss.MaybeOwnedVectorUInt8_owner_set) - c_ptr = property(_swigfaiss.MaybeOwnedVectorUInt8_c_ptr_get, _swigfaiss.MaybeOwnedVectorUInt8_c_ptr_set) - c_size = property(_swigfaiss.MaybeOwnedVectorUInt8_c_size_get, _swigfaiss.MaybeOwnedVectorUInt8_c_size_set) - - def __init__(self, *args): - _swigfaiss.MaybeOwnedVectorUInt8_swiginit(self, _swigfaiss.new_MaybeOwnedVectorUInt8(*args)) - - @staticmethod - def create_view(address, n_elements, owner): - return _swigfaiss.MaybeOwnedVectorUInt8_create_view(address, n_elements, owner) - - def data(self, *args): - return _swigfaiss.MaybeOwnedVectorUInt8_data(self, *args) - - def size(self): - return _swigfaiss.MaybeOwnedVectorUInt8_size(self) - - def byte_size(self): - return _swigfaiss.MaybeOwnedVectorUInt8_byte_size(self) - - def at(self, *args): - return _swigfaiss.MaybeOwnedVectorUInt8_at(self, *args) - - def begin(self, *args): - return _swigfaiss.MaybeOwnedVectorUInt8_begin(self, *args) - - def end(self, *args): - return _swigfaiss.MaybeOwnedVectorUInt8_end(self, *args) - - def erase(self, begin, end): - return _swigfaiss.MaybeOwnedVectorUInt8_erase(self, begin, end) - - def clear(self): - return _swigfaiss.MaybeOwnedVectorUInt8_clear(self) - - def resize(self, *args): - return _swigfaiss.MaybeOwnedVectorUInt8_resize(self, *args) - __swig_destroy__ = _swigfaiss.delete_MaybeOwnedVectorUInt8 - -# Register MaybeOwnedVectorUInt8 in _swigfaiss: -_swigfaiss.MaybeOwnedVectorUInt8_swigregister(MaybeOwnedVectorUInt8) -class MaybeOwnedVectorInt32(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - is_owned = property(_swigfaiss.MaybeOwnedVectorInt32_is_owned_get, _swigfaiss.MaybeOwnedVectorInt32_is_owned_set) - owned_data = property(_swigfaiss.MaybeOwnedVectorInt32_owned_data_get, _swigfaiss.MaybeOwnedVectorInt32_owned_data_set) - view_data = property(_swigfaiss.MaybeOwnedVectorInt32_view_data_get, _swigfaiss.MaybeOwnedVectorInt32_view_data_set) - view_size = property(_swigfaiss.MaybeOwnedVectorInt32_view_size_get, _swigfaiss.MaybeOwnedVectorInt32_view_size_set) - owner = property(_swigfaiss.MaybeOwnedVectorInt32_owner_get, _swigfaiss.MaybeOwnedVectorInt32_owner_set) - c_ptr = property(_swigfaiss.MaybeOwnedVectorInt32_c_ptr_get, _swigfaiss.MaybeOwnedVectorInt32_c_ptr_set) - c_size = property(_swigfaiss.MaybeOwnedVectorInt32_c_size_get, _swigfaiss.MaybeOwnedVectorInt32_c_size_set) - - def __init__(self, *args): - _swigfaiss.MaybeOwnedVectorInt32_swiginit(self, _swigfaiss.new_MaybeOwnedVectorInt32(*args)) - - @staticmethod - def create_view(address, n_elements, owner): - return _swigfaiss.MaybeOwnedVectorInt32_create_view(address, n_elements, owner) - - def data(self, *args): - return _swigfaiss.MaybeOwnedVectorInt32_data(self, *args) - - def size(self): - return _swigfaiss.MaybeOwnedVectorInt32_size(self) - - def byte_size(self): - return _swigfaiss.MaybeOwnedVectorInt32_byte_size(self) - - def at(self, *args): - return _swigfaiss.MaybeOwnedVectorInt32_at(self, *args) - - def begin(self, *args): - return _swigfaiss.MaybeOwnedVectorInt32_begin(self, *args) - - def end(self, *args): - return _swigfaiss.MaybeOwnedVectorInt32_end(self, *args) - - def erase(self, begin, end): - return _swigfaiss.MaybeOwnedVectorInt32_erase(self, begin, end) - - def clear(self): - return _swigfaiss.MaybeOwnedVectorInt32_clear(self) - - def resize(self, *args): - return _swigfaiss.MaybeOwnedVectorInt32_resize(self, *args) - __swig_destroy__ = _swigfaiss.delete_MaybeOwnedVectorInt32 - -# Register MaybeOwnedVectorInt32 in _swigfaiss: -_swigfaiss.MaybeOwnedVectorInt32_swigregister(MaybeOwnedVectorInt32) -class MaybeOwnedVectorFloat32(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - is_owned = property(_swigfaiss.MaybeOwnedVectorFloat32_is_owned_get, _swigfaiss.MaybeOwnedVectorFloat32_is_owned_set) - owned_data = property(_swigfaiss.MaybeOwnedVectorFloat32_owned_data_get, _swigfaiss.MaybeOwnedVectorFloat32_owned_data_set) - view_data = property(_swigfaiss.MaybeOwnedVectorFloat32_view_data_get, _swigfaiss.MaybeOwnedVectorFloat32_view_data_set) - view_size = property(_swigfaiss.MaybeOwnedVectorFloat32_view_size_get, _swigfaiss.MaybeOwnedVectorFloat32_view_size_set) - owner = property(_swigfaiss.MaybeOwnedVectorFloat32_owner_get, _swigfaiss.MaybeOwnedVectorFloat32_owner_set) - c_ptr = property(_swigfaiss.MaybeOwnedVectorFloat32_c_ptr_get, _swigfaiss.MaybeOwnedVectorFloat32_c_ptr_set) - c_size = property(_swigfaiss.MaybeOwnedVectorFloat32_c_size_get, _swigfaiss.MaybeOwnedVectorFloat32_c_size_set) - - def __init__(self, *args): - _swigfaiss.MaybeOwnedVectorFloat32_swiginit(self, _swigfaiss.new_MaybeOwnedVectorFloat32(*args)) - - @staticmethod - def create_view(address, n_elements, owner): - return _swigfaiss.MaybeOwnedVectorFloat32_create_view(address, n_elements, owner) - - def data(self, *args): - return _swigfaiss.MaybeOwnedVectorFloat32_data(self, *args) - - def size(self): - return _swigfaiss.MaybeOwnedVectorFloat32_size(self) - - def byte_size(self): - return _swigfaiss.MaybeOwnedVectorFloat32_byte_size(self) - - def at(self, *args): - return _swigfaiss.MaybeOwnedVectorFloat32_at(self, *args) - - def begin(self, *args): - return _swigfaiss.MaybeOwnedVectorFloat32_begin(self, *args) - - def end(self, *args): - return _swigfaiss.MaybeOwnedVectorFloat32_end(self, *args) - - def erase(self, begin, end): - return _swigfaiss.MaybeOwnedVectorFloat32_erase(self, begin, end) - - def clear(self): - return _swigfaiss.MaybeOwnedVectorFloat32_clear(self) - - def resize(self, *args): - return _swigfaiss.MaybeOwnedVectorFloat32_resize(self, *args) - __swig_destroy__ = _swigfaiss.delete_MaybeOwnedVectorFloat32 - -# Register MaybeOwnedVectorFloat32 in _swigfaiss: -_swigfaiss.MaybeOwnedVectorFloat32_swigregister(MaybeOwnedVectorFloat32) - -def CMin_uint16_partition_fuzzy(*args): - return _swigfaiss.CMin_uint16_partition_fuzzy(*args) - -def CMax_uint16_partition_fuzzy(*args): - return _swigfaiss.CMax_uint16_partition_fuzzy(*args) - -def merge_knn_results_CMin(*args): - return _swigfaiss.merge_knn_results_CMin(*args) - -def merge_knn_results_CMax(*args): - return _swigfaiss.merge_knn_results_CMax(*args) -class MapLong2Long(object): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - map = property(_swigfaiss.MapLong2Long_map_get, _swigfaiss.MapLong2Long_map_set) - - def add(self, n, keys, vals): - return _swigfaiss.MapLong2Long_add(self, n, keys, vals) - - def search(self, key): - return _swigfaiss.MapLong2Long_search(self, key) - - def search_multiple(self, n, keys, vals): - return _swigfaiss.MapLong2Long_search_multiple(self, n, keys, vals) - - def __init__(self): - _swigfaiss.MapLong2Long_swiginit(self, _swigfaiss.new_MapLong2Long()) - __swig_destroy__ = _swigfaiss.delete_MapLong2Long - -# Register MapLong2Long in _swigfaiss: -_swigfaiss.MapLong2Long_swigregister(MapLong2Long) - -def omp_set_num_threads(num_threads): - return _swigfaiss.omp_set_num_threads(num_threads) - -def omp_get_max_threads(): - return _swigfaiss.omp_get_max_threads() - -def memcpy(dest, src, n): - return _swigfaiss.memcpy(dest, src, n) -class PythonInterruptCallback(InterruptCallback): - thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") - __repr__ = _swig_repr - - def want_interrupt(self): - return _swigfaiss.PythonInterruptCallback_want_interrupt(self) - - @staticmethod - def reset(): - return _swigfaiss.PythonInterruptCallback_reset() - - def __init__(self): - _swigfaiss.PythonInterruptCallback_swiginit(self, _swigfaiss.new_PythonInterruptCallback()) - __swig_destroy__ = _swigfaiss.delete_PythonInterruptCallback - -# Register PythonInterruptCallback in _swigfaiss: -_swigfaiss.PythonInterruptCallback_swigregister(PythonInterruptCallback) - -def swig_ptr(a): - return _swigfaiss.swig_ptr(a) - -def rev_swig_ptr(*args): - return _swigfaiss.rev_swig_ptr(*args) - -def cast_integer_to_uint8_ptr(x): - return _swigfaiss.cast_integer_to_uint8_ptr(x) - -def cast_integer_to_float_ptr(x): - return _swigfaiss.cast_integer_to_float_ptr(x) - -def cast_integer_to_idx_t_ptr(x): - return _swigfaiss.cast_integer_to_idx_t_ptr(x) - -def cast_integer_to_int_ptr(x): - return _swigfaiss.cast_integer_to_int_ptr(x) - -def cast_integer_to_void_ptr(x): - return _swigfaiss.cast_integer_to_void_ptr(x) - -def swig_version(): - return _swigfaiss.swig_version() - diff --git a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/DELVEWHEEL b/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/DELVEWHEEL deleted file mode 100644 index c0f4d26fb619814fc17f5b02fe06b058cd462c1d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/DELVEWHEEL +++ /dev/null @@ -1,2 +0,0 @@ -Version: 1.13.0 -Arguments: ['C:\\Users\\runneradmin\\AppData\\Local\\Temp\\cibw-run-fm8v1x01\\cp310-win_amd64\\build\\venv\\Scripts\\delvewheel', 'repair', '-w', 'C:\\Users\\runneradmin\\AppData\\Local\\Temp\\cibw-run-fm8v1x01\\cp310-win_amd64\\repaired_wheel', 'C:\\Users\\runneradmin\\AppData\\Local\\Temp\\cibw-run-fm8v1x01\\cp310-win_amd64\\built_wheel\\faiss_cpu-1.15.0-cp310-cp310-win_amd64.whl', '--add-path', 'C:/openblas/bin', '--ignore-in-wheel'] diff --git a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/INSTALLER deleted file mode 100644 index a1b589e38a32041e49332e5e81c2d363dc418d68..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/METADATA deleted file mode 100644 index f66dea23bff9ff6d129443e0e42eddda31e96244..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/METADATA +++ /dev/null @@ -1,124 +0,0 @@ -Metadata-Version: 2.4 -Name: faiss-cpu -Version: 1.15.0 -Summary: A library for efficient similarity search and clustering of dense vectors. -Keywords: search,nearest-neighbors,clustering,vectors,similarity -Author: Meta AI Research -License-Expression: MIT -License-File: LICENSE -License-File: THIRD_PARTY_NOTICES -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Science/Research -Classifier: Operating System :: MacOS :: MacOS X -Classifier: Operating System :: Microsoft :: Windows -Classifier: Operating System :: POSIX :: Linux -Classifier: Programming Language :: C++ -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence -Project-URL: Homepage, https://github.com/facebookresearch/faiss -Project-URL: Documentation, https://github.com/facebookresearch/faiss/wiki -Project-URL: Repository, https://github.com/facebookresearch/faiss -Project-URL: Issues, https://github.com/facebookresearch/faiss/issues -Requires-Python: >=3.10 -Requires-Dist: numpy>=1.25 -Requires-Dist: packaging -Description-Content-Type: text/markdown - -# Faiss - -Faiss is a library for efficient similarity search and clustering of dense vectors. It contains algorithms that search in sets of vectors of any size, up to ones that possibly do not fit in RAM. It also contains supporting code for evaluation and parameter tuning. Faiss is written in C++ with complete wrappers for Python/numpy. Some of the most useful algorithms are implemented on the GPU. It is developed primarily at Meta's [Fundamental AI Research](https://ai.facebook.com/) group. - -## News - -See [CHANGELOG.md](CHANGELOG.md) for detailed information about latest features. - -## Introduction - -Faiss contains several methods for similarity search. It assumes that the instances are represented as vectors and are identified by an integer, and that the vectors can be compared with L2 (Euclidean) distances or dot products. Vectors that are similar to a query vector are those that have the lowest L2 distance or the highest dot product with the query vector. It also supports cosine similarity, since this is a dot product on normalized vectors. - -Some of the methods, like those based on binary vectors and compact quantization codes, solely use a compressed representation of the vectors and do not require to keep the original vectors. This generally comes at the cost of a less precise search but these methods can scale to billions of vectors in main memory on a single server. Other methods, like HNSW and NSG add an indexing structure on top of the raw vectors to make searching more efficient. - -The GPU implementation can accept input from either CPU or GPU memory. On a server with GPUs, the GPU indexes can be used a drop-in replacement for the CPU indexes (e.g., replace `IndexFlatL2` with `GpuIndexFlatL2`) and copies to/from GPU memory are handled automatically. Results will be faster however if both input and output remain resident on the GPU. Both single and multi-GPU usage is supported. - -## Installing - -Faiss comes with precompiled libraries for Anaconda in Python, see [faiss-cpu](https://anaconda.org/pytorch/faiss-cpu), [faiss-gpu](https://anaconda.org/pytorch/faiss-gpu) and [faiss-gpu-cuvs](https://anaconda.org/pytorch/faiss-gpu-cuvs). The library is mostly implemented in C++, the only dependency is a [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) implementation. Optional GPU support is provided via CUDA or AMD ROCm, and the Python interface is also optional. The backend GPU implementations of NVIDIA [cuVS](https://github.com/rapidsai/cuvs) can also be enabled optionally. It compiles with cmake. See [INSTALL.md](INSTALL.md) for details. - -## How Faiss works - -Faiss is built around an index type that stores a set of vectors, and provides a function to search in them with L2 and/or dot product vector comparison. Some index types are simple baselines, such as exact search. Most of the available indexing structures correspond to various trade-offs with respect to - -- search time -- search quality -- memory used per index vector -- training time -- adding time -- need for external data for unsupervised training - -The optional GPU implementation provides what is likely (as of March 2017) the fastest exact and approximate (compressed-domain) nearest neighbor search implementation for high-dimensional vectors, fastest Lloyd's k-means, and fastest small k-selection algorithm known. [The implementation is detailed here](https://arxiv.org/abs/1702.08734). - -## Full documentation of Faiss - -The following are entry points for documentation: - -- the full documentation can be found on the [wiki page](https://github.com/facebookresearch/faiss/wiki), including a [tutorial](https://github.com/facebookresearch/faiss/wiki/Getting-started), a [FAQ](https://github.com/facebookresearch/faiss/wiki/FAQ) and a [troubleshooting section](https://github.com/facebookresearch/faiss/wiki/Troubleshooting) -- the [doxygen documentation](https://faiss.ai/) gives per-class information extracted from code comments -- to reproduce results from our research papers, [Polysemous codes](https://arxiv.org/abs/1609.01882) and [Billion-scale similarity search with GPUs](https://arxiv.org/abs/1702.08734), refer to the [benchmarks README](benchs/README.md). For [ -Link and code: Fast indexing with graphs and compact regression codes](https://arxiv.org/abs/1804.09996), see the [link_and_code README](benchs/link_and_code) - -## Authors - -The main authors of Faiss are: -- [Hervé Jégou](https://github.com/jegou) initiated the Faiss project and wrote its first implementation -- [Matthijs Douze](https://github.com/mdouze) implemented most of the CPU Faiss -- [Jeff Johnson](https://github.com/wickedfoo) implemented all of the GPU Faiss -- [Lucas Hosseini](https://github.com/beauby) implemented the binary indexes and the build system -- [Chengqi Deng](https://github.com/KinglittleQ) implemented NSG, NNdescent and much of the additive quantization code. -- [Alexandr Guzhva](https://github.com/alexanderguzhva) many optimizations: SIMD, memory allocation and layout, fast decoding kernels for vector codecs, etc. -- [Gergely Szilvasy](https://github.com/algoriddle) build system, benchmarking framework. - -## Reference - -References to cite when you use Faiss in a research paper: -``` -@article{douze2024faiss, - title={The Faiss library}, - author={Matthijs Douze and Alexandr Guzhva and Chengqi Deng and Jeff Johnson and Gergely Szilvasy and Pierre-Emmanuel Mazaré and Maria Lomeli and Lucas Hosseini and Hervé Jégou}, - year={2024}, - eprint={2401.08281}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` -For the GPU version of Faiss, please cite: -``` -@article{johnson2019billion, - title={Billion-scale similarity search with {GPUs}}, - author={Johnson, Jeff and Douze, Matthijs and J{\'e}gou, Herv{\'e}}, - journal={IEEE Transactions on Big Data}, - volume={7}, - number={3}, - pages={535--547}, - year={2019}, - publisher={IEEE} -} -``` - -## Join the Faiss community - -For public discussion of Faiss or for questions, visit https://github.com/facebookresearch/faiss/discussions. - -We monitor the [issues page](https://github.com/facebookresearch/faiss/issues) of the repository. -You can report bugs, ask questions, etc. - -## Legal - -Faiss is MIT-licensed, refer to the [LICENSE file](https://github.com/facebookresearch/faiss/blob/main/LICENSE) in the top level directory. - -Copyright © Meta Platforms, Inc. diff --git a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/RECORD deleted file mode 100644 index 18483778f5424ddda09f9bc444351b0a2fd8952e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/RECORD +++ /dev/null @@ -1,64 +0,0 @@ -faiss/__init__.py,sha256=wZrkF7fk6Ha-tz-lcuWdIL95lPJ5Qdg5InA1ZIb28yA,21606 -faiss/__init__.pyi,sha256=LKajJAT1Mc1iVj2Z4Ne63HwY_9AFF0ExvbfWOxU8fsY,153231 -faiss/__pycache__/__init__.cpython-310.pyc,, -faiss/__pycache__/array_conversions.cpython-310.pyc,, -faiss/__pycache__/class_wrappers.cpython-310.pyc,, -faiss/__pycache__/extra_wrappers.cpython-310.pyc,, -faiss/__pycache__/gpu_wrappers.cpython-310.pyc,, -faiss/__pycache__/loader.cpython-310.pyc,, -faiss/__pycache__/swigfaiss.cpython-310.pyc,, -faiss/_swigfaiss.pyd,sha256=Y2lul2uzYKAbAbdHWz7UEPaTMWni2DByJXpXKNq1wSA,5585920 -faiss/array_conversions.py,sha256=z62k0IUN5KRkWlY1V6veS9N9loAzCxISZjk-jczeMy0,6057 -faiss/class_wrappers.py,sha256=LZMddaM_AgjTYt8rHTmDkq0_0lcepCfz6NU2IazEAOk,56980 -faiss/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -faiss/contrib/__pycache__/__init__.cpython-310.pyc,, -faiss/contrib/__pycache__/big_batch_search.cpython-310.pyc,, -faiss/contrib/__pycache__/client_server.cpython-310.pyc,, -faiss/contrib/__pycache__/clustering.cpython-310.pyc,, -faiss/contrib/__pycache__/datasets.cpython-310.pyc,, -faiss/contrib/__pycache__/evaluation.cpython-310.pyc,, -faiss/contrib/__pycache__/exhaustive_search.cpython-310.pyc,, -faiss/contrib/__pycache__/factory_tools.cpython-310.pyc,, -faiss/contrib/__pycache__/inspect_tools.cpython-310.pyc,, -faiss/contrib/__pycache__/ivf_tools.cpython-310.pyc,, -faiss/contrib/__pycache__/ondisk.cpython-310.pyc,, -faiss/contrib/__pycache__/rpc.cpython-310.pyc,, -faiss/contrib/__pycache__/torch_utils.cpython-310.pyc,, -faiss/contrib/__pycache__/vecs_io.cpython-310.pyc,, -faiss/contrib/big_batch_search.py,sha256=gfN4CdJdCzLb9Zc23A9XtXvl9FBWyVUJmeRKPv4QUFg,18657 -faiss/contrib/client_server.py,sha256=yLg5cCBsoyGrD4a3hu2TTp8L_44yBnSe2ALtLrjK6mo,2776 -faiss/contrib/clustering.py,sha256=3OO7x8ArPxVSW3PqjTitZW3sDQd8nahllKNzCUWQvmw,16451 -faiss/contrib/datasets.py,sha256=XuDvIxPkCCZeVAntozaVBEHVaN4W08abQqGUb6CPHdY,17865 -faiss/contrib/evaluation.py,sha256=se3-wu_Fu09HmHb_4Lx4x-pTpPiJLuHnXBBViNs1vWo,15433 -faiss/contrib/exhaustive_search.py,sha256=SLa7UjPlpjJxuIqrDZLJd27ud5y7N1TVSFHLyqBIpsE,12899 -faiss/contrib/factory_tools.py,sha256=xoQUH6CB_a4MAYvCkH60tfDg41l_r2dgApylL72_aNk,7054 -faiss/contrib/inspect_tools.py,sha256=2y01Uhb4-w4_77VdBJur_zK_BBqIkF9vQ7tMLtRMQSQ,3787 -faiss/contrib/ivf_tools.py,sha256=NrYKr_sFv4mgCVaNSFIqzvNb7HAnixp1pM2IuvgWALU,4986 -faiss/contrib/ondisk.py,sha256=0birRO7TNgtC4_1vYBFMIhoDqlv6qJ3bC6lpwUIfyy8,2160 -faiss/contrib/rpc.py,sha256=9Th07LYKO8duE-mtAOi00Y0UYO39tyYYbNEqHmXX-Jo,7647 -faiss/contrib/torch/README.md,sha256=vF49uS9H9pGeDefHK-YrMM5gmRs3vqo4VlV2xlNRJZw,218 -faiss/contrib/torch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -faiss/contrib/torch/__pycache__/__init__.cpython-310.pyc,, -faiss/contrib/torch/__pycache__/clustering.cpython-310.pyc,, -faiss/contrib/torch/__pycache__/quantization.cpython-310.pyc,, -faiss/contrib/torch/clustering.py,sha256=dODq1Oe4lbycNbXGWtZ3_ecCB4edqqwjq39-FCbYMJw,1735 -faiss/contrib/torch/quantization.py,sha256=7Jqz_GvSAXCPQIAFItxuVcZzBu8iigTWDD9zW298l00,2893 -faiss/contrib/torch_utils.py,sha256=FnkWPE_YjoatkVR0q-dCpmC2iVDBmKszCU-HEybF6dc,29739 -faiss/contrib/vecs_io.py,sha256=gGCGQvuaHg912LzWJVFUpsxAcOiR48e0hnMZ0sJMPZ8,5697 -faiss/extra_wrappers.py,sha256=zlvrVDF3sXJ6Ti8M5I67pV0yv7s5fEI5zAS1JHSsrQk,24351 -faiss/faiss.dll,sha256=7kapTdGqyWfBAJGiOF2TNAYRqJzuLH4o5YGFh2K3OUM,7428608 -faiss/gpu_wrappers.py,sha256=DQv-oO3vbH63QhQe-liTZOduNnzh8MQKv6oATxd2e_g,9563 -faiss/loader.py,sha256=o-9WTRHz0OkEv_WTy99TkhljYaRfvSZBnBObuYQSpck,8632 -faiss/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -faiss/swigfaiss.py,sha256=q_LKOsvhMPvKj55u19HRd8fsCSxnEyCY7X-gsOOtQGE,671346 -faiss_cpu-1.15.0.dist-info/DELVEWHEEL,sha256=uWkITleavrRf42UvFkpWLaESYG3duCgZVOiabHntYrc,457 -faiss_cpu-1.15.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -faiss_cpu-1.15.0.dist-info/METADATA,sha256=MFkMJeZsHYx2v2M79Yb-8Yg1T7Iq4ksQveaj9TpqzNw,7785 -faiss_cpu-1.15.0.dist-info/RECORD,, -faiss_cpu-1.15.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -faiss_cpu-1.15.0.dist-info/WHEEL,sha256=dLsqnLqoa03X-KKRvIZetJerE-pAfZT9U_wzO7zFOgQ,105 -faiss_cpu-1.15.0.dist-info/licenses/LICENSE,sha256=8IqUv7C9wdfxy1m62vD4xVGxDbhxGFYi2_H9A2r_qcs,1107 -faiss_cpu-1.15.0.dist-info/licenses/THIRD_PARTY_NOTICES,sha256=xZTQJum-hkwSSD99Fj9lsADyOQnTOzKc3-_Vqc9T1fY,22241 -faiss_cpu.libs/libopenblas.dll,sha256=6CTPn8IuWUmAfOmVoy5BOjRfuX5j5tEM-_QaqGOCGTw,51076488 -faiss_cpu.libs/msvcp140.dll,sha256=pMIim9wqKmMKzcCVtNhgCOXD47x3cxdDVPPaT1vrnN4,575056 -faiss_cpu.libs/vcomp140.dll,sha256=-W86FNiNiEbzHzqzikkDBM59bk9w-uQwTGPlnHrqLTA,213072 diff --git a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/WHEEL deleted file mode 100644 index 1b26578e474ada8147cf896584d64b24f79eea22..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/WHEEL +++ /dev/null @@ -1,5 +0,0 @@ -Wheel-Version: 1.0 -Generator: scikit-build-core 1.0.3 -Root-Is-Purelib: false -Tag: cp310-cp310-win_amd64 - diff --git a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/licenses/LICENSE b/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/licenses/LICENSE deleted file mode 100644 index a5d83a84b7140fd65fe29238b11c559d317ed460..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) Facebook, Inc. and its affiliates. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/licenses/THIRD_PARTY_NOTICES b/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/licenses/THIRD_PARTY_NOTICES deleted file mode 100644 index 93ddde874eeb6cb115d6e1900fbdf6b11afd88cc..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss_cpu-1.15.0.dist-info/licenses/THIRD_PARTY_NOTICES +++ /dev/null @@ -1,410 +0,0 @@ -Third-Party Notices for faiss pip wheels -======================================== - -The faiss pip wheels bundle the following third-party libraries as binary -dependencies. Their licenses are reproduced below. - - -1. OpenBLAS (Linux, Windows wheels) -------------------------------------------- - -Copyright (c) 2011-2014, The OpenBLAS Project -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - 3. Neither the name of the OpenBLAS project nor the names of - its contributors may be used to endorse or promote products - derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -License: BSD-3-Clause -Source: https://github.com/OpenMathLib/OpenBLAS - - -2. LLVM OpenMP Runtime (libomp) (macOS wheels) ------------------------------------------------ - -============================================================================== -The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: -============================================================================== - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - ----- LLVM Exceptions to the Apache 2.0 License ---- - -As an exception, if, as a result of your compiling your source code, portions -of this Software are embedded into an Object form of such source code, you -may redistribute such embedded portions in such Object form without complying -with the conditions of Sections 4(a), 4(b) and 4(d) of the License. - -In addition, if you combine or link compiled forms of this Software with -software that is licensed under the GPLv2 ("Combined Software") and if a -court of competent jurisdiction determines that the patent provision (Section -3), the indemnity provision (Section 9) or other Section of the License -conflicts with the conditions of the GPLv2, you may retroactively and -prospectively choose to deem waived or otherwise exclude such Section(s) of -the License, but only in their entirety and only with respect to the Combined -Software. - -============================================================================== -Software from third parties included in the LLVM Project: -============================================================================== -The LLVM Project contains third party software which is under different license -terms. All such code will be identified clearly using at least one of two -mechanisms: -1) It will be in a separate directory tree with its own `LICENSE.txt` or - `LICENSE` file at the top containing the specific license and restrictions - which apply to that software, or -2) It will contain specific license and restriction terms at the top of every - file. - -============================================================================== -Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): -============================================================================== - -The software contained in this directory tree is dual licensed under both the -University of Illinois "BSD-Like" license and the MIT license. As a user of -this code you may choose to use it under either license. As a contributor, -you agree to allow your code to be used under both. The full text of the -relevant licenses is included below. - -In addition, a license agreement from the copyright/patent holders of the -software contained in this directory tree is included below. - -============================================================================== - -University of Illinois/NCSA -Open Source License - -Copyright (c) 1997-2019 Intel Corporation - -All rights reserved. - -Developed by: - OpenMP Runtime Team - Intel Corporation - http://www.openmprtl.org - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal with -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimers. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimers in the - documentation and/or other materials provided with the distribution. - - * Neither the names of Intel Corporation OpenMP Runtime Team nor the - names of its contributors may be used to endorse or promote products - derived from this Software without specific prior written permission. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -SOFTWARE. - -============================================================================== - -Copyright (c) 1997-2019 Intel Corporation - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -============================================================================== - -Intel Corporation - -Software Grant License Agreement ("Agreement") - -Except for the license granted herein to you, Intel Corporation ("Intel") reserves -all right, title, and interest in and to the Software (defined below). - -Definition - -"Software" means the code and documentation as well as any original work of -authorship, including any modifications or additions to an existing work, that -is intentionally submitted by Intel to llvm.org (http://llvm.org) ("LLVM") for -inclusion in, or documentation of, any of the products owned or managed by LLVM -(the "Work"). For the purposes of this definition, "submitted" means any form of -electronic, verbal, or written communication sent to LLVM or its -representatives, including but not limited to communication on electronic -mailing lists, source code control systems, and issue tracking systems that are -managed by, or on behalf of, LLVM for the purpose of discussing and improving -the Work, but excluding communication that is conspicuously marked otherwise. - -1. Grant of Copyright License. Subject to the terms and conditions of this - Agreement, Intel hereby grants to you and to recipients of the Software - distributed by LLVM a perpetual, worldwide, non-exclusive, no-charge, - royalty-free, irrevocable copyright license to reproduce, prepare derivative - works of, publicly display, publicly perform, sublicense, and distribute the - Software and such derivative works. - -2. Grant of Patent License. Subject to the terms and conditions of this - Agreement, Intel hereby grants you and to recipients of the Software - distributed by LLVM a perpetual, worldwide, non-exclusive, no-charge, - royalty-free, irrevocable (except as stated in this section) patent license - to make, have made, use, offer to sell, sell, import, and otherwise transfer - the Work, where such license applies only to those patent claims licensable - by Intel that are necessarily infringed by Intel's Software alone or by - combination of the Software with the Work to which such Software was - submitted. If any entity institutes patent litigation against Intel or any - other entity (including a cross-claim or counterclaim in a lawsuit) alleging - that Intel's Software, or the Work to which Intel has contributed constitutes - direct or contributory patent infringement, then any patent licenses granted - to that entity under this Agreement for the Software or Work shall terminate - as of the date such litigation is filed. - -Unless required by applicable law or agreed to in writing, the software is -provided on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -either express or implied, including, without limitation, any warranties or -conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A -PARTICULAR PURPOSE. - -============================================================================== -License: Apache-2.0 WITH LLVM-exception -Source: https://github.com/llvm/llvm-project diff --git a/bundle/python-cpu/Lib/site-packages/faiss_cpu.libs/libopenblas.dll b/bundle/python-cpu/Lib/site-packages/faiss_cpu.libs/libopenblas.dll deleted file mode 100644 index 5ec205e59ce6c6043edc9039da018f47020a1406..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss_cpu.libs/libopenblas.dll +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e824cf9fc22e5949807ce995a32e413a345fb97e63e6d10cfbf41aa86382193c -size 51076488 diff --git a/bundle/python-cpu/Lib/site-packages/faiss_cpu.libs/msvcp140.dll b/bundle/python-cpu/Lib/site-packages/faiss_cpu.libs/msvcp140.dll deleted file mode 100644 index e71a456c461eebca84058e6f2f8b5002f689dd08..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss_cpu.libs/msvcp140.dll +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a4c2229bdc2a2a630acdc095b4d86008e5c3e3bc7773174354f3da4f5beb9cde -size 575056 diff --git a/bundle/python-cpu/Lib/site-packages/faiss_cpu.libs/vcomp140.dll b/bundle/python-cpu/Lib/site-packages/faiss_cpu.libs/vcomp140.dll deleted file mode 100644 index 87f5e96904482931c93deddd53ca96f7d03f3e2b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/faiss_cpu.libs/vcomp140.dll +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f96f3a14d88d8846f31f3ab38a490304ce7d6e4f70fae4304c63e59c7aea2d30 -size 213072 diff --git a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/METADATA deleted file mode 100644 index 8bffeff5e31968851d860a97c45a0cc59d372aa4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/METADATA +++ /dev/null @@ -1,617 +0,0 @@ -Metadata-Version: 2.4 -Name: fastapi -Version: 0.141.1 -Summary: FastAPI framework, high performance, easy to learn, fast to code, ready for production -Author-Email: =?utf-8?q?Sebasti=C3=A1n_Ram=C3=ADrez?= -License-Expression: MIT -License-File: LICENSE -Classifier: Intended Audience :: Information Technology -Classifier: Intended Audience :: System Administrators -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python -Classifier: Topic :: Internet -Classifier: Topic :: Software Development :: Libraries :: Application Frameworks -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Topic :: Software Development :: Libraries -Classifier: Topic :: Software Development -Classifier: Typing :: Typed -Classifier: Development Status :: 4 - Beta -Classifier: Environment :: Web Environment -Classifier: Framework :: AsyncIO -Classifier: Framework :: FastAPI -Classifier: Framework :: Pydantic -Classifier: Framework :: Pydantic :: 2 -Classifier: Intended Audience :: Developers -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers -Classifier: Topic :: Internet :: WWW/HTTP -Project-URL: Homepage, https://github.com/fastapi/fastapi -Project-URL: Documentation, https://fastapi.tiangolo.com/ -Project-URL: Repository, https://github.com/fastapi/fastapi -Project-URL: Issues, https://github.com/fastapi/fastapi/issues -Project-URL: Changelog, https://fastapi.tiangolo.com/release-notes/ -Requires-Python: >=3.10 -Requires-Dist: starlette>=0.46.0 -Requires-Dist: pydantic>=2.9.0 -Requires-Dist: typing-extensions>=4.8.0 -Requires-Dist: typing-inspection>=0.4.2 -Requires-Dist: annotated-doc>=0.0.2 -Provides-Extra: standard -Requires-Dist: fastapi-cli[standard]>=0.0.32; extra == "standard" -Requires-Dist: fastar>=0.9.0; extra == "standard" -Requires-Dist: httpx<1.0.0,>=0.23.0; extra == "standard" -Requires-Dist: jinja2>=3.1.5; extra == "standard" -Requires-Dist: python-multipart>=0.0.18; extra == "standard" -Requires-Dist: email-validator>=2.0.0; extra == "standard" -Requires-Dist: uvicorn[standard]>=0.12.0; extra == "standard" -Requires-Dist: pydantic-settings>=2.0.0; extra == "standard" -Requires-Dist: pydantic-extra-types>=2.0.0; extra == "standard" -Provides-Extra: standard-no-fastapi-cloud-cli -Requires-Dist: fastapi-cli[standard-no-fastapi-cloud-cli]>=0.0.32; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: httpx<1.0.0,>=0.23.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: jinja2>=3.1.5; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: python-multipart>=0.0.18; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: email-validator>=2.0.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: uvicorn[standard]>=0.12.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: pydantic-settings>=2.0.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: pydantic-extra-types>=2.0.0; extra == "standard-no-fastapi-cloud-cli" -Provides-Extra: all -Requires-Dist: fastapi-cli[standard]>=0.0.32; extra == "all" -Requires-Dist: httpx<1.0.0,>=0.23.0; extra == "all" -Requires-Dist: jinja2>=3.1.5; extra == "all" -Requires-Dist: python-multipart>=0.0.18; extra == "all" -Requires-Dist: itsdangerous>=1.1.0; extra == "all" -Requires-Dist: pyyaml>=5.3.1; extra == "all" -Requires-Dist: email-validator>=2.0.0; extra == "all" -Requires-Dist: uvicorn[standard]>=0.12.0; extra == "all" -Requires-Dist: pydantic-settings>=2.0.0; extra == "all" -Requires-Dist: pydantic-extra-types>=2.0.0; extra == "all" -Description-Content-Type: text/markdown - -

- FastAPI -

-

- FastAPI framework, high performance, easy to learn, fast to code, ready for production -

-

- - Test - - - Coverage - - - Package version - - - Supported Python versions - -

- ---- - -**Documentation**: [https://fastapi.tiangolo.com](https://fastapi.tiangolo.com) - -**Source Code**: [https://github.com/fastapi/fastapi](https://github.com/fastapi/fastapi) - ---- - -FastAPI is a modern, fast (high-performance), web framework for building APIs with Python based on standard Python type hints. - -The key features are: - -* **Fast**: Very high performance, on par with **NodeJS** and **Go** (thanks to Starlette and Pydantic). [One of the fastest Python frameworks available](#performance). -* **Fast to code**: Increase the speed to develop features by about 200% to 300%. * -* **Fewer bugs**: Reduce about 40% of human (developer) induced errors. * -* **Intuitive**: Great editor support. Completion everywhere. Less time debugging. -* **Easy**: Designed to be easy to use and learn. Less time reading docs. -* **Short**: Minimize code duplication. Multiple features from each parameter declaration. Fewer bugs. -* **Robust**: Get production-ready code. With automatic interactive documentation. -* **Standards-based**: Based on (and fully compatible with) the open standards for APIs: [OpenAPI](https://github.com/OAI/OpenAPI-Specification) (previously known as Swagger) and [JSON Schema](https://json-schema.org/). - -* estimation based on tests conducted by an internal development team, building production applications. - -## Sponsors - - -### Keystone Sponsor - - - -### Gold Sponsors - - - - - - - - - - -### Silver Sponsors - - - - - - - - - - -[Other sponsors](https://fastapi.tiangolo.com/fastapi-people/#sponsors) - -## Opinions - - - -
- -"_[...] I'm using **FastAPI** a ton these days. [...] I'm actually planning to use it for all of my team's **ML services at Microsoft**. Some of them are getting integrated into the core **Windows** product and some **Office** products._" - -
Kabir Khan - Microsoft (ref)
- ---- - -"_We adopted the **FastAPI** library to spawn a **REST** server that can be queried to obtain **predictions**. [for Ludwig]_" - -
Piero Molino, Yaroslav Dudin, and Sai Sumanth Miryala - Uber (ref)
- ---- - -"_**Netflix** is pleased to announce the open-source release of our **crisis management** orchestration framework: **Dispatch**! [built with **FastAPI**]_" - -
Kevin Glisson, Marc Vilanova, Forest Monsen - Netflix (ref)
- ---- - -"_If anyone is looking to build a production Python API, I would highly recommend **FastAPI**. It is **beautifully designed**, **simple to use** and **highly scalable**, it has become a **key component** in our API first development strategy and is driving many automations and services such as our Virtual TAC Engineer._" - -
Deon Pillsbury - Cisco (ref)
- ---- - -
- -## FastAPI Conf - -[**FastAPI Conf '26**](https://fastapiconf.com) is happening on **October 28, 2026** in **Amsterdam, NL**. All about FastAPI, right from the source. 🎤 - -FastAPI Conf '26 - October 28, 2026 - Amsterdam, NL - -## FastAPI mini documentary - -There's a [FastAPI mini documentary](https://www.youtube.com/watch?v=mpR8ngthqiE) released at the end of 2025, you can watch it online: - -FastAPI Mini Documentary - -## **Typer**, the FastAPI of CLIs - - - -If you are building a CLI app to be used in the terminal instead of a web API, check out [**Typer**](https://typer.tiangolo.com/). - -**Typer** is FastAPI's little sibling. And it's intended to be the **FastAPI of CLIs**. ⌨️ 🚀 - -## Requirements - -FastAPI stands on the shoulders of giants: - -* [Starlette](https://starlette.dev/) for the web parts. -* [Pydantic](https://pydantic.dev/docs/) for the data parts. - -## Installation - -First, [install `uv`](https://docs.astral.sh/uv/getting-started/installation/), and then add FastAPI to your project: - -
- -```console -$ uv add "fastapi[standard]" - ----> 100% -``` - -
- -**Note**: Make sure you put `"fastapi[standard]"` in quotes to ensure it works in all terminals. - -If you prefer to use `pip`, install `fastapi[standard]` inside a virtual environment. See the [installation guide](tutorial/#install-fastapi) for the alternative steps. - -## Example - -### Create it - -Create a file `main.py` with: - -```Python -from fastapi import FastAPI - -app = FastAPI() - - -@app.get("/") -def read_root(): - return {"Hello": "World"} - - -@app.get("/items/{item_id}") -def read_item(item_id: int, q: str | None = None): - return {"item_id": item_id, "q": q} -``` - -
-Or use async def... - -If your code uses `async` / `await`, use `async def`: - -```Python hl_lines="7 12" -from fastapi import FastAPI - -app = FastAPI() - - -@app.get("/") -async def read_root(): - return {"Hello": "World"} - - -@app.get("/items/{item_id}") -async def read_item(item_id: int, q: str | None = None): - return {"item_id": item_id, "q": q} -``` - -**Note**: - -If you don't know, check the _"In a hurry?"_ section about [`async` and `await` in the docs](https://fastapi.tiangolo.com/async/#in-a-hurry). - -
- -### Run it - -Run the server with: - -
- -```console -$ uv run fastapi dev - - ╭────────── FastAPI CLI - Development mode ───────────╮ - │ │ - │ Serving at: http://127.0.0.1:8000 │ - │ │ - │ API docs: http://127.0.0.1:8000/docs │ - │ │ - │ Running in development mode, for production use: │ - │ │ - │ fastapi run │ - │ │ - ╰─────────────────────────────────────────────────────╯ - -INFO: Will watch for changes in these directories: ['/home/user/code/awesomeapp'] -INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) -INFO: Started reloader process [2248755] using WatchFiles -INFO: Started server process [2248757] -INFO: Waiting for application startup. -INFO: Application startup complete. -``` - -
- -
-About the command fastapi dev... - -The command `fastapi dev` reads your `main.py` file automatically, detects the **FastAPI** app in it, and starts a server using [Uvicorn](https://uvicorn.dev). - -By default, `fastapi dev` will start with auto-reload enabled for local development. - -You can read more about it in the [FastAPI CLI docs](https://fastapi.tiangolo.com/fastapi-cli/). - -
- -### Check it - -Open your browser at [http://127.0.0.1:8000/items/5?q=somequery](http://127.0.0.1:8000/items/5?q=somequery). - -You will see the JSON response as: - -```JSON -{"item_id": 5, "q": "somequery"} -``` - -You already created an API that: - -* Receives HTTP requests in the _paths_ `/` and `/items/{item_id}`. -* Both _paths_ take `GET` operations (also known as HTTP _methods_). -* The _path_ `/items/{item_id}` has a _path parameter_ `item_id` that should be an `int`. -* The _path_ `/items/{item_id}` has an optional `str` _query parameter_ `q`. - -### Interactive API docs - -Now go to [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs). - -You will see the automatic interactive API documentation (provided by [Swagger UI](https://github.com/swagger-api/swagger-ui)): - -![Swagger UI](https://fastapi.tiangolo.com/img/index/index-01-swagger-ui-simple.png) - -### Alternative API docs - -And now, go to [http://127.0.0.1:8000/redoc](http://127.0.0.1:8000/redoc). - -You will see the alternative automatic documentation (provided by [ReDoc](https://github.com/Redocly/redoc)): - -![ReDoc](https://fastapi.tiangolo.com/img/index/index-02-redoc-simple.png) - -## Example upgrade - -Now modify the file `main.py` to receive a body from a `PUT` request. - -Declare the body using standard Python types, thanks to Pydantic. - -```Python hl_lines="2 7-10 23-25" -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - price: float - is_offer: bool | None = None - - -@app.get("/") -def read_root(): - return {"Hello": "World"} - - -@app.get("/items/{item_id}") -def read_item(item_id: int, q: str | None = None): - return {"item_id": item_id, "q": q} - - -@app.put("/items/{item_id}") -def update_item(item_id: int, item: Item): - return {"item_name": item.name, "item_id": item_id} -``` - -The `fastapi dev` server should reload automatically. - -### Interactive API docs upgrade - -Now go to [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs). - -* The interactive API documentation will be automatically updated, including the new body: - -![Swagger UI](https://fastapi.tiangolo.com/img/index/index-03-swagger-02.png) - -* Click on the button "Try it out", it allows you to fill the parameters and directly interact with the API: - -![Swagger UI interaction](https://fastapi.tiangolo.com/img/index/index-04-swagger-03.png) - -* Then click on the "Execute" button, the user interface will communicate with your API, send the parameters, get the results and show them on the screen: - -![Swagger UI interaction](https://fastapi.tiangolo.com/img/index/index-05-swagger-04.png) - -### Alternative API docs upgrade - -And now, go to [http://127.0.0.1:8000/redoc](http://127.0.0.1:8000/redoc). - -* The alternative documentation will also reflect the new query parameter and body: - -![ReDoc](https://fastapi.tiangolo.com/img/index/index-06-redoc-02.png) - -### Recap - -In summary, you declare **once** the types of parameters, body, etc. as function parameters. - -You do that with standard modern Python types. - -You don't have to learn a new syntax, the methods or classes of a specific library, etc. - -Just standard **Python**. - -For example, for an `int`: - -```Python -item_id: int -``` - -or for a more complex `Item` model: - -```Python -item: Item -``` - -...and with that single declaration you get: - -* Editor support, including: - * Completion. - * Type checks. -* Validation of data: - * Automatic and clear errors when the data is invalid. - * Validation even for deeply nested JSON objects. -* Conversion of input data: coming from the network to Python data and types. Reading from: - * JSON. - * Path parameters. - * Query parameters. - * Cookies. - * Headers. - * Forms. - * Files. -* Conversion of output data: converting from Python data and types to network data (as JSON): - * Convert Python types (`str`, `int`, `float`, `bool`, `list`, etc). - * `datetime` objects. - * `UUID` objects. - * Database models. - * ...and many more. -* Automatic interactive API documentation, including 2 alternative user interfaces: - * Swagger UI. - * ReDoc. - ---- - -Coming back to the previous code example, **FastAPI** will: - -* Validate that there is an `item_id` in the path for `GET` and `PUT` requests. -* Validate that the `item_id` is of type `int` for `GET` and `PUT` requests. - * If it is not, the client will see a useful, clear error. -* Check if there is an optional query parameter named `q` (as in `http://127.0.0.1:8000/items/foo?q=somequery`) for `GET` requests. - * As the `q` parameter is declared with `= None`, it is optional. - * Without the `None` it would be required (as is the body in the case with `PUT`). -* For `PUT` requests to `/items/{item_id}`, read the body as JSON: - * Check that it has a required attribute `name` that should be a `str`. - * Check that it has a required attribute `price` that has to be a `float`. - * Check that it has an optional attribute `is_offer`, that should be a `bool`, if present. - * All this would also work for deeply nested JSON objects. -* Convert from and to JSON automatically. -* Document everything with OpenAPI, that can be used by: - * Interactive documentation systems. - * Automatic client code generation systems, for many languages. -* Provide 2 interactive documentation web interfaces directly. - ---- - -We just scratched the surface, but you already get the idea of how it all works. - -Try changing the line with: - -```Python - return {"item_name": item.name, "item_id": item_id} -``` - -...from: - -```Python - ... "item_name": item.name ... -``` - -...to: - -```Python - ... "item_price": item.price ... -``` - -...and see how your editor will auto-complete the attributes and know their types: - -![editor support](https://fastapi.tiangolo.com/img/vscode-completion.png) - -For a more complete example including more features, see the Tutorial - User Guide. - -**Spoiler alert**: the tutorial - user guide includes: - -* Declaration of **parameters** from other different places such as: **headers**, **cookies**, **form fields** and **files**. -* How to set **validation constraints** such as `maximum_length` or `regex`. -* A very powerful and easy to use **Dependency Injection** system. -* Security and authentication, including support for **OAuth2** with **JWT tokens** and **HTTP Basic** auth. -* More advanced (but equally easy) techniques for declaring **deeply nested JSON models** (thanks to Pydantic). -* **GraphQL** integration with [Strawberry](https://strawberry.rocks) and other libraries. -* Many extra features (thanks to Starlette) such as: - * **WebSockets** - * extremely easy tests based on HTTPX and `pytest` - * **CORS** - * **Cookie Sessions** - * ...and more. - -### Deploy your app (optional) - -You can optionally deploy your FastAPI app to [FastAPI Cloud](https://fastapicloud.com) with a single command. 🚀 - -
- -```console -$ uv run fastapi deploy - -Deploying to FastAPI Cloud... - -✅ Deployment successful! - -🐔 Ready the chicken! Your app is ready at https://myapp.fastapicloud.dev -``` - -
- -The CLI will automatically detect your FastAPI application and deploy it to the cloud. If you are not logged in, your browser will open to complete the authentication process. - -That's it! Now you can access your app at that URL. ✨ - -#### About FastAPI Cloud - -**[FastAPI Cloud](https://fastapicloud.com)** is built by the same author and team behind **FastAPI**. - -It streamlines the process of **building**, **deploying**, and **accessing** an API with minimal effort. - -It brings the same **developer experience** of building apps with FastAPI to **deploying** them to the cloud. 🎉 - -FastAPI Cloud is the primary sponsor and funding provider for the *FastAPI and friends* open source projects. ✨ - -#### Deploy to other cloud providers - -FastAPI is open source and based on standards. You can deploy FastAPI apps to any cloud provider you choose. - -Follow your cloud provider's guides to deploy FastAPI apps with them. 🤓 - -## Performance - -Independent TechEmpower benchmarks show **FastAPI** applications running under Uvicorn as [one of the fastest Python frameworks available](https://www.techempower.com/benchmarks/#section=test&runid=7464e520-0dc2-473d-bd34-dbdfd7e85911&hw=ph&test=query&l=zijzen-7), only below Starlette and Uvicorn themselves (used internally by FastAPI). (*) - -To understand more about it, see the section [Benchmarks](https://fastapi.tiangolo.com/benchmarks/). - -## Dependencies - -FastAPI depends on Pydantic and Starlette. - -### `standard` Dependencies - -When you install FastAPI with `uv add "fastapi[standard]"` it comes with the `standard` group of optional dependencies: - -Used by Pydantic: - -* [`email-validator`](https://github.com/JoshData/python-email-validator) - for email validation. - -Used by Starlette: - -* [`httpx`](https://www.python-httpx.org) - Required if you want to use the `TestClient`. -* [`jinja2`](https://jinja.palletsprojects.com) - Required if you want to use the default template configuration. -* [`python-multipart`](https://github.com/Kludex/python-multipart) - Required if you want to support form "parsing", with `request.form()`. - -Used by FastAPI: - -* [`uvicorn`](https://uvicorn.dev) - for the server that loads and serves your application. This includes `uvicorn[standard]`, which includes some dependencies (e.g. `uvloop`) needed for high performance serving. -* `fastapi-cli[standard]` - to provide the `fastapi` command. - * This includes `fastapi-cloud-cli`, which allows you to deploy your FastAPI application to [FastAPI Cloud](https://fastapicloud.com). - -### Without `standard` Dependencies - -If you don't want to include the `standard` optional dependencies, you can install with `uv add fastapi` instead of `uv add "fastapi[standard]"`. - -### Without `fastapi-cloud-cli` - -If you want to install FastAPI with the standard dependencies but without the `fastapi-cloud-cli`, you can install with `uv add "fastapi[standard-no-fastapi-cloud-cli]"`. - -### Additional Optional Dependencies - -There are some additional dependencies you might want to install. - -Additional optional Pydantic dependencies: - -* [`pydantic-settings`](https://pydantic.dev/docs/validation/latest/concepts/pydantic_settings/) - for settings management. -* [`pydantic-extra-types`](https://github.com/pydantic/pydantic-extra-types) - for extra types to be used with Pydantic. - -Additional optional FastAPI dependencies: - -* [`orjson`](https://github.com/ijl/orjson) - Required if you want to use `ORJSONResponse`. -* [`ujson`](https://github.com/ultrajson/ultrajson) - Required if you want to use `UJSONResponse`. - -## License - -This project is licensed under the terms of the MIT license. diff --git a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/RECORD deleted file mode 100644 index b9bff9646af1c91e9ca7599b7ea4bcce5f461c85..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/RECORD +++ /dev/null @@ -1,64 +0,0 @@ -../../Scripts/fastapi.exe,sha256=PTYuF1w1y-RUqjyl5MJyhWNpL1Sx9uf1yr2oId1l7Po,46080 -fastapi-0.141.1.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -fastapi-0.141.1.dist-info/METADATA,sha256=EBDClUe0AmvbPNHEBuBXC515u0SLWmH6TVu1pesLdSw,27029 -fastapi-0.141.1.dist-info/RECORD,, -fastapi-0.141.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi-0.141.1.dist-info/WHEEL,sha256=VP-D4TPS230sME9Z3vb3INXvo1yt0924YRm5AOsk_dE,90 -fastapi-0.141.1.dist-info/entry_points.txt,sha256=GCf-WbIZxyGT4MUmrPGj1cOHYZoGsNPHAvNkT6hnGeA,61 -fastapi-0.141.1.dist-info/licenses/LICENSE,sha256=Tsif_IFIW5f-xYSy1KlhAy7v_oNEU4lP2cEnSQbMdE4,1086 -fastapi/.agents/skills/fastapi/SKILL.md,sha256=mJ5RNq0BXBdcyihmnQUb_vLiDa0TqDhTg4hWSGB_XrY,10111 -fastapi/.agents/skills/fastapi/references/dependencies.md,sha256=i2txoD-hRoCQWoH1PxiDuQwqt6xl7vp3wmZMLQON5Gk,3268 -fastapi/.agents/skills/fastapi/references/other-tools.md,sha256=oskZlYkCdGgezS9mIfqW5kImCGtOsgPUa46Sd81H-P0,1527 -fastapi/.agents/skills/fastapi/references/path-operations.md,sha256=7Y0qIRI7z2PdHpqmHMjk4zqHjLvx0JSYpVakIDAplco,1583 -fastapi/.agents/skills/fastapi/references/pydantic.md,sha256=hAq5YnOYYa7S6neAXcRpecmpar2PwAQ_GXs71dY46Es,1844 -fastapi/.agents/skills/fastapi/references/responses.md,sha256=fJIVCLvQk2lzAcqWEgOJds-1tlvcLcQqDALigJ02Uqc,1856 -fastapi/.agents/skills/fastapi/references/streaming.md,sha256=bHaIKnwbTkd7TUVQm_uxapUnAlEG4rdvALV9koD5ypI,2581 -fastapi/__init__.py,sha256=A2QG2fYV0hWsknlMOMeZ7ua0mJNWWc4DNd6_1HgV17k,1081 -fastapi/__main__.py,sha256=bKePXLdO4SsVSM6r9SVoLickJDcR2c0cTOxZRKq26YQ,37 -fastapi/_compat/__init__.py,sha256=PYOR-8vJ5va4Qjl810FcQmJbqmpyyeQMoZ9R86CeE2U,2095 -fastapi/_compat/shared.py,sha256=lv0zlY5SSt8gx-TDpHL9NNGy9Qf16RBEK3-OXc7_LOQ,7323 -fastapi/_compat/v2.py,sha256=sDGyi1iKSFW9LuJ7n4B61-1yrQI1NHRSwsFRRLuFJ8k,17601 -fastapi/applications.py,sha256=ONzLGbKgsLmEyNdUEmOEKVSjfAh8-WtEY66heHPGGDw,183840 -fastapi/background.py,sha256=TADzAethOAaqpVvckYuTT3c4O9N1HaFQysFCPt0MsgU,1820 -fastapi/cli.py,sha256=OYhZb0NR_deuT5ofyPF2NoNBzZDNOP8Salef2nk-HqA,418 -fastapi/concurrency.py,sha256=xHGDEOQAA6cvFEDX46oq3r2t1Zd4sVvreaRgdIE4juM,1489 -fastapi/datastructures.py,sha256=XPugnojHc4N07eF5Xp1TTndP2gMv1fG4k-0L4Vdr7Kc,5321 -fastapi/dependencies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi/dependencies/models.py,sha256=3EpGqAA2d885jr1ENd-Pj8e1b80uGYPtMiYjnUHbp3c,8092 -fastapi/dependencies/utils.py,sha256=E2kzdauV4yQk5UNMIfkoRjY0r17hJhzz4EAxOBvfR9k,39180 -fastapi/encoders.py,sha256=TMCSMOymQ1iS9rwlohheIU2-3-mU1RA_62OtJpE3yq0,11713 -fastapi/exception_handlers.py,sha256=YVcT8Zy021VYYeecgdyh5YEUjEIHKcLspbkSf4OfbJI,1275 -fastapi/exceptions.py,sha256=fUNOBRdIsULU0TnO8aNBg3HoJeyJxXp9UV6D8wYu9lI,7453 -fastapi/logger.py,sha256=I9NNi3ov8AcqbsbC9wl1X-hdItKgYt2XTrx1f99Zpl4,54 -fastapi/middleware/__init__.py,sha256=oQDxiFVcc1fYJUOIFvphnK7pTT5kktmfL32QXpBFvvo,58 -fastapi/middleware/asyncexitstack.py,sha256=RKGlQpGzg3GLosqVhrxBy_NCZ9qJS7zQeNHt5Y3x-00,637 -fastapi/middleware/cors.py,sha256=ynwjWQZoc_vbhzZ3_ZXceoaSrslHFHPdoM52rXr0WUU,79 -fastapi/middleware/gzip.py,sha256=xM5PcsH8QlAimZw4VDvcmTnqQamslThsfe3CVN2voa0,79 -fastapi/middleware/httpsredirect.py,sha256=rL8eXMnmLijwVkH7_400zHri1AekfeBd6D6qs8ix950,115 -fastapi/middleware/trustedhost.py,sha256=eE5XGRxGa7c5zPnMJDGp3BxaL25k5iVQlhnv-Pk0Pss,109 -fastapi/middleware/wsgi.py,sha256=a_FMDoeTwcdig9wdAGumIH82oDFfuj4pxtQxLbAw2Ns,107 -fastapi/openapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi/openapi/constants.py,sha256=adGzmis1L1HJRTE3kJ5fmHS_Noq6tIY6pWv_SFzoFDU,153 -fastapi/openapi/docs.py,sha256=PcjH0Sn-yp97O0exXincG8epsY7ATyJC5662bK1DRao,12425 -fastapi/openapi/models.py,sha256=twjJWGf6lcKRJZgaZCcnQ0Ten1hDhcyCLgbr7tXIP30,14608 -fastapi/openapi/utils.py,sha256=gdzqK4eEuxpoQmyZyVJfRGNLWDkWqJSjx9kkpyiG9Sc,29347 -fastapi/param_functions.py,sha256=4fTCVlvEDbAQr7gvWYkOX8hktdl2Iwk-kv4FaTz7P8w,69596 -fastapi/params.py,sha256=1fNNSK5J7PM5Fw-F5_9uOysA2NgC-61mb3LlrBWpqJM,26205 -fastapi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi/requests.py,sha256=zayepKFcienBllv3snmWI20Gk0oHNVLU4DDhqXBb4LU,142 -fastapi/responses.py,sha256=BM7JtiZ_G4j2Z7QvHUzV1KtZPnjOoCfX35RqYXlZDh0,4144 -fastapi/routing.py,sha256=ex72X7ayCURdxDvgcKIzJLeHmu-ca59j5paCNLdyO1U,255632 -fastapi/security/__init__.py,sha256=bO8pNmxqVRXUjfl2mOKiVZLn0FpBQ61VUYVjmppnbJw,881 -fastapi/security/api_key.py,sha256=4CNLNVAStOsMhytH9C5EOUEOZrtLg_IpMQS_HcRDP4M,9793 -fastapi/security/base.py,sha256=dl4pvbC-RxjfbWgPtCWd8MVU-7CB2SZ22rJDXVCXO6c,141 -fastapi/security/http.py,sha256=Z0xALDqwgJZRAaDs40Sa68rAnjFzEL99UEmO5PJzTKA,13410 -fastapi/security/oauth2.py,sha256=sSqW4tbvoHaWNld46TYatta5rgFfjod4LhxmedXzkI8,24178 -fastapi/security/open_id_connect_url.py,sha256=V8WLPEsEq_WJlIjPwJO2vCoJWGLu-VTt1-N2H7aV-D4,3136 -fastapi/security/utils.py,sha256=E9YIoez-H2k1oBLEdxqJEi8sV1umunJYKjQHvZG3FRY,261 -fastapi/sse.py,sha256=27Z0q8AO7ExKQWKQOEpTg94isE_NVcuHA33-YL9-Pr0,7083 -fastapi/staticfiles.py,sha256=iirGIt3sdY2QZXd36ijs3Cj-T0FuGFda3cd90kM9Ikw,69 -fastapi/templating.py,sha256=4zsuTWgcjcEainMJFAlW6-gnslm6AgOS1SiiDWfmQxk,76 -fastapi/testclient.py,sha256=nBvaAmX66YldReJNZXPOk1sfuo2Q6hs8bOvIaCep6LQ,66 -fastapi/types.py,sha256=g2tD842BUHC2C3_P8P06albQ4MhCb9RybrSmp5rODgU,438 -fastapi/utils.py,sha256=DX0VrnMwfVsZxRz8IitQ42c2---fDzmFZkeRMZR2UMo,4341 -fastapi/websockets.py,sha256=419uncYObEKZG0YcrXscfQQYLSWoE10jqxVMetGdR98,222 diff --git a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/WHEEL deleted file mode 100644 index e651d8efb5fb543a06227b0ebfab1007f9e19326..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: pdm-backend (2.4.9) -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/entry_points.txt b/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/entry_points.txt deleted file mode 100644 index b81849e1ef82b793f6f81bb970cdc3cb791f5776..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/entry_points.txt +++ /dev/null @@ -1,5 +0,0 @@ -[console_scripts] -fastapi = fastapi.cli:main - -[gui_scripts] - diff --git a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/licenses/LICENSE b/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/licenses/LICENSE deleted file mode 100644 index 3e92463e6bd522a2a21e5f0a80d8089d6c4be20d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi-0.141.1.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Sebastián Ramírez - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/SKILL.md b/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/SKILL.md deleted file mode 100644 index fc35b97ed9dc2ec0db51dee69c4ba358c5c8f3b8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/SKILL.md +++ /dev/null @@ -1,321 +0,0 @@ ---- -name: fastapi -description: FastAPI best practices and conventions. Use when working with FastAPI APIs, Pydantic models, dependencies, streaming responses including Server-Sent Events (SSE), and serving frontend apps. Keeps FastAPI code clean and up to date with the latest features and patterns. ---- - -# FastAPI - -Official FastAPI skill to write code with best practices, keeping up to date with new versions and features. - -## Quick Reference - -* Serve frontend apps: use `app.frontend()` or `router.frontend()` for built frontend assets; see [Serve Frontend Apps](#serve-frontend-apps). -* Server-Sent Events (SSE): use `response_class=EventSourceResponse` and `yield`; see [Streaming](#streaming-json-lines-sse-bytes) and [the streaming reference](references/streaming.md). -* JSON Lines and byte streaming: see [the streaming reference](references/streaming.md). -* Dependencies: use `Annotated[..., Depends(...)]`; see [Dependency Injection](#dependency-injection) and [the dependency injection reference](references/dependencies.md) for `yield`, scopes, and class dependencies. -* Response models: prefer return types; use `response_model` when the public response schema differs from the internal return value; see [the response reference](references/responses.md). -* Pydantic models: do not use ellipsis or `RootModel`; see [the Pydantic reference](references/pydantic.md). -* Routing: declare router-level prefix, tags, and shared dependencies on the `APIRouter`; see [the path operation reference](references/path-operations.md). -* Tooling and related libraries: use uv, Ruff, ty, Asyncer, SQLModel, and HTTPX when applicable; see [the other tools reference](references/other-tools.md). - -## Use the `fastapi` CLI - -Run the development server on localhost with reload: - -```bash -fastapi dev -``` - -Run the production server: - -```bash -fastapi run -``` - -Prefer declaring the entrypoint in `pyproject.toml`: - -```toml -[tool.fastapi] -entrypoint = "my_app.main:app" -``` - -When adding the entrypoint is not possible, or the user explicitly asks not to, pass the app file path: - -```bash -fastapi dev my_app/main.py -``` - -## Use `Annotated` - -Always prefer the `Annotated` style for parameter and dependency declarations. It keeps function signatures working in other contexts, respects the types, and allows reusability. - -Use `Annotated` for parameter declarations, including `Path`, `Query`, `Header`, etc.: - -```python -from typing import Annotated - -from fastapi import FastAPI, Path, Query - -app = FastAPI() - - -@app.get("/items/{item_id}") -async def read_item( - item_id: Annotated[int, Path(ge=1, description="The item ID")], - q: Annotated[str | None, Query(max_length=50)] = None, -): - return {"message": "Hello World"} -``` - -Use `Annotated` for dependencies with `Depends()`. Unless asked not to, create a new type alias for the dependency to allow reusing it: - -```python -from typing import Annotated - -from fastapi import Depends, FastAPI - -app = FastAPI() - - -def get_current_user(): - return {"username": "johndoe"} - - -CurrentUserDep = Annotated[dict, Depends(get_current_user)] - - -@app.get("/items/") -async def read_item(current_user: CurrentUserDep): - return {"message": "Hello World"} -``` - -## Do not use Ellipsis for *path operations* or Pydantic models - -Do not use `...` as a default value for required parameters or model fields. It's not needed and not recommended. - -```python -from typing import Annotated - -from fastapi import FastAPI, Query -from pydantic import BaseModel, Field - -app = FastAPI() - - -class Item(BaseModel): - name: str - description: str | None = None - price: float = Field(gt=0) - - -@app.post("/items/") -async def create_item(item: Item, project_id: Annotated[int, Query()]): - return item -``` - -See [the Pydantic reference](references/pydantic.md) for more details. - -## Return Type or Response Model - -When possible, include a return type. It will be used to validate, filter, document, and serialize the response. - -```python -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - description: str | None = None - - -@app.get("/items/me") -async def get_item() -> Item: - return Item(name="Plumbus", description="All-purpose home device") -``` - -Return types or response models filter data to avoid exposing sensitive information, and they let Pydantic serialize the data on the Rust side for performance. - -Use `response_model` when the type you return is not the same as the public schema you want to validate, filter, document, and serialize. See [the response reference](references/responses.md). - -## Performance - -Do not use `ORJSONResponse` or `UJSONResponse`, they are deprecated. - -Instead, declare a return type or response model. Pydantic will handle the data serialization on the Rust side. - -## Including Routers - -When declaring routers, prefer to add router-level parameters like prefix, tags, and shared dependencies to the router itself instead of in `include_router()`. - -```python -from fastapi import APIRouter, Depends, FastAPI - -app = FastAPI() - - -def get_current_user(): - return {"username": "johndoe"} - - -router = APIRouter( - prefix="/items", - tags=["items"], - dependencies=[Depends(get_current_user)], -) - - -@router.get("/") -async def list_items(): - return [] - - -app.include_router(router) -``` - -See [the path operation reference](references/path-operations.md) for more routing patterns. - -## Serve Frontend Apps - -Use `app.frontend()` to serve a built static frontend app, for example a directory generated by Vite, Astro, Angular, Svelte, Vue, or a similar tool. - -```python -from fastapi import FastAPI - -app = FastAPI() - -app.frontend("/", directory="dist") -``` - -Use `router.frontend()` when the frontend belongs to an `APIRouter`; normal router prefix behavior applies when the router is included. - -```python -from fastapi import APIRouter, FastAPI - -app = FastAPI() -router = APIRouter(prefix="/admin") - -router.frontend("/", directory="admin-dist") -app.include_router(router) -``` - -`app.frontend()` and `router.frontend()` are low-priority routes: regular API routes are matched first, then frontend files and client-side routing fallbacks. Use this for single-page apps and built frontend assets instead of mounting `StaticFiles` manually. - -## Dependency Injection - -Use dependencies when the logic can't be declared in Pydantic validation, depends on external resources, needs cleanup with `yield`, or is shared across endpoints. - -Apply shared dependencies at the router level via `dependencies=[Depends(...)]`. - -See [the dependency injection reference](references/dependencies.md) for detailed patterns including `yield` with `scope`, and class dependencies. - -## Async vs Sync *path operations* - -Use `async` *path operations* only when fully certain that the logic called inside is compatible with async and await, and that it doesn't block. - -```python -from fastapi import FastAPI - -app = FastAPI() - - -@app.get("/async-items/") -async def read_async_items(): - data = await some_async_library.fetch_items() - return data - - -@app.get("/items/") -def read_items(): - data = some_blocking_library.fetch_items() - return data -``` - -In case of doubt, or by default, use regular `def` functions. They will be run in a threadpool so they don't block the event loop. The same rules apply to dependencies. - -Make sure blocking code is not run inside of `async` functions. The logic will work, but will damage performance heavily. - -When needing to mix blocking and async code, see Asyncer in [the other tools reference](references/other-tools.md). - -## Streaming (JSON Lines, SSE, bytes) - -To stream Server-Sent Events, use `response_class=EventSourceResponse` and `yield` items from the endpoint. - -```python -from collections.abc import AsyncIterable - -from fastapi import FastAPI -from fastapi.sse import EventSourceResponse, ServerSentEvent - -app = FastAPI() - - -@app.get("/events", response_class=EventSourceResponse) -async def stream_events() -> AsyncIterable[ServerSentEvent]: - yield ServerSentEvent(data={"status": "started"}, event="status", id="1") -``` - -Plain objects are automatically JSON-serialized as `data:` fields. Use `ServerSentEvent` for full control over SSE fields (`event`, `id`, `retry`, `comment`) and `raw_data` for pre-formatted strings. - -See [the streaming reference](references/streaming.md) for JSON Lines, Server-Sent Events (`EventSourceResponse`, `ServerSentEvent`), and byte streaming (`StreamingResponse`) patterns. - -## Tooling - -See [the other tools reference](references/other-tools.md) for details on uv, Ruff, ty for package management, linting, type checking, formatting, etc. - -## Other Libraries - -See [the other tools reference](references/other-tools.md) for details on other libraries: - -* Asyncer for handling async and await, concurrency, mixing async and blocking code, prefer it over AnyIO or asyncio. -* SQLModel for working with SQL databases, prefer it over SQLAlchemy. -* HTTPX for interacting with HTTP (other APIs), prefer it over Requests. - -## Do not use Pydantic RootModels - -Do not use Pydantic `RootModel`; instead use regular type annotations with `Annotated` and Pydantic validation utilities. - -```python -from typing import Annotated - -from fastapi import Body, FastAPI -from pydantic import Field - -app = FastAPI() - - -@app.post("/items/") -async def create_items(items: Annotated[list[int], Field(min_length=1), Body()]): - return items -``` - -FastAPI supports these type annotations and will create a Pydantic `TypeAdapter` for them, so types work normally without custom wrapper models. See [the Pydantic reference](references/pydantic.md). - -## Use one HTTP operation per function - -Don't mix HTTP operations in a single function. Having one function per HTTP operation helps separate concerns and organize the code. - -```python -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - - -@app.get("/items/") -async def list_items(): - return [] - - -@app.post("/items/") -async def create_item(item: Item): - return item -``` - -See [the path operation reference](references/path-operations.md) for more examples. diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/dependencies.md b/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/dependencies.md deleted file mode 100644 index a562dd9469851c085b4591ce748ca6689864a1d3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/dependencies.md +++ /dev/null @@ -1,142 +0,0 @@ -# Dependency Injection - -Use dependencies when: - -* They can't be declared in Pydantic validation and require additional logic -* The logic depends on external resources or could block in any other way -* Other dependencies need their results (it's a sub-dependency) -* The logic can be shared by multiple endpoints to do things like error early, handle authentication, etc. -* They need to handle cleanup (e.g., DB sessions, file handles), using dependencies with `yield` -* Their logic needs input data from the request, like headers, query parameters, etc. - -## Dependencies with `yield` and `scope` - -When using dependencies with `yield`, they can have a `scope` that defines when the exit code is run. - -Use the default scope `"request"` to run the exit code after the response is sent back. - -```python -from typing import Annotated - -from fastapi import Depends, FastAPI - -app = FastAPI() - - -def get_db(): - db = DBSession() - try: - yield db - finally: - db.close() - - -DBDep = Annotated[DBSession, Depends(get_db)] - - -@app.get("/items/") -async def read_items(db: DBDep): - return db.query(Item).all() -``` - -Use the scope `"function"` when they should run the exit code after the response data is generated but before the response is sent back to the client. - -```python -from typing import Annotated - -from fastapi import Depends, FastAPI - -app = FastAPI() - - -def get_username(): - try: - yield "Rick" - finally: - print("Clean up before response is sent") - -UserNameDep = Annotated[str, Depends(get_username, scope="function")] - -@app.get("/users/me") -def get_user_me(username: UserNameDep): - return username -``` - -## Class Dependencies - -Avoid creating class dependencies when possible. - -If a class is needed, instead create a regular function dependency that returns a class instance. - -Do this: - -```python -from dataclasses import dataclass -from typing import Annotated - -from fastapi import Depends, FastAPI - -app = FastAPI() - - -@dataclass -class DatabasePaginator: - offset: int = 0 - limit: int = 100 - q: str | None = None - - def get_page(self) -> dict: - # Simulate a page of data - return { - "offset": self.offset, - "limit": self.limit, - "q": self.q, - "items": [], - } - - -def get_db_paginator( - offset: int = 0, limit: int = 100, q: str | None = None -) -> DatabasePaginator: - return DatabasePaginator(offset=offset, limit=limit, q=q) - - -PaginatorDep = Annotated[DatabasePaginator, Depends(get_db_paginator)] - - -@app.get("/items/") -async def read_items(paginator: PaginatorDep): - return paginator.get_page() -``` - -instead of this: - -```python -# DO NOT DO THIS -from typing import Annotated - -from fastapi import Depends, FastAPI - -app = FastAPI() - - -class DatabasePaginator: - def __init__(self, offset: int = 0, limit: int = 100, q: str | None = None): - self.offset = offset - self.limit = limit - self.q = q - - def get_page(self) -> dict: - # Simulate a page of data - return { - "offset": self.offset, - "limit": self.limit, - "q": self.q, - "items": [], - } - - -@app.get("/items/") -async def read_items(paginator: Annotated[DatabasePaginator, Depends()]): - return paginator.get_page() -``` diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/other-tools.md b/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/other-tools.md deleted file mode 100644 index b5b58cfd62e330e738b25399175f9e5c921cd09d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/other-tools.md +++ /dev/null @@ -1,76 +0,0 @@ -# Other Tools - -## uv - -If uv is available, use it to manage dependencies. - -## Ruff - -If Ruff is available, use it to lint and format the code. Consider enabling the FastAPI rules. - -## ty - -If ty is available, use it to check types. - -## Asyncer - -When needing to run blocking code inside of async functions, or async code inside of blocking functions, suggest using Asyncer. - -Prefer it over AnyIO or asyncio. - -Install: - -```bash -uv add asyncer -``` - -Run blocking sync code inside of async with `asyncify()`: - -```python -from asyncer import asyncify -from fastapi import FastAPI - -app = FastAPI() - - -def do_blocking_work(name: str) -> str: - # Some blocking I/O operation - return f"Hello {name}" - - -@app.get("/items/") -async def read_items(): - result = await asyncify(do_blocking_work)(name="World") - return {"message": result} -``` - -And run async code inside of blocking sync code with `syncify()`: - -```python -from asyncer import syncify -from fastapi import FastAPI - -app = FastAPI() - - -async def do_async_work(name: str) -> str: - return f"Hello {name}" - - -@app.get("/items/") -def read_items(): - result = syncify(do_async_work)(name="World") - return {"message": result} -``` - -## SQLModel for SQL databases - -When working with SQL databases, prefer using SQLModel as it is integrated with Pydantic and will allow declaring data validation with the same models. - -Prefer it over SQLAlchemy. - -## HTTPX - -Use HTTPX for handling HTTP communication (e.g. with other APIs). It supports sync and async usage. - -Prefer it over Requests. diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/path-operations.md b/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/path-operations.md deleted file mode 100644 index 1292c17742536ea255813563d4757502bf209b2d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/path-operations.md +++ /dev/null @@ -1,93 +0,0 @@ -# Path Operations and Routing - -## Including Routers - -When declaring routers, prefer to add router-level parameters like prefix, tags, and shared dependencies to the router itself instead of in `include_router()`. - -Do this: - -```python -from fastapi import APIRouter, FastAPI - -app = FastAPI() - -router = APIRouter(prefix="/items", tags=["items"]) - - -@router.get("/") -async def list_items(): - return [] - - -app.include_router(router) -``` - -Instead of: - -```python -# DO NOT DO THIS -from fastapi import APIRouter, FastAPI - -app = FastAPI() - -router = APIRouter() - - -@router.get("/") -async def list_items(): - return [] - - -app.include_router(router, prefix="/items", tags=["items"]) -``` - -There could be exceptions, but try to follow this convention. - -Apply shared dependencies at the router level via `dependencies=[Depends(...)]`. - -## Use one HTTP operation per function - -Don't mix HTTP operations in a single function. Having one function per HTTP operation helps separate concerns and organize the code. - -Do this: - -```python -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - - -@app.get("/items/") -async def list_items(): - return [] - - -@app.post("/items/") -async def create_item(item: Item): - return item -``` - -Instead of: - -```python -# DO NOT DO THIS -from fastapi import FastAPI, Request -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - - -@app.api_route("/items/", methods=["GET", "POST"]) -async def handle_items(request: Request): - if request.method == "GET": - return [] -``` diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/pydantic.md b/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/pydantic.md deleted file mode 100644 index fadf99c1a9d52110d2039cc03b0ab80a95875e82..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/pydantic.md +++ /dev/null @@ -1,93 +0,0 @@ -# Pydantic - -## Do not use Ellipsis - -Do not use `...` as a default value for required parameters or model fields. It's not needed and not recommended. - -Do this, without Ellipsis (`...`): - -```python -from typing import Annotated - -from fastapi import FastAPI, Query -from pydantic import BaseModel, Field - -app = FastAPI() - - -class Item(BaseModel): - name: str - description: str | None = None - price: float = Field(gt=0) - - -@app.post("/items/") -async def create_item(item: Item, project_id: Annotated[int, Query()]): - return item -``` - -Instead of: - -```python -# DO NOT DO THIS -from typing import Annotated - -from fastapi import FastAPI, Query -from pydantic import BaseModel, Field - -app = FastAPI() - - -class Item(BaseModel): - name: str = ... - description: str | None = None - price: float = Field(..., gt=0) - - -@app.post("/items/") -async def create_item(item: Item, project_id: Annotated[int, Query(...)]): - return item -``` - -## Do not use Pydantic RootModels - -Do not use Pydantic `RootModel`; instead use regular type annotations with `Annotated` and Pydantic validation utilities. - -For example, for a list with validations: - -```python -from typing import Annotated - -from fastapi import Body, FastAPI -from pydantic import Field - -app = FastAPI() - - -@app.post("/items/") -async def create_items(items: Annotated[list[int], Field(min_length=1), Body()]): - return items -``` - -Instead of: - -```python -# DO NOT DO THIS -from typing import Annotated - -from fastapi import FastAPI -from pydantic import Field, RootModel - -app = FastAPI() - - -class ItemList(RootModel[Annotated[list[int], Field(min_length=1)]]): - pass - - -@app.post("/items/") -async def create_items(items: ItemList): - return items -``` - -FastAPI supports these type annotations and will create a Pydantic `TypeAdapter` for them, so types work normally without custom wrapper models. diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/responses.md b/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/responses.md deleted file mode 100644 index 09081236a0ac06a15cc08d1eb25b01e00f69e564..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/responses.md +++ /dev/null @@ -1,79 +0,0 @@ -# Responses - -## Return Type or Response Model - -When possible, include a return type. It will be used to validate, filter, document, and serialize the response. - -```python -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - description: str | None = None - - -@app.get("/items/me") -async def get_item() -> Item: - return Item(name="Plumbus", description="All-purpose home device") -``` - -Return types or response models filter data to avoid exposing sensitive information. They also let Pydantic serialize data on the Rust side for performance. - -The return type doesn't have to be a Pydantic model. It can be a different type, like a list of integers, a dict, etc. - -## When to use `response_model` - -If the return type is not the same as the type that you want to use to validate, filter, or serialize, use the `response_model` parameter on the decorator. - -```python -from typing import Any - -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - description: str | None = None - - -@app.get("/items/me", response_model=Item) -async def get_item() -> Any: - return {"name": "Foo", "description": "A very nice Item"} -``` - -This is particularly useful when filtering data to expose only the public fields and avoid exposing sensitive information. - -```python -from typing import Any - -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class InternalItem(BaseModel): - name: str - description: str | None = None - secret_key: str - - -class Item(BaseModel): - name: str - description: str | None = None - - -@app.get("/items/me", response_model=Item) -async def get_item() -> Any: - item = InternalItem( - name="Foo", description="A very nice Item", secret_key="supersecret" - ) - return item -``` diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/streaming.md b/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/streaming.md deleted file mode 100644 index 0832eedcb92b747444e4fcc406c845ed51fa21bd..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/.agents/skills/fastapi/references/streaming.md +++ /dev/null @@ -1,105 +0,0 @@ -# Streaming - -## Stream JSON Lines - -To stream JSON Lines, declare the return type and use `yield` to return the data. - -```python -@app.get("/items/stream") -async def stream_items() -> AsyncIterable[Item]: - for item in items: - yield item -``` - -## Server-Sent Events (SSE) - -To stream Server-Sent Events, use `response_class=EventSourceResponse` and `yield` items from the endpoint. - -Plain objects are automatically JSON-serialized as `data:` fields, declare the return type so the serialization is done by Pydantic: - -```python -from collections.abc import AsyncIterable - -from fastapi import FastAPI -from fastapi.sse import EventSourceResponse -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - price: float - - -@app.get("/items/stream", response_class=EventSourceResponse) -async def stream_items() -> AsyncIterable[Item]: - yield Item(name="Plumbus", price=32.99) - yield Item(name="Portal Gun", price=999.99) -``` - -For full control over SSE fields (`event`, `id`, `retry`, `comment`), yield `ServerSentEvent` instances: - -```python -from collections.abc import AsyncIterable - -from fastapi import FastAPI -from fastapi.sse import EventSourceResponse, ServerSentEvent - -app = FastAPI() - - -@app.get("/events", response_class=EventSourceResponse) -async def stream_events() -> AsyncIterable[ServerSentEvent]: - yield ServerSentEvent(data={"status": "started"}, event="status", id="1") - yield ServerSentEvent(data={"progress": 50}, event="progress", id="2") -``` - -Use `raw_data` instead of `data` to send pre-formatted strings without JSON encoding: - -```python -yield ServerSentEvent(raw_data="plain text line", event="log") -``` - -## Stream bytes - -To stream bytes, declare a `response_class=` of `StreamingResponse` or a sub-class, and use `yield` to return the data. - -```python -from fastapi import FastAPI -from fastapi.responses import StreamingResponse -from app.utils import read_image - -app = FastAPI() - - -class PNGStreamingResponse(StreamingResponse): - media_type = "image/png" - -@app.get("/image", response_class=PNGStreamingResponse) -def stream_image_no_async_no_annotation(): - with read_image() as image_file: - yield from image_file -``` - -prefer this over returning a `StreamingResponse` directly: - -```python -# DO NOT DO THIS - -import anyio -from fastapi import FastAPI -from fastapi.responses import StreamingResponse -from app.utils import read_image - -app = FastAPI() - - -class PNGStreamingResponse(StreamingResponse): - media_type = "image/png" - - -@app.get("/") -async def main(): - return PNGStreamingResponse(read_image()) -``` diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/__init__.py b/bundle/python-cpu/Lib/site-packages/fastapi/__init__.py deleted file mode 100644 index 1ca29aa85e498ee348e860504c5f50b117e82221..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -"""FastAPI framework, high performance, easy to learn, fast to code, ready for production""" - -__version__ = "0.141.1" - -from starlette import status as status - -from .applications import FastAPI as FastAPI -from .background import BackgroundTasks as BackgroundTasks -from .datastructures import UploadFile as UploadFile -from .exceptions import HTTPException as HTTPException -from .exceptions import WebSocketException as WebSocketException -from .param_functions import Body as Body -from .param_functions import Cookie as Cookie -from .param_functions import Depends as Depends -from .param_functions import File as File -from .param_functions import Form as Form -from .param_functions import Header as Header -from .param_functions import Path as Path -from .param_functions import Query as Query -from .param_functions import Security as Security -from .requests import Request as Request -from .responses import Response as Response -from .routing import APIRouter as APIRouter -from .websockets import WebSocket as WebSocket -from .websockets import WebSocketDisconnect as WebSocketDisconnect diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/__main__.py b/bundle/python-cpu/Lib/site-packages/fastapi/__main__.py deleted file mode 100644 index fc36465f5f40701bf333de8811d76d3484f211e6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from fastapi.cli import main - -main() diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/_compat/__init__.py b/bundle/python-cpu/Lib/site-packages/fastapi/_compat/__init__.py deleted file mode 100644 index 4581c38c88ede0d70f9c40a67fbc9ee79529cffd..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/_compat/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -from .shared import PYDANTIC_VERSION_MINOR_TUPLE as PYDANTIC_VERSION_MINOR_TUPLE -from .shared import annotation_is_pydantic_v1 as annotation_is_pydantic_v1 -from .shared import field_annotation_is_scalar as field_annotation_is_scalar -from .shared import ( - field_annotation_is_scalar_sequence as field_annotation_is_scalar_sequence, -) -from .shared import field_annotation_is_sequence as field_annotation_is_sequence -from .shared import ( - is_bytes_or_nonable_bytes_annotation as is_bytes_or_nonable_bytes_annotation, -) -from .shared import is_bytes_sequence_annotation as is_bytes_sequence_annotation -from .shared import is_pydantic_v1_model_instance as is_pydantic_v1_model_instance -from .shared import ( - is_uploadfile_or_nonable_uploadfile_annotation as is_uploadfile_or_nonable_uploadfile_annotation, -) -from .shared import ( - is_uploadfile_sequence_annotation as is_uploadfile_sequence_annotation, -) -from .shared import lenient_issubclass as lenient_issubclass -from .shared import sequence_types as sequence_types -from .shared import value_is_sequence as value_is_sequence -from .v2 import ModelField as ModelField -from .v2 import PydanticSchemaGenerationError as PydanticSchemaGenerationError -from .v2 import RequiredParam as RequiredParam -from .v2 import Undefined as Undefined -from .v2 import Url as Url -from .v2 import copy_field_info as copy_field_info -from .v2 import create_body_model as create_body_model -from .v2 import evaluate_forwardref as evaluate_forwardref -from .v2 import get_cached_model_fields as get_cached_model_fields -from .v2 import get_definitions as get_definitions -from .v2 import get_flat_models_from_fields as get_flat_models_from_fields -from .v2 import get_missing_field_error as get_missing_field_error -from .v2 import get_model_name_map as get_model_name_map -from .v2 import get_schema_from_model_field as get_schema_from_model_field -from .v2 import is_scalar_field as is_scalar_field -from .v2 import serialize_sequence_value as serialize_sequence_value -from .v2 import ( - with_info_plain_validator_function as with_info_plain_validator_function, -) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/_compat/shared.py b/bundle/python-cpu/Lib/site-packages/fastapi/_compat/shared.py deleted file mode 100644 index 8d4720a2ed3a31531efaf3e0bc3cf4b2a3763ac5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/_compat/shared.py +++ /dev/null @@ -1,222 +0,0 @@ -import types -import typing -import warnings -from collections import deque -from collections.abc import Mapping, Sequence -from dataclasses import is_dataclass -from typing import ( - Annotated, - Any, - TypeGuard, - TypeVar, - Union, - get_args, - get_origin, -) - -from fastapi.types import UnionType -from pydantic import BaseModel -from pydantic.version import VERSION as PYDANTIC_VERSION -from starlette.datastructures import UploadFile - -_T = TypeVar("_T") - -# Copy from Pydantic: pydantic/_internal/_typing_extra.py -WithArgsTypes: tuple[Any, ...] = ( - typing._GenericAlias, # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] - types.GenericAlias, - types.UnionType, -) # pyright: ignore[reportAttributeAccessIssue] - -PYDANTIC_VERSION_MINOR_TUPLE = tuple(int(x) for x in PYDANTIC_VERSION.split(".")[:2]) - - -sequence_annotation_to_type = { - Sequence: list, - list: list, - tuple: tuple, - set: set, - frozenset: frozenset, - deque: deque, -} - -sequence_types: tuple[type[Any], ...] = tuple(sequence_annotation_to_type.keys()) - - -# Copy of Pydantic: pydantic/_internal/_utils.py with added TypeGuard -def lenient_issubclass( - cls: Any, class_or_tuple: type[_T] | tuple[type[_T], ...] | None -) -> TypeGuard[type[_T]]: - try: - return isinstance(cls, type) and issubclass(cls, class_or_tuple) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - except TypeError: # pragma: no cover - if isinstance(cls, WithArgsTypes): - return False - raise # pragma: no cover - - -def _annotation_is_sequence(annotation: type[Any] | None) -> bool: - if lenient_issubclass(annotation, (str, bytes)): - return False - return lenient_issubclass(annotation, sequence_types) - - -def field_annotation_is_sequence(annotation: type[Any] | None) -> bool: - origin = get_origin(annotation) - - if origin is Annotated: - return field_annotation_is_sequence(get_args(annotation)[0]) - - if origin is Union or origin is UnionType: - for arg in get_args(annotation): - if field_annotation_is_sequence(arg): - return True - return False - return _annotation_is_sequence(annotation) or _annotation_is_sequence( - get_origin(annotation) - ) - - -def value_is_sequence(value: Any) -> bool: - return isinstance(value, sequence_types) and not isinstance(value, (str, bytes)) - - -def _annotation_is_complex(annotation: type[Any] | None) -> bool: - return ( - lenient_issubclass(annotation, (BaseModel, Mapping, UploadFile)) - or _annotation_is_sequence(annotation) - or is_dataclass(annotation) - ) - - -def field_annotation_is_complex(annotation: type[Any] | None) -> bool: - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - return any(field_annotation_is_complex(arg) for arg in get_args(annotation)) - - if origin is Annotated: - return field_annotation_is_complex(get_args(annotation)[0]) - - return ( - _annotation_is_complex(annotation) - or _annotation_is_complex(origin) - or hasattr(origin, "__pydantic_core_schema__") - or hasattr(origin, "__get_pydantic_core_schema__") - ) - - -def field_annotation_is_scalar(annotation: Any) -> bool: - # handle Ellipsis here to make tuple[int, ...] work nicely - return annotation is Ellipsis or not field_annotation_is_complex(annotation) - - -def field_annotation_is_scalar_sequence(annotation: type[Any] | None) -> bool: - origin = get_origin(annotation) - - if origin is Annotated: - return field_annotation_is_scalar_sequence(get_args(annotation)[0]) - - if origin is Union or origin is UnionType: - at_least_one_scalar_sequence = False - for arg in get_args(annotation): - if field_annotation_is_scalar_sequence(arg): - at_least_one_scalar_sequence = True - continue - elif not field_annotation_is_scalar(arg): - return False - return at_least_one_scalar_sequence - return field_annotation_is_sequence(annotation) and all( - field_annotation_is_scalar(sub_annotation) - for sub_annotation in get_args(annotation) - ) - - -def is_bytes_or_nonable_bytes_annotation(annotation: Any) -> bool: - if lenient_issubclass(annotation, bytes): - return True - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - for arg in get_args(annotation): - if lenient_issubclass(arg, bytes): - return True - return False - - -def is_uploadfile_or_nonable_uploadfile_annotation(annotation: Any) -> bool: - if lenient_issubclass(annotation, UploadFile): - return True - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - for arg in get_args(annotation): - if lenient_issubclass(arg, UploadFile): - return True - return False - - -def is_bytes_sequence_annotation(annotation: Any) -> bool: - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - at_least_one = False - for arg in get_args(annotation): - if is_bytes_sequence_annotation(arg): - at_least_one = True - continue - return at_least_one - return field_annotation_is_sequence(annotation) and all( - is_bytes_or_nonable_bytes_annotation(sub_annotation) - for sub_annotation in get_args(annotation) - ) - - -def is_uploadfile_sequence_annotation(annotation: Any) -> bool: - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - at_least_one = False - for arg in get_args(annotation): - if is_uploadfile_sequence_annotation(arg): - at_least_one = True - continue - return at_least_one - return field_annotation_is_sequence(annotation) and all( - is_uploadfile_or_nonable_uploadfile_annotation(sub_annotation) - for sub_annotation in get_args(annotation) - ) - - -def is_pydantic_v1_model_instance(obj: Any) -> bool: - # TODO: remove this function once the required version of Pydantic fully - # removes pydantic.v1 - try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) - from pydantic import v1 - except ImportError: # pragma: no cover - return False - return isinstance(obj, v1.BaseModel) - - -def is_pydantic_v1_model_class(cls: Any) -> bool: - # TODO: remove this function once the required version of Pydantic fully - # removes pydantic.v1 - try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) - from pydantic import v1 - except ImportError: # pragma: no cover - return False - return lenient_issubclass(cls, v1.BaseModel) - - -def annotation_is_pydantic_v1(annotation: Any) -> bool: - if is_pydantic_v1_model_class(annotation): - return True - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - for arg in get_args(annotation): - if is_pydantic_v1_model_class(arg): - return True - if field_annotation_is_sequence(annotation): - for sub_annotation in get_args(annotation): - if annotation_is_pydantic_v1(sub_annotation): - return True - return False diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/_compat/v2.py b/bundle/python-cpu/Lib/site-packages/fastapi/_compat/v2.py deleted file mode 100644 index 7be686d8655a9e7720ff0d751f82283cac20db98..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/_compat/v2.py +++ /dev/null @@ -1,493 +0,0 @@ -import re -import warnings -from collections.abc import Sequence -from copy import copy -from dataclasses import dataclass, is_dataclass -from enum import Enum -from functools import lru_cache -from typing import ( - Annotated, - Any, - Literal, - Union, - cast, - get_args, - get_origin, -) - -from fastapi._compat import lenient_issubclass, shared -from fastapi.openapi.constants import REF_TEMPLATE -from fastapi.types import IncEx, ModelNameMap, UnionType -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, create_model -from pydantic import PydanticSchemaGenerationError as PydanticSchemaGenerationError -from pydantic import PydanticUndefinedAnnotation as PydanticUndefinedAnnotation -from pydantic import ValidationError as ValidationError -from pydantic._internal import _typing_extra as _pydantic_typing_extra -from pydantic._internal._schema_generation_shared import ( # type: ignore[attr-defined] - GetJsonSchemaHandler as GetJsonSchemaHandler, -) -from pydantic.fields import FieldInfo as FieldInfo -from pydantic.json_schema import GenerateJsonSchema as _GenerateJsonSchema -from pydantic.json_schema import JsonSchemaValue as JsonSchemaValue -from pydantic_core import CoreSchema as CoreSchema -from pydantic_core import PydanticUndefined -from pydantic_core import Url as Url -from pydantic_core.core_schema import ( - with_info_plain_validator_function as with_info_plain_validator_function, -) - -RequiredParam = PydanticUndefined -Undefined = PydanticUndefined - - -def evaluate_forwardref( - value: Any, - globalns: dict[str, Any] | None = None, - localns: dict[str, Any] | None = None, -) -> Any: - # eval_type_lenient has been deprecated since Pydantic v2.10.0b1 (PR #10530) - try_eval_type = getattr(_pydantic_typing_extra, "try_eval_type", None) - if try_eval_type is not None: - return try_eval_type(value, globalns, localns)[0] - return _pydantic_typing_extra.eval_type_lenient( # ty: ignore[deprecated] - value, globalns, localns - ) - - -class GenerateJsonSchema(_GenerateJsonSchema): - # TODO: remove when this is merged (or equivalent): https://github.com/pydantic/pydantic/pull/12841 - # and dropping support for any version of Pydantic before that one (so, in a very long time) - def bytes_schema(self, schema: CoreSchema) -> JsonSchemaValue: - json_schema = {"type": "string", "contentMediaType": "application/octet-stream"} - bytes_mode = ( - self._config.ser_json_bytes - if self.mode == "serialization" - else self._config.val_json_bytes - ) - if bytes_mode == "base64": - json_schema["contentEncoding"] = "base64" - self.update_with_validations(json_schema, schema, self.ValidationsMapping.bytes) - return json_schema - - -# TODO: remove when dropping support for Pydantic < v2.12.3 -_Attrs = { - "default": ..., - "default_factory": None, - "alias": None, - "alias_priority": None, - "validation_alias": None, - "serialization_alias": None, - "title": None, - "field_title_generator": None, - "description": None, - "examples": None, - "exclude": None, - "exclude_if": None, - "discriminator": None, - "deprecated": None, - "json_schema_extra": None, - "frozen": None, - "validate_default": None, - "repr": True, - "init": None, - "init_var": None, - "kw_only": None, -} - - -# TODO: remove when dropping support for Pydantic < v2.12.3 -def asdict(field_info: FieldInfo) -> dict[str, Any]: - attributes = {} - for attr in _Attrs: - value = getattr(field_info, attr, Undefined) - if value is not Undefined: - attributes[attr] = value - return { - "annotation": field_info.annotation, - "metadata": field_info.metadata, - "attributes": attributes, - } - - -@dataclass -class ModelField: - field_info: FieldInfo - name: str - mode: Literal["validation", "serialization"] = "validation" - config: ConfigDict | None = None - - @property - def alias(self) -> str: - a = self.field_info.alias - return a if a is not None else self.name - - @property - def validation_alias(self) -> str | None: - va = self.field_info.validation_alias - if isinstance(va, str) and va: - return va - return None - - @property - def serialization_alias(self) -> str | None: - sa = self.field_info.serialization_alias - return sa or None - - @property - def default(self) -> Any: - return self.get_default() - - def __post_init__(self) -> None: - with warnings.catch_warnings(): - # Pydantic >= 2.12.0 warns about field specific metadata that is unused - # (e.g. `TypeAdapter(Annotated[int, Field(alias='b')])`). In some cases, we - # end up building the type adapter from a model field annotation so we - # need to ignore the warning: - if shared.PYDANTIC_VERSION_MINOR_TUPLE >= (2, 12): - from pydantic.warnings import UnsupportedFieldAttributeWarning - - warnings.simplefilter( - "ignore", category=UnsupportedFieldAttributeWarning - ) - # TODO: remove after setting the min Pydantic to v2.12.3 - # that adds asdict(), and use self.field_info.asdict() instead - field_dict = asdict(self.field_info) - annotated_args = ( - field_dict["annotation"], - *field_dict["metadata"], - # this FieldInfo needs to be created again so that it doesn't include - # the old field info metadata and only the rest of the attributes - Field(**field_dict["attributes"]), - ) - self._type_adapter: TypeAdapter[Any] = TypeAdapter( - Annotated[annotated_args], # ty: ignore[invalid-type-form] - config=self.config, - ) - - def get_default(self) -> Any: - if self.field_info.is_required(): - return Undefined - return self.field_info.get_default(call_default_factory=True) - - def validate( - self, - value: Any, - values: dict[str, Any] = {}, # noqa: B006 - *, - loc: tuple[int | str, ...] = (), - ) -> tuple[Any, list[dict[str, Any]]]: - try: - return ( - self._type_adapter.validate_python(value, from_attributes=True), - [], - ) - except ValidationError as exc: - return None, _regenerate_error_with_loc( - errors=exc.errors(include_url=False), loc_prefix=loc - ) - - def serialize( - self, - value: Any, - *, - mode: Literal["json", "python"] = "json", - include: IncEx | None = None, - exclude: IncEx | None = None, - by_alias: bool = True, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - ) -> Any: - # What calls this code passes a value that already called - # self._type_adapter.validate_python(value) - return self._type_adapter.dump_python( - value, - mode=mode, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - - def serialize_json( - self, - value: Any, - *, - include: IncEx | None = None, - exclude: IncEx | None = None, - by_alias: bool = True, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - ) -> bytes: - # What calls this code passes a value that already called - # self._type_adapter.validate_python(value) - # This uses Pydantic's dump_json() which serializes directly to JSON - # bytes in one pass (via Rust), avoiding the intermediate Python dict - # step of dump_python(mode="json") + json.dumps(). - return self._type_adapter.dump_json( - value, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - - def __hash__(self) -> int: - # Each ModelField is unique for our purposes, to allow making a dict from - # ModelField to its JSON Schema. - return id(self) - - -def _has_computed_fields(field: ModelField) -> bool: - computed_fields = field._type_adapter.core_schema.get("schema", {}).get( - "computed_fields", [] - ) - return len(computed_fields) > 0 - - -def get_schema_from_model_field( - *, - field: ModelField, - model_name_map: ModelNameMap, - field_mapping: dict[ - tuple[ModelField, Literal["validation", "serialization"]], JsonSchemaValue - ], - separate_input_output_schemas: bool = True, -) -> dict[str, Any]: - override_mode: Literal["validation"] | None = ( - None - if (separate_input_output_schemas or _has_computed_fields(field)) - else "validation" - ) - field_alias = ( - (field.validation_alias or field.alias) - if field.mode == "validation" - else (field.serialization_alias or field.alias) - ) - - # This expects that GenerateJsonSchema was already used to generate the definitions - json_schema = field_mapping[(field, override_mode or field.mode)] - if "$ref" not in json_schema: - # TODO remove when deprecating Pydantic v1 - # Ref: https://github.com/pydantic/pydantic/blob/d61792cc42c80b13b23e3ffa74bc37ec7c77f7d1/pydantic/schema.py#L207 - json_schema["title"] = field.field_info.title or field_alias.title().replace( - "_", " " - ) - return json_schema - - -def get_definitions( - *, - fields: Sequence[ModelField], - model_name_map: ModelNameMap, - separate_input_output_schemas: bool = True, -) -> tuple[ - dict[tuple[ModelField, Literal["validation", "serialization"]], JsonSchemaValue], - dict[str, dict[str, Any]], -]: - schema_generator = GenerateJsonSchema(ref_template=REF_TEMPLATE) - validation_fields = [field for field in fields if field.mode == "validation"] - serialization_fields = [field for field in fields if field.mode == "serialization"] - flat_validation_models = get_flat_models_from_fields( - validation_fields, known_models=set() - ) - flat_serialization_models = get_flat_models_from_fields( - serialization_fields, known_models=set() - ) - flat_validation_model_fields = [ - ModelField( - field_info=FieldInfo(annotation=model), - name=model.__name__, - mode="validation", - ) - for model in flat_validation_models - ] - flat_serialization_model_fields = [ - ModelField( - field_info=FieldInfo(annotation=model), - name=model.__name__, - mode="serialization", - ) - for model in flat_serialization_models - ] - flat_model_fields = flat_validation_model_fields + flat_serialization_model_fields - input_types = {f.field_info.annotation for f in fields} - unique_flat_model_fields = { - f for f in flat_model_fields if f.field_info.annotation not in input_types - } - inputs = [ - ( - field, - ( - field.mode - if (separate_input_output_schemas or _has_computed_fields(field)) - else "validation" - ), - field._type_adapter.core_schema, - ) - for field in list(fields) + list(unique_flat_model_fields) - ] - field_mapping, definitions = schema_generator.generate_definitions(inputs=inputs) - for item_def in cast(dict[str, dict[str, Any]], definitions).values(): - if "description" in item_def: - item_description = cast(str, item_def["description"]).split("\f")[0] - item_def["description"] = item_description - # definitions: dict[DefsRef, dict[str, Any]] - # but mypy complains about general str in other places that are not declared as - # DefsRef, although DefsRef is just str: - # DefsRef = NewType('DefsRef', str) - # So, a cast to simplify the types here - return field_mapping, cast(dict[str, dict[str, Any]], definitions) - - -def is_scalar_field(field: ModelField) -> bool: - from fastapi import params - - return shared.field_annotation_is_scalar( - field.field_info.annotation - ) and not isinstance(field.field_info, params.Body) - - -def copy_field_info(*, field_info: FieldInfo, annotation: Any) -> FieldInfo: - cls = type(field_info) - merged_field_info = cls.from_annotation(annotation) - new_field_info = copy(field_info) - new_field_info.metadata = merged_field_info.metadata - new_field_info.annotation = merged_field_info.annotation - return new_field_info - - -def serialize_sequence_value(*, field: ModelField, value: Any) -> Sequence[Any]: - origin_type = get_origin(field.field_info.annotation) or field.field_info.annotation - if origin_type is Union or origin_type is UnionType: # Handle optional sequences - union_args = get_args(field.field_info.annotation) - for union_arg in union_args: - if union_arg is type(None): - continue - origin_type = get_origin(union_arg) or union_arg - break - assert issubclass(origin_type, shared.sequence_types) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - return shared.sequence_annotation_to_type[origin_type](value) # type: ignore[no-any-return,index] # ty: ignore[invalid-return-type] - - -def get_missing_field_error(loc: tuple[int | str, ...]) -> dict[str, Any]: - error = ValidationError.from_exception_data( - "Field required", [{"type": "missing", "loc": loc, "input": {}}] - ).errors(include_url=False)[0] - error["input"] = None - return error # type: ignore[return-value] # ty: ignore[invalid-return-type] - - -def create_body_model( - *, fields: Sequence[ModelField], model_name: str -) -> type[BaseModel]: - field_params = {f.name: (f.field_info.annotation, f.field_info) for f in fields} - BodyModel: type[BaseModel] = create_model(model_name, **field_params) # type: ignore[call-overload] # ty: ignore[no-matching-overload] - return BodyModel - - -def get_model_fields(model: type[BaseModel]) -> list[ModelField]: - model_fields: list[ModelField] = [] - for name, field_info in model.model_fields.items(): - type_ = field_info.annotation - if lenient_issubclass(type_, (BaseModel, dict)) or is_dataclass(type_): - model_config = None - else: - model_config = model.model_config - model_fields.append( - ModelField( - field_info=field_info, - name=name, - config=model_config, - ) - ) - return model_fields - - -@lru_cache -def get_cached_model_fields(model: type[BaseModel]) -> list[ModelField]: - return get_model_fields(model) - - -# Duplicate of several schema functions from Pydantic v1 to make them compatible with -# Pydantic v2 and allow mixing the models - -TypeModelOrEnum = type["BaseModel"] | type[Enum] -TypeModelSet = set[TypeModelOrEnum] - - -def normalize_name(name: str) -> str: - return re.sub(r"[^a-zA-Z0-9.\-_]", "_", name) - - -def get_model_name_map(unique_models: TypeModelSet) -> dict[TypeModelOrEnum, str]: - name_model_map = {} - for model in unique_models: - model_name = normalize_name(model.__name__) - name_model_map[model_name] = model - return {v: k for k, v in name_model_map.items()} - - -def get_flat_models_from_model( - model: type["BaseModel"], known_models: TypeModelSet | None = None -) -> TypeModelSet: - known_models = known_models or set() - fields = get_model_fields(model) - get_flat_models_from_fields(fields, known_models=known_models) - return known_models - - -def get_flat_models_from_annotation( - annotation: Any, known_models: TypeModelSet -) -> TypeModelSet: - origin = get_origin(annotation) - if origin is not None: - for arg in get_args(annotation): - if lenient_issubclass(arg, (BaseModel, Enum)): - if arg not in known_models: - known_models.add(arg) # type: ignore[arg-type] - if lenient_issubclass(arg, BaseModel): - get_flat_models_from_model(arg, known_models=known_models) - else: - get_flat_models_from_annotation(arg, known_models=known_models) - return known_models - - -def get_flat_models_from_field( - field: ModelField, known_models: TypeModelSet -) -> TypeModelSet: - field_type = field.field_info.annotation - if lenient_issubclass(field_type, BaseModel): - if field_type in known_models: - return known_models - known_models.add(field_type) - get_flat_models_from_model(field_type, known_models=known_models) - elif lenient_issubclass(field_type, Enum): - known_models.add(field_type) - else: - get_flat_models_from_annotation(field_type, known_models=known_models) - return known_models - - -def get_flat_models_from_fields( - fields: Sequence[ModelField], known_models: TypeModelSet -) -> TypeModelSet: - for field in fields: - get_flat_models_from_field(field, known_models=known_models) - return known_models - - -def _regenerate_error_with_loc( - *, errors: Sequence[Any], loc_prefix: tuple[str | int, ...] -) -> list[dict[str, Any]]: - updated_loc_errors: list[Any] = [ - {**err, "loc": loc_prefix + err.get("loc", ())} for err in errors - ] - - return updated_loc_errors diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/applications.py b/bundle/python-cpu/Lib/site-packages/fastapi/applications.py deleted file mode 100644 index b5fc76d1607622648ccc0eff63a6b08e65d42afa..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/applications.py +++ /dev/null @@ -1,4774 +0,0 @@ -import os -from collections.abc import Awaitable, Callable, Coroutine, Sequence -from enum import Enum -from typing import Annotated, Any, Literal, TypeVar - -from annotated_doc import Doc -from fastapi import routing -from fastapi.datastructures import Default, DefaultPlaceholder -from fastapi.exception_handlers import ( - http_exception_handler, - request_validation_exception_handler, - websocket_request_validation_exception_handler, -) -from fastapi.exceptions import RequestValidationError, WebSocketRequestValidationError -from fastapi.logger import logger -from fastapi.middleware.asyncexitstack import AsyncExitStackMiddleware -from fastapi.openapi.docs import ( - get_redoc_html, - get_swagger_ui_html, - get_swagger_ui_oauth2_redirect_html, -) -from fastapi.openapi.utils import get_openapi -from fastapi.params import Depends -from fastapi.types import DecoratedCallable, IncEx -from fastapi.utils import generate_unique_id -from starlette.applications import Starlette -from starlette.datastructures import State -from starlette.exceptions import HTTPException -from starlette.middleware import Middleware -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.middleware.errors import ServerErrorMiddleware -from starlette.middleware.exceptions import ExceptionMiddleware -from starlette.requests import Request -from starlette.responses import HTMLResponse, JSONResponse, Response -from starlette.routing import BaseRoute -from starlette.types import ASGIApp, ExceptionHandler, Lifespan, Receive, Scope, Send -from typing_extensions import deprecated - -AppType = TypeVar("AppType", bound="FastAPI") - - -class FastAPI(Starlette): - """ - `FastAPI` app class, the main entrypoint to use FastAPI. - - Read more in the - [FastAPI docs for First Steps](https://fastapi.tiangolo.com/tutorial/first-steps/). - - ## Example - - ```python - from fastapi import FastAPI - - app = FastAPI() - ``` - """ - - def __init__( - self: AppType, - *, - debug: Annotated[ - bool, - Doc( - """ - Boolean indicating if debug tracebacks should be returned on server - errors. - - Read more in the - [Starlette docs for Applications](https://starlette.dev/applications/#starlette.applications.Starlette). - """ - ), - ] = False, - routes: Annotated[ - list[BaseRoute] | None, - Doc( - """ - **Note**: you probably shouldn't use this parameter, it is inherited - from Starlette and supported for compatibility. - - --- - - A list of routes to serve incoming HTTP and WebSocket requests. - """ - ), - deprecated( - """ - You normally wouldn't use this parameter with FastAPI, it is inherited - from Starlette and supported for compatibility. - - In FastAPI, you normally would use the *path operation methods*, - like `app.get()`, `app.post()`, etc. - """ - ), - ] = None, - title: Annotated[ - str, - Doc( - """ - The title of the API. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(title="ChimichangApp") - ``` - """ - ), - ] = "FastAPI", - summary: Annotated[ - str | None, - Doc( - """ - A short summary of the API. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(summary="Deadpond's favorite app. Nuff said.") - ``` - """ - ), - ] = None, - description: Annotated[ - str, - Doc( - ''' - A description of the API. Supports Markdown (using - [CommonMark syntax](https://commonmark.org/)). - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI( - description=""" - ChimichangApp API helps you do awesome stuff. 🚀 - - ## Items - - You can **read items**. - - ## Users - - You will be able to: - - * **Create users** (_not implemented_). - * **Read users** (_not implemented_). - - """ - ) - ``` - ''' - ), - ] = "", - version: Annotated[ - str, - Doc( - """ - The version of the API. - - **Note** This is the version of your application, not the version of - the OpenAPI specification nor the version of FastAPI being used. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(version="0.0.1") - ``` - """ - ), - ] = "0.1.0", - openapi_url: Annotated[ - str | None, - Doc( - """ - The URL where the OpenAPI schema will be served from. - - If you set it to `None`, no OpenAPI schema will be served publicly, and - the default automatic endpoints `/docs` and `/redoc` will also be - disabled. - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#openapi-url). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(openapi_url="/api/v1/openapi.json") - ``` - """ - ), - ] = "/openapi.json", - openapi_tags: Annotated[ - list[dict[str, Any]] | None, - Doc( - """ - A list of tags used by OpenAPI, these are the same `tags` you can set - in the *path operations*, like: - - * `@app.get("/users/", tags=["users"])` - * `@app.get("/items/", tags=["items"])` - - The order of the tags can be used to specify the order shown in - tools like Swagger UI, used in the automatic path `/docs`. - - It's not required to specify all the tags used. - - The tags that are not declared MAY be organized randomly or based - on the tools' logic. Each tag name in the list MUST be unique. - - The value of each item is a `dict` containing: - - * `name`: The name of the tag. - * `description`: A short description of the tag. - [CommonMark syntax](https://commonmark.org/) MAY be used for rich - text representation. - * `externalDocs`: Additional external documentation for this tag. If - provided, it would contain a `dict` with: - * `description`: A short description of the target documentation. - [CommonMark syntax](https://commonmark.org/) MAY be used for - rich text representation. - * `url`: The URL for the target documentation. Value MUST be in - the form of a URL. - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-tags). - - **Example** - - ```python - from fastapi import FastAPI - - tags_metadata = [ - { - "name": "users", - "description": "Operations with users. The **login** logic is also here.", - }, - { - "name": "items", - "description": "Manage items. So _fancy_ they have their own docs.", - "externalDocs": { - "description": "Items external docs", - "url": "https://fastapi.tiangolo.com/", - }, - }, - ] - - app = FastAPI(openapi_tags=tags_metadata) - ``` - """ - ), - ] = None, - servers: Annotated[ - list[dict[str, str | Any]] | None, - Doc( - """ - A `list` of `dict`s with connectivity information to a target server. - - You would use it, for example, if your application is served from - different domains and you want to use the same Swagger UI in the - browser to interact with each of them (instead of having multiple - browser tabs open). Or if you want to leave fixed the possible URLs. - - If the servers `list` is not provided, or is an empty `list`, the - `servers` property in the generated OpenAPI will be: - - * a `dict` with a `url` value of the application's mounting point - (`root_path`) if it's different from `/`. - * otherwise, the `servers` property will be omitted from the OpenAPI - schema. - - Each item in the `list` is a `dict` containing: - - * `url`: A URL to the target host. This URL supports Server Variables - and MAY be relative, to indicate that the host location is relative - to the location where the OpenAPI document is being served. Variable - substitutions will be made when a variable is named in `{`brackets`}`. - * `description`: An optional string describing the host designated by - the URL. [CommonMark syntax](https://commonmark.org/) MAY be used for - rich text representation. - * `variables`: A `dict` between a variable name and its value. The value - is used for substitution in the server's URL template. - - Read more in the - [FastAPI docs for Behind a Proxy](https://fastapi.tiangolo.com/advanced/behind-a-proxy/#additional-servers). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI( - servers=[ - {"url": "https://stag.example.com", "description": "Staging environment"}, - {"url": "https://prod.example.com", "description": "Production environment"}, - ] - ) - ``` - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of global dependencies, they will be applied to each - *path operation*, including in sub-routers. - - Read more about it in the - [FastAPI docs for Global Dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/global-dependencies/). - - **Example** - - ```python - from fastapi import Depends, FastAPI - - from .dependencies import func_dep_1, func_dep_2 - - app = FastAPI(dependencies=[Depends(func_dep_1), Depends(func_dep_2)]) - ``` - """ - ), - ] = None, - default_response_class: Annotated[ - type[Response], - Doc( - """ - The default response class to be used. - - Read more in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#default-response-class). - - **Example** - - ```python - from fastapi import FastAPI - from fastapi.responses import ORJSONResponse - - app = FastAPI(default_response_class=ORJSONResponse) - ``` - """ - ), - ] = Default(JSONResponse), - redirect_slashes: Annotated[ - bool, - Doc( - """ - Whether to detect and redirect slashes in URLs when the client doesn't - use the same format. - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(redirect_slashes=True) # the default - - @app.get("/items/") - async def read_items(): - return [{"item_id": "Foo"}] - ``` - - With this app, if a client goes to `/items` (without a trailing slash), - they will be automatically redirected with an HTTP status code of 307 - to `/items/`. - """ - ), - ] = True, - docs_url: Annotated[ - str | None, - Doc( - """ - The path to the automatic interactive API documentation. - It is handled in the browser by Swagger UI. - - The default URL is `/docs`. You can disable it by setting it to `None`. - - If `openapi_url` is set to `None`, this will be automatically disabled. - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#docs-urls). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(docs_url="/documentation", redoc_url=None) - ``` - """ - ), - ] = "/docs", - redoc_url: Annotated[ - str | None, - Doc( - """ - The path to the alternative automatic interactive API documentation - provided by ReDoc. - - The default URL is `/redoc`. You can disable it by setting it to `None`. - - If `openapi_url` is set to `None`, this will be automatically disabled. - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#docs-urls). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(docs_url="/documentation", redoc_url="redocumentation") - ``` - """ - ), - ] = "/redoc", - swagger_ui_oauth2_redirect_url: Annotated[ - str | None, - Doc( - """ - The OAuth2 redirect endpoint for the Swagger UI. - - By default it is `/docs/oauth2-redirect`. - - This is only used if you use OAuth2 (with the "Authorize" button) - with Swagger UI. - """ - ), - ] = "/docs/oauth2-redirect", - swagger_ui_init_oauth: Annotated[ - dict[str, Any] | None, - Doc( - """ - OAuth2 configuration for the Swagger UI, by default shown at `/docs`. - - Read more about the available configuration options in the - [Swagger UI docs](https://swagger.io/docs/open-source-tools/swagger-ui/usage/oauth2/). - """ - ), - ] = None, - middleware: Annotated[ - Sequence[Middleware] | None, - Doc( - """ - List of middleware to be added when creating the application. - - In FastAPI you would normally do this with `app.add_middleware()` - instead. - - Read more in the - [FastAPI docs for Middleware](https://fastapi.tiangolo.com/tutorial/middleware/). - """ - ), - ] = None, - exception_handlers: Annotated[ - dict[ - int | type[Exception], - Callable[[Request, Any], Coroutine[Any, Any, Response]], - ] - | None, - Doc( - """ - A dictionary with handlers for exceptions. - - In FastAPI, you would normally use the decorator - `@app.exception_handler()`. - - Read more in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/). - """ - ), - ] = None, - on_startup: Annotated[ - Sequence[Callable[[], Any]] | None, - Doc( - """ - A list of startup event handler functions. - - You should instead use the `lifespan` handlers. - - Read more in the [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - on_shutdown: Annotated[ - Sequence[Callable[[], Any]] | None, - Doc( - """ - A list of shutdown event handler functions. - - You should instead use the `lifespan` handlers. - - Read more in the - [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - lifespan: Annotated[ - Lifespan[AppType] | None, - Doc( - """ - A `Lifespan` context manager handler. This replaces `startup` and - `shutdown` functions with a single context manager. - - Read more in the - [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - terms_of_service: Annotated[ - str | None, - Doc( - """ - A URL to the Terms of Service for your API. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more at the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - app = FastAPI(terms_of_service="http://example.com/terms/") - ``` - """ - ), - ] = None, - contact: Annotated[ - dict[str, str | Any] | None, - Doc( - """ - A dictionary with the contact information for the exposed API. - - It can contain several fields. - - * `name`: (`str`) The name of the contact person/organization. - * `url`: (`str`) A URL pointing to the contact information. MUST be in - the format of a URL. - * `email`: (`str`) The email address of the contact person/organization. - MUST be in the format of an email address. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more at the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - app = FastAPI( - contact={ - "name": "Deadpoolio the Amazing", - "url": "http://x-force.example.com/contact/", - "email": "dp@x-force.example.com", - } - ) - ``` - """ - ), - ] = None, - license_info: Annotated[ - dict[str, str | Any] | None, - Doc( - """ - A dictionary with the license information for the exposed API. - - It can contain several fields. - - * `name`: (`str`) **REQUIRED** (if a `license_info` is set). The - license name used for the API. - * `identifier`: (`str`) An [SPDX](https://spdx.dev/) license expression - for the API. The `identifier` field is mutually exclusive of the `url` - field. Available since OpenAPI 3.1.0, FastAPI 0.99.0. - * `url`: (`str`) A URL to the license used for the API. This MUST be - the format of a URL. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more at the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - app = FastAPI( - license_info={ - "name": "Apache 2.0", - "url": "https://www.apache.org/licenses/LICENSE-2.0.html", - } - ) - ``` - """ - ), - ] = None, - openapi_prefix: Annotated[ - str, - Doc( - """ - A URL prefix for the OpenAPI URL. - """ - ), - deprecated( - """ - "openapi_prefix" has been deprecated in favor of "root_path", which - follows more closely the ASGI standard, is simpler, and more - automatic. - """ - ), - ] = "", - root_path: Annotated[ - str, - Doc( - """ - A path prefix handled by a proxy that is not seen by the application - but is seen by external clients, which affects things like Swagger UI. - - Read more about it at the - [FastAPI docs for Behind a Proxy](https://fastapi.tiangolo.com/advanced/behind-a-proxy/). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(root_path="/api/v1") - ``` - """ - ), - ] = "", - root_path_in_servers: Annotated[ - bool, - Doc( - """ - To disable automatically generating the URLs in the `servers` field - in the autogenerated OpenAPI using the `root_path`. - - Read more about it in the - [FastAPI docs for Behind a Proxy](https://fastapi.tiangolo.com/advanced/behind-a-proxy/#disable-automatic-server-from-root-path). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(root_path_in_servers=False) - ``` - """ - ), - ] = True, - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses to be shown in OpenAPI. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Additional Responses in OpenAPI](https://fastapi.tiangolo.com/advanced/additional-responses/). - - And in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - OpenAPI callbacks that should apply to all *path operations*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - webhooks: Annotated[ - routing.APIRouter | None, - Doc( - """ - Add OpenAPI webhooks. This is similar to `callbacks` but it doesn't - depend on specific *path operations*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - **Note**: This is available since OpenAPI 3.1.0, FastAPI 0.99.0. - - Read more about it in the - [FastAPI docs for OpenAPI Webhooks](https://fastapi.tiangolo.com/advanced/openapi-webhooks/). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark all *path operations* as deprecated. You probably don't need it, - but it's available. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#deprecate-a-path-operation). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) all the *path operations* in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - swagger_ui_parameters: Annotated[ - dict[str, Any] | None, - Doc( - """ - Parameters to configure Swagger UI, the autogenerated interactive API - documentation (by default at `/docs`). - - Read more about it in the - [FastAPI docs about how to Configure Swagger UI](https://fastapi.tiangolo.com/how-to/configure-swagger-ui/). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - separate_input_output_schemas: Annotated[ - bool, - Doc( - """ - Whether to generate separate OpenAPI schemas for request body and - response body when the results would be more precise. - - This is particularly useful when automatically generating clients. - - For example, if you have a model like: - - ```python - from pydantic import BaseModel - - class Item(BaseModel): - name: str - tags: list[str] = [] - ``` - - When `Item` is used for input, a request body, `tags` is not required, - the client doesn't have to provide it. - - But when using `Item` for output, for a response body, `tags` is always - available because it has a default value, even if it's just an empty - list. So, the client should be able to always expect it. - - In this case, there would be two different schemas, one for input and - another one for output. - - Read more about it in the - [FastAPI docs about how to separate schemas for input and output](https://fastapi.tiangolo.com/how-to/separate-openapi-schemas) - """ - ), - ] = True, - openapi_external_docs: Annotated[ - dict[str, Any] | None, - Doc( - """ - This field allows you to provide additional external documentation links. - If provided, it must be a dictionary containing: - - * `description`: A brief description of the external documentation. - * `url`: The URL pointing to the external documentation. The value **MUST** - be a valid URL format. - - **Example**: - - ```python - from fastapi import FastAPI - - external_docs = { - "description": "Detailed API Reference", - "url": "https://example.com/api-docs", - } - - app = FastAPI(openapi_external_docs=external_docs) - ``` - """ - ), - ] = None, - strict_content_type: Annotated[ - bool, - Doc( - """ - Enable strict checking for request Content-Type headers. - - When `True` (the default), requests with a body that do not include - a `Content-Type` header will **not** be parsed as JSON. - - This prevents potential cross-site request forgery (CSRF) attacks - that exploit the browser's ability to send requests without a - Content-Type header, bypassing CORS preflight checks. In particular - applicable for apps that need to be run locally (in localhost). - - When `False`, requests without a `Content-Type` header will have - their body parsed as JSON, which maintains compatibility with - certain clients that don't send `Content-Type` headers. - - Read more about it in the - [FastAPI docs for Strict Content-Type](https://fastapi.tiangolo.com/advanced/strict-content-type/). - """ - ), - ] = True, - **extra: Annotated[ - Any, - Doc( - """ - Extra keyword arguments to be stored in the app, not used by FastAPI - anywhere. - """ - ), - ], - ) -> None: - self.debug = debug - self.title = title - self.summary = summary - self.description = description - self.version = version - self.terms_of_service = terms_of_service - self.contact = contact - self.license_info = license_info - self.openapi_url = openapi_url - self.openapi_tags = openapi_tags - self.root_path_in_servers = root_path_in_servers - self.docs_url = docs_url - self.redoc_url = redoc_url - self.swagger_ui_oauth2_redirect_url = swagger_ui_oauth2_redirect_url - self.swagger_ui_init_oauth = swagger_ui_init_oauth - self.swagger_ui_parameters = swagger_ui_parameters - self.servers = servers or [] - self.separate_input_output_schemas = separate_input_output_schemas - self.openapi_external_docs = openapi_external_docs - self.extra = extra - self.openapi_version: Annotated[ - str, - Doc( - """ - The version string of OpenAPI. - - FastAPI will generate OpenAPI version 3.1.0, and will output that as - the OpenAPI version. But some tools, even though they might be - compatible with OpenAPI 3.1.0, might not recognize it as a valid. - - So you could override this value to trick those tools into using - the generated OpenAPI. Have in mind that this is a hack. But if you - avoid using features added in OpenAPI 3.1.0, it might work for your - use case. - - This is not passed as a parameter to the `FastAPI` class to avoid - giving the false idea that FastAPI would generate a different OpenAPI - schema. It is only available as an attribute. - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI() - - app.openapi_version = "3.0.2" - ``` - """ - ), - ] = "3.1.0" - self.openapi_schema: dict[str, Any] | None = None - self._openapi_routes_version: int | None = None - if self.openapi_url: - assert self.title, "A title must be provided for OpenAPI, e.g.: 'My API'" - assert self.version, "A version must be provided for OpenAPI, e.g.: '2.1.0'" - # TODO: remove when discarding the openapi_prefix parameter - if openapi_prefix: - logger.warning( - '"openapi_prefix" has been deprecated in favor of "root_path", which ' - "follows more closely the ASGI standard, is simpler, and more " - "automatic. Check the docs at " - "https://fastapi.tiangolo.com/advanced/sub-applications/" - ) - self.webhooks: Annotated[ - routing.APIRouter, - Doc( - """ - The `app.webhooks` attribute is an `APIRouter` with the *path - operations* that will be used just for documentation of webhooks. - - Read more about it in the - [FastAPI docs for OpenAPI Webhooks](https://fastapi.tiangolo.com/advanced/openapi-webhooks/). - """ - ), - ] = webhooks or routing.APIRouter() - self.root_path = root_path or openapi_prefix - self.state: Annotated[ - State, - Doc( - """ - A state object for the application. This is the same object for the - entire application, it doesn't change from request to request. - - You normally wouldn't use this in FastAPI, for most of the cases you - would instead use FastAPI dependencies. - - This is simply inherited from Starlette. - - Read more about it in the - [Starlette docs for Applications](https://starlette.dev/applications/#storing-state-on-the-app-instance). - """ - ), - ] = State() - self.dependency_overrides: Annotated[ - dict[Callable[..., Any], Callable[..., Any]], - Doc( - """ - A dictionary with overrides for the dependencies. - - Each key is the original dependency callable, and the value is the - actual dependency that should be called. - - This is for testing, to replace expensive dependencies with testing - versions. - - Read more about it in the - [FastAPI docs for Testing Dependencies with Overrides](https://fastapi.tiangolo.com/advanced/testing-dependencies/). - """ - ), - ] = {} - self.router: routing.APIRouter = routing.APIRouter( - routes=routes, - redirect_slashes=redirect_slashes, - dependency_overrides_provider=self, - on_startup=on_startup, - on_shutdown=on_shutdown, - lifespan=lifespan, - default_response_class=default_response_class, - dependencies=dependencies, - callbacks=callbacks, - deprecated=deprecated, - include_in_schema=include_in_schema, - responses=responses, - generate_unique_id_function=generate_unique_id_function, - strict_content_type=strict_content_type, - ) - self.exception_handlers: dict[ - Any, Callable[[Request, Any], Response | Awaitable[Response]] - ] = {} if exception_handlers is None else dict(exception_handlers) - self.exception_handlers.setdefault(HTTPException, http_exception_handler) - self.exception_handlers.setdefault( - RequestValidationError, request_validation_exception_handler - ) - - # Starlette still has incorrect type specification for the handlers - self.exception_handlers.setdefault( - WebSocketRequestValidationError, - websocket_request_validation_exception_handler, # type: ignore[arg-type] - ) # ty: ignore[no-matching-overload] - - self.user_middleware: list[Middleware] = ( - [] if middleware is None else list(middleware) - ) - self.middleware_stack: ASGIApp | None = None - self.setup() - - def build_middleware_stack(self) -> ASGIApp: - # Duplicate/override from Starlette to add AsyncExitStackMiddleware - # inside of ExceptionMiddleware, inside of custom user middlewares - debug = self.debug - error_handler = None - exception_handlers: dict[Any, ExceptionHandler] = {} - - for key, value in self.exception_handlers.items(): - if key in (500, Exception): - error_handler = value - else: - exception_handlers[key] = value - - middleware = ( - [Middleware(ServerErrorMiddleware, handler=error_handler, debug=debug)] - + self.user_middleware - + [ - Middleware( - ExceptionMiddleware, - handlers=exception_handlers, - debug=debug, - ), - # Add FastAPI-specific AsyncExitStackMiddleware for closing files. - # Before this was also used for closing dependencies with yield but - # those now have their own AsyncExitStack, to properly support - # streaming responses while keeping compatibility with the previous - # versions (as of writing 0.117.1) that allowed doing - # except HTTPException inside a dependency with yield. - # This needs to happen after user middlewares because those create a - # new contextvars context copy by using a new AnyIO task group. - # This AsyncExitStack preserves the context for contextvars, not - # strictly necessary for closing files but it was one of the original - # intentions. - # If the AsyncExitStack lived outside of the custom middlewares and - # contextvars were set, for example in a dependency with 'yield' - # in that internal contextvars context, the values would not be - # available in the outer context of the AsyncExitStack. - # By placing the middleware and the AsyncExitStack here, inside all - # user middlewares, the same context is used. - # This is currently not needed, only for closing files, but used to be - # important when dependencies with yield were closed here. - Middleware(AsyncExitStackMiddleware), - ] - ) - - app = self.router - for cls, args, kwargs in reversed(middleware): - app = cls(app, *args, **kwargs) - return app - - def openapi(self) -> dict[str, Any]: - """ - Generate the OpenAPI schema of the application. This is called by FastAPI - internally. - - The first time it is called it stores the result in the attribute - `app.openapi_schema`, and next times it is called, it just returns that same - result. To avoid the cost of generating the schema every time. - - If you need to modify the generated OpenAPI schema, you could modify it. - - Read more in the - [FastAPI docs for OpenAPI](https://fastapi.tiangolo.com/how-to/extending-openapi/). - """ - routes_version = self.router._get_routes_version() - if not self.openapi_schema or self._openapi_routes_version != routes_version: - self.openapi_schema = get_openapi( - title=self.title, - version=self.version, - openapi_version=self.openapi_version, - summary=self.summary, - description=self.description, - terms_of_service=self.terms_of_service, - contact=self.contact, - license_info=self.license_info, - routes=self.routes, - webhooks=self.webhooks.routes, - tags=self.openapi_tags, - servers=self.servers, - separate_input_output_schemas=self.separate_input_output_schemas, - external_docs=self.openapi_external_docs, - ) - self._openapi_routes_version = routes_version - return self.openapi_schema - - def setup(self) -> None: - if self.openapi_url: - - async def openapi(req: Request) -> JSONResponse: - root_path = req.scope.get("root_path", "").rstrip("/") - schema = self.openapi() - if root_path and self.root_path_in_servers: - server_urls = {s.get("url") for s in schema.get("servers", [])} - if root_path not in server_urls: - schema = dict(schema) - schema["servers"] = [{"url": root_path}] + schema.get( - "servers", [] - ) - return JSONResponse(schema) - - self.add_route(self.openapi_url, openapi, include_in_schema=False) - if self.openapi_url and self.docs_url: - - async def swagger_ui_html(req: Request) -> HTMLResponse: - root_path = req.scope.get("root_path", "").rstrip("/") - openapi_url = root_path + self.openapi_url - oauth2_redirect_url = self.swagger_ui_oauth2_redirect_url - if oauth2_redirect_url: - oauth2_redirect_url = root_path + oauth2_redirect_url - return get_swagger_ui_html( - openapi_url=openapi_url, - title=f"{self.title} - Swagger UI", - oauth2_redirect_url=oauth2_redirect_url, - init_oauth=self.swagger_ui_init_oauth, - swagger_ui_parameters=self.swagger_ui_parameters, - ) - - self.add_route(self.docs_url, swagger_ui_html, include_in_schema=False) - - if self.swagger_ui_oauth2_redirect_url: - - async def swagger_ui_redirect(req: Request) -> HTMLResponse: - return get_swagger_ui_oauth2_redirect_html() - - self.add_route( - self.swagger_ui_oauth2_redirect_url, - swagger_ui_redirect, - include_in_schema=False, - ) - if self.openapi_url and self.redoc_url: - - async def redoc_html(req: Request) -> HTMLResponse: - root_path = req.scope.get("root_path", "").rstrip("/") - openapi_url = root_path + self.openapi_url - return get_redoc_html( - openapi_url=openapi_url, title=f"{self.title} - ReDoc" - ) - - self.add_route(self.redoc_url, redoc_html, include_in_schema=False) - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if self.root_path: - scope["root_path"] = self.root_path - await super().__call__(scope, receive, send) - - def add_api_route( - self, - path: str, - endpoint: Callable[..., Any], - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - methods: list[str] | None = None, - operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] | DefaultPlaceholder = Default(JSONResponse), - name: str | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[[routing.APIRoute], str] = Default( - generate_unique_id - ), - ) -> None: - self.router.add_api_route( - path, - endpoint=endpoint, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=methods, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def frontend( - self, - path: Annotated[ - str, - Doc( - """ - The URL path prefix where the frontend build should be served. - """ - ), - ], - *, - directory: Annotated[ - str | os.PathLike[str], - Doc( - """ - The directory containing the static frontend build output. - """ - ), - ], - fallback: Annotated[ - Literal["auto", "index.html", "404.html"] | None, - Doc( - """ - The fallback file behavior for missing frontend paths. - """ - ), - ] = "auto", - check_dir: Annotated[ - bool | Literal["auto"], - Doc( - """ - Check that the frontend directory exists when the app is created. When - set to `"auto"`, skip the check with a warning when `FASTAPI_ENV` is - `"development"`, and check it otherwise. The `fastapi dev` command - sets `FASTAPI_ENV` to `"development"` if it is not already set. - """ - ), - ] = "auto", - ) -> None: - """ - Serve a static frontend build as low-priority routes. - - Use this for frontend tools that build static files into a directory, - such as `dist`. **FastAPI** path operations are checked first, and - the frontend files are checked only if no normal route matched. - - A typical project could look like this: - - ```text - . - ├── pyproject.toml - ├── app - │ ├── __init__.py - │ └── main.py - └── dist - ├── index.html - └── assets - └── app.js - ``` - - Then in `app/main.py`: - - ```python - from fastapi import FastAPI - - app = FastAPI() - app.frontend("/", directory="dist") - ``` - """ - check_dir = routing._resolve_frontend_check_dir( - directory=directory, check_dir=check_dir - ) - self.router.frontend( - path, - directory=directory, - fallback=fallback, - check_dir=check_dir, - ) - - def api_route( - self, - path: str, - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - methods: list[str] | None = None, - operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] = Default(JSONResponse), - name: str | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[[routing.APIRoute], str] = Default( - generate_unique_id - ), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.router.add_api_route( - path, - func, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=methods, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - return func - - return decorator - - def add_api_websocket_route( - self, - path: str, - endpoint: Callable[..., Any], - name: str | None = None, - *, - dependencies: Sequence[Depends] | None = None, - ) -> None: - self.router.add_api_websocket_route( - path, - endpoint, - name=name, - dependencies=dependencies, - ) - - def websocket( - self, - path: Annotated[ - str, - Doc( - """ - WebSocket path. - """ - ), - ], - name: Annotated[ - str | None, - Doc( - """ - A name for the WebSocket. Only used internally. - """ - ), - ] = None, - *, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be used for this - WebSocket. - - Read more about it in the - [FastAPI docs for WebSockets](https://fastapi.tiangolo.com/advanced/websockets/). - """ - ), - ] = None, - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Decorate a WebSocket function. - - Read more about it in the - [FastAPI docs for WebSockets](https://fastapi.tiangolo.com/advanced/websockets/). - - **Example** - - ```python - from fastapi import FastAPI, WebSocket - - app = FastAPI() - - @app.websocket("/ws") - async def websocket_endpoint(websocket: WebSocket): - await websocket.accept() - while True: - data = await websocket.receive_text() - await websocket.send_text(f"Message text was: {data}") - ``` - """ - - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_api_websocket_route( - path, - func, - name=name, - dependencies=dependencies, - ) - return func - - return decorator - - def include_router( - self, - router: Annotated[routing.APIRouter, Doc("The `APIRouter` to include.")], - *, - prefix: Annotated[str, Doc("An optional path prefix for the router.")] = "", - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to all the *path operations* in this - router. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to all the - *path operations* in this router. - - Read more about it in the - [FastAPI docs for Bigger Applications - Multiple Files](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - - **Example** - - ```python - from fastapi import Depends, FastAPI - - from .dependencies import get_token_header - from .internal import admin - - app = FastAPI() - - app.include_router( - admin.router, - dependencies=[Depends(get_token_header)], - ) - ``` - """ - ), - ] = None, - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses to be shown in OpenAPI. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Additional Responses in OpenAPI](https://fastapi.tiangolo.com/advanced/additional-responses/). - - And in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark all the *path operations* in this router as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - **Example** - - ```python - from fastapi import FastAPI - - from .internal import old_api - - app = FastAPI() - - app.include_router( - old_api.router, - deprecated=True, - ) - ``` - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include (or not) all the *path operations* in this router in the - generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - **Example** - - ```python - from fastapi import FastAPI - - from .internal import old_api - - app = FastAPI() - - app.include_router( - old_api.router, - include_in_schema=False, - ) - ``` - """ - ), - ] = True, - default_response_class: Annotated[ - type[Response], - Doc( - """ - Default response class to be used for the *path operations* in this - router. - - Read more in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#default-response-class). - - **Example** - - ```python - from fastapi import FastAPI - from fastapi.responses import ORJSONResponse - - from .internal import old_api - - app = FastAPI() - - app.include_router( - old_api.router, - default_response_class=ORJSONResponse, - ) - ``` - """ - ), - ] = Default(JSONResponse), - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> None: - """ - Include an `APIRouter` in the same app. - - Read more about it in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/). - - ## Example - - ```python - from fastapi import FastAPI - - from .users import users_router - - app = FastAPI() - - app.include_router(users_router) - ``` - """ - self.router.include_router( - router, - prefix=prefix, - tags=tags, - dependencies=dependencies, - responses=responses, - deprecated=deprecated, - include_in_schema=include_in_schema, - default_response_class=default_response_class, - callbacks=callbacks, - generate_unique_id_function=generate_unique_id_function, - ) - - def get( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP GET operation. - - ## Example - - ```python - from fastapi import FastAPI - - app = FastAPI() - - @app.get("/items/") - def read_items(): - return [{"name": "Empanada"}, {"name": "Arepa"}] - ``` - """ - return self.router.get( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def put( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP PUT operation. - - ## Example - - ```python - from fastapi import FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - - @app.put("/items/{item_id}") - def replace_item(item_id: str, item: Item): - return {"message": "Item replaced", "id": item_id} - ``` - """ - return self.router.put( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def post( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP POST operation. - - ## Example - - ```python - from fastapi import FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - - @app.post("/items/") - def create_item(item: Item): - return {"message": "Item created"} - ``` - """ - return self.router.post( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def delete( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP DELETE operation. - - ## Example - - ```python - from fastapi import FastAPI - - app = FastAPI() - - @app.delete("/items/{item_id}") - def delete_item(item_id: str): - return {"message": "Item deleted"} - ``` - """ - return self.router.delete( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def options( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP OPTIONS operation. - - ## Example - - ```python - from fastapi import FastAPI - - app = FastAPI() - - @app.options("/items/") - def get_item_options(): - return {"additions": ["Aji", "Guacamole"]} - ``` - """ - return self.router.options( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def head( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP HEAD operation. - - ## Example - - ```python - from fastapi import FastAPI, Response - - app = FastAPI() - - @app.head("/items/", status_code=204) - def get_items_headers(response: Response): - response.headers["X-Cat-Dog"] = "Alone in the world" - ``` - """ - return self.router.head( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def patch( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP PATCH operation. - - ## Example - - ```python - from fastapi import FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - - @app.patch("/items/") - def update_item(item: Item): - return {"message": "Item updated in place"} - ``` - """ - return self.router.patch( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def trace( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP TRACE operation. - - ## Example - - ```python - from fastapi import FastAPI - - app = FastAPI() - - @app.trace("/items/{item_id}") - def trace_item(item_id: str): - return None - ``` - """ - return self.router.trace( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def websocket_route( - self, path: str, name: str | None = None - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.router.add_websocket_route(path, func, name=name) - return func - - return decorator - - @deprecated( - """ - on_event is deprecated, use lifespan event handlers instead. - - Read more about it in the - [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/). - """ - ) - def on_event( - self, - event_type: Annotated[ - str, - Doc( - """ - The type of event. `startup` or `shutdown`. - """ - ), - ], - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add an event handler for the application. - - `on_event` is deprecated, use `lifespan` event handlers instead. - - Read more about it in the - [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/#alternative-events-deprecated). - """ - return self.router.on_event(event_type) # ty: ignore[deprecated] - - def middleware( - self, - middleware_type: Annotated[ - str, - Doc( - """ - The type of middleware. Currently only supports `http`. - """ - ), - ], - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a middleware to the application. - - Read more about it in the - [FastAPI docs for Middleware](https://fastapi.tiangolo.com/tutorial/middleware/). - - ## Example - - ```python - import time - from typing import Awaitable, Callable - - from fastapi import FastAPI, Request, Response - - app = FastAPI() - - - @app.middleware("http") - async def add_process_time_header( - request: Request, call_next: Callable[[Request], Awaitable[Response]] - ) -> Response: - start_time = time.time() - response = await call_next(request) - process_time = time.time() - start_time - response.headers["X-Process-Time"] = str(process_time) - return response - ``` - """ - - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_middleware(BaseHTTPMiddleware, dispatch=func) - return func - - return decorator - - def exception_handler( - self, - exc_class_or_status_code: Annotated[ - int | type[Exception], - Doc( - """ - The Exception class this would handle, or a status code. - """ - ), - ], - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add an exception handler to the app. - - Read more about it in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/). - - ## Example - - ```python - from fastapi import FastAPI, Request - from fastapi.responses import JSONResponse - - - class UnicornException(Exception): - def __init__(self, name: str): - self.name = name - - - app = FastAPI() - - - @app.exception_handler(UnicornException) - async def unicorn_exception_handler(request: Request, exc: UnicornException): - return JSONResponse( - status_code=418, - content={"message": f"Oops! {exc.name} did something. There goes a rainbow..."}, - ) - ``` - """ - - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_exception_handler(exc_class_or_status_code, func) - return func - - return decorator diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/background.py b/bundle/python-cpu/Lib/site-packages/fastapi/background.py deleted file mode 100644 index 7677058c438105d599233d441e8e7de81225a697..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/background.py +++ /dev/null @@ -1,61 +0,0 @@ -from collections.abc import Callable -from typing import Annotated, Any - -from annotated_doc import Doc -from starlette.background import BackgroundTasks as StarletteBackgroundTasks -from typing_extensions import ParamSpec - -P = ParamSpec("P") - - -class BackgroundTasks(StarletteBackgroundTasks): - """ - A collection of background tasks that will be called after a response has been - sent to the client. - - Read more about it in the - [FastAPI docs for Background Tasks](https://fastapi.tiangolo.com/tutorial/background-tasks/). - - ## Example - - ```python - from fastapi import BackgroundTasks, FastAPI - - app = FastAPI() - - - def write_notification(email: str, message=""): - with open("log.txt", mode="w") as email_file: - content = f"notification for {email}: {message}" - email_file.write(content) - - - @app.post("/send-notification/{email}") - async def send_notification(email: str, background_tasks: BackgroundTasks): - background_tasks.add_task(write_notification, email, message="some notification") - return {"message": "Notification sent in the background"} - ``` - """ - - def add_task( - self, - func: Annotated[ - Callable[P, Any], - Doc( - """ - The function to call after the response is sent. - - It can be a regular `def` function or an `async def` function. - """ - ), - ], - *args: P.args, - **kwargs: P.kwargs, - ) -> None: - """ - Add a function to be called in the background after the response is sent. - - Read more about it in the - [FastAPI docs for Background Tasks](https://fastapi.tiangolo.com/tutorial/background-tasks/). - """ - return super().add_task(func, *args, **kwargs) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/cli.py b/bundle/python-cpu/Lib/site-packages/fastapi/cli.py deleted file mode 100644 index 8d3301e9daf73b474162d712f4b87f54ccd97a16..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/cli.py +++ /dev/null @@ -1,13 +0,0 @@ -try: - from fastapi_cli.cli import main as cli_main - -except ImportError: # pragma: no cover - cli_main = None # type: ignore - - -def main() -> None: - if not cli_main: # type: ignore[truthy-function] - message = 'To use the fastapi command, please install "fastapi[standard]":\n\n\tpip install "fastapi[standard]"\n' - print(message) - raise RuntimeError(message) # noqa: B904 - cli_main() diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/concurrency.py b/bundle/python-cpu/Lib/site-packages/fastapi/concurrency.py deleted file mode 100644 index 76a5a2eb128bc8c22d61d986ab2cdc434178dd24..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/concurrency.py +++ /dev/null @@ -1,41 +0,0 @@ -from collections.abc import AsyncGenerator -from contextlib import AbstractContextManager -from contextlib import asynccontextmanager as asynccontextmanager -from typing import TypeVar - -import anyio.to_thread -from anyio import CapacityLimiter -from starlette.concurrency import iterate_in_threadpool as iterate_in_threadpool # noqa -from starlette.concurrency import run_in_threadpool as run_in_threadpool # noqa -from starlette.concurrency import ( # noqa - run_until_first_complete as run_until_first_complete, -) - -_T = TypeVar("_T") - - -@asynccontextmanager -async def contextmanager_in_threadpool( - cm: AbstractContextManager[_T], -) -> AsyncGenerator[_T, None]: - # blocking __exit__ from running waiting on a free thread - # can create race conditions/deadlocks if the context manager itself - # has its own internal pool (e.g. a database connection pool) - # to avoid this we let __exit__ run without a capacity limit - # since we're creating a new limiter for each call, any non-zero limit - # works (1 is arbitrary) - exit_limiter = CapacityLimiter(1) - try: - yield await run_in_threadpool(cm.__enter__) - except Exception as e: - ok = bool( - await anyio.to_thread.run_sync( - cm.__exit__, type(e), e, e.__traceback__, limiter=exit_limiter - ) - ) - if not ok: - raise e - else: - await anyio.to_thread.run_sync( - cm.__exit__, None, None, None, limiter=exit_limiter - ) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/datastructures.py b/bundle/python-cpu/Lib/site-packages/fastapi/datastructures.py deleted file mode 100644 index 1da784cf0927ed55ec6abeb051d89a6ce1e90630..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/datastructures.py +++ /dev/null @@ -1,186 +0,0 @@ -from collections.abc import Callable, Mapping -from typing import ( - Annotated, - Any, - BinaryIO, - TypeVar, - cast, -) - -from annotated_doc import Doc -from pydantic import GetJsonSchemaHandler -from starlette.datastructures import URL as URL # noqa: F401 -from starlette.datastructures import Address as Address # noqa: F401 -from starlette.datastructures import FormData as FormData # noqa: F401 -from starlette.datastructures import Headers as Headers # noqa: F401 -from starlette.datastructures import QueryParams as QueryParams # noqa: F401 -from starlette.datastructures import State as State # noqa: F401 -from starlette.datastructures import UploadFile as StarletteUploadFile - - -class UploadFile(StarletteUploadFile): - """ - A file uploaded in a request. - - Define it as a *path operation function* (or dependency) parameter. - - If you are using a regular `def` function, you can use the `upload_file.file` - attribute to access the raw standard Python file (blocking, not async), useful and - needed for non-async code. - - Read more about it in the - [FastAPI docs for Request Files](https://fastapi.tiangolo.com/tutorial/request-files/). - - ## Example - - ```python - from typing import Annotated - - from fastapi import FastAPI, File, UploadFile - - app = FastAPI() - - - @app.post("/files/") - async def create_file(file: Annotated[bytes, File()]): - return {"file_size": len(file)} - - - @app.post("/uploadfile/") - async def create_upload_file(file: UploadFile): - return {"filename": file.filename} - ``` - """ - - file: Annotated[ - BinaryIO, - Doc("The standard Python file object (non-async)."), - ] - filename: Annotated[str | None, Doc("The original file name.")] - size: Annotated[int | None, Doc("The size of the file in bytes.")] - headers: Annotated[Headers, Doc("The headers of the request.")] - content_type: Annotated[ - str | None, Doc("The content type of the request, from the headers.") - ] - - async def write( - self, - data: Annotated[ - bytes, - Doc( - """ - The bytes to write to the file. - """ - ), - ], - ) -> None: - """ - Write some bytes to the file. - - You normally wouldn't use this from a file you read in a request. - - To be awaitable, compatible with async, this is run in threadpool. - """ - return await super().write(data) - - async def read( - self, - size: Annotated[ - int, - Doc( - """ - The number of bytes to read from the file. - """ - ), - ] = -1, - ) -> bytes: - """ - Read some bytes from the file. - - To be awaitable, compatible with async, this is run in threadpool. - """ - return await super().read(size) - - async def seek( - self, - offset: Annotated[ - int, - Doc( - """ - The position in bytes to seek to in the file. - """ - ), - ], - ) -> None: - """ - Move to a position in the file. - - Any next read or write will be done from that position. - - To be awaitable, compatible with async, this is run in threadpool. - """ - return await super().seek(offset) - - async def close(self) -> None: - """ - Close the file. - - To be awaitable, compatible with async, this is run in threadpool. - """ - return await super().close() - - @classmethod - def _validate(cls, __input_value: Any, _: Any) -> "UploadFile": - if not isinstance(__input_value, StarletteUploadFile): - raise ValueError(f"Expected UploadFile, received: {type(__input_value)}") - return cast(UploadFile, __input_value) - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: Mapping[str, Any], handler: GetJsonSchemaHandler - ) -> dict[str, Any]: - return {"type": "string", "contentMediaType": "application/octet-stream"} - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: Callable[[Any], Mapping[str, Any]] - ) -> Mapping[str, Any]: - from ._compat.v2 import with_info_plain_validator_function - - return with_info_plain_validator_function(cls._validate) - - -class DefaultPlaceholder: - """ - You shouldn't use this class directly. - - It's used internally to recognize when a default value has been overwritten, even - if the overridden default value was truthy. - """ - - def __init__(self, value: Any): - self.value = value - - def __bool__(self) -> bool: - return bool(self.value) - - def __eq__(self, o: object) -> bool: - return isinstance(o, DefaultPlaceholder) and o.value == self.value - - -DefaultType = TypeVar("DefaultType") - - -def Default(value: DefaultType) -> DefaultType: - """ - You shouldn't use this function directly. - - It's used internally to recognize when a default value has been overwritten, even - if the overridden default value was truthy. - """ - return DefaultPlaceholder(value) # type: ignore - - -# Sentinel for "parameter not provided" in Param/FieldInfo. -# Typed as None to satisfy ty -_Unset = Default(None) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/dependencies/__init__.py b/bundle/python-cpu/Lib/site-packages/fastapi/dependencies/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/dependencies/models.py b/bundle/python-cpu/Lib/site-packages/fastapi/dependencies/models.py deleted file mode 100644 index 5c7cbc82ba12e30d18887f170ca1704b59c8f915..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/dependencies/models.py +++ /dev/null @@ -1,234 +0,0 @@ -import inspect -import sys -from collections.abc import Callable -from dataclasses import dataclass, field -from functools import lru_cache, partial -from typing import Any, Literal - -from fastapi._compat import ModelField -from fastapi.security.base import SecurityBase -from fastapi.types import DependencyCacheKey - -if sys.version_info >= (3, 13): # pragma: no cover - from inspect import iscoroutinefunction -else: # pragma: no cover - from asyncio import iscoroutinefunction - - -def _unwrapped_call(call: Callable[..., Any] | None) -> Any: - if call is None: - return call # pragma: no cover - unwrapped = inspect.unwrap(_impartial(call)) - return unwrapped - - -def _impartial(func: Callable[..., Any]) -> Callable[..., Any]: - while isinstance(func, partial): - func = func.func - return func - - -@dataclass(slots=True) -class Dependant: - path_params: list[ModelField] = field(default_factory=list) - query_params: list[ModelField] = field(default_factory=list) - header_params: list[ModelField] = field(default_factory=list) - cookie_params: list[ModelField] = field(default_factory=list) - body_params: list[ModelField] = field(default_factory=list) - dependencies: list["Dependant"] = field(default_factory=list) - name: str | None = None - call: Callable[..., Any] | None = None - request_param_name: str | None = None - websocket_param_name: str | None = None - http_connection_param_name: str | None = None - response_param_name: str | None = None - background_tasks_param_name: str | None = None - security_scopes_param_name: str | None = None - own_oauth_scopes: list[str] | None = None - parent_oauth_scopes: list[str] | None = None - use_cache: bool = True - path: str | None = None - scope: Literal["function", "request"] | None = None - - -_UsesScopesCache = dict[int, tuple[Dependant, bool]] -_CALLABLE_CLASSIFICATION_CACHE_SIZE = 4096 - - -class _CallIdentity: - __slots__ = ("call",) - - def __init__(self, call: Callable[..., Any]) -> None: - self.call = call - - def __hash__(self) -> int: - return id(self.call) - - def __eq__(self, other: object) -> bool: - return isinstance(other, _CallIdentity) and self.call is other.call - - -def _get_oauth_scopes(*, dependant: Dependant) -> list[str]: - scopes = ( - dependant.parent_oauth_scopes.copy() if dependant.parent_oauth_scopes else [] - ) - # This doesn't use a set to preserve order, just in case - for scope in dependant.own_oauth_scopes or []: - if scope not in scopes: - scopes.append(scope) - return scopes - - -def _get_cache_key( - *, - dependant: Dependant, - uses_scopes_cache: _UsesScopesCache | None = None, -) -> DependencyCacheKey: - scopes_for_cache = ( - tuple(sorted(set(_get_oauth_scopes(dependant=dependant)))) - if _uses_scopes(dependant=dependant, cache=uses_scopes_cache) - else () - ) - return ( - dependant.call, - scopes_for_cache, - _get_computed_scope(dependant=dependant) or "", - ) - - -def _uses_scopes( - *, dependant: Dependant, cache: _UsesScopesCache | None = None -) -> bool: - if cache is None: - cache = {} - cache_key = id(dependant) - cached = cache.get(cache_key) - if cached is not None and cached[0] is dependant: - return cached[1] - if dependant.own_oauth_scopes: - result = True - elif dependant.security_scopes_param_name is not None: - result = True - elif _is_security_scheme(dependant=dependant): - result = True - else: - result = any( - _uses_scopes(dependant=sub_dep, cache=cache) - for sub_dep in dependant.dependencies - ) - cache[cache_key] = (dependant, result) - return result - - -def _is_security_scheme(*, dependant: Dependant) -> bool: - if dependant.call is None: - return False # pragma: no cover - unwrapped = _unwrapped_call(dependant.call) - return isinstance(unwrapped, SecurityBase) - - -def _get_security_scheme(*, dependant: Dependant) -> SecurityBase: - # Mainly to get the type of SecurityBase, but it's the same dependant.call - unwrapped = _unwrapped_call(dependant.call) - assert isinstance(unwrapped, SecurityBase) - return unwrapped - - -@lru_cache(maxsize=_CALLABLE_CLASSIFICATION_CACHE_SIZE) -def _is_gen_callable_cached(call_identity: _CallIdentity) -> bool: - call = call_identity.call - if inspect.isgeneratorfunction(_impartial(call)) or inspect.isgeneratorfunction( - _unwrapped_call(call) - ): - return True - if inspect.isclass(_unwrapped_call(call)): - return False - dunder_call = getattr(_impartial(call), "__call__", None) # noqa: B004 - if dunder_call is None: - return False # pragma: no cover - if inspect.isgeneratorfunction( - _impartial(dunder_call) - ) or inspect.isgeneratorfunction(_unwrapped_call(dunder_call)): - return True - dunder_unwrapped_call = getattr(_unwrapped_call(call), "__call__", None) # noqa: B004 - if dunder_unwrapped_call is None: - return False # pragma: no cover - return inspect.isgeneratorfunction( - _impartial(dunder_unwrapped_call) - ) or inspect.isgeneratorfunction(_unwrapped_call(dunder_unwrapped_call)) - - -def _is_gen_callable(call: Callable[..., Any] | None) -> bool: - if call is None: - return False # pragma: no cover - return _is_gen_callable_cached(_CallIdentity(call)) - - -@lru_cache(maxsize=_CALLABLE_CLASSIFICATION_CACHE_SIZE) -def _is_async_gen_callable_cached(call_identity: _CallIdentity) -> bool: - call = call_identity.call - if inspect.isasyncgenfunction(_impartial(call)) or inspect.isasyncgenfunction( - _unwrapped_call(call) - ): - return True - if inspect.isclass(_unwrapped_call(call)): - return False - dunder_call = getattr(_impartial(call), "__call__", None) # noqa: B004 - if dunder_call is None: - return False # pragma: no cover - if inspect.isasyncgenfunction( - _impartial(dunder_call) - ) or inspect.isasyncgenfunction(_unwrapped_call(dunder_call)): - return True - dunder_unwrapped_call = getattr(_unwrapped_call(call), "__call__", None) # noqa: B004 - if dunder_unwrapped_call is None: - return False # pragma: no cover - return inspect.isasyncgenfunction( - _impartial(dunder_unwrapped_call) - ) or inspect.isasyncgenfunction(_unwrapped_call(dunder_unwrapped_call)) - - -def _is_async_gen_callable(call: Callable[..., Any] | None) -> bool: - if call is None: - return False # pragma: no cover - return _is_async_gen_callable_cached(_CallIdentity(call)) - - -@lru_cache(maxsize=_CALLABLE_CLASSIFICATION_CACHE_SIZE) -def _is_coroutine_callable_cached(call_identity: _CallIdentity) -> bool: - call = call_identity.call - if inspect.isroutine(_impartial(call)) and iscoroutinefunction(_impartial(call)): - return True - if inspect.isroutine(_unwrapped_call(call)) and iscoroutinefunction( - _unwrapped_call(call) - ): - return True - if inspect.isclass(_unwrapped_call(call)): - return False - dunder_call = getattr(_impartial(call), "__call__", None) # noqa: B004 - if dunder_call is None: - return False # pragma: no cover - if iscoroutinefunction(_impartial(dunder_call)) or iscoroutinefunction( - _unwrapped_call(dunder_call) - ): - return True - dunder_unwrapped_call = getattr(_unwrapped_call(call), "__call__", None) # noqa: B004 - if dunder_unwrapped_call is None: - return False # pragma: no cover - return iscoroutinefunction( - _impartial(dunder_unwrapped_call) - ) or iscoroutinefunction(_unwrapped_call(dunder_unwrapped_call)) - - -def _is_coroutine_callable(call: Callable[..., Any] | None) -> bool: - if call is None: - return False # pragma: no cover - return _is_coroutine_callable_cached(_CallIdentity(call)) - - -def _get_computed_scope(*, dependant: Dependant) -> str | None: - if dependant.scope: - return dependant.scope - if _is_gen_callable(dependant.call) or _is_async_gen_callable(dependant.call): - return "request" - return None diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/dependencies/utils.py b/bundle/python-cpu/Lib/site-packages/fastapi/dependencies/utils.py deleted file mode 100644 index 01820e6a4cf477a22fffeaad1b308489d6a4a0a8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/dependencies/utils.py +++ /dev/null @@ -1,1053 +0,0 @@ -import dataclasses -import inspect -import sys -from collections.abc import ( - AsyncGenerator, - AsyncIterable, - AsyncIterator, - Callable, - Generator, - Iterable, - Iterator, - Mapping, - Sequence, -) -from contextlib import AsyncExitStack, contextmanager -from copy import copy, deepcopy -from dataclasses import dataclass -from typing import ( - Annotated, - Any, - ForwardRef, - Literal, - Union, - cast, - get_args, - get_origin, -) - -from fastapi import params -from fastapi._compat import ( - ModelField, - RequiredParam, - Undefined, - copy_field_info, - create_body_model, - evaluate_forwardref, - field_annotation_is_scalar, - field_annotation_is_scalar_sequence, - field_annotation_is_sequence, - get_cached_model_fields, - get_missing_field_error, - is_bytes_or_nonable_bytes_annotation, - is_bytes_sequence_annotation, - is_scalar_field, - is_uploadfile_or_nonable_uploadfile_annotation, - is_uploadfile_sequence_annotation, - lenient_issubclass, - sequence_types, - serialize_sequence_value, - value_is_sequence, -) -from fastapi.background import BackgroundTasks -from fastapi.concurrency import ( - asynccontextmanager, - contextmanager_in_threadpool, -) -from fastapi.dependencies.models import ( - Dependant, - _get_cache_key, - _get_computed_scope, - _get_oauth_scopes, - _is_async_gen_callable, - _is_coroutine_callable, - _is_gen_callable, - _UsesScopesCache, -) -from fastapi.exceptions import DependencyScopeError -from fastapi.logger import logger -from fastapi.security.oauth2 import SecurityScopes -from fastapi.types import DependencyCacheKey -from fastapi.utils import create_model_field, get_path_param_names -from pydantic import BaseModel, Json -from pydantic.fields import FieldInfo -from starlette.background import BackgroundTasks as StarletteBackgroundTasks -from starlette.concurrency import run_in_threadpool -from starlette.datastructures import ( - FormData, - Headers, - ImmutableMultiDict, - QueryParams, - UploadFile, -) -from starlette.requests import HTTPConnection, Request -from starlette.responses import Response -from starlette.websockets import WebSocket -from typing_inspection.typing_objects import is_typealiastype - -multipart_not_installed_error = ( - 'Form data requires "python-multipart" to be installed. \n' - 'You can install "python-multipart" with: \n\n' - "pip install python-multipart\n" -) -multipart_incorrect_install_error = ( - 'Form data requires "python-multipart" to be installed. ' - 'It seems you installed "multipart" instead. \n' - 'You can remove "multipart" with: \n\n' - "pip uninstall multipart\n\n" - 'And then install "python-multipart" with: \n\n' - "pip install python-multipart\n" -) - - -def ensure_multipart_is_installed() -> None: - try: - from python_multipart import __version__ - - # Import an attribute that can be mocked/deleted in testing - assert __version__ > "0.0.12" - except (ImportError, AssertionError): - try: - # __version__ is available in both multiparts, and can be mocked - from multipart import ( # type: ignore[no-redef,import-untyped] - __version__, - ) - - assert __version__ - try: - # parse_options_header is only available in the right multipart - from multipart.multipart import ( # type: ignore[import-untyped] - parse_options_header, - ) - - assert parse_options_header - except ImportError: - logger.error(multipart_incorrect_install_error) - raise RuntimeError(multipart_incorrect_install_error) from None - except ImportError: - logger.error(multipart_not_installed_error) - raise RuntimeError(multipart_not_installed_error) from None - - -def get_parameterless_sub_dependant(*, depends: params.Depends, path: str) -> Dependant: - assert callable(depends.dependency), ( - "A parameter-less dependency must have a callable dependency" - ) - own_oauth_scopes: list[str] = [] - if isinstance(depends, params.Security) and depends.scopes: - own_oauth_scopes.extend(depends.scopes) - return get_dependant( - path=path, - call=depends.dependency, - scope=depends.scope, - own_oauth_scopes=own_oauth_scopes, - ) - - -def _get_flat_body_params(dependant: Dependant) -> list[ModelField]: - body_params: list[ModelField] = [] - dependants = [dependant] - while dependants: - current_dependant = dependants.pop() - body_params.extend(current_dependant.body_params) - dependants.extend(reversed(current_dependant.dependencies)) - return body_params - - -def _get_flat_fields_from_params(fields: list[ModelField]) -> list[ModelField]: - if not fields: - return fields - first_field = fields[0] - if len(fields) == 1 and lenient_issubclass( - first_field.field_info.annotation, BaseModel - ): - fields_to_extract = get_cached_model_fields(first_field.field_info.annotation) - return fields_to_extract - return fields - - -def get_flat_params(dependant: Dependant) -> list[ModelField]: - path_params: list[ModelField] = [] - query_params: list[ModelField] = [] - header_params: list[ModelField] = [] - cookie_params: list[ModelField] = [] - visited: list[DependencyCacheKey] = [] - uses_scopes_cache: _UsesScopesCache = {} - dependants = [dependant] - while dependants: - current_dependant = dependants.pop() - cache_key = _get_cache_key( - dependant=current_dependant, - uses_scopes_cache=uses_scopes_cache, - ) - if cache_key in visited: - continue - visited.append(cache_key) - path_params.extend(current_dependant.path_params) - query_params.extend(current_dependant.query_params) - header_params.extend(current_dependant.header_params) - cookie_params.extend(current_dependant.cookie_params) - dependants.extend(reversed(current_dependant.dependencies)) - path_params = _get_flat_fields_from_params(path_params) - query_params = _get_flat_fields_from_params(query_params) - header_params = _get_flat_fields_from_params(header_params) - cookie_params = _get_flat_fields_from_params(cookie_params) - return path_params + query_params + header_params + cookie_params - - -def _get_signature(call: Callable[..., Any]) -> inspect.Signature: - try: - signature = inspect.signature(call, eval_str=True) - except NameError: - # Handle type annotations with if TYPE_CHECKING, not used by FastAPI - # e.g. dependency return types - if sys.version_info >= (3, 14): - from annotationlib import Format - - signature = inspect.signature(call, annotation_format=Format.FORWARDREF) - else: - signature = inspect.signature(call) - return signature - - -def get_typed_signature(call: Callable[..., Any]) -> inspect.Signature: - signature = _get_signature(call) - unwrapped = inspect.unwrap(call) - globalns = getattr(unwrapped, "__globals__", {}) - typed_params = [ - inspect.Parameter( - name=param.name, - kind=param.kind, - default=param.default, - annotation=get_typed_annotation(param.annotation, globalns), - ) - for param in signature.parameters.values() - ] - typed_signature = inspect.Signature(typed_params) - return typed_signature - - -def get_typed_annotation(annotation: Any, globalns: dict[str, Any]) -> Any: - if isinstance(annotation, str): - annotation = ForwardRef(annotation) - annotation = evaluate_forwardref(annotation, globalns, globalns) - if annotation is type(None): - return None - return annotation - - -def get_typed_return_annotation(call: Callable[..., Any]) -> Any: - signature = _get_signature(call) - unwrapped = inspect.unwrap(call) - annotation = signature.return_annotation - - if annotation is inspect.Signature.empty: - return None - - globalns = getattr(unwrapped, "__globals__", {}) - return get_typed_annotation(annotation, globalns) - - -_STREAM_ORIGINS = { - AsyncIterable, - AsyncIterator, - AsyncGenerator, - Iterable, - Iterator, - Generator, -} - - -def get_stream_item_type(annotation: Any) -> Any | None: - origin = get_origin(annotation) - if origin is not None and origin in _STREAM_ORIGINS: - type_args = get_args(annotation) - if type_args: - return type_args[0] - return Any - return None - - -def get_dependant( - *, - path: str, - call: Callable[..., Any], - name: str | None = None, - own_oauth_scopes: list[str] | None = None, - parent_oauth_scopes: list[str] | None = None, - use_cache: bool = True, - scope: Literal["function", "request"] | None = None, -) -> Dependant: - dependant = Dependant( - call=call, - name=name, - path=path, - use_cache=use_cache, - scope=scope, - own_oauth_scopes=own_oauth_scopes, - parent_oauth_scopes=parent_oauth_scopes, - ) - current_scopes = (parent_oauth_scopes or []) + (own_oauth_scopes or []) - path_param_names = get_path_param_names(path) - endpoint_signature = get_typed_signature(call) - signature_params = endpoint_signature.parameters - for param_name, param in signature_params.items(): - is_path_param = param_name in path_param_names - param_details = analyze_param( - param_name=param_name, - annotation=param.annotation, - value=param.default, - is_path_param=is_path_param, - ) - if param_details.depends is not None: - assert param_details.depends.dependency - if ( - ( - _is_gen_callable(dependant.call) - or _is_async_gen_callable(dependant.call) - ) - and _get_computed_scope(dependant=dependant) == "request" - and param_details.depends.scope == "function" - ): - assert dependant.call - call_name = getattr(dependant.call, "__name__", "") - raise DependencyScopeError( - f'The dependency "{call_name}" has a scope of ' - '"request", it cannot depend on dependencies with scope "function".' - ) - sub_own_oauth_scopes: list[str] = [] - if isinstance(param_details.depends, params.Security): - if param_details.depends.scopes: - sub_own_oauth_scopes = list(param_details.depends.scopes) - sub_dependant = get_dependant( - path=path, - call=param_details.depends.dependency, - name=param_name, - own_oauth_scopes=sub_own_oauth_scopes, - parent_oauth_scopes=current_scopes, - use_cache=param_details.depends.use_cache, - scope=param_details.depends.scope, - ) - dependant.dependencies.append(sub_dependant) - continue - if add_non_field_param_to_dependency( - param_name=param_name, - type_annotation=param_details.type_annotation, - dependant=dependant, - ): - assert param_details.field is None, ( - f"Cannot specify multiple FastAPI annotations for {param_name!r}" - ) - continue - assert param_details.field is not None - if isinstance(param_details.field.field_info, params.Body): - dependant.body_params.append(param_details.field) - else: - add_param_to_fields(field=param_details.field, dependant=dependant) - return dependant - - -def add_non_field_param_to_dependency( - *, param_name: str, type_annotation: Any, dependant: Dependant -) -> bool | None: - if lenient_issubclass(type_annotation, Request): - dependant.request_param_name = param_name - return True - elif lenient_issubclass(type_annotation, WebSocket): - dependant.websocket_param_name = param_name - return True - elif lenient_issubclass(type_annotation, HTTPConnection): - dependant.http_connection_param_name = param_name - return True - elif lenient_issubclass(type_annotation, Response): - dependant.response_param_name = param_name - return True - elif lenient_issubclass(type_annotation, StarletteBackgroundTasks): - dependant.background_tasks_param_name = param_name - return True - elif lenient_issubclass(type_annotation, SecurityScopes): - dependant.security_scopes_param_name = param_name - return True - return None - - -@dataclass -class ParamDetails: - type_annotation: Any - depends: params.Depends | None - field: ModelField | None - - -def analyze_param( - *, - param_name: str, - annotation: Any, - value: Any, - is_path_param: bool, -) -> ParamDetails: - field_info = None - depends = None - type_annotation: Any = Any - use_annotation: Any = Any - if is_typealiastype(annotation): - # unpack in case PEP 695 type syntax is used - annotation = annotation.__value__ - if annotation is not inspect.Signature.empty: - use_annotation = annotation - type_annotation = annotation - # Extract Annotated info - if get_origin(use_annotation) is Annotated: - annotated_args = get_args(annotation) - type_annotation = annotated_args[0] - fastapi_annotations = [ - arg - for arg in annotated_args[1:] - if isinstance(arg, (FieldInfo, params.Depends)) - ] - fastapi_specific_annotations = [ - arg - for arg in fastapi_annotations - if isinstance( - arg, - ( - params.Param, - params.Body, - params.Depends, - ), - ) - ] - if fastapi_specific_annotations: - fastapi_annotation: FieldInfo | params.Depends | None = ( - fastapi_specific_annotations[-1] - ) - else: - fastapi_annotation = None - # Set default for Annotated FieldInfo - if isinstance(fastapi_annotation, FieldInfo): - # Copy `field_info` because we mutate `field_info.default` below. - field_info = copy_field_info( - field_info=fastapi_annotation, - annotation=use_annotation, - ) - assert ( - field_info.default == Undefined or field_info.default == RequiredParam - ), ( - f"`{field_info.__class__.__name__}` default value cannot be set in" - f" `Annotated` for {param_name!r}. Set the default value with `=` instead." - ) - if value is not inspect.Signature.empty: - assert not is_path_param, "Path parameters cannot have default values" - field_info.default = value - else: - field_info.default = RequiredParam - # Get Annotated Depends - elif isinstance(fastapi_annotation, params.Depends): - depends = fastapi_annotation - # Get Depends from default value - if isinstance(value, params.Depends): - assert depends is None, ( - "Cannot specify `Depends` in `Annotated` and default value" - f" together for {param_name!r}" - ) - assert field_info is None, ( - "Cannot specify a FastAPI annotation in `Annotated` and `Depends` as a" - f" default value together for {param_name!r}" - ) - depends = value - # Get FieldInfo from default value - elif isinstance(value, FieldInfo): - assert field_info is None, ( - "Cannot specify FastAPI annotations in `Annotated` and default value" - f" together for {param_name!r}" - ) - field_info = value - if isinstance(field_info, FieldInfo): - field_info.annotation = type_annotation - - # Get Depends from type annotation - if depends is not None and depends.dependency is None: - # Copy `depends` before mutating it - depends = copy(depends) - depends = dataclasses.replace(depends, dependency=type_annotation) - - # Handle non-param type annotations like Request - # Only apply special handling when there's no explicit Depends - if there's a Depends, - # the dependency will be called and its return value used instead of the special injection - if depends is None and lenient_issubclass( - type_annotation, - ( - Request, - WebSocket, - HTTPConnection, - Response, - StarletteBackgroundTasks, - SecurityScopes, - ), - ): - assert field_info is None, ( - f"Cannot specify FastAPI annotation for type {type_annotation!r}" - ) - # Handle default assignations, neither field_info nor depends was not found in Annotated nor default value - elif field_info is None and depends is None: - default_value = value if value is not inspect.Signature.empty else RequiredParam - if is_path_param: - # We might check here that `default_value is RequiredParam`, but the fact is that the same - # parameter might sometimes be a path parameter and sometimes not. See - # `tests/test_infer_param_optionality.py` for an example. - field_info = params.Path(annotation=use_annotation) - elif is_uploadfile_or_nonable_uploadfile_annotation( - type_annotation - ) or is_uploadfile_sequence_annotation(type_annotation): - field_info = params.File(annotation=use_annotation, default=default_value) - elif not field_annotation_is_scalar(annotation=type_annotation): - field_info = params.Body(annotation=use_annotation, default=default_value) - else: - field_info = params.Query(annotation=use_annotation, default=default_value) - - field = None - # It's a field_info, not a dependency - if field_info is not None: - # Handle field_info.in_ - if is_path_param: - assert isinstance(field_info, params.Path), ( - f"Cannot use `{field_info.__class__.__name__}` for path param" - f" {param_name!r}" - ) - elif ( - isinstance(field_info, params.Param) - and getattr(field_info, "in_", None) is None - ): - field_info.in_ = params.ParamTypes.query - use_annotation_from_field_info = use_annotation - if isinstance(field_info, params.Form): - ensure_multipart_is_installed() - if not field_info.alias and getattr(field_info, "convert_underscores", None): - alias = param_name.replace("_", "-") - else: - alias = field_info.alias or param_name - field_info.alias = alias - field = create_model_field( - name=param_name, - type_=use_annotation_from_field_info, - default=field_info.default, - alias=alias, - field_info=field_info, - ) - if is_path_param: - assert is_scalar_field(field=field), ( - "Path params must be of one of the supported types" - ) - elif isinstance(field_info, params.Query): - assert ( - is_scalar_field(field) - or field_annotation_is_scalar_sequence(field.field_info.annotation) - or lenient_issubclass(field.field_info.annotation, BaseModel) - ), f"Query parameter {param_name!r} must be one of the supported types" - - return ParamDetails(type_annotation=type_annotation, depends=depends, field=field) - - -def add_param_to_fields(*, field: ModelField, dependant: Dependant) -> None: - field_info = field.field_info - field_info_in = getattr(field_info, "in_", None) - if field_info_in == params.ParamTypes.path: - dependant.path_params.append(field) - elif field_info_in == params.ParamTypes.query: - dependant.query_params.append(field) - elif field_info_in == params.ParamTypes.header: - dependant.header_params.append(field) - else: - assert field_info_in == params.ParamTypes.cookie, ( - f"non-body parameters must be in path, query, header or cookie: {field.name}" - ) - dependant.cookie_params.append(field) - - -async def _solve_generator( - *, dependant: Dependant, stack: AsyncExitStack, sub_values: dict[str, Any] -) -> Any: - assert dependant.call - if _is_async_gen_callable(dependant.call): - cm = asynccontextmanager(dependant.call)(**sub_values) - elif _is_gen_callable(dependant.call): - cm = contextmanager_in_threadpool(contextmanager(dependant.call)(**sub_values)) - return await stack.enter_async_context(cm) - - -@dataclass -class SolvedDependency: - values: dict[str, Any] - errors: list[Any] - background_tasks: StarletteBackgroundTasks | None - response: Response - dependency_cache: dict[DependencyCacheKey, Any] - - -async def solve_dependencies( - *, - request: Request | WebSocket, - dependant: Dependant, - body: dict[str, Any] | FormData | bytes | None = None, - background_tasks: StarletteBackgroundTasks | None = None, - response: Response | None = None, - dependency_overrides_provider: Any | None = None, - dependency_cache: dict[DependencyCacheKey, Any] | None = None, - # TODO: remove this parameter later, no longer used, not removing it yet as some - # people might be monkey patching this function (although that's not supported) - async_exit_stack: AsyncExitStack, - embed_body_fields: bool, - _uses_scopes_cache: _UsesScopesCache | None = None, -) -> SolvedDependency: - request_astack = request.scope.get("fastapi_inner_astack") - assert isinstance(request_astack, AsyncExitStack), ( - "fastapi_inner_astack not found in request scope" - ) - function_astack = request.scope.get("fastapi_function_astack") - assert isinstance(function_astack, AsyncExitStack), ( - "fastapi_function_astack not found in request scope" - ) - values: dict[str, Any] = {} - errors: list[Any] = [] - if response is None: - response = Response() - del response.headers["content-length"] - response.status_code = None # type: ignore - if dependency_cache is None: - dependency_cache = {} - if _uses_scopes_cache is None: - _uses_scopes_cache = {} - for sub_dependant in dependant.dependencies: - sub_dependant.call = cast(Callable[..., Any], sub_dependant.call) - call = sub_dependant.call - use_sub_dependant = sub_dependant - if ( - dependency_overrides_provider - and dependency_overrides_provider.dependency_overrides - ): - original_call = sub_dependant.call - call = getattr( - dependency_overrides_provider, "dependency_overrides", {} - ).get(original_call, original_call) - use_path: str = sub_dependant.path # type: ignore - use_sub_dependant = get_dependant( - path=use_path, - call=call, - name=sub_dependant.name, - parent_oauth_scopes=_get_oauth_scopes(dependant=sub_dependant), - scope=sub_dependant.scope, - ) - - solved_result = await solve_dependencies( - request=request, - dependant=use_sub_dependant, - body=body, - background_tasks=background_tasks, - response=response, - dependency_overrides_provider=dependency_overrides_provider, - dependency_cache=dependency_cache, - async_exit_stack=async_exit_stack, - embed_body_fields=embed_body_fields, - _uses_scopes_cache=_uses_scopes_cache, - ) - background_tasks = solved_result.background_tasks - if solved_result.errors: - errors.extend(solved_result.errors) - continue - sub_dependant_cache_key = _get_cache_key( - dependant=sub_dependant, - uses_scopes_cache=_uses_scopes_cache, - ) - if sub_dependant.use_cache and sub_dependant_cache_key in dependency_cache: - solved = dependency_cache[sub_dependant_cache_key] - elif _is_gen_callable(use_sub_dependant.call) or _is_async_gen_callable( - use_sub_dependant.call - ): - use_astack = request_astack - if sub_dependant.scope == "function": - use_astack = function_astack - solved = await _solve_generator( - dependant=use_sub_dependant, - stack=use_astack, - sub_values=solved_result.values, - ) - elif _is_coroutine_callable(use_sub_dependant.call): - solved = await call(**solved_result.values) - else: - solved = await run_in_threadpool(call, **solved_result.values) - if sub_dependant.name is not None: - values[sub_dependant.name] = solved - if sub_dependant_cache_key not in dependency_cache: - dependency_cache[sub_dependant_cache_key] = solved - path_values, path_errors = request_params_to_args( - dependant.path_params, request.path_params - ) - query_values, query_errors = request_params_to_args( - dependant.query_params, request.query_params - ) - header_values, header_errors = request_params_to_args( - dependant.header_params, request.headers - ) - cookie_values, cookie_errors = request_params_to_args( - dependant.cookie_params, request.cookies - ) - values.update(path_values) - values.update(query_values) - values.update(header_values) - values.update(cookie_values) - errors += path_errors + query_errors + header_errors + cookie_errors - if dependant.body_params: - ( - body_values, - body_errors, - ) = await request_body_to_args( # body_params checked above - body_fields=dependant.body_params, - received_body=body, - embed_body_fields=embed_body_fields, - ) - values.update(body_values) - errors.extend(body_errors) - if dependant.http_connection_param_name: - values[dependant.http_connection_param_name] = request - if dependant.request_param_name and isinstance(request, Request): - values[dependant.request_param_name] = request - elif dependant.websocket_param_name and isinstance(request, WebSocket): - values[dependant.websocket_param_name] = request - if dependant.background_tasks_param_name: - if background_tasks is None: - background_tasks = BackgroundTasks() - values[dependant.background_tasks_param_name] = background_tasks - if dependant.response_param_name: - values[dependant.response_param_name] = response - if dependant.security_scopes_param_name: - values[dependant.security_scopes_param_name] = SecurityScopes( - scopes=_get_oauth_scopes(dependant=dependant) - ) - return SolvedDependency( - values=values, - errors=errors, - background_tasks=background_tasks, - response=response, - dependency_cache=dependency_cache, - ) - - -def _validate_value_with_model_field( - *, field: ModelField, value: Any, values: dict[str, Any], loc: tuple[str, ...] -) -> tuple[Any, list[Any]]: - if value is None: - if field.field_info.is_required(): - return None, [get_missing_field_error(loc=loc)] - else: - return deepcopy(field.default), [] - return field.validate(value, values, loc=loc) - - -def _is_json_field(field: ModelField) -> bool: - return any(type(item) is Json for item in field.field_info.metadata) - - -def _get_multidict_value( - field: ModelField, values: Mapping[str, Any], alias: str | None = None -) -> Any: - alias = alias or get_validation_alias(field) - if ( - (not _is_json_field(field)) - and field_annotation_is_sequence(field.field_info.annotation) - and isinstance(values, (ImmutableMultiDict, Headers)) - ): - value = values.getlist(alias) - else: - value = values.get(alias, None) - if ( - value is None - or ( - isinstance(field.field_info, params.Form) - and isinstance(value, str) # For type checks - and value == "" - ) - or ( - field_annotation_is_sequence(field.field_info.annotation) - and len(value) == 0 - ) - ): - if field.field_info.is_required(): - return - else: - return deepcopy(field.default) - return value - - -def request_params_to_args( - fields: Sequence[ModelField], - received_params: Mapping[str, Any] | QueryParams | Headers, -) -> tuple[dict[str, Any], list[Any]]: - values: dict[str, Any] = {} - errors: list[dict[str, Any]] = [] - - if not fields: - return values, errors - - first_field = fields[0] - fields_to_extract = fields - single_not_embedded_field = False - default_convert_underscores = True - if len(fields) == 1 and lenient_issubclass( - first_field.field_info.annotation, BaseModel - ): - fields_to_extract = get_cached_model_fields(first_field.field_info.annotation) - single_not_embedded_field = True - # If headers are in a Pydantic model, the way to disable convert_underscores - # would be with Header(convert_underscores=False) at the Pydantic model level - default_convert_underscores = getattr( - first_field.field_info, "convert_underscores", True - ) - - params_to_process: dict[str, Any] = {} - - processed_keys = set() - - for field in fields_to_extract: - alias = None - if isinstance(received_params, Headers): - # Handle fields extracted from a Pydantic Model for a header, each field - # doesn't have a FieldInfo of type Header with the default convert_underscores=True - convert_underscores = getattr( - field.field_info, "convert_underscores", default_convert_underscores - ) - if convert_underscores: - alias = get_validation_alias(field) - if alias == field.name: - alias = alias.replace("_", "-") - value = _get_multidict_value(field, received_params, alias=alias) - if value is not None: - params_to_process[get_validation_alias(field)] = value - processed_keys.add(alias or get_validation_alias(field)) - # For headers with convert_underscores=True, mark both the converted - # header name and the original field alias as processed to avoid - # accepting the original alias as an extra header. - processed_keys.add(get_validation_alias(field)) - - for key in received_params.keys(): - if key not in processed_keys: - if isinstance(received_params, (ImmutableMultiDict, Headers)): - value = received_params.getlist(key) - if isinstance(value, list) and (len(value) == 1): - params_to_process[key] = value[0] - else: - params_to_process[key] = value - else: - params_to_process[key] = received_params.get(key) - - if single_not_embedded_field: - field_info = first_field.field_info - assert isinstance(field_info, params.Param), ( - "Params must be subclasses of Param" - ) - loc: tuple[str, ...] = (field_info.in_.value,) - v_, errors_ = _validate_value_with_model_field( - field=first_field, value=params_to_process, values=values, loc=loc - ) - return {first_field.name: v_}, errors_ - - for field in fields: - value = _get_multidict_value(field, received_params) - field_info = field.field_info - assert isinstance(field_info, params.Param), ( - "Params must be subclasses of Param" - ) - loc = (field_info.in_.value, get_validation_alias(field)) - v_, errors_ = _validate_value_with_model_field( - field=field, value=value, values=values, loc=loc - ) - if errors_: - errors.extend(errors_) - else: - values[field.name] = v_ - return values, errors - - -def is_union_of_base_models(field_type: Any) -> bool: - """Check if field type is a Union where all members are BaseModel subclasses.""" - from fastapi.types import UnionType - - origin = get_origin(field_type) - - # Check if it's a Union type (covers both typing.Union and types.UnionType in Python 3.10+) - if origin is not Union and origin is not UnionType: - return False - - union_args = get_args(field_type) - - for arg in union_args: - if not lenient_issubclass(arg, BaseModel): - return False - - return True - - -def _should_embed_body_fields(fields: list[ModelField]) -> bool: - if not fields: - return False - # More than one dependency could have the same field, it would show up as multiple - # fields but it's the same one, so count them by name - body_param_names_set = {field.name for field in fields} - # A top level field has to be a single field, not multiple - if len(body_param_names_set) > 1: - return True - first_field = fields[0] - # If it explicitly specifies it is embedded, it has to be embedded - if getattr(first_field.field_info, "embed", None): - return True - # If it's a Form (or File) field, it has to be a BaseModel (or a union of BaseModels) to be top level - # otherwise it has to be embedded, so that the key value pair can be extracted - if ( - isinstance(first_field.field_info, params.Form) - and not lenient_issubclass(first_field.field_info.annotation, BaseModel) - and not is_union_of_base_models(first_field.field_info.annotation) - ): - return True - return False - - -async def _extract_form_body( - body_fields: list[ModelField], - received_body: FormData, -) -> dict[str, Any]: - values = {} - - for field in body_fields: - value = _get_multidict_value(field, received_body) - field_info = field.field_info - if ( - isinstance(field_info, params.File) - and is_bytes_or_nonable_bytes_annotation(field.field_info.annotation) - and isinstance(value, UploadFile) - ): - value = await value.read() - elif ( - is_bytes_sequence_annotation(field.field_info.annotation) - and isinstance(field_info, params.File) - and value_is_sequence(value) - ): - # For types - assert isinstance(value, sequence_types) - results: list[bytes | str] = [] - for sub_value in value: - results.append(await sub_value.read()) - value = serialize_sequence_value(field=field, value=results) - if value is not None: - values[get_validation_alias(field)] = value - field_aliases = {get_validation_alias(field) for field in body_fields} - for key in received_body.keys(): - if key not in field_aliases: - param_values = received_body.getlist(key) - if len(param_values) == 1: - values[key] = param_values[0] - else: - values[key] = param_values - return values - - -async def request_body_to_args( - body_fields: list[ModelField], - received_body: dict[str, Any] | FormData | bytes | None, - embed_body_fields: bool, -) -> tuple[dict[str, Any], list[dict[str, Any]]]: - values: dict[str, Any] = {} - errors: list[dict[str, Any]] = [] - assert body_fields, "request_body_to_args() should be called with fields" - single_not_embedded_field = len(body_fields) == 1 and not embed_body_fields - first_field = body_fields[0] - body_to_process = received_body - - fields_to_extract: list[ModelField] = body_fields - - if ( - single_not_embedded_field - and lenient_issubclass(first_field.field_info.annotation, BaseModel) - and isinstance(received_body, FormData) - ): - fields_to_extract = get_cached_model_fields(first_field.field_info.annotation) - - if isinstance(received_body, FormData): - body_to_process = await _extract_form_body(fields_to_extract, received_body) - - if single_not_embedded_field: - loc: tuple[str, ...] = ("body",) - v_, errors_ = _validate_value_with_model_field( - field=first_field, value=body_to_process, values=values, loc=loc - ) - return {first_field.name: v_}, errors_ - for field in body_fields: - loc = ("body", get_validation_alias(field)) - value: Any | None = None - if body_to_process is not None and not isinstance(body_to_process, bytes): - try: - value = body_to_process.get(get_validation_alias(field)) - # If the received body is a list, not a dict - except AttributeError: - errors.append(get_missing_field_error(loc)) - continue - v_, errors_ = _validate_value_with_model_field( - field=field, value=value, values=values, loc=loc - ) - if errors_: - errors.extend(errors_) - else: - values[field.name] = v_ - return values, errors - - -def _get_body_field( - *, body_params: list[ModelField], name: str, embed_body_fields: bool -) -> ModelField | None: - """ - Get a ModelField representing the request body for a path operation, combining - all body parameters into a single field if necessary. - - Used to check if it's form data (with `isinstance(body_field, params.Form)`) - or JSON and to generate the JSON Schema for a request body. - - This is **not** used to validate/parse the request body, that's done with each - individual body parameter. - """ - if not body_params: - return None - first_param = body_params[0] - if not embed_body_fields: - return first_param - model_name = "Body_" + name - BodyModel = create_body_model(fields=body_params, model_name=model_name) - required = any(True for f in body_params if f.field_info.is_required()) - BodyFieldInfo_kwargs: dict[str, Any] = { - "annotation": BodyModel, - "alias": "body", - } - if not required: - BodyFieldInfo_kwargs["default"] = None - if any(isinstance(f.field_info, params.File) for f in body_params): - BodyFieldInfo: type[params.Body] = params.File - elif any(isinstance(f.field_info, params.Form) for f in body_params): - BodyFieldInfo = params.Form - else: - BodyFieldInfo = params.Body - - body_param_media_types = [ - f.field_info.media_type - for f in body_params - if isinstance(f.field_info, params.Body) - ] - if len(set(body_param_media_types)) == 1: - BodyFieldInfo_kwargs["media_type"] = body_param_media_types[0] - final_field = create_model_field( - name="body", - type_=BodyModel, - alias="body", - field_info=BodyFieldInfo(**BodyFieldInfo_kwargs), - ) - return final_field - - -def get_validation_alias(field: ModelField) -> str: - va = getattr(field, "validation_alias", None) - return va or field.alias diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/encoders.py b/bundle/python-cpu/Lib/site-packages/fastapi/encoders.py deleted file mode 100644 index e578768dac5b33556087b42bd6a2e0782a9d5379..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/encoders.py +++ /dev/null @@ -1,366 +0,0 @@ -import dataclasses -import datetime -from collections import defaultdict, deque -from collections.abc import Callable -from decimal import Decimal -from enum import Enum -from ipaddress import ( - IPv4Address, - IPv4Interface, - IPv4Network, - IPv6Address, - IPv6Interface, - IPv6Network, -) -from pathlib import Path, PurePath -from re import Pattern -from types import GeneratorType -from typing import Annotated, Any -from uuid import UUID - -from annotated_doc import Doc -from fastapi.exceptions import PydanticV1NotSupportedError -from fastapi.types import IncEx -from pydantic import BaseModel -from pydantic.networks import AnyUrl, NameEmail -from pydantic.types import SecretBytes, SecretStr -from pydantic_core import PydanticUndefinedType - -from ._compat import ( - Url, - is_pydantic_v1_model_instance, -) - -try: - # pydantic.color.Color is deprecated since v2.0b3, but supporting for bwd-compat - from pydantic.color import Color # ty: ignore[deprecated] -except ImportError: # pragma: no cover - - class Color: # type: ignore[no-redef] - pass - - -try: - # Supporting the new Color format for newer versions of Pydantic - from pydantic_extra_types.color import Color as PyExtraColor -except ImportError: # pragma: no cover - - class PyExtraColor: # type: ignore[no-redef] - pass - - -# Taken from Pydantic v1 as is -def isoformat(o: datetime.date | datetime.time) -> str: - return o.isoformat() - - -# Adapted from Pydantic v1 -# TODO: pv2 should this return strings instead? -def decimal_encoder(dec_value: Decimal) -> int | float: - """ - Encodes a Decimal as int if there's no exponent, otherwise float - - This is useful when we use ConstrainedDecimal to represent Numeric(x,0) - where an integer (but not int typed) is used. Encoding this as a float - results in failed round-tripping between encode and parse. - Our Id type is a prime example of this. - - >>> decimal_encoder(Decimal("1.0")) - 1.0 - - >>> decimal_encoder(Decimal("1")) - 1 - - >>> decimal_encoder(Decimal("NaN")) - nan - """ - exponent = dec_value.as_tuple().exponent - if isinstance(exponent, int) and exponent >= 0: - return int(dec_value) - else: - return float(dec_value) - - -ENCODERS_BY_TYPE: dict[type[Any], Callable[[Any], Any]] = { - bytes: lambda o: o.decode(), - Color: str, - PyExtraColor: str, - datetime.date: isoformat, - datetime.datetime: isoformat, - datetime.time: isoformat, - datetime.timedelta: lambda td: td.total_seconds(), - Decimal: decimal_encoder, - Enum: lambda o: o.value, - frozenset: list, - deque: list, - GeneratorType: list, - IPv4Address: str, - IPv4Interface: str, - IPv4Network: str, - IPv6Address: str, - IPv6Interface: str, - IPv6Network: str, - NameEmail: str, - Path: str, - Pattern: lambda o: o.pattern, - SecretBytes: str, - SecretStr: str, - set: list, - UUID: str, - Url: str, - AnyUrl: str, -} - - -def generate_encoders_by_class_tuples( - type_encoder_map: dict[Any, Callable[[Any], Any]], -) -> dict[Callable[[Any], Any], tuple[Any, ...]]: - encoders_by_class_tuples: dict[Callable[[Any], Any], tuple[Any, ...]] = defaultdict( - tuple - ) - for type_, encoder in type_encoder_map.items(): - encoders_by_class_tuples[encoder] += (type_,) - return encoders_by_class_tuples - - -encoders_by_class_tuples = generate_encoders_by_class_tuples(ENCODERS_BY_TYPE) - - -def jsonable_encoder( - obj: Annotated[ - Any, - Doc( - """ - The input object to convert to JSON. - """ - ), - ], - include: Annotated[ - IncEx | None, - Doc( - """ - Pydantic's `include` parameter, passed to Pydantic models to set the - fields to include. - """ - ), - ] = None, - exclude: Annotated[ - IncEx | None, - Doc( - """ - Pydantic's `exclude` parameter, passed to Pydantic models to set the - fields to exclude. - """ - ), - ] = None, - by_alias: Annotated[ - bool, - Doc( - """ - Pydantic's `by_alias` parameter, passed to Pydantic models to define if - the output should use the alias names (when provided) or the Python - attribute names. In an API, if you set an alias, it's probably because you - want to use it in the result, so you probably want to leave this set to - `True`. - """ - ), - ] = True, - exclude_unset: Annotated[ - bool, - Doc( - """ - Pydantic's `exclude_unset` parameter, passed to Pydantic models to define - if it should exclude from the output the fields that were not explicitly - set (and that only had their default values). - """ - ), - ] = False, - exclude_defaults: Annotated[ - bool, - Doc( - """ - Pydantic's `exclude_defaults` parameter, passed to Pydantic models to define - if it should exclude from the output the fields that had the same default - value, even when they were explicitly set. - """ - ), - ] = False, - exclude_none: Annotated[ - bool, - Doc( - """ - Pydantic's `exclude_none` parameter, passed to Pydantic models to define - if it should exclude from the output any fields that have a `None` value. - """ - ), - ] = False, - custom_encoder: Annotated[ - dict[Any, Callable[[Any], Any]] | None, - Doc( - """ - Pydantic's `custom_encoder` parameter, passed to Pydantic models to define - a custom encoder. - """ - ), - ] = None, - sqlalchemy_safe: Annotated[ - bool, - Doc( - """ - Exclude from the output any fields that start with the name `_sa`. - - This is mainly a hack for compatibility with SQLAlchemy objects, they - store internal SQLAlchemy-specific state in attributes named with `_sa`, - and those objects can't (and shouldn't be) serialized to JSON. - """ - ), - ] = True, -) -> Any: - """ - Convert any object to something that can be encoded in JSON. - - This is used internally by FastAPI to make sure anything you return can be - encoded as JSON before it is sent to the client. - - You can also use it yourself, for example to convert objects before saving them - in a database that supports only JSON. - - Read more about it in the - [FastAPI docs for JSON Compatible Encoder](https://fastapi.tiangolo.com/tutorial/encoder/). - """ - custom_encoder = custom_encoder or {} - if custom_encoder: - if type(obj) in custom_encoder: - return custom_encoder[type(obj)](obj) - else: - for encoder_type, encoder_instance in custom_encoder.items(): - if isinstance(obj, encoder_type): - return encoder_instance(obj) - if include is not None and not isinstance(include, (set, dict)): - include = set(include) # type: ignore[assignment] # ty: ignore[invalid-assignment] - if exclude is not None and not isinstance(exclude, (set, dict)): - exclude = set(exclude) # type: ignore[assignment] # ty: ignore[invalid-assignment] - if isinstance(obj, BaseModel): - obj_dict = obj.model_dump( - mode="json", - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_none=exclude_none, - exclude_defaults=exclude_defaults, - ) - return jsonable_encoder( - obj_dict, - exclude_none=exclude_none, - exclude_defaults=exclude_defaults, - sqlalchemy_safe=sqlalchemy_safe, - ) - if dataclasses.is_dataclass(obj): - assert not isinstance(obj, type) - obj_dict = dataclasses.asdict(obj) - return jsonable_encoder( - obj_dict, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - custom_encoder=custom_encoder, - sqlalchemy_safe=sqlalchemy_safe, - ) - if isinstance(obj, Enum): - return obj.value - if isinstance(obj, PurePath): - return str(obj) - if isinstance(obj, (str, int, float, type(None))): - return obj - if isinstance(obj, PydanticUndefinedType): - return None - if isinstance(obj, dict): - encoded_dict = {} - allowed_keys = set(obj.keys()) - if include is not None: - allowed_keys &= set(include) - if exclude is not None: - allowed_keys -= set(exclude) - for key, value in obj.items(): - if ( - ( - not sqlalchemy_safe - or (not isinstance(key, str)) - or (not key.startswith("_sa")) - ) - and (value is not None or not exclude_none) - and key in allowed_keys - ): - encoded_key = jsonable_encoder( - key, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - custom_encoder=custom_encoder, - sqlalchemy_safe=sqlalchemy_safe, - ) - encoded_value = jsonable_encoder( - value, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - custom_encoder=custom_encoder, - sqlalchemy_safe=sqlalchemy_safe, - ) - encoded_dict[encoded_key] = encoded_value - return encoded_dict - if isinstance(obj, (list, set, frozenset, GeneratorType, tuple, deque)): - encoded_list = [] - for item in obj: - encoded_list.append( - jsonable_encoder( - item, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - custom_encoder=custom_encoder, - sqlalchemy_safe=sqlalchemy_safe, - ) - ) - return encoded_list - - if type(obj) in ENCODERS_BY_TYPE: - return ENCODERS_BY_TYPE[type(obj)](obj) - for encoder, classes_tuple in encoders_by_class_tuples.items(): - if isinstance(obj, classes_tuple): - return encoder(obj) - if is_pydantic_v1_model_instance(obj): - raise PydanticV1NotSupportedError( - "pydantic.v1 models are no longer supported by FastAPI." - f" Please update the model {obj!r}." - ) - try: - data = dict(obj) - except Exception as e: - errors: list[Exception] = [] - errors.append(e) - try: - data = vars(obj) - except Exception as e: - errors.append(e) - raise ValueError(errors) from e - return jsonable_encoder( - data, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - custom_encoder=custom_encoder, - sqlalchemy_safe=sqlalchemy_safe, - ) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/exception_handlers.py b/bundle/python-cpu/Lib/site-packages/fastapi/exception_handlers.py deleted file mode 100644 index 475dd7bdd9891a7595b6df5db97cd3840179c8fa..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/exception_handlers.py +++ /dev/null @@ -1,34 +0,0 @@ -from fastapi.encoders import jsonable_encoder -from fastapi.exceptions import RequestValidationError, WebSocketRequestValidationError -from fastapi.utils import is_body_allowed_for_status_code -from fastapi.websockets import WebSocket -from starlette.exceptions import HTTPException -from starlette.requests import Request -from starlette.responses import JSONResponse, Response -from starlette.status import WS_1008_POLICY_VIOLATION - - -async def http_exception_handler(request: Request, exc: HTTPException) -> Response: - headers = getattr(exc, "headers", None) - if not is_body_allowed_for_status_code(exc.status_code): - return Response(status_code=exc.status_code, headers=headers) - return JSONResponse( - {"detail": exc.detail}, status_code=exc.status_code, headers=headers - ) - - -async def request_validation_exception_handler( - request: Request, exc: RequestValidationError -) -> JSONResponse: - return JSONResponse( - status_code=422, - content={"detail": jsonable_encoder(exc.errors())}, - ) - - -async def websocket_request_validation_exception_handler( - websocket: WebSocket, exc: WebSocketRequestValidationError -) -> None: - await websocket.close( - code=WS_1008_POLICY_VIOLATION, reason=jsonable_encoder(exc.errors()) - ) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/exceptions.py b/bundle/python-cpu/Lib/site-packages/fastapi/exceptions.py deleted file mode 100644 index d7065c52fe20220e12b7d20db4da7cbeadaf171a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/exceptions.py +++ /dev/null @@ -1,256 +0,0 @@ -from collections.abc import Mapping, Sequence -from typing import Annotated, Any, TypedDict - -from annotated_doc import Doc -from pydantic import BaseModel, create_model -from starlette.exceptions import HTTPException as StarletteHTTPException -from starlette.exceptions import WebSocketException as StarletteWebSocketException - - -class EndpointContext(TypedDict, total=False): - function: str - path: str - file: str - line: int - - -class HTTPException(StarletteHTTPException): - """ - An HTTP exception you can raise in your own code to show errors to the client. - - This is for client errors, invalid authentication, invalid data, etc. Not for server - errors in your code. - - Read more about it in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/). - - ## Example - - ```python - from fastapi import FastAPI, HTTPException - - app = FastAPI() - - items = {"foo": "The Foo Wrestlers"} - - - @app.get("/items/{item_id}") - async def read_item(item_id: str): - if item_id not in items: - raise HTTPException(status_code=404, detail="Item not found") - return {"item": items[item_id]} - ``` - """ - - def __init__( - self, - status_code: Annotated[ - int, - Doc( - """ - HTTP status code to send to the client. - - Read more about it in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/#use-httpexception) - """ - ), - ], - detail: Annotated[ - Any, - Doc( - """ - Any data to be sent to the client in the `detail` key of the JSON - response. - - Read more about it in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/#use-httpexception) - """ - ), - ] = None, - headers: Annotated[ - Mapping[str, str] | None, - Doc( - """ - Any headers to send to the client in the response. - - Read more about it in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/#add-custom-headers) - - """ - ), - ] = None, - ) -> None: - super().__init__(status_code=status_code, detail=detail, headers=headers) - - -class WebSocketException(StarletteWebSocketException): - """ - A WebSocket exception you can raise in your own code to show errors to the client. - - This is for client errors, invalid authentication, invalid data, etc. Not for server - errors in your code. - - Read more about it in the - [FastAPI docs for WebSockets](https://fastapi.tiangolo.com/advanced/websockets/). - - ## Example - - ```python - from typing import Annotated - - from fastapi import ( - Cookie, - FastAPI, - WebSocket, - WebSocketException, - status, - ) - - app = FastAPI() - - @app.websocket("/items/{item_id}/ws") - async def websocket_endpoint( - *, - websocket: WebSocket, - session: Annotated[str | None, Cookie()] = None, - item_id: str, - ): - if session is None: - raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION) - await websocket.accept() - while True: - data = await websocket.receive_text() - await websocket.send_text(f"Session cookie is: {session}") - await websocket.send_text(f"Message text was: {data}, for item ID: {item_id}") - ``` - """ - - def __init__( - self, - code: Annotated[ - int, - Doc( - """ - A closing code from the - [valid codes defined in the specification](https://datatracker.ietf.org/doc/html/rfc6455#section-7.4.1). - """ - ), - ], - reason: Annotated[ - str | None, - Doc( - """ - The reason to close the WebSocket connection. - - It is UTF-8-encoded data. The interpretation of the reason is up to the - application, it is not specified by the WebSocket specification. - - It could contain text that could be human-readable or interpretable - by the client code, etc. - """ - ), - ] = None, - ) -> None: - super().__init__(code=code, reason=reason) - - -RequestErrorModel: type[BaseModel] = create_model("Request") -WebSocketErrorModel: type[BaseModel] = create_model("WebSocket") - - -class FastAPIError(RuntimeError): - """ - A generic, FastAPI-specific error. - """ - - -class DependencyScopeError(FastAPIError): - """ - A dependency declared that it depends on another dependency with an invalid - (narrower) scope. - """ - - -class ValidationException(Exception): - def __init__( - self, - errors: Sequence[Any], - *, - endpoint_ctx: EndpointContext | None = None, - ) -> None: - self._errors = errors - self.endpoint_ctx = endpoint_ctx - - ctx = endpoint_ctx or {} - self.endpoint_function = ctx.get("function") - self.endpoint_path = ctx.get("path") - self.endpoint_file = ctx.get("file") - self.endpoint_line = ctx.get("line") - - def errors(self) -> Sequence[Any]: - return self._errors - - def _format_endpoint_context(self) -> str: - if not (self.endpoint_file and self.endpoint_line and self.endpoint_function): - if self.endpoint_path: - return f"\n Endpoint: {self.endpoint_path}" - return "" - - context = f'\n File "{self.endpoint_file}", line {self.endpoint_line}, in {self.endpoint_function}' - if self.endpoint_path: - context += f"\n {self.endpoint_path}" - return context - - def __str__(self) -> str: - message = f"{len(self._errors)} validation error{'s' if len(self._errors) != 1 else ''}:\n" - for err in self._errors: - message += f" {err}\n" - message += self._format_endpoint_context() - return message.rstrip() - - -class RequestValidationError(ValidationException): - def __init__( - self, - errors: Sequence[Any], - *, - body: Any = None, - endpoint_ctx: EndpointContext | None = None, - ) -> None: - super().__init__(errors, endpoint_ctx=endpoint_ctx) - self.body = body - - -class WebSocketRequestValidationError(ValidationException): - def __init__( - self, - errors: Sequence[Any], - *, - endpoint_ctx: EndpointContext | None = None, - ) -> None: - super().__init__(errors, endpoint_ctx=endpoint_ctx) - - -class ResponseValidationError(ValidationException): - def __init__( - self, - errors: Sequence[Any], - *, - body: Any = None, - endpoint_ctx: EndpointContext | None = None, - ) -> None: - super().__init__(errors, endpoint_ctx=endpoint_ctx) - self.body = body - - -class PydanticV1NotSupportedError(FastAPIError): - """ - A pydantic.v1 model is used, which is no longer supported. - """ - - -class FastAPIDeprecationWarning(UserWarning): - """ - A custom deprecation warning as DeprecationWarning is ignored - Ref: https://sethmlarson.dev/deprecations-via-warnings-dont-work-for-python-libraries - """ diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/logger.py b/bundle/python-cpu/Lib/site-packages/fastapi/logger.py deleted file mode 100644 index 5b2c4ad5250b589aa0c8f8d1cc9125b91b10edb0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/logger.py +++ /dev/null @@ -1,3 +0,0 @@ -import logging - -logger = logging.getLogger("fastapi") diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/__init__.py b/bundle/python-cpu/Lib/site-packages/fastapi/middleware/__init__.py deleted file mode 100644 index 620296d5ad6ca2cc49eb5d0dc140bcbc3204e9b4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.middleware import Middleware as Middleware diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/asyncexitstack.py b/bundle/python-cpu/Lib/site-packages/fastapi/middleware/asyncexitstack.py deleted file mode 100644 index 4ce3f5a625548a00514f872d1653194bd3669a73..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/asyncexitstack.py +++ /dev/null @@ -1,18 +0,0 @@ -from contextlib import AsyncExitStack - -from starlette.types import ASGIApp, Receive, Scope, Send - - -# Used mainly to close files after the request is done, dependencies are closed -# in their own AsyncExitStack -class AsyncExitStackMiddleware: - def __init__( - self, app: ASGIApp, context_name: str = "fastapi_middleware_astack" - ) -> None: - self.app = app - self.context_name = context_name - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - async with AsyncExitStack() as stack: - scope[self.context_name] = stack - await self.app(scope, receive, send) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/cors.py b/bundle/python-cpu/Lib/site-packages/fastapi/middleware/cors.py deleted file mode 100644 index 8dfaad0dbb3ff5300cccb2023748cd30f54bc920..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/cors.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.middleware.cors import CORSMiddleware as CORSMiddleware # noqa diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/gzip.py b/bundle/python-cpu/Lib/site-packages/fastapi/middleware/gzip.py deleted file mode 100644 index bbeb2cc7861a735d6cd5c0e29aeb6dbf8457023a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/gzip.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.middleware.gzip import GZipMiddleware as GZipMiddleware # noqa diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/httpsredirect.py b/bundle/python-cpu/Lib/site-packages/fastapi/middleware/httpsredirect.py deleted file mode 100644 index b7a3d8e078574e87dc6e345d621f5a596c3bdc1e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/httpsredirect.py +++ /dev/null @@ -1,3 +0,0 @@ -from starlette.middleware.httpsredirect import ( # noqa - HTTPSRedirectMiddleware as HTTPSRedirectMiddleware, -) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/trustedhost.py b/bundle/python-cpu/Lib/site-packages/fastapi/middleware/trustedhost.py deleted file mode 100644 index 08d7e035315677856fd2cd0be2044689b57619bf..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/trustedhost.py +++ /dev/null @@ -1,3 +0,0 @@ -from starlette.middleware.trustedhost import ( # noqa - TrustedHostMiddleware as TrustedHostMiddleware, -) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/wsgi.py b/bundle/python-cpu/Lib/site-packages/fastapi/middleware/wsgi.py deleted file mode 100644 index 69e4dcab96370cac0ab93039a1eb9376d1659120..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/middleware/wsgi.py +++ /dev/null @@ -1,3 +0,0 @@ -from starlette.middleware.wsgi import ( - WSGIMiddleware as WSGIMiddleware, -) # pragma: no cover # noqa diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/openapi/__init__.py b/bundle/python-cpu/Lib/site-packages/fastapi/openapi/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/openapi/constants.py b/bundle/python-cpu/Lib/site-packages/fastapi/openapi/constants.py deleted file mode 100644 index d724ee3cfdbcda1c39f39511046c7a884186ca98..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/openapi/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -METHODS_WITH_BODY = {"GET", "HEAD", "POST", "PUT", "DELETE", "PATCH"} -REF_PREFIX = "#/components/schemas/" -REF_TEMPLATE = "#/components/schemas/{model}" diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/openapi/docs.py b/bundle/python-cpu/Lib/site-packages/fastapi/openapi/docs.py deleted file mode 100644 index 0d9242f9fa6a5212114b8f4036adfaf0e518020f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/openapi/docs.py +++ /dev/null @@ -1,389 +0,0 @@ -import json -from typing import Annotated, Any - -from annotated_doc import Doc -from fastapi.encoders import jsonable_encoder -from starlette.responses import HTMLResponse - - -def _html_safe_json(value: Any) -> str: - """Serialize a value to JSON with HTML special characters escaped. - - This prevents injection when the JSON is embedded inside a - - - - - """ - return HTMLResponse(html) - - -def get_redoc_html( - *, - openapi_url: Annotated[ - str, - Doc( - """ - The OpenAPI URL that ReDoc should load and use. - - This is normally done automatically by FastAPI using the default URL - `/openapi.json`. - - Read more about it in the - [FastAPI docs for Conditional OpenAPI](https://fastapi.tiangolo.com/how-to/conditional-openapi/#conditional-openapi-from-settings-and-env-vars) - """ - ), - ], - title: Annotated[ - str, - Doc( - """ - The HTML `` content, normally shown in the browser tab. - - Read more about it in the - [FastAPI docs for Custom Docs UI Static Assets](https://fastapi.tiangolo.com/how-to/custom-docs-ui-assets/) - """ - ), - ], - redoc_js_url: Annotated[ - str, - Doc( - """ - The URL to use to load the ReDoc JavaScript. - - It is normally set to a CDN URL. - - Read more about it in the - [FastAPI docs for Custom Docs UI Static Assets](https://fastapi.tiangolo.com/how-to/custom-docs-ui-assets/) - """ - ), - ] = "https://cdn.jsdelivr.net/npm/redoc@2/bundles/redoc.standalone.js", - redoc_favicon_url: Annotated[ - str, - Doc( - """ - The URL of the favicon to use. It is normally shown in the browser tab. - """ - ), - ] = "https://fastapi.tiangolo.com/img/favicon.png", - with_google_fonts: Annotated[ - bool, - Doc( - """ - Load and use Google Fonts. - """ - ), - ] = True, -) -> HTMLResponse: - """ - Generate and return the HTML response that loads ReDoc for the alternative - API docs (normally served at `/redoc`). - - You would only call this function yourself if you needed to override some parts, - for example the URLs to use to load ReDoc's JavaScript and CSS. - - Read more about it in the - [FastAPI docs for Custom Docs UI Static Assets (Self-Hosting)](https://fastapi.tiangolo.com/how-to/custom-docs-ui-assets/). - """ - html = f""" - <!DOCTYPE html> - <html> - <head> - <title>{title} - - - - """ - if with_google_fonts: - html += """ - - """ - html += f""" - - - - - - - - - - - """ - return HTMLResponse(html) - - -def get_swagger_ui_oauth2_redirect_html() -> HTMLResponse: - """ - Generate the HTML response with the OAuth2 redirection for Swagger UI. - - You normally don't need to use or change this. - """ - # copied from https://github.com/swagger-api/swagger-ui/blob/v4.14.0/dist/oauth2-redirect.html - html = """ - - - - Swagger UI: OAuth2 Redirect - - - - - - """ - return HTMLResponse(content=html) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/openapi/models.py b/bundle/python-cpu/Lib/site-packages/fastapi/openapi/models.py deleted file mode 100644 index ca26bf931e196fa9c5ef367838c98bdff730b230..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/openapi/models.py +++ /dev/null @@ -1,435 +0,0 @@ -from collections.abc import Callable, Iterable, Mapping -from enum import Enum -from typing import Annotated, Any, Literal, Optional, Union - -from fastapi._compat import with_info_plain_validator_function -from fastapi.logger import logger -from pydantic import ( - AnyUrl, - BaseModel, - Field, - GetJsonSchemaHandler, -) -from typing_extensions import TypedDict -from typing_extensions import deprecated as typing_deprecated - -try: - import email_validator - - assert email_validator # make autoflake ignore the unused import - from pydantic import EmailStr -except ImportError: # pragma: no cover - - class EmailStr(str): # type: ignore[no-redef] - @classmethod - def __get_validators__(cls) -> Iterable[Callable[..., Any]]: - yield cls.validate - - @classmethod - def validate(cls, v: Any) -> str: - logger.warning( - "email-validator not installed, email fields will be treated as str.\n" - "To install, run: pip install email-validator" - ) - return str(v) - - @classmethod - def _validate(cls, __input_value: Any, _: Any) -> str: - logger.warning( - "email-validator not installed, email fields will be treated as str.\n" - "To install, run: pip install email-validator" - ) - return str(__input_value) - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: Mapping[str, Any], handler: GetJsonSchemaHandler - ) -> dict[str, Any]: - return {"type": "string", "format": "email"} - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: Callable[[Any], Mapping[str, Any]] - ) -> Mapping[str, Any]: - return with_info_plain_validator_function(cls._validate) - - -class BaseModelWithConfig(BaseModel): - model_config = {"extra": "allow"} - - -class Contact(BaseModelWithConfig): - name: str | None = None - url: AnyUrl | None = None - email: EmailStr | None = None - - -class License(BaseModelWithConfig): - name: str - identifier: str | None = None - url: AnyUrl | None = None - - -class Info(BaseModelWithConfig): - title: str - summary: str | None = None - description: str | None = None - termsOfService: str | None = None - contact: Contact | None = None - license: License | None = None - version: str - - -class ServerVariable(BaseModelWithConfig): - enum: Annotated[list[str] | None, Field(min_length=1)] = None - default: str - description: str | None = None - - -class Server(BaseModelWithConfig): - url: AnyUrl | str - description: str | None = None - variables: dict[str, ServerVariable] | None = None - - -class Reference(BaseModel): - ref: str = Field(alias="$ref") - - -class Discriminator(BaseModel): - propertyName: str - mapping: dict[str, str] | None = None - - -class XML(BaseModelWithConfig): - name: str | None = None - namespace: str | None = None - prefix: str | None = None - attribute: bool | None = None - wrapped: bool | None = None - - -class ExternalDocumentation(BaseModelWithConfig): - description: str | None = None - url: AnyUrl - - -# Ref JSON Schema 2020-12: https://json-schema.org/draft/2020-12/json-schema-validation#name-type -SchemaType = Literal[ - "array", "boolean", "integer", "null", "number", "object", "string" -] - - -class Schema(BaseModelWithConfig): - # Ref: JSON Schema 2020-12: https://json-schema.org/draft/2020-12/json-schema-core.html#name-the-json-schema-core-vocabu - # Core Vocabulary - schema_: str | None = Field(default=None, alias="$schema") - vocabulary: str | None = Field(default=None, alias="$vocabulary") - id: str | None = Field(default=None, alias="$id") - anchor: str | None = Field(default=None, alias="$anchor") - dynamicAnchor: str | None = Field(default=None, alias="$dynamicAnchor") - ref: str | None = Field(default=None, alias="$ref") - dynamicRef: str | None = Field(default=None, alias="$dynamicRef") - defs: dict[str, "SchemaOrBool"] | None = Field(default=None, alias="$defs") - comment: str | None = Field(default=None, alias="$comment") - # Ref: JSON Schema 2020-12: https://json-schema.org/draft/2020-12/json-schema-core.html#name-a-vocabulary-for-applying-s - # A Vocabulary for Applying Subschemas - allOf: list["SchemaOrBool"] | None = None - anyOf: list["SchemaOrBool"] | None = None - oneOf: list["SchemaOrBool"] | None = None - not_: Optional["SchemaOrBool"] = Field(default=None, alias="not") - if_: Optional["SchemaOrBool"] = Field(default=None, alias="if") - then: Optional["SchemaOrBool"] = None - else_: Optional["SchemaOrBool"] = Field(default=None, alias="else") - dependentSchemas: dict[str, "SchemaOrBool"] | None = None - prefixItems: list["SchemaOrBool"] | None = None - items: Optional["SchemaOrBool"] = None - contains: Optional["SchemaOrBool"] = None - properties: dict[str, "SchemaOrBool"] | None = None - patternProperties: dict[str, "SchemaOrBool"] | None = None - additionalProperties: Optional["SchemaOrBool"] = None - propertyNames: Optional["SchemaOrBool"] = None - unevaluatedItems: Optional["SchemaOrBool"] = None - unevaluatedProperties: Optional["SchemaOrBool"] = None - # Ref: JSON Schema Validation 2020-12: https://json-schema.org/draft/2020-12/json-schema-validation.html#name-a-vocabulary-for-structural - # A Vocabulary for Structural Validation - type: SchemaType | list[SchemaType] | None = None - enum: list[Any] | None = None - const: Any | None = None - multipleOf: float | None = Field(default=None, gt=0) - maximum: float | None = None - exclusiveMaximum: float | None = None - minimum: float | None = None - exclusiveMinimum: float | None = None - maxLength: int | None = Field(default=None, ge=0) - minLength: int | None = Field(default=None, ge=0) - pattern: str | None = None - maxItems: int | None = Field(default=None, ge=0) - minItems: int | None = Field(default=None, ge=0) - uniqueItems: bool | None = None - maxContains: int | None = Field(default=None, ge=0) - minContains: int | None = Field(default=None, ge=0) - maxProperties: int | None = Field(default=None, ge=0) - minProperties: int | None = Field(default=None, ge=0) - required: list[str] | None = None - dependentRequired: dict[str, set[str]] | None = None - # Ref: JSON Schema Validation 2020-12: https://json-schema.org/draft/2020-12/json-schema-validation.html#name-vocabularies-for-semantic-c - # Vocabularies for Semantic Content With "format" - format: str | None = None - # Ref: JSON Schema Validation 2020-12: https://json-schema.org/draft/2020-12/json-schema-validation.html#name-a-vocabulary-for-the-conten - # A Vocabulary for the Contents of String-Encoded Data - contentEncoding: str | None = None - contentMediaType: str | None = None - contentSchema: Optional["SchemaOrBool"] = None - # Ref: JSON Schema Validation 2020-12: https://json-schema.org/draft/2020-12/json-schema-validation.html#name-a-vocabulary-for-basic-meta - # A Vocabulary for Basic Meta-Data Annotations - title: str | None = None - description: str | None = None - default: Any | None = None - deprecated: bool | None = None - readOnly: bool | None = None - writeOnly: bool | None = None - examples: list[Any] | None = None - # Ref: OpenAPI 3.1.0: https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.1.0.md#schema-object - # Schema Object - discriminator: Discriminator | None = None - xml: XML | None = None - externalDocs: ExternalDocumentation | None = None - example: Annotated[ - Any | None, - typing_deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = None - - -# Ref: https://json-schema.org/draft/2020-12/json-schema-core.html#name-json-schema-documents -# A JSON Schema MUST be an object or a boolean. -SchemaOrBool = Schema | bool - - -class Example(TypedDict, total=False): - summary: str | None - description: str | None - value: Any | None - externalValue: AnyUrl | None - - __pydantic_config__ = {"extra": "allow"} # type: ignore[misc] # ty: ignore[invalid-typed-dict-statement] - - -class ParameterInType(Enum): - query = "query" - header = "header" - path = "path" - cookie = "cookie" - - -class Encoding(BaseModelWithConfig): - contentType: str | None = None - headers: dict[str, Union["Header", Reference]] | None = None - style: str | None = None - explode: bool | None = None - allowReserved: bool | None = None - - -class MediaType(BaseModelWithConfig): - schema_: Schema | Reference | None = Field(default=None, alias="schema") - example: Any | None = None - examples: dict[str, Example | Reference] | None = None - encoding: dict[str, Encoding] | None = None - - -class ParameterBase(BaseModelWithConfig): - description: str | None = None - required: bool | None = None - deprecated: bool | None = None - # Serialization rules for simple scenarios - style: str | None = None - explode: bool | None = None - allowReserved: bool | None = None - schema_: Schema | Reference | None = Field(default=None, alias="schema") - example: Any | None = None - examples: dict[str, Example | Reference] | None = None - # Serialization rules for more complex scenarios - content: dict[str, MediaType] | None = None - - -class Parameter(ParameterBase): - name: str - in_: ParameterInType = Field(alias="in") - - -class Header(ParameterBase): - pass - - -class RequestBody(BaseModelWithConfig): - description: str | None = None - content: dict[str, MediaType] - required: bool | None = None - - -class Link(BaseModelWithConfig): - operationRef: str | None = None - operationId: str | None = None - parameters: dict[str, Any | str] | None = None - requestBody: Any | str | None = None - description: str | None = None - server: Server | None = None - - -class Response(BaseModelWithConfig): - description: str - headers: dict[str, Header | Reference] | None = None - content: dict[str, MediaType] | None = None - links: dict[str, Link | Reference] | None = None - - -class Operation(BaseModelWithConfig): - tags: list[str] | None = None - summary: str | None = None - description: str | None = None - externalDocs: ExternalDocumentation | None = None - operationId: str | None = None - parameters: list[Parameter | Reference] | None = None - requestBody: RequestBody | Reference | None = None - # Using Any for Specification Extensions - responses: dict[str, Response | Any] | None = None - callbacks: dict[str, dict[str, "PathItem"] | Reference] | None = None - deprecated: bool | None = None - security: list[dict[str, list[str]]] | None = None - servers: list[Server] | None = None - - -class PathItem(BaseModelWithConfig): - ref: str | None = Field(default=None, alias="$ref") - summary: str | None = None - description: str | None = None - get: Operation | None = None - put: Operation | None = None - post: Operation | None = None - delete: Operation | None = None - options: Operation | None = None - head: Operation | None = None - patch: Operation | None = None - trace: Operation | None = None - servers: list[Server] | None = None - parameters: list[Parameter | Reference] | None = None - - -class SecuritySchemeType(Enum): - apiKey = "apiKey" - http = "http" - oauth2 = "oauth2" - openIdConnect = "openIdConnect" - - -class SecurityBase(BaseModelWithConfig): - type_: SecuritySchemeType = Field(alias="type") - description: str | None = None - - -class APIKeyIn(Enum): - query = "query" - header = "header" - cookie = "cookie" - - -class APIKey(SecurityBase): - type_: SecuritySchemeType = Field(default=SecuritySchemeType.apiKey, alias="type") - in_: APIKeyIn = Field(alias="in") - name: str - - -class HTTPBase(SecurityBase): - type_: SecuritySchemeType = Field(default=SecuritySchemeType.http, alias="type") - scheme: str - - -class HTTPBearer(HTTPBase): - scheme: Literal["bearer"] = "bearer" - bearerFormat: str | None = None - - -class OAuthFlow(BaseModelWithConfig): - refreshUrl: str | None = None - scopes: dict[str, str] = {} - - -class OAuthFlowImplicit(OAuthFlow): - authorizationUrl: str - - -class OAuthFlowPassword(OAuthFlow): - tokenUrl: str - - -class OAuthFlowClientCredentials(OAuthFlow): - tokenUrl: str - - -class OAuthFlowAuthorizationCode(OAuthFlow): - authorizationUrl: str - tokenUrl: str - - -class OAuthFlows(BaseModelWithConfig): - implicit: OAuthFlowImplicit | None = None - password: OAuthFlowPassword | None = None - clientCredentials: OAuthFlowClientCredentials | None = None - authorizationCode: OAuthFlowAuthorizationCode | None = None - - -class OAuth2(SecurityBase): - type_: SecuritySchemeType = Field(default=SecuritySchemeType.oauth2, alias="type") - flows: OAuthFlows - - -class OpenIdConnect(SecurityBase): - type_: SecuritySchemeType = Field( - default=SecuritySchemeType.openIdConnect, alias="type" - ) - openIdConnectUrl: str - - -SecurityScheme = APIKey | HTTPBase | OAuth2 | OpenIdConnect | HTTPBearer - - -class Components(BaseModelWithConfig): - schemas: dict[str, Schema | Reference] | None = None - responses: dict[str, Response | Reference] | None = None - parameters: dict[str, Parameter | Reference] | None = None - examples: dict[str, Example | Reference] | None = None - requestBodies: dict[str, RequestBody | Reference] | None = None - headers: dict[str, Header | Reference] | None = None - securitySchemes: dict[str, SecurityScheme | Reference] | None = None - links: dict[str, Link | Reference] | None = None - # Using Any for Specification Extensions - callbacks: dict[str, dict[str, PathItem] | Reference | Any] | None = None - pathItems: dict[str, PathItem | Reference] | None = None - - -class Tag(BaseModelWithConfig): - name: str - description: str | None = None - externalDocs: ExternalDocumentation | None = None - - -class OpenAPI(BaseModelWithConfig): - openapi: str - info: Info - jsonSchemaDialect: str | None = None - servers: list[Server] | None = None - # Using Any for Specification Extensions - paths: dict[str, PathItem | Any] | None = None - webhooks: dict[str, PathItem | Reference] | None = None - components: Components | None = None - security: list[dict[str, list[str]]] | None = None - tags: list[Tag] | None = None - externalDocs: ExternalDocumentation | None = None - - -Schema.model_rebuild() -Operation.model_rebuild() -Encoding.model_rebuild() diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/openapi/utils.py b/bundle/python-cpu/Lib/site-packages/fastapi/openapi/utils.py deleted file mode 100644 index 74caac44a4187fa6646b5053dd5ca83641ee26df..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/openapi/utils.py +++ /dev/null @@ -1,679 +0,0 @@ -import copy -import http.client -import inspect -import warnings -from collections.abc import Sequence -from dataclasses import dataclass, field -from typing import Any, Literal, cast - -from fastapi import routing -from fastapi._compat import ( - ModelField, - get_definitions, - get_flat_models_from_fields, - get_model_name_map, - get_schema_from_model_field, - lenient_issubclass, -) -from fastapi.datastructures import DefaultPlaceholder, _Unset -from fastapi.dependencies.models import ( - Dependant, - _get_cache_key, - _get_oauth_scopes, - _get_security_scheme, - _is_security_scheme, - _UsesScopesCache, -) -from fastapi.dependencies.utils import ( - _get_flat_fields_from_params, - get_flat_params, - get_validation_alias, -) -from fastapi.encoders import jsonable_encoder -from fastapi.exceptions import FastAPIDeprecationWarning -from fastapi.openapi.constants import METHODS_WITH_BODY, REF_PREFIX -from fastapi.openapi.models import OpenAPI -from fastapi.params import Body, ParamTypes -from fastapi.responses import Response -from fastapi.sse import _SSE_EVENT_SCHEMA -from fastapi.types import DependencyCacheKey, ModelNameMap -from fastapi.utils import ( - deep_dict_update, - generate_operation_id_for_path, - is_body_allowed_for_status_code, -) -from pydantic import BaseModel -from starlette.responses import JSONResponse -from starlette.routing import BaseRoute - -validation_error_definition = { - "title": "ValidationError", - "type": "object", - "properties": { - "loc": { - "title": "Location", - "type": "array", - "items": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, - }, - "msg": {"title": "Message", "type": "string"}, - "type": {"title": "Error Type", "type": "string"}, - "input": {"title": "Input"}, - "ctx": {"title": "Context", "type": "object"}, - }, - "required": ["loc", "msg", "type"], -} - -validation_error_response_definition = { - "title": "HTTPValidationError", - "type": "object", - "properties": { - "detail": { - "title": "Detail", - "type": "array", - "items": {"$ref": REF_PREFIX + "ValidationError"}, - } - }, -} - -status_code_ranges: dict[str, str] = { - "1XX": "Information", - "2XX": "Success", - "3XX": "Redirection", - "4XX": "Client Error", - "5XX": "Server Error", - "DEFAULT": "Default Response", -} - - -@dataclass -class _OpenAPIDependencyData: - path_params: list[ModelField] = field(default_factory=list) - query_params: list[ModelField] = field(default_factory=list) - header_params: list[ModelField] = field(default_factory=list) - cookie_params: list[ModelField] = field(default_factory=list) - security_dependencies: list[tuple[Dependant, list[str]]] = field( - default_factory=list - ) - - -def _get_openapi_dependency_data(dependant: Dependant) -> _OpenAPIDependencyData: - dependency_data = _OpenAPIDependencyData() - visited: list[DependencyCacheKey] = [] - uses_scopes_cache: _UsesScopesCache = {} - dependants: list[tuple[Dependant, list[str], bool]] = [(dependant, [], True)] - while dependants: - current_dependant, parent_oauth_scopes, is_root = dependants.pop() - cache_key = _get_cache_key( - dependant=current_dependant, - uses_scopes_cache=uses_scopes_cache, - ) - if cache_key in visited: - continue - visited.append(cache_key) - dependency_data.path_params.extend(current_dependant.path_params) - dependency_data.query_params.extend(current_dependant.query_params) - dependency_data.header_params.extend(current_dependant.header_params) - dependency_data.cookie_params.extend(current_dependant.cookie_params) - oauth_scopes = parent_oauth_scopes.copy() - for scope in _get_oauth_scopes(dependant=current_dependant): - if scope not in oauth_scopes: - oauth_scopes.append(scope) - if not is_root and _is_security_scheme(dependant=current_dependant): - dependency_data.security_dependencies.append( - (current_dependant, oauth_scopes) - ) - dependants.extend( - (sub_dependant, oauth_scopes, False) - for sub_dependant in reversed(current_dependant.dependencies) - ) - return dependency_data - - -def _get_openapi_security_definitions( - security_dependencies: list[tuple[Dependant, list[str]]], -) -> tuple[dict[str, Any], list[dict[str, Any]]]: - security_definitions = {} - # Use a dict to merge scopes for same security scheme - operation_security_dict: dict[str, list[str]] = {} - for security_dependency, oauth_scopes in security_dependencies: - security_scheme = _get_security_scheme(dependant=security_dependency) - security_definition = jsonable_encoder( - security_scheme.model, - by_alias=True, - exclude_none=True, - ) - security_name = security_scheme.scheme_name - security_definitions[security_name] = security_definition - # Merge scopes for the same security scheme - if security_name not in operation_security_dict: - operation_security_dict[security_name] = [] - for scope in oauth_scopes: - if scope not in operation_security_dict[security_name]: - operation_security_dict[security_name].append(scope) - operation_security = [ - {name: scopes} for name, scopes in operation_security_dict.items() - ] - return security_definitions, operation_security - - -def _get_openapi_operation_parameters( - *, - dependency_data: _OpenAPIDependencyData, - model_name_map: ModelNameMap, - field_mapping: dict[ - tuple[ModelField, Literal["validation", "serialization"]], dict[str, Any] - ], - separate_input_output_schemas: bool = True, -) -> list[dict[str, Any]]: - parameters = [] - path_params = _get_flat_fields_from_params(dependency_data.path_params) - query_params = _get_flat_fields_from_params(dependency_data.query_params) - header_params = _get_flat_fields_from_params(dependency_data.header_params) - cookie_params = _get_flat_fields_from_params(dependency_data.cookie_params) - parameter_groups = [ - (ParamTypes.path, path_params), - (ParamTypes.query, query_params), - (ParamTypes.header, header_params), - (ParamTypes.cookie, cookie_params), - ] - default_convert_underscores = True - if len(dependency_data.header_params) == 1: - first_field = dependency_data.header_params[0] - if lenient_issubclass(first_field.field_info.annotation, BaseModel): - default_convert_underscores = getattr( - first_field.field_info, "convert_underscores", True - ) - for param_type, param_group in parameter_groups: - for param in param_group: - field_info = param.field_info - # field_info = cast(Param, field_info) - if not getattr(field_info, "include_in_schema", True): - continue - param_schema = get_schema_from_model_field( - field=param, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - name = get_validation_alias(param) - convert_underscores = getattr( - param.field_info, - "convert_underscores", - default_convert_underscores, - ) - if ( - param_type == ParamTypes.header - and name == param.name - and convert_underscores - ): - name = param.name.replace("_", "-") - - parameter = { - "name": name, - "in": param_type.value, - "required": param.field_info.is_required(), - "schema": param_schema, - } - if field_info.description: - parameter["description"] = field_info.description - openapi_examples = getattr(field_info, "openapi_examples", None) - example = getattr(field_info, "example", None) - if openapi_examples: - parameter["examples"] = jsonable_encoder(openapi_examples) - elif example is not _Unset: - parameter["example"] = jsonable_encoder(example) - if getattr(field_info, "deprecated", None): - parameter["deprecated"] = True - parameters.append(parameter) - return parameters - - -def get_openapi_operation_request_body( - *, - body_field: ModelField | None, - model_name_map: ModelNameMap, - field_mapping: dict[ - tuple[ModelField, Literal["validation", "serialization"]], dict[str, Any] - ], - separate_input_output_schemas: bool = True, -) -> dict[str, Any] | None: - if not body_field: - return None - assert isinstance(body_field, ModelField) - body_schema = get_schema_from_model_field( - field=body_field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - field_info = cast(Body, body_field.field_info) - request_media_type = field_info.media_type - required = body_field.field_info.is_required() - request_body_oai: dict[str, Any] = {} - if required: - request_body_oai["required"] = required - request_media_content: dict[str, Any] = {"schema": body_schema} - if field_info.openapi_examples: - request_media_content["examples"] = jsonable_encoder( - field_info.openapi_examples - ) - elif field_info.example is not _Unset: - request_media_content["example"] = jsonable_encoder(field_info.example) - request_body_oai["content"] = {request_media_type: request_media_content} - return request_body_oai - - -def generate_operation_id( - *, route: routing._APIRouteLike, method: str -) -> str: # pragma: nocover - warnings.warn( - message="fastapi.openapi.utils.generate_operation_id() was deprecated, " - "it is not used internally, and will be removed soon", - category=FastAPIDeprecationWarning, - stacklevel=2, - ) - if route.operation_id: - return route.operation_id - path: str = route.path_format - return generate_operation_id_for_path(name=route.name, path=path, method=method) - - -def generate_operation_summary(*, route: routing._APIRouteLike, method: str) -> str: - if route.summary: - return route.summary - return route.name.replace("_", " ").title() - - -def get_openapi_operation_metadata( - *, route: routing._APIRouteLike, method: str, operation_ids: set[str] -) -> dict[str, Any]: - operation: dict[str, Any] = {} - if route.tags: - operation["tags"] = route.tags - operation["summary"] = generate_operation_summary(route=route, method=method) - if route.description: - operation["description"] = route.description - operation_id = route.operation_id or route.unique_id - if operation_id in operation_ids: - endpoint_name = getattr(route.endpoint, "__name__", "") - message = f"Duplicate Operation ID {operation_id} for function {endpoint_name}" - file_name = getattr(route.endpoint, "__globals__", {}).get("__file__") - if file_name: - message += f" at {file_name}" - warnings.warn(message, stacklevel=1) - operation_ids.add(operation_id) - operation["operationId"] = operation_id - if route.deprecated: - operation["deprecated"] = route.deprecated - return operation - - -def get_openapi_path( - *, - route: routing._APIRouteLike, - operation_ids: set[str], - model_name_map: ModelNameMap, - field_mapping: dict[ - tuple[ModelField, Literal["validation", "serialization"]], dict[str, Any] - ], - separate_input_output_schemas: bool = True, -) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: - path = {} - security_schemes: dict[str, Any] = {} - definitions: dict[str, Any] = {} - assert route.methods is not None, "Methods must be a list" - if isinstance(route.response_class, DefaultPlaceholder): - current_response_class: type[Response] = route.response_class.value - else: - current_response_class = route.response_class - assert current_response_class, "A response class is needed to generate OpenAPI" - route_response_media_type: str | None = current_response_class.media_type - if route.include_in_schema: - dependency_data = _get_openapi_dependency_data(route.dependant) - all_route_params = [ - field - for fields in ( - dependency_data.path_params, - dependency_data.query_params, - dependency_data.header_params, - dependency_data.cookie_params, - ) - for field in _get_flat_fields_from_params(fields) - ] - for method in route.methods: - operation = get_openapi_operation_metadata( - route=route, method=method, operation_ids=operation_ids - ) - parameters: list[dict[str, Any]] = [] - security_definitions, operation_security = ( - _get_openapi_security_definitions( - security_dependencies=dependency_data.security_dependencies - ) - ) - if operation_security: - operation.setdefault("security", []).extend(operation_security) - if security_definitions: - security_schemes.update(security_definitions) - operation_parameters = _get_openapi_operation_parameters( - dependency_data=dependency_data, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - parameters.extend(operation_parameters) - if parameters: - all_parameters = { - (param["in"], param["name"]): param for param in parameters - } - required_parameters = { - (param["in"], param["name"]): param - for param in parameters - if param.get("required") - } - # Make sure required definitions of the same parameter take precedence - # over non-required definitions - all_parameters.update(required_parameters) - operation["parameters"] = list(all_parameters.values()) - if method in METHODS_WITH_BODY: - request_body_oai = get_openapi_operation_request_body( - body_field=route.body_field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - if request_body_oai: - operation["requestBody"] = request_body_oai - if route.callbacks: - callbacks = {} - for callback in route.callbacks: - if isinstance(callback, routing.APIRoute): - ( - cb_path, - cb_security_schemes, - cb_definitions, - ) = get_openapi_path( - route=cast(routing._APIRouteLike, callback), - operation_ids=operation_ids, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - callbacks[callback.name] = {callback.path: cb_path} - operation["callbacks"] = callbacks - if route.status_code is not None: - status_code = str(route.status_code) - else: - # It would probably make more sense for all response classes to have an - # explicit default status_code, and to extract it from them, instead of - # doing this inspection tricks, that would probably be in the future - # TODO: probably make status_code a default class attribute for all - # responses in Starlette - response_signature = inspect.signature(current_response_class.__init__) - status_code_param = response_signature.parameters.get("status_code") - if status_code_param is not None: - if isinstance(status_code_param.default, int): - status_code = str(status_code_param.default) - operation.setdefault("responses", {}).setdefault(status_code, {})[ - "description" - ] = route.response_description - if is_body_allowed_for_status_code(route.status_code): - # Check for JSONL streaming (generator endpoints) - if route.is_json_stream: - jsonl_content: dict[str, Any] = {} - if route.stream_item_field: - item_schema = get_schema_from_model_field( - field=route.stream_item_field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - jsonl_content["itemSchema"] = item_schema - else: - jsonl_content["itemSchema"] = {} - operation.setdefault("responses", {}).setdefault( - status_code, {} - ).setdefault("content", {})["application/jsonl"] = jsonl_content - elif route.is_sse_stream: - sse_content: dict[str, Any] = {} - item_schema = copy.deepcopy(_SSE_EVENT_SCHEMA) - if route.stream_item_field: - content_schema = get_schema_from_model_field( - field=route.stream_item_field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - item_schema["required"] = ["data"] - item_schema["properties"]["data"] = { - "type": "string", - "contentMediaType": "application/json", - "contentSchema": content_schema, - } - sse_content["itemSchema"] = item_schema - operation.setdefault("responses", {}).setdefault( - status_code, {} - ).setdefault("content", {})["text/event-stream"] = sse_content - elif route_response_media_type: - response_schema = {"type": "string"} - if lenient_issubclass(current_response_class, JSONResponse): - if route.response_field: - response_schema = get_schema_from_model_field( - field=route.response_field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - else: - response_schema = {} - operation.setdefault("responses", {}).setdefault( - status_code, {} - ).setdefault("content", {}).setdefault( - route_response_media_type, {} - )["schema"] = response_schema - if route.responses: - operation_responses = operation.setdefault("responses", {}) - for ( - additional_status_code, - additional_response, - ) in route.responses.items(): - process_response = copy.deepcopy(additional_response) - process_response.pop("model", None) - status_code_key = str(additional_status_code).upper() - if status_code_key == "DEFAULT": - status_code_key = "default" - openapi_response = operation_responses.setdefault( - status_code_key, {} - ) - assert isinstance(process_response, dict), ( - "An additional response must be a dict" - ) - field = route.response_fields.get(additional_status_code) - additional_field_schema: dict[str, Any] | None = None - if field: - additional_field_schema = get_schema_from_model_field( - field=field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - media_type = route_response_media_type or "application/json" - additional_schema = ( - process_response.setdefault("content", {}) - .setdefault(media_type, {}) - .setdefault("schema", {}) - ) - deep_dict_update(additional_schema, additional_field_schema) - status_text: str | None = status_code_ranges.get( - str(additional_status_code).upper() - ) or http.client.responses.get(int(additional_status_code)) - description = ( - process_response.get("description") - or openapi_response.get("description") - or status_text - or "Additional Response" - ) - deep_dict_update(openapi_response, process_response) - openapi_response["description"] = description - http422 = "422" - if (all_route_params or route.body_field) and not any( - status in operation["responses"] - for status in [http422, "4XX", "default"] - ): - operation["responses"][http422] = { - "description": "Validation Error", - "content": { - "application/json": { - "schema": {"$ref": REF_PREFIX + "HTTPValidationError"} - } - }, - } - if "ValidationError" not in definitions: - definitions.update( - { - "ValidationError": validation_error_definition, - "HTTPValidationError": validation_error_response_definition, - } - ) - if route.openapi_extra: - deep_dict_update(operation, route.openapi_extra) - path[method.lower()] = operation - return path, security_schemes, definitions - - -def _get_api_route_for_openapi( - route_context: routing.RouteContext, -) -> routing._APIRouteLike | None: - if isinstance(route_context.original_route, routing.APIRoute): - return cast(routing._APIRouteLike, route_context) - return None - - -def get_fields_from_routes( - routes: Sequence[BaseRoute | routing.RouteContext], -) -> list[ModelField]: - body_fields_from_routes: list[ModelField] = [] - responses_from_routes: list[ModelField] = [] - request_fields_from_routes: list[ModelField] = [] - callback_flat_models: list[ModelField] = [] - for route_context in routing.iter_route_contexts(routes): - api_route = _get_api_route_for_openapi(route_context) - if api_route is None: - continue - if api_route.include_in_schema: - if api_route.body_field: - assert isinstance(api_route.body_field, ModelField), ( - "A request body must be a Pydantic Field" - ) - body_fields_from_routes.append(api_route.body_field) - if api_route.response_field: - responses_from_routes.append(api_route.response_field) - if api_route.response_fields: - responses_from_routes.extend(api_route.response_fields.values()) - if api_route.stream_item_field: - responses_from_routes.append(api_route.stream_item_field) - if api_route.callbacks: - callback_flat_models.extend(get_fields_from_routes(api_route.callbacks)) - params = get_flat_params(api_route.dependant) - request_fields_from_routes.extend(params) - - flat_models = callback_flat_models + list( - body_fields_from_routes + responses_from_routes + request_fields_from_routes - ) - return flat_models - - -def get_openapi( - *, - title: str, - version: str, - openapi_version: str = "3.1.0", - summary: str | None = None, - description: str | None = None, - routes: Sequence[BaseRoute | routing.RouteContext], - webhooks: Sequence[BaseRoute | routing.RouteContext] | None = None, - tags: list[dict[str, Any]] | None = None, - servers: list[dict[str, str | Any]] | None = None, - terms_of_service: str | None = None, - contact: dict[str, str | Any] | None = None, - license_info: dict[str, str | Any] | None = None, - separate_input_output_schemas: bool = True, - external_docs: dict[str, Any] | None = None, -) -> dict[str, Any]: - info: dict[str, Any] = {"title": title, "version": version} - if summary: - info["summary"] = summary - if description: - info["description"] = description - if terms_of_service: - info["termsOfService"] = terms_of_service - if contact: - info["contact"] = contact - if license_info: - info["license"] = license_info - output: dict[str, Any] = {"openapi": openapi_version, "info": info} - if servers: - output["servers"] = servers - components: dict[str, dict[str, Any]] = {} - paths: dict[str, dict[str, Any]] = {} - webhook_paths: dict[str, dict[str, Any]] = {} - operation_ids: set[str] = set() - all_fields = get_fields_from_routes(list(routes) + list(webhooks or [])) - flat_models = get_flat_models_from_fields(all_fields, known_models=set()) - model_name_map = get_model_name_map(flat_models) - field_mapping, definitions = get_definitions( - fields=all_fields, - model_name_map=model_name_map, - separate_input_output_schemas=separate_input_output_schemas, - ) - for route_context in routing.iter_route_contexts(routes): - api_route = _get_api_route_for_openapi(route_context) - if api_route is not None: - result = get_openapi_path( - route=api_route, - operation_ids=operation_ids, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - if result: - path, security_schemes, path_definitions = result - if path: - paths.setdefault(api_route.path_format, {}).update(path) - if security_schemes: - components.setdefault("securitySchemes", {}).update( - security_schemes - ) - if path_definitions: - definitions.update(path_definitions) - for webhook_context in routing.iter_route_contexts(webhooks or []): - api_webhook = _get_api_route_for_openapi(webhook_context) - if api_webhook is not None: - result = get_openapi_path( - route=api_webhook, - operation_ids=operation_ids, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - if result: - path, security_schemes, path_definitions = result - if path: - webhook_paths.setdefault(api_webhook.path_format, {}).update(path) - if security_schemes: - components.setdefault("securitySchemes", {}).update( - security_schemes - ) - if path_definitions: - definitions.update(path_definitions) - if definitions: - components["schemas"] = {k: definitions[k] for k in sorted(definitions)} - if components: - output["components"] = components - output["paths"] = paths - if webhook_paths: - output["webhooks"] = webhook_paths - if tags: - output["tags"] = tags - if external_docs: - output["externalDocs"] = external_docs - return jsonable_encoder(OpenAPI(**output), by_alias=True, exclude_none=True) # type: ignore[no-any-return] diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/param_functions.py b/bundle/python-cpu/Lib/site-packages/fastapi/param_functions.py deleted file mode 100644 index 1856178fcb5c88a46899a7a490fbb0c26d314da1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/param_functions.py +++ /dev/null @@ -1,2460 +0,0 @@ -from collections.abc import Callable, Sequence -from typing import Annotated, Any, Literal - -from annotated_doc import Doc -from fastapi import params -from fastapi._compat import Undefined -from fastapi.datastructures import _Unset -from fastapi.openapi.models import Example -from pydantic import AliasChoices, AliasPath -from typing_extensions import deprecated - - -def Path( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = ..., - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - - Read more about it in the - [FastAPI docs for Path Parameters and Numeric Validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#declare-metadata) - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - """ - Declare a path parameter for a *path operation*. - - Read more about it in the - [FastAPI docs for Path Parameters and Numeric Validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/). - - ```python - from typing import Annotated - - from fastapi import FastAPI, Path - - app = FastAPI() - - - @app.get("/items/{item_id}") - async def read_items( - item_id: Annotated[int, Path(title="The ID of the item to get")], - ): - return {"item_id": item_id} - ``` - """ - return params.Path( - default=default, - default_factory=default_factory, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Query( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#alternative-old-query-as-the-default-value) - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#alias-parameters) - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#declare-more-metadata) - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#declare-more-metadata) - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/) - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/) - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#add-regular-expressions - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#deprecating-parameters) - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.Query( - default=default, - default_factory=default_factory, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Header( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - convert_underscores: Annotated[ - bool, - Doc( - """ - Automatically convert underscores to hyphens in the parameter field name. - - Read more about it in the - [FastAPI docs for Header Parameters](https://fastapi.tiangolo.com/tutorial/header-params/#automatic-conversion) - """ - ), - ] = True, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.Header( - default=default, - default_factory=default_factory, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - convert_underscores=convert_underscores, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Cookie( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.Cookie( - default=default, - default_factory=default_factory, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Body( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - embed: Annotated[ - bool | None, - Doc( - """ - When `embed` is `True`, the parameter will be expected in a JSON body as a - key instead of being the JSON body itself. - - This happens automatically when more than one `Body` parameter is declared. - - Read more about it in the - [FastAPI docs for Body - Multiple Parameters](https://fastapi.tiangolo.com/tutorial/body-multiple-params/#embed-a-single-body-parameter). - """ - ), - ] = None, - media_type: Annotated[ - str, - Doc( - """ - The media type of this parameter field. Changing it would affect the - generated OpenAPI, but currently it doesn't affect the parsing of the data. - """ - ), - ] = "application/json", - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.Body( - default=default, - default_factory=default_factory, - embed=embed, - media_type=media_type, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Form( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - media_type: Annotated[ - str, - Doc( - """ - The media type of this parameter field. Changing it would affect the - generated OpenAPI, but currently it doesn't affect the parsing of the data. - """ - ), - ] = "application/x-www-form-urlencoded", - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.Form( - default=default, - default_factory=default_factory, - media_type=media_type, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def File( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - media_type: Annotated[ - str, - Doc( - """ - The media type of this parameter field. Changing it would affect the - generated OpenAPI, but currently it doesn't affect the parsing of the data. - """ - ), - ] = "multipart/form-data", - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.File( - default=default, - default_factory=default_factory, - media_type=media_type, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Depends( # noqa: N802 - dependency: Annotated[ - Callable[..., Any] | None, - Doc( - """ - A "dependable" callable (like a function). - - Don't call it directly, FastAPI will call it for you, just pass the object - directly. - - Read more about it in the - [FastAPI docs for Dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/) - """ - ), - ] = None, - *, - use_cache: Annotated[ - bool, - Doc( - """ - By default, after a dependency is called the first time in a request, if - the dependency is declared again for the rest of the request (for example - if the dependency is needed by several dependencies), the value will be - re-used for the rest of the request. - - Set `use_cache` to `False` to disable this behavior and ensure the - dependency is called again (if declared more than once) in the same request. - - Read more about it in the - [FastAPI docs about sub-dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/sub-dependencies/#using-the-same-dependency-multiple-times) - """ - ), - ] = True, - scope: Annotated[ - Literal["function", "request"] | None, - Doc( - """ - Mainly for dependencies with `yield`, define when the dependency function - should start (the code before `yield`) and when it should end (the code - after `yield`). - - * `"function"`: start the dependency before the *path operation function* - that handles the request, end the dependency after the *path operation - function* ends, but **before** the response is sent back to the client. - So, the dependency function will be executed **around** the *path operation - **function***. - * `"request"`: start the dependency before the *path operation function* - that handles the request (similar to when using `"function"`), but end - **after** the response is sent back to the client. So, the dependency - function will be executed **around** the **request** and response cycle. - - Read more about it in the - [FastAPI docs for FastAPI Dependencies with yield](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-with-yield/#early-exit-and-scope) - """ - ), - ] = None, -) -> Any: - """ - Declare a FastAPI dependency. - - It takes a single "dependable" callable (like a function). - - Don't call it directly, FastAPI will call it for you. - - Read more about it in the - [FastAPI docs for Dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/). - - **Example** - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - - app = FastAPI() - - - async def common_parameters(q: str | None = None, skip: int = 0, limit: int = 100): - return {"q": q, "skip": skip, "limit": limit} - - - @app.get("/items/") - async def read_items(commons: Annotated[dict, Depends(common_parameters)]): - return commons - ``` - """ - return params.Depends(dependency=dependency, use_cache=use_cache, scope=scope) - - -def Security( # noqa: N802 - dependency: Annotated[ - Callable[..., Any] | None, - Doc( - """ - A "dependable" callable (like a function). - - Don't call it directly, FastAPI will call it for you, just pass the object - directly. - - Read more about it in the - [FastAPI docs for Dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/) - """ - ), - ] = None, - *, - scopes: Annotated[ - Sequence[str] | None, - Doc( - """ - OAuth2 scopes required for the *path operation* that uses this Security - dependency. - - The term "scope" comes from the OAuth2 specification, it seems to be - intentionally vague and interpretable. It normally refers to permissions, - in cases to roles. - - These scopes are integrated with OpenAPI (and the API docs at `/docs`). - So they are visible in the OpenAPI specification. - - Read more about it in the - [FastAPI docs about OAuth2 scopes](https://fastapi.tiangolo.com/advanced/security/oauth2-scopes/) - """ - ), - ] = None, - use_cache: Annotated[ - bool, - Doc( - """ - By default, after a dependency is called the first time in a request, if - the dependency is declared again for the rest of the request (for example - if the dependency is needed by several dependencies), the value will be - re-used for the rest of the request. - - Set `use_cache` to `False` to disable this behavior and ensure the - dependency is called again (if declared more than once) in the same request. - - Read more about it in the - [FastAPI docs about sub-dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/sub-dependencies/#using-the-same-dependency-multiple-times) - """ - ), - ] = True, -) -> Any: - """ - Declare a FastAPI Security dependency. - - The only difference with a regular dependency is that it can declare OAuth2 - scopes that will be integrated with OpenAPI and the automatic UI docs (by default - at `/docs`). - - It takes a single "dependable" callable (like a function). - - Don't call it directly, FastAPI will call it for you. - - Read more about it in the - [FastAPI docs for Security](https://fastapi.tiangolo.com/tutorial/security/) and - in the - [FastAPI docs for OAuth2 scopes](https://fastapi.tiangolo.com/advanced/security/oauth2-scopes/). - - **Example** - - ```python - from typing import Annotated - - from fastapi import Security, FastAPI - - from .db import User - from .security import get_current_active_user - - app = FastAPI() - - @app.get("/users/me/items/") - async def read_own_items( - current_user: Annotated[User, Security(get_current_active_user, scopes=["items"])] - ): - return [{"item_id": "Foo", "owner": current_user.username}] - ``` - """ - return params.Security(dependency=dependency, scopes=scopes, use_cache=use_cache) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/params.py b/bundle/python-cpu/Lib/site-packages/fastapi/params.py deleted file mode 100644 index d3f2ae175b1fd3f0dc95c2be70ec7c546040f6ae..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/params.py +++ /dev/null @@ -1,754 +0,0 @@ -import warnings -from collections.abc import Callable, Sequence -from dataclasses import dataclass -from enum import Enum -from typing import Annotated, Any, Literal - -from fastapi.exceptions import FastAPIDeprecationWarning -from fastapi.openapi.models import Example -from pydantic import AliasChoices, AliasPath -from pydantic.fields import FieldInfo -from typing_extensions import deprecated - -from ._compat import ( - Undefined, -) -from .datastructures import _Unset - - -class ParamTypes(Enum): - query = "query" - header = "header" - path = "path" - cookie = "cookie" - - -class Param(FieldInfo): # type: ignore[misc] # ty: ignore[subclass-of-final-class] - in_: ParamTypes - - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - if example is not _Unset: - warnings.warn( - "`example` has been deprecated, please use `examples` instead", - category=FastAPIDeprecationWarning, - stacklevel=4, - ) - self.example = example - self.include_in_schema = include_in_schema - self.openapi_examples = openapi_examples - kwargs = dict( - default=default, - default_factory=default_factory, - alias=alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - discriminator=discriminator, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - **extra, - ) - if examples is not None: - kwargs["examples"] = examples - if regex is not None: - warnings.warn( - "`regex` has been deprecated, please use `pattern` instead", - category=FastAPIDeprecationWarning, - stacklevel=4, - ) - current_json_schema_extra = json_schema_extra or extra - kwargs["deprecated"] = deprecated - - if serialization_alias in (_Unset, None) and isinstance(alias, str): - serialization_alias = alias - if validation_alias in (_Unset, None): - validation_alias = alias - kwargs.update( - { - "annotation": annotation, - "alias_priority": alias_priority, - "validation_alias": validation_alias, - "serialization_alias": serialization_alias, - "strict": strict, - "json_schema_extra": current_json_schema_extra, - } - ) - kwargs["pattern"] = pattern or regex - - use_kwargs = {k: v for k, v in kwargs.items() if v is not _Unset} - - super().__init__(**use_kwargs) # ty: ignore[invalid-argument-type] - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.default})" - - -class Path(Param): # type: ignore[misc] - in_ = ParamTypes.path - - def __init__( - self, - default: Any = ..., - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - assert default is ..., "Path parameters cannot have a default value" - self.in_ = self.in_ - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -class Query(Param): # type: ignore[misc] - in_ = ParamTypes.query - - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -class Header(Param): # type: ignore[misc] - in_ = ParamTypes.header - - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - convert_underscores: bool = True, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - self.convert_underscores = convert_underscores - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -class Cookie(Param): # type: ignore[misc] - in_ = ParamTypes.cookie - - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -class Body(FieldInfo): # type: ignore[misc] # ty: ignore[subclass-of-final-class] - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - embed: bool | None = None, - media_type: str = "application/json", - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - self.embed = embed - self.media_type = media_type - if example is not _Unset: - warnings.warn( - "`example` has been deprecated, please use `examples` instead", - category=FastAPIDeprecationWarning, - stacklevel=4, - ) - self.example = example - self.include_in_schema = include_in_schema - self.openapi_examples = openapi_examples - kwargs = dict( - default=default, - default_factory=default_factory, - alias=alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - discriminator=discriminator, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - **extra, - ) - if examples is not None: - kwargs["examples"] = examples - if regex is not None: - warnings.warn( - "`regex` has been deprecated, please use `pattern` instead", - category=FastAPIDeprecationWarning, - stacklevel=4, - ) - current_json_schema_extra = json_schema_extra or extra - kwargs["deprecated"] = deprecated - if serialization_alias in (_Unset, None) and isinstance(alias, str): - serialization_alias = alias - if validation_alias in (_Unset, None): - validation_alias = alias - kwargs.update( - { - "annotation": annotation, - "alias_priority": alias_priority, - "validation_alias": validation_alias, - "serialization_alias": serialization_alias, - "strict": strict, - "json_schema_extra": current_json_schema_extra, - } - ) - kwargs["pattern"] = pattern or regex - - use_kwargs = {k: v for k, v in kwargs.items() if v is not _Unset} - - super().__init__(**use_kwargs) # ty: ignore[invalid-argument-type] - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.default})" - - -class Form(Body): # type: ignore[misc] - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - media_type: str = "application/x-www-form-urlencoded", - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - media_type=media_type, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -class File(Form): # type: ignore[misc] - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - media_type: str = "multipart/form-data", - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - media_type=media_type, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -@dataclass(frozen=True) -class Depends: - dependency: Callable[..., Any] | None = None - use_cache: bool = True - scope: Literal["function", "request"] | None = None - - -@dataclass(frozen=True) -class Security(Depends): - scopes: Sequence[str] | None = None diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/py.typed b/bundle/python-cpu/Lib/site-packages/fastapi/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/requests.py b/bundle/python-cpu/Lib/site-packages/fastapi/requests.py deleted file mode 100644 index d16552c0a9535e1c0bd7f701987301681832eba5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/requests.py +++ /dev/null @@ -1,2 +0,0 @@ -from starlette.requests import HTTPConnection as HTTPConnection # noqa: F401 -from starlette.requests import Request as Request # noqa: F401 diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/responses.py b/bundle/python-cpu/Lib/site-packages/fastapi/responses.py deleted file mode 100644 index 29df4b7a614b411237700d25c4ac27b985811730..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/responses.py +++ /dev/null @@ -1,98 +0,0 @@ -import importlib -from typing import Any, Protocol, cast - -from fastapi.exceptions import FastAPIDeprecationWarning -from fastapi.sse import EventSourceResponse as EventSourceResponse # noqa -from starlette.responses import FileResponse as FileResponse # noqa -from starlette.responses import HTMLResponse as HTMLResponse # noqa -from starlette.responses import JSONResponse as JSONResponse # noqa -from starlette.responses import PlainTextResponse as PlainTextResponse # noqa -from starlette.responses import RedirectResponse as RedirectResponse # noqa -from starlette.responses import Response as Response # noqa -from starlette.responses import StreamingResponse as StreamingResponse # noqa -from typing_extensions import deprecated - - -class _UjsonModule(Protocol): - def dumps(self, __obj: Any, *, ensure_ascii: bool = ...) -> str: ... - - -class _OrjsonModule(Protocol): - OPT_NON_STR_KEYS: int - OPT_SERIALIZE_NUMPY: int - - def dumps(self, __obj: Any, *, option: int = ...) -> bytes: ... - - -try: - ujson = cast(_UjsonModule, importlib.import_module("ujson")) -except ModuleNotFoundError: # pragma: nocover - ujson = None # type: ignore[assignment] - - -try: - orjson = cast(_OrjsonModule, importlib.import_module("orjson")) -except ModuleNotFoundError: # pragma: nocover - orjson = None # type: ignore[assignment] - - -@deprecated( - "UJSONResponse is deprecated, FastAPI now serializes data directly to JSON " - "bytes via Pydantic when a return type or response model is set, which is " - "faster and doesn't need a custom response class. Read more in the FastAPI " - "docs: https://fastapi.tiangolo.com/advanced/custom-response/#orjson-or-response-model " - "and https://fastapi.tiangolo.com/tutorial/response-model/", - category=FastAPIDeprecationWarning, - stacklevel=2, -) -class UJSONResponse(JSONResponse): - """JSON response using the ujson library to serialize data to JSON. - - **Deprecated**: `UJSONResponse` is deprecated. FastAPI now serializes data - directly to JSON bytes via Pydantic when a return type or response model is - set, which is faster and doesn't need a custom response class. - - Read more in the - [FastAPI docs for Custom Response](https://fastapi.tiangolo.com/advanced/custom-response/#orjson-or-response-model) - and the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - - **Note**: `ujson` is not included with FastAPI and must be installed - separately, e.g. `pip install ujson`. - """ - - def render(self, content: Any) -> bytes: - assert ujson is not None, "ujson must be installed to use UJSONResponse" - return ujson.dumps(content, ensure_ascii=False).encode("utf-8") - - -@deprecated( - "ORJSONResponse is deprecated, FastAPI now serializes data directly to JSON " - "bytes via Pydantic when a return type or response model is set, which is " - "faster and doesn't need a custom response class. Read more in the FastAPI " - "docs: https://fastapi.tiangolo.com/advanced/custom-response/#orjson-or-response-model " - "and https://fastapi.tiangolo.com/tutorial/response-model/", - category=FastAPIDeprecationWarning, - stacklevel=2, -) -class ORJSONResponse(JSONResponse): - """JSON response using the orjson library to serialize data to JSON. - - **Deprecated**: `ORJSONResponse` is deprecated. FastAPI now serializes data - directly to JSON bytes via Pydantic when a return type or response model is - set, which is faster and doesn't need a custom response class. - - Read more in the - [FastAPI docs for Custom Response](https://fastapi.tiangolo.com/advanced/custom-response/#orjson-or-response-model) - and the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - - **Note**: `orjson` is not included with FastAPI and must be installed - separately, e.g. `pip install orjson`. - """ - - def render(self, content: Any) -> bytes: - assert orjson is not None, "orjson must be installed to use ORJSONResponse" - return orjson.dumps( - content, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SERIALIZE_NUMPY - ) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/routing.py b/bundle/python-cpu/Lib/site-packages/fastapi/routing.py deleted file mode 100644 index dc48a33d63ed9aae3c1b4a5d8abd2ab4f724c908..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/routing.py +++ /dev/null @@ -1,6447 +0,0 @@ -import contextlib -import copy -import email.message -import errno -import functools -import inspect -import json -import os -import stat -import threading -import types -import warnings -from collections.abc import ( - AsyncIterator, - Awaitable, - Callable, - Collection, - Coroutine, - Generator, - Iterator, - Mapping, - Sequence, -) -from contextlib import ( - AbstractAsyncContextManager, - AbstractContextManager, - AsyncExitStack, - asynccontextmanager, -) -from contextvars import ContextVar -from dataclasses import dataclass, field -from enum import Enum, IntEnum -from typing import ( - Annotated, - Any, - Literal, - Protocol, - TypeVar, - cast, -) - -import anyio -from annotated_doc import Doc -from anyio.abc import ObjectReceiveStream -from fastapi import params -from fastapi._compat import ( - ModelField, - Undefined, - lenient_issubclass, -) -from fastapi.datastructures import Default, DefaultPlaceholder -from fastapi.dependencies.models import ( - Dependant, - _is_async_gen_callable, - _is_coroutine_callable, - _is_gen_callable, -) -from fastapi.dependencies.utils import ( - SolvedDependency, - _get_body_field, - _get_flat_body_params, - _should_embed_body_fields, - get_dependant, - get_parameterless_sub_dependant, - get_stream_item_type, - get_typed_return_annotation, - solve_dependencies, -) -from fastapi.encoders import jsonable_encoder -from fastapi.exceptions import ( - EndpointContext, - FastAPIError, - RequestValidationError, - ResponseValidationError, - WebSocketRequestValidationError, -) -from fastapi.sse import ( - _PING_INTERVAL, - KEEPALIVE_COMMENT, - EventSourceResponse, - ServerSentEvent, - format_sse_event, -) -from fastapi.types import DecoratedCallable, IncEx -from fastapi.utils import ( - create_model_field, - generate_unique_id, - get_value_or_default, - is_body_allowed_for_status_code, -) -from starlette import routing -from starlette._exception_handler import wrap_app_handling_exceptions -from starlette._utils import get_route_path, is_async_callable -from starlette.concurrency import iterate_in_threadpool, run_in_threadpool -from starlette.datastructures import URL, FormData, URLPath -from starlette.exceptions import HTTPException -from starlette.requests import Request -from starlette.responses import ( - JSONResponse, - PlainTextResponse, - RedirectResponse, - Response, - StreamingResponse, -) -from starlette.routing import ( - BaseRoute, - Match, - NoMatchFound, - compile_path, - get_name, -) -from starlette.routing import Mount as Mount # noqa -from starlette.staticfiles import StaticFiles -from starlette.types import AppType, ASGIApp, Lifespan, Receive, Scope, Send -from starlette.websockets import WebSocket -from typing_extensions import deprecated - - -# Copy of starlette.routing.request_response modified to include the -# dependencies' AsyncExitStack -def request_response( - func: Callable[[Request], Awaitable[Response] | Response], -) -> ASGIApp: - """ - Takes a function or coroutine `func(request) -> response`, - and returns an ASGI application. - """ - f: Callable[[Request], Awaitable[Response]] = ( - func # type: ignore[assignment] - if is_async_callable(func) - else functools.partial(run_in_threadpool, func) # type: ignore[call-arg] - ) # ty: ignore[invalid-assignment] - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - request = Request(scope, receive, send) - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - # Starts customization - response_awaited = False - async with AsyncExitStack() as request_stack: - scope["fastapi_inner_astack"] = request_stack - async with AsyncExitStack() as function_stack: - scope["fastapi_function_astack"] = function_stack - response = await f(request) - await response(scope, receive, send) - # Continues customization - response_awaited = True - if not response_awaited: - raise FastAPIError( - "Response not awaited. There's a high chance that the " - "application code is raising an exception and a dependency with yield " - "has a block with a bare except, or a block with except Exception, " - "and is not raising the exception again. Read more about it in the " - "docs: https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-with-yield/#dependencies-with-yield-and-except" - ) - - # Same as in Starlette - await wrap_app_handling_exceptions(app, request)(scope, receive, send) - - return app - - -# Copy of starlette.routing.websocket_session modified to include the -# dependencies' AsyncExitStack -def websocket_session( - func: Callable[[WebSocket], Awaitable[None]], -) -> ASGIApp: - """ - Takes a coroutine `func(session)`, and returns an ASGI application. - """ - # assert asyncio.iscoroutinefunction(func), "WebSocket endpoints must be async" - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - session = WebSocket(scope, receive=receive, send=send) - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - async with AsyncExitStack() as request_stack: - scope["fastapi_inner_astack"] = request_stack - async with AsyncExitStack() as function_stack: - scope["fastapi_function_astack"] = function_stack - await func(session) - - # Same as in Starlette - await wrap_app_handling_exceptions(app, session)(scope, receive, send) - - return app - - -_T = TypeVar("_T") - - -# Vendored from starlette.routing to avoid importing private symbols -class _AsyncLiftContextManager(AbstractAsyncContextManager[_T]): - """ - Wraps a synchronous context manager to make it async. - - This is vendored from Starlette to avoid importing private symbols. - """ - - def __init__(self, cm: AbstractContextManager[_T]) -> None: - self._cm = cm - - async def __aenter__(self) -> _T: - return self._cm.__enter__() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: types.TracebackType | None, - ) -> bool | None: - return self._cm.__exit__(exc_type, exc_value, traceback) - - -# Vendored from starlette.routing to avoid importing private symbols -def _wrap_gen_lifespan_context( - lifespan_context: Callable[[Any], Generator[Any, Any, Any]], -) -> Callable[[Any], AbstractAsyncContextManager[Any]]: - """ - Wrap a generator-based lifespan context into an async context manager. - - This is vendored from Starlette to avoid importing private symbols. - """ - cmgr = contextlib.contextmanager(lifespan_context) - - @functools.wraps(cmgr) - def wrapper(app: Any) -> _AsyncLiftContextManager[Any]: - return _AsyncLiftContextManager(cmgr(app)) - - return wrapper - - -def _merge_lifespan_context( - original_context: Lifespan[Any], nested_context: Lifespan[Any] -) -> Lifespan[Any]: - @asynccontextmanager - async def merged_lifespan( - app: AppType, - ) -> AsyncIterator[Mapping[str, Any] | None]: - async with original_context(app) as maybe_original_state: - async with nested_context(app) as maybe_nested_state: - if maybe_nested_state is None and maybe_original_state is None: - yield None # old ASGI compatibility - else: - yield {**(maybe_nested_state or {}), **(maybe_original_state or {})} - - return merged_lifespan # type: ignore[return-value] # ty: ignore[invalid-return-type] - - -class _DefaultLifespan: - """ - Default lifespan context manager that runs on_startup and on_shutdown handlers. - - This is a copy of the Starlette _DefaultLifespan class that was removed - in Starlette. FastAPI keeps it to maintain backward compatibility with - on_startup and on_shutdown event handlers. - - Ref: https://github.com/Kludex/starlette/pull/3117 - """ - - def __init__(self, router: "APIRouter") -> None: - self._router = router - - async def __aenter__(self) -> None: - await self._router._startup() - - async def __aexit__(self, *exc_info: object) -> None: - await self._router._shutdown() - - def __call__(self: _T, app: object) -> _T: - return self - - -# Cache for endpoint context to avoid re-extracting on every request -_endpoint_context_cache: dict[int, EndpointContext] = {} - - -def _extract_endpoint_context(func: Any) -> EndpointContext: - """Extract endpoint context with caching to avoid repeated file I/O.""" - func_id = id(func) - - if func_id in _endpoint_context_cache: - return _endpoint_context_cache[func_id] - - try: - ctx: EndpointContext = {} - - if (source_file := inspect.getsourcefile(func)) is not None: - ctx["file"] = source_file - if (line_number := inspect.getsourcelines(func)[1]) is not None: - ctx["line"] = line_number - if (func_name := getattr(func, "__name__", None)) is not None: - ctx["function"] = func_name - except Exception: - ctx = EndpointContext() - - _endpoint_context_cache[func_id] = ctx - return ctx - - -async def serialize_response( - *, - field: ModelField | None = None, - response_content: Any, - include: IncEx | None = None, - exclude: IncEx | None = None, - by_alias: bool = True, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - is_coroutine: bool = True, - endpoint_ctx: EndpointContext | None = None, - dump_json: bool = False, -) -> Any: - if field: - if is_coroutine: - value, errors = field.validate(response_content, {}, loc=("response",)) - else: - value, errors = await run_in_threadpool( - field.validate, response_content, {}, loc=("response",) - ) - if errors: - ctx = endpoint_ctx or EndpointContext() - raise ResponseValidationError( - errors=errors, - body=response_content, - endpoint_ctx=ctx, - ) - serializer = field.serialize_json if dump_json else field.serialize - return serializer( - value, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - - else: - return jsonable_encoder(response_content) - - -async def run_endpoint_function( - *, dependant: Dependant, values: dict[str, Any], is_coroutine: bool -) -> Any: - # Only called by get_request_handler. Has been split into its own function to - # facilitate profiling endpoints, since inner functions are harder to profile. - assert dependant.call is not None, "dependant.call must be a function" - - if is_coroutine: - return await dependant.call(**values) - else: - return await run_in_threadpool(dependant.call, **values) - - -def _build_response_args( - *, status_code: int | None, solved_result: Any -) -> dict[str, Any]: - response_args: dict[str, Any] = { - "background": solved_result.background_tasks, - } - # If status_code was set, use it, otherwise use the default from the - # response class, in the case of redirect it's 307 - current_status_code = ( - status_code if status_code else solved_result.response.status_code - ) - if current_status_code is not None: - response_args["status_code"] = current_status_code - if solved_result.response.status_code: - response_args["status_code"] = solved_result.response.status_code - return response_args - - -def get_request_handler( - dependant: Dependant, - body_field: ModelField | None = None, - status_code: int | None = None, - response_class: type[Response] | DefaultPlaceholder = Default(JSONResponse), - response_field: ModelField | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - dependency_overrides_provider: Any | None = None, - embed_body_fields: bool = False, - strict_content_type: bool | DefaultPlaceholder = Default(True), - stream_item_field: ModelField | None = None, - is_json_stream: bool = False, -) -> Callable[[Request], Coroutine[Any, Any, Response]]: - assert dependant.call is not None, "dependant.call must be a function" - is_coroutine = _is_coroutine_callable(dependant.call) - is_body_form = body_field and isinstance(body_field.field_info, params.Form) - if isinstance(response_class, DefaultPlaceholder): - actual_response_class: type[Response] = response_class.value - else: - actual_response_class = response_class - is_sse_stream = lenient_issubclass(actual_response_class, EventSourceResponse) - if isinstance(strict_content_type, DefaultPlaceholder): - actual_strict_content_type: bool = strict_content_type.value - else: - actual_strict_content_type = strict_content_type - - async def app(request: Request) -> Response: - response: Response | None = None - file_stack = request.scope.get("fastapi_middleware_astack") - assert isinstance(file_stack, AsyncExitStack), ( - "fastapi_middleware_astack not found in request scope" - ) - - # Extract endpoint context for error messages - endpoint_ctx = ( - _extract_endpoint_context(dependant.call) - if dependant.call - else EndpointContext() - ) - - if dependant.path: - # For mounted sub-apps, include the mount path prefix - mount_path = request.scope.get("root_path", "").rstrip("/") - endpoint_ctx["path"] = f"{request.method} {mount_path}{dependant.path}" - - # Read body and auto-close files - try: - body: Any = None - if body_field: - if is_body_form: - body = await request.form() - file_stack.push_async_callback(body.close) - else: - body_bytes = await request.body() - if body_bytes: - json_body: Any = Undefined - content_type_value = request.headers.get("content-type") - if not content_type_value: - if not actual_strict_content_type: - json_body = await request.json() - else: - message = email.message.Message() - message["content-type"] = content_type_value - if message.get_content_maintype() == "application": - subtype = message.get_content_subtype() - if subtype == "json" or subtype.endswith("+json"): - json_body = await request.json() - if json_body != Undefined: - body = json_body - else: - body = body_bytes - except json.JSONDecodeError as e: - validation_error = RequestValidationError( - [ - { - "type": "json_invalid", - "loc": ("body", e.pos), - "msg": "JSON decode error", - "input": {}, - "ctx": {"error": e.msg}, - } - ], - body=e.doc, - endpoint_ctx=endpoint_ctx, - ) - raise validation_error from e - except HTTPException: - # If a middleware raises an HTTPException, it should be raised again - raise - except Exception as e: - http_error = HTTPException( - status_code=400, detail="There was an error parsing the body" - ) - raise http_error from e - - # Solve dependencies and run path operation function, auto-closing dependencies - errors: list[Any] = [] - async_exit_stack = request.scope.get("fastapi_inner_astack") - assert isinstance(async_exit_stack, AsyncExitStack), ( - "fastapi_inner_astack not found in request scope" - ) - solved_result = await solve_dependencies( - request=request, - dependant=dependant, - body=cast(dict[str, Any] | FormData | bytes | None, body), - dependency_overrides_provider=dependency_overrides_provider, - async_exit_stack=async_exit_stack, - embed_body_fields=embed_body_fields, - ) - errors = solved_result.errors - assert dependant.call # For types - if not errors: - # Shared serializer for stream items (JSONL and SSE). - # Validates against stream_item_field when set, then - # serializes to JSON bytes. - def _serialize_data(data: Any) -> bytes: - if stream_item_field: - value, errors_ = stream_item_field.validate( - data, {}, loc=("response",) - ) - if errors_: - ctx = endpoint_ctx or EndpointContext() - raise ResponseValidationError( - errors=errors_, - body=data, - endpoint_ctx=ctx, - ) - return stream_item_field.serialize_json( - value, - include=response_model_include, - exclude=response_model_exclude, - by_alias=response_model_by_alias, - exclude_unset=response_model_exclude_unset, - exclude_defaults=response_model_exclude_defaults, - exclude_none=response_model_exclude_none, - ) - else: - data = jsonable_encoder(data) - return json.dumps(data).encode("utf-8") - - if is_sse_stream: - # Generator endpoint: stream as Server-Sent Events - gen = dependant.call(**solved_result.values) - - def _serialize_sse_item(item: Any) -> bytes: - if isinstance(item, ServerSentEvent): - # User controls the event structure. - # Serialize the data payload if present. - # For ServerSentEvent items we skip stream_item_field - # validation (the user may mix types intentionally). - if item.raw_data is not None: - data_str: str | None = item.raw_data - elif item.data is not None: - if hasattr(item.data, "model_dump_json"): - data_str = item.data.model_dump_json() - else: - data_str = json.dumps(jsonable_encoder(item.data)) - else: - data_str = None - return format_sse_event( - data_str=data_str, - event=item.event, - id=item.id, - retry=item.retry, - comment=item.comment, - ) - else: - # Plain object: validate + serialize via - # stream_item_field (if set) and wrap in data field - return format_sse_event( - data_str=_serialize_data(item).decode("utf-8") - ) - - if _is_async_gen_callable(dependant.call): - sse_aiter: AsyncIterator[Any] = gen.__aiter__() - else: - sse_aiter = iterate_in_threadpool(gen) - - @asynccontextmanager - async def _sse_producer_cm() -> AsyncIterator[ - ObjectReceiveStream[bytes] - ]: - # Use a memory stream to decouple generator iteration - # from the keepalive timer. A producer task pulls items - # from the generator independently, so - # `anyio.fail_after` never wraps the generator's - # `__anext__` directly - avoiding CancelledError that - # would finalize the generator and also working for sync - # generators running in a thread pool. - # - # This context manager is entered on the request-scoped - # AsyncExitStack so its __aexit__ (which cancels the - # task group) is called by the exit stack after the - # streaming response completes — not by async generator - # finalization via GeneratorExit. - # Ref: https://peps.python.org/pep-0789/ - send_stream, receive_stream = anyio.create_memory_object_stream[ - bytes - ](max_buffer_size=1) - - async def _producer() -> None: - async with send_stream: - async for raw_item in sse_aiter: - await send_stream.send(_serialize_sse_item(raw_item)) - - send_keepalive, receive_keepalive = ( - anyio.create_memory_object_stream[bytes](max_buffer_size=1) - ) - - async def _keepalive_inserter() -> None: - """Read from the producer and forward to the output, - inserting keepalive comments on timeout.""" - async with send_keepalive, receive_stream: - try: - while True: - try: - with anyio.fail_after(_PING_INTERVAL): - data = await receive_stream.receive() - await send_keepalive.send(data) - except TimeoutError: - await send_keepalive.send(KEEPALIVE_COMMENT) - except anyio.EndOfStream: - pass - - async with anyio.create_task_group() as tg: - tg.start_soon(_producer) - tg.start_soon(_keepalive_inserter) - yield receive_keepalive - tg.cancel_scope.cancel() - - # Enter the SSE context manager on the request-scoped - # exit stack. The stack outlives the streaming response, - # so __aexit__ runs via proper structured teardown, not - # via GeneratorExit thrown into an async generator. - sse_receive_stream = await async_exit_stack.enter_async_context( - _sse_producer_cm() - ) - # Ensure the receive stream is closed when the exit stack - # unwinds, preventing ResourceWarning from __del__. - async_exit_stack.push_async_callback(sse_receive_stream.aclose) - - async def _sse_with_checkpoints( - stream: ObjectReceiveStream[bytes], - ) -> AsyncIterator[bytes]: - async for data in stream: - yield data - # Guarantee a checkpoint so cancellation can be - # delivered even when the producer is faster than - # the consumer and receive() never suspends. - await anyio.sleep(0) - - sse_stream_content: AsyncIterator[bytes] | Iterator[bytes] = ( - _sse_with_checkpoints(sse_receive_stream) - ) - - response_args = _build_response_args( - status_code=status_code, solved_result=solved_result - ) - response = StreamingResponse( - sse_stream_content, - media_type="text/event-stream", - **response_args, - ) - response.headers["Cache-Control"] = "no-cache" - # For Nginx proxies to not buffer server sent events - response.headers["X-Accel-Buffering"] = "no" - response.headers.raw.extend(solved_result.response.headers.raw) - elif is_json_stream: - # Generator endpoint: stream as JSONL - gen = dependant.call(**solved_result.values) - - def _serialize_item(item: Any) -> bytes: - return _serialize_data(item) + b"\n" - - if _is_async_gen_callable(dependant.call): - - async def _async_stream_jsonl() -> AsyncIterator[bytes]: - async for item in gen: - yield _serialize_item(item) - # To allow for cancellation to trigger - # Ref: https://github.com/fastapi/fastapi/issues/14680 - await anyio.sleep(0) - - jsonl_stream_content: AsyncIterator[bytes] | Iterator[bytes] = ( - _async_stream_jsonl() - ) - else: - - def _sync_stream_jsonl() -> Iterator[bytes]: - for item in gen: # ty: ignore[not-iterable] - yield _serialize_item(item) - - jsonl_stream_content = _sync_stream_jsonl() - - response_args = _build_response_args( - status_code=status_code, solved_result=solved_result - ) - response = StreamingResponse( - jsonl_stream_content, - media_type="application/jsonl", - **response_args, - ) - response.headers.raw.extend(solved_result.response.headers.raw) - elif _is_async_gen_callable(dependant.call) or _is_gen_callable( - dependant.call - ): - # Raw streaming with explicit response_class (e.g. StreamingResponse) - gen = dependant.call(**solved_result.values) - if _is_async_gen_callable(dependant.call): - - async def _async_stream_raw( - async_gen: AsyncIterator[Any], - ) -> AsyncIterator[Any]: - async for chunk in async_gen: - yield chunk - # To allow for cancellation to trigger - # Ref: https://github.com/fastapi/fastapi/issues/14680 - await anyio.sleep(0) - - gen = _async_stream_raw(gen) - response_args = _build_response_args( - status_code=status_code, solved_result=solved_result - ) - response = actual_response_class(content=gen, **response_args) - response.headers.raw.extend(solved_result.response.headers.raw) - else: - raw_response = await run_endpoint_function( - dependant=dependant, - values=solved_result.values, - is_coroutine=is_coroutine, - ) - if isinstance(raw_response, Response): - if raw_response.background is None: - raw_response.background = solved_result.background_tasks - response = raw_response - else: - response_args = _build_response_args( - status_code=status_code, solved_result=solved_result - ) - # Use the fast path (dump_json) when no custom response - # class was set and a response field with a TypeAdapter - # exists. Serializes directly to JSON bytes via Pydantic's - # Rust core, skipping the intermediate Python dict + - # json.dumps() step. - use_dump_json = response_field is not None and isinstance( - response_class, DefaultPlaceholder - ) - content = await serialize_response( - field=response_field, - response_content=raw_response, - include=response_model_include, - exclude=response_model_exclude, - by_alias=response_model_by_alias, - exclude_unset=response_model_exclude_unset, - exclude_defaults=response_model_exclude_defaults, - exclude_none=response_model_exclude_none, - is_coroutine=is_coroutine, - endpoint_ctx=endpoint_ctx, - dump_json=use_dump_json, - ) - if use_dump_json: - response = Response( - content=content, - media_type="application/json", - **response_args, - ) - else: - response = actual_response_class(content, **response_args) - if not is_body_allowed_for_status_code(response.status_code): - response.body = b"" - response.headers.raw.extend(solved_result.response.headers.raw) - if errors: - validation_error = RequestValidationError( - errors, body=body, endpoint_ctx=endpoint_ctx - ) - raise validation_error - - # Return response - assert response - return response - - return app - - -def get_websocket_app( - dependant: Dependant, - dependency_overrides_provider: Any | None = None, - embed_body_fields: bool = False, -) -> Callable[[WebSocket], Coroutine[Any, Any, Any]]: - async def app(websocket: WebSocket) -> None: - endpoint_ctx = ( - _extract_endpoint_context(dependant.call) - if dependant.call - else EndpointContext() - ) - if dependant.path: - # For mounted sub-apps, include the mount path prefix - mount_path = websocket.scope.get("root_path", "").rstrip("/") - endpoint_ctx["path"] = f"WS {mount_path}{dependant.path}" - async_exit_stack = websocket.scope.get("fastapi_inner_astack") - assert isinstance(async_exit_stack, AsyncExitStack), ( - "fastapi_inner_astack not found in request scope" - ) - solved_result = await solve_dependencies( - request=websocket, - dependant=dependant, - dependency_overrides_provider=dependency_overrides_provider, - async_exit_stack=async_exit_stack, - embed_body_fields=embed_body_fields, - ) - if solved_result.errors: - raise WebSocketRequestValidationError( - solved_result.errors, - endpoint_ctx=endpoint_ctx, - ) - assert dependant.call is not None, "dependant.call must be a function" - await dependant.call(**solved_result.values) - - return app - - -class APIWebSocketRoute(routing.WebSocketRoute): - def __init__( - self, - path: str, - endpoint: Callable[..., Any], - *, - name: str | None = None, - dependencies: Sequence[params.Depends] | None = None, - dependency_overrides_provider: Any | None = None, - ) -> None: - self.path = path - self.endpoint = endpoint - self.name = get_name(endpoint) if name is None else name - self.dependencies = list(dependencies or []) - self.path_regex, self.path_format, self.param_convertors = compile_path(path) - ( - self.dependant, - _, - self._embed_body_fields, - ) = _build_dependant_with_parameterless_dependencies( - path=self.path_format, - call=self.endpoint, - dependencies=self.dependencies, - ) - self.app = websocket_session( - get_websocket_app( - dependant=self.dependant, - dependency_overrides_provider=dependency_overrides_provider, - embed_body_fields=self._embed_body_fields, - ) - ) - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - match, child_scope = super().matches(scope) - if match != Match.NONE: - child_scope["route"] = self - return match, child_scope - - -_FASTAPI_SCOPE_KEY = "fastapi" -_FASTAPI_EFFECTIVE_ROUTE_CONTEXT_KEY = "effective_route_context" -_FASTAPI_FRONTEND_PATH_KEY = "frontend_path" -_FASTAPI_FRONTEND_SPECIFICITY_KEY = "frontend_specificity" -_FASTAPI_INCLUDED_ROUTER_KEY = "included_router" -_effective_route_context_var: ContextVar[Any | None] = ContextVar( - "fastapi_effective_route_context", default=None -) -_SCOPE_MISSING = object() - - -def _frontend_dependency_endpoint() -> None: - pass # pragma: no cover - - -def _build_dependant_with_parameterless_dependencies( - *, - path: str, - call: Callable[..., Any], - dependencies: Sequence[params.Depends], -) -> tuple[Dependant, list[ModelField], bool]: - dependant = get_dependant(path=path, call=call, scope="function") - for depends in dependencies[::-1]: - dependant.dependencies.insert( - 0, - get_parameterless_sub_dependant(depends=depends, path=path), - ) - body_params = _get_flat_body_params(dependant) - embed_body_fields = _should_embed_body_fields(body_params) - return dependant, body_params, embed_body_fields - - -class _RouteWithPath(Protocol): - path: str - - -def _get_fastapi_scope(scope: Scope) -> dict[str, Any]: - fastapi_scope = scope.setdefault(_FASTAPI_SCOPE_KEY, {}) - assert isinstance(fastapi_scope, dict) - return fastapi_scope - - -def _update_scope(scope: Scope, child_scope: Scope) -> None: - fastapi_child_scope = child_scope.get(_FASTAPI_SCOPE_KEY) - for key, value in child_scope.items(): - if key != _FASTAPI_SCOPE_KEY: - scope[key] = value - if isinstance(fastapi_child_scope, dict): - _get_fastapi_scope(scope).update(fastapi_child_scope) - - -def _get_scope_effective_route_context(scope: Scope) -> Any | None: - return scope.get(_FASTAPI_SCOPE_KEY, {}).get(_FASTAPI_EFFECTIVE_ROUTE_CONTEXT_KEY) - - -def _get_scope_included_router(scope: Scope) -> Any | None: - return scope.get(_FASTAPI_SCOPE_KEY, {}).get(_FASTAPI_INCLUDED_ROUTER_KEY) - - -def _frontend_scope_specificity(scope: Scope) -> int | None: - specificity = scope.get(_FASTAPI_SCOPE_KEY, {}).get( - _FASTAPI_FRONTEND_SPECIFICITY_KEY - ) - if isinstance(specificity, int): - return specificity - return None - - -def _restore_fastapi_scope_key(scope: Scope, key: str, previous: Any) -> None: - fastapi_scope = scope.get(_FASTAPI_SCOPE_KEY) - if not isinstance(fastapi_scope, dict): - return - if previous is _SCOPE_MISSING: - fastapi_scope.pop(key, None) - else: - fastapi_scope[key] = previous - - -class _APIRouteLike(Protocol): - path: str - endpoint: Callable[..., Any] - stream_item_type: Any | None - response_model: Any - summary: str | None - response_description: str - deprecated: bool | None - operation_id: str | None - response_model_include: IncEx | None - response_model_exclude: IncEx | None - response_model_by_alias: bool - response_model_exclude_unset: bool - response_model_exclude_defaults: bool - response_model_exclude_none: bool - include_in_schema: bool - response_class: type[Response] | DefaultPlaceholder - dependency_overrides_provider: Any | None - callbacks: list[BaseRoute] | None - openapi_extra: dict[str, Any] | None - generate_unique_id_function: Callable[[Any], str] | DefaultPlaceholder - strict_content_type: bool | DefaultPlaceholder - tags: list[str | Enum] - responses: dict[int | str, dict[str, Any]] - name: str - path_regex: Any - path_format: str - param_convertors: dict[str, Any] - methods: set[str] - unique_id: str - status_code: int | None - response_field: ModelField | None - stream_item_field: ModelField | None - dependencies: list[params.Depends] - description: str - response_fields: dict[int | str, ModelField] - dependant: Dependant - _embed_body_fields: bool - body_field: ModelField | None - is_sse_stream: bool - is_json_stream: bool - - -def _populate_api_route_state( - route: _APIRouteLike, - path: str, - endpoint: Callable[..., Any], - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[params.Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - name: str | None = None, - methods: set[str] | list[str] | None = None, - operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] | DefaultPlaceholder = Default(JSONResponse), - dependency_overrides_provider: Any | None = None, - callbacks: list[BaseRoute] | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[[Any], str] | DefaultPlaceholder = Default( - generate_unique_id - ), - strict_content_type: bool | DefaultPlaceholder = Default(True), - stream_item_type: Any | None = None, -) -> None: - route.path = path - route.endpoint = endpoint - route.stream_item_type = stream_item_type - route.summary = summary - route.response_description = response_description - route.deprecated = deprecated - route.operation_id = operation_id - route.response_model_include = response_model_include - route.response_model_exclude = response_model_exclude - route.response_model_by_alias = response_model_by_alias - route.response_model_exclude_unset = response_model_exclude_unset - route.response_model_exclude_defaults = response_model_exclude_defaults - route.response_model_exclude_none = response_model_exclude_none - route.include_in_schema = include_in_schema - route.response_class = response_class - route.dependency_overrides_provider = dependency_overrides_provider - route.callbacks = callbacks - route.openapi_extra = openapi_extra - route.generate_unique_id_function = generate_unique_id_function - route.strict_content_type = strict_content_type - route.tags = tags or [] - route.responses = responses or {} - route.name = get_name(endpoint) if name is None else name - route.path_regex, route.path_format, route.param_convertors = compile_path(path) - if methods is None: - methods = ["GET"] - route.methods = {method.upper() for method in methods} - if isinstance(generate_unique_id_function, DefaultPlaceholder): - current_generate_unique_id: Callable[[Any], str] = ( - generate_unique_id_function.value - ) - else: - current_generate_unique_id = generate_unique_id_function - route.unique_id = route.operation_id or current_generate_unique_id(route) - # normalize enums e.g. http.HTTPStatus - if isinstance(status_code, IntEnum): - status_code = int(status_code) - route.status_code = status_code - route.dependencies = list(dependencies or []) - route.description = description or inspect.cleandoc(route.endpoint.__doc__ or "") - # if a "form feed" character (page break) is found in the description text, - # truncate description text to the content preceding the first "form feed" - route.description = route.description.split("\f")[0].strip() - response_fields = {} - for additional_status_code, response in route.responses.items(): - assert isinstance(response, dict), "An additional response must be a dict" - model = response.get("model") - if model: - assert is_body_allowed_for_status_code(additional_status_code), ( - f"Status code {additional_status_code} must not have a response body" - ) - response_name = f"Response_{additional_status_code}_{route.unique_id}" - response_field = create_model_field( - name=response_name, type_=model, mode="serialization" - ) - response_fields[additional_status_code] = response_field - if response_fields: - route.response_fields = response_fields - else: - route.response_fields = {} - - assert callable(endpoint), "An endpoint must be a callable" - ( - route.dependant, - body_params, - route._embed_body_fields, - ) = _build_dependant_with_parameterless_dependencies( - path=route.path_format, - call=route.endpoint, - dependencies=route.dependencies, - ) - route.body_field = _get_body_field( - body_params=body_params, - name=route.unique_id, - embed_body_fields=route._embed_body_fields, - ) - # Detect generator endpoints that should stream as JSONL or SSE - is_generator = _is_async_gen_callable(route.dependant.call) or _is_gen_callable( - route.dependant.call - ) - route.is_sse_stream = is_generator and lenient_issubclass( - response_class, EventSourceResponse - ) - route.is_json_stream = is_generator and isinstance( - response_class, DefaultPlaceholder - ) - if isinstance(response_model, DefaultPlaceholder): - return_annotation = get_typed_return_annotation(endpoint) - if lenient_issubclass(return_annotation, Response): - response_model = None - else: - stream_item = get_stream_item_type(return_annotation) - if stream_item is not None and is_generator: - # Extract item type for JSONL or SSE streaming for - # generator endpoints when response_class is - # DefaultPlaceholder (JSONL) or EventSourceResponse (SSE). - # ServerSentEvent is excluded: it's a transport - # wrapper, not a data model, so it shouldn't feed - # into validation or OpenAPI schema generation. - if ( - isinstance(response_class, DefaultPlaceholder) - or lenient_issubclass(response_class, EventSourceResponse) - ) and not lenient_issubclass(stream_item, ServerSentEvent): - route.stream_item_type = stream_item - response_model = None - else: - response_model = return_annotation - route.response_model = response_model - if route.response_model: - assert is_body_allowed_for_status_code(status_code), ( - f"Status code {status_code} must not have a response body" - ) - response_name = "Response_" + route.unique_id - route.response_field = create_model_field( - name=response_name, - type_=route.response_model, - mode="serialization", - ) - else: - route.response_field = None - if route.stream_item_type: - stream_item_name = "StreamItem_" + route.unique_id - route.stream_item_field = create_model_field( - name=stream_item_name, - type_=route.stream_item_type, - mode="serialization", - ) - else: - route.stream_item_field = None - - -class APIRoute(routing.Route): - stream_item_type: Any | None - response_model: Any - summary: str | None - response_description: str - deprecated: bool | None - operation_id: str | None - response_model_include: IncEx | None - response_model_exclude: IncEx | None - response_model_by_alias: bool - response_model_exclude_unset: bool - response_model_exclude_defaults: bool - response_model_exclude_none: bool - include_in_schema: bool - response_class: type[Response] | DefaultPlaceholder - dependency_overrides_provider: Any | None - callbacks: list[BaseRoute] | None - openapi_extra: dict[str, Any] | None - generate_unique_id_function: Callable[[Any], str] | DefaultPlaceholder - strict_content_type: bool | DefaultPlaceholder - tags: list[str | Enum] - responses: dict[int | str, dict[str, Any]] - unique_id: str - status_code: int | None - response_field: ModelField | None - stream_item_field: ModelField | None - dependencies: list[params.Depends] - description: str - response_fields: dict[int | str, ModelField] - dependant: Dependant - _embed_body_fields: bool - body_field: ModelField | None - is_sse_stream: bool - is_json_stream: bool - - def __init__( - self, - path: str, - endpoint: Callable[..., Any], - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[params.Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - name: str | None = None, - methods: set[str] | list[str] | None = None, - operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] | DefaultPlaceholder = Default(JSONResponse), - dependency_overrides_provider: Any | None = None, - callbacks: list[BaseRoute] | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[["APIRoute"], str] - | DefaultPlaceholder = Default(generate_unique_id), - strict_content_type: bool | DefaultPlaceholder = Default(True), - ) -> None: - _populate_api_route_state( - cast(_APIRouteLike, self), - path, - endpoint, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - name=name, - methods=methods, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - dependency_overrides_provider=dependency_overrides_provider, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - strict_content_type=strict_content_type, - ) - self.app = request_response(self.get_route_handler()) - - def get_route_handler(self) -> Callable[[Request], Coroutine[Any, Any, Response]]: - route = cast(_APIRouteLike, self) - # TODO: Replace or deprecate this no-scope hook so included-route - # effective context can be passed explicitly instead of via ContextVar. - effective_context = _effective_route_context_var.get() - if effective_context is not None and effective_context.original_route is self: - route = cast(_APIRouteLike, effective_context) - return get_request_handler( - dependant=route.dependant, - body_field=route.body_field, - status_code=route.status_code, - response_class=route.response_class, - response_field=route.response_field, - response_model_include=route.response_model_include, - response_model_exclude=route.response_model_exclude, - response_model_by_alias=route.response_model_by_alias, - response_model_exclude_unset=route.response_model_exclude_unset, - response_model_exclude_defaults=route.response_model_exclude_defaults, - response_model_exclude_none=route.response_model_exclude_none, - dependency_overrides_provider=route.dependency_overrides_provider, - embed_body_fields=route._embed_body_fields, - strict_content_type=route.strict_content_type, - stream_item_field=route.stream_item_field, - is_json_stream=route.is_json_stream, - ) - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - effective_context = _get_scope_effective_route_context(scope) - if effective_context is not None and effective_context.original_route is self: - match, child_scope = effective_context.matches(scope) - else: - match, child_scope = super().matches(scope) - if match != Match.NONE: - child_scope["route"] = self - return match, child_scope - - async def handle(self, scope: Scope, receive: Receive, send: Send) -> None: - effective_context = _get_scope_effective_route_context(scope) - if effective_context is not None and effective_context.original_route is self: - methods = effective_context.methods - if methods and scope["method"] not in methods: - headers = {"Allow": ", ".join(methods)} - if "app" in scope: - raise HTTPException(status_code=405, headers=headers) - response = PlainTextResponse( - "Method Not Allowed", status_code=405, headers=headers - ) - await response(scope, receive, send) - return - token = _effective_route_context_var.set(effective_context) - try: - app = request_response(self.get_route_handler()) - finally: - _effective_route_context_var.reset(token) - await app(scope, receive, send) - return - await super().handle(scope, receive, send) - - -@dataclass -class _RouterIncludeContext: - included_router: "APIRouter" - prefix: str = "" - tags: list[str | Enum] = field(default_factory=list) - dependencies: list[params.Depends] = field(default_factory=list) - default_response_class: type[Response] | DefaultPlaceholder = field( - default_factory=lambda: Default(JSONResponse) - ) - responses: dict[int | str, dict[str, Any]] = field(default_factory=dict) - callbacks: list[BaseRoute] = field(default_factory=list) - deprecated: bool | None = None - include_in_schema: bool = True - generate_unique_id_function: Callable[[APIRoute], str] | DefaultPlaceholder = field( - default_factory=lambda: Default(generate_unique_id) - ) - strict_content_type: bool | DefaultPlaceholder = field( - default_factory=lambda: Default(True) - ) - dependency_overrides_provider: Any | None = None - - @classmethod - def for_include( - cls, - *, - parent_router: "APIRouter", - included_router: "APIRouter", - prefix: str = "", - tags: list[str | Enum] | None = None, - dependencies: Sequence[params.Depends] | None = None, - default_response_class: type[Response] | DefaultPlaceholder = Default( - JSONResponse - ), - responses: dict[int | str, dict[str, Any]] | None = None, - callbacks: list[BaseRoute] | None = None, - deprecated: bool | None = None, - include_in_schema: bool = True, - generate_unique_id_function: Callable[[APIRoute], str] - | DefaultPlaceholder = Default(generate_unique_id), - ) -> "_RouterIncludeContext": - return cls( - included_router=included_router, - prefix=parent_router.prefix + prefix, - tags=[*parent_router.tags, *(tags or [])], - dependencies=[*parent_router.dependencies, *(dependencies or [])], - default_response_class=get_value_or_default( - default_response_class, parent_router.default_response_class - ), - responses={**parent_router.responses, **(responses or {})}, - callbacks=[*parent_router.callbacks, *(callbacks or [])], - deprecated=deprecated or parent_router.deprecated, - include_in_schema=parent_router.include_in_schema and include_in_schema, - generate_unique_id_function=get_value_or_default( - generate_unique_id_function, parent_router.generate_unique_id_function - ), - strict_content_type=parent_router.strict_content_type, - dependency_overrides_provider=parent_router.dependency_overrides_provider, - ) - - def combine( - self, child_context: "_RouterIncludeContext" - ) -> "_RouterIncludeContext": - return _RouterIncludeContext( - included_router=child_context.included_router, - prefix=self.prefix + child_context.prefix, - tags=[*self.tags, *child_context.tags], - dependencies=[*self.dependencies, *child_context.dependencies], - default_response_class=get_value_or_default( - child_context.default_response_class, self.default_response_class - ), - responses={**self.responses, **child_context.responses}, - callbacks=[*self.callbacks, *child_context.callbacks], - deprecated=self.deprecated or child_context.deprecated, - include_in_schema=self.include_in_schema - and child_context.include_in_schema, - generate_unique_id_function=get_value_or_default( - child_context.generate_unique_id_function, - self.generate_unique_id_function, - ), - strict_content_type=get_value_or_default( - child_context.strict_content_type, self.strict_content_type - ), - dependency_overrides_provider=self.dependency_overrides_provider, - ) - - def path_for(self, route: _RouteWithPath) -> str: - return self.prefix + route.path - - -@dataclass -class _EffectiveRouteContext: - original_route: BaseRoute - starlette_route: BaseRoute | None = None - frontend_prefix: str = "" - path: str = "" - endpoint: Callable[..., Any] | None = None - stream_item_type: Any | None = None - response_model: Any = None - summary: str | None = None - response_description: str = "Successful Response" - deprecated: bool | None = None - operation_id: str | None = None - response_model_include: IncEx | None = None - response_model_exclude: IncEx | None = None - response_model_by_alias: bool = True - response_model_exclude_unset: bool = False - response_model_exclude_defaults: bool = False - response_model_exclude_none: bool = False - include_in_schema: bool = True - response_class: type[Response] | DefaultPlaceholder = field( - default_factory=lambda: Default(JSONResponse) - ) - dependency_overrides_provider: Any | None = None - callbacks: list[BaseRoute] | None = None - openapi_extra: dict[str, Any] | None = None - generate_unique_id_function: Callable[[Any], str] | DefaultPlaceholder = field( - default_factory=lambda: Default(generate_unique_id) - ) - strict_content_type: bool | DefaultPlaceholder = field( - default_factory=lambda: Default(True) - ) - tags: list[str | Enum] = field(default_factory=list) - responses: dict[int | str, dict[str, Any]] = field(default_factory=dict) - name: str = "" - path_regex: Any = None - path_format: str = "" - param_convertors: dict[str, Any] = field(default_factory=dict) - methods: set[str] = field(default_factory=set) - unique_id: str = "" - status_code: int | None = None - response_field: ModelField | None = None - stream_item_field: ModelField | None = None - dependencies: list[params.Depends] = field(default_factory=list) - description: str = "" - response_fields: dict[int | str, ModelField] = field(default_factory=dict) - dependant: Dependant | None = None - _embed_body_fields: bool = False - body_field: ModelField | None = None - is_sse_stream: bool = False - is_json_stream: bool = False - - @classmethod - def from_api_route( - cls, - *, - original_route: APIRoute, - include_context: _RouterIncludeContext, - ) -> "_EffectiveRouteContext": - route = cast(_APIRouteLike, original_route) - context = cls(original_route=original_route) - _populate_api_route_state( - cast(_APIRouteLike, context), - include_context.path_for(original_route), - route.endpoint, - response_model=route.response_model, - status_code=route.status_code, - tags=[*include_context.tags, *route.tags], - dependencies=[*include_context.dependencies, *route.dependencies], - summary=route.summary, - description=route.description, - response_description=route.response_description, - responses={**include_context.responses, **route.responses}, - deprecated=route.deprecated or include_context.deprecated, - methods=route.methods, - operation_id=route.operation_id, - response_model_include=route.response_model_include, - response_model_exclude=route.response_model_exclude, - response_model_by_alias=route.response_model_by_alias, - response_model_exclude_unset=route.response_model_exclude_unset, - response_model_exclude_defaults=route.response_model_exclude_defaults, - response_model_exclude_none=route.response_model_exclude_none, - include_in_schema=route.include_in_schema - and include_context.include_in_schema, - response_class=get_value_or_default( - route.response_class, - include_context.included_router.default_response_class, - include_context.default_response_class, - ), - name=route.name, - dependency_overrides_provider=include_context.dependency_overrides_provider, - callbacks=[*include_context.callbacks, *(route.callbacks or [])], - openapi_extra=route.openapi_extra, - generate_unique_id_function=get_value_or_default( - route.generate_unique_id_function, - include_context.included_router.generate_unique_id_function, - include_context.generate_unique_id_function, - ), - strict_content_type=get_value_or_default( - route.strict_content_type, - include_context.included_router.strict_content_type, - include_context.strict_content_type, - ), - stream_item_type=route.stream_item_type, - ) - return context - - @classmethod - def from_frontend_route_group( - cls, - *, - original_route: "_FrontendRouteGroup", - include_context: _RouterIncludeContext, - ) -> "_EffectiveRouteContext": - dependencies = [*include_context.dependencies, *original_route.dependencies] - context = cls( - original_route=original_route, - frontend_prefix=include_context.prefix, - dependencies=dependencies, - dependency_overrides_provider=include_context.dependency_overrides_provider, - ) - ( - context.dependant, - _, - context._embed_body_fields, - ) = _build_dependant_with_parameterless_dependencies( - path="", - call=_frontend_dependency_endpoint, - dependencies=dependencies, - ) - return context - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - if isinstance(self.original_route, _FrontendRouteGroup): - return self.original_route.matches_with_prefix(scope, self.frontend_prefix) - if not isinstance(self.original_route, APIRoute): - assert self.starlette_route is not None - return self.starlette_route.matches(scope) - if scope["type"] != "http": - return Match.NONE, {} - route_path = get_route_path(scope) - match = self.path_regex.match(route_path) - if not match: - return Match.NONE, {} - matched_params = match.groupdict() - for key, value in matched_params.items(): - matched_params[key] = self.param_convertors[key].convert(value) - path_params = dict(scope.get("path_params", {})) - path_params.update(matched_params) - child_scope = {"endpoint": self.endpoint, "path_params": path_params} - methods = self.methods - if methods and scope["method"] not in methods: - return Match.PARTIAL, child_scope - return Match.FULL, child_scope - - def url_path_for(self, name: str, /, **path_params: Any) -> Any: - if not isinstance(self.original_route, APIRoute): - assert self.starlette_route is not None - return self.starlette_route.url_path_for(name, **path_params) - seen_params = set(path_params.keys()) - param_convertors = self.param_convertors - expected_params = set(param_convertors.keys()) - if name != self.name or seen_params != expected_params: - raise routing.NoMatchFound(name, path_params) - path, remaining_params = routing.replace_params( - self.path_format, param_convertors, path_params - ) - assert not remaining_params - return URLPath(path=path, protocol="http") - - -@dataclass(frozen=True) -class RouteContext: - route: BaseRoute - _route_context: _EffectiveRouteContext | None = field(default=None, repr=False) - - @property - def original_route(self) -> BaseRoute: - if self._route_context is not None: - return self._route_context.original_route - return self.route - - @property - def _effective_route(self) -> BaseRoute | _EffectiveRouteContext: - if self._route_context is not None: - return self._route_context - return self.route - - @property - def path(self) -> str | None: - return getattr(self._effective_route, "path", None) - - @property - def path_format(self) -> str | None: - return getattr(self._effective_route, "path_format", None) - - @property - def name(self) -> str | None: - return getattr(self._effective_route, "name", None) - - @property - def methods(self) -> set[str] | None: - return getattr(self._effective_route, "methods", None) - - @property - def endpoint(self) -> Callable[..., Any] | None: - return getattr(self._effective_route, "endpoint", None) - - def __getattr__(self, name: str) -> Any: - return getattr(self._effective_route, name) - - -@dataclass -class _IncludedRouter(BaseRoute): - original_router: "APIRouter" - include_context: _RouterIncludeContext - _effective_routes_lock: Any = field( - default_factory=threading.Lock, repr=False, compare=False - ) - _effective_candidates: list["_EffectiveRouteContext | _IncludedRouter"] = field( - default_factory=list - ) - _effective_candidates_version: int | None = None - _effective_low_priority_routes: list["_EffectiveRouteContext"] = field( - default_factory=list - ) - _effective_low_priority_routes_version: int | None = None - - def effective_candidates(self) -> list["_EffectiveRouteContext | _IncludedRouter"]: - routes_version = self.original_router._get_routes_version() - if routes_version == self._effective_candidates_version: - return self._effective_candidates - with self._effective_routes_lock: - routes_version = self.original_router._get_routes_version() - if routes_version == self._effective_candidates_version: - return self._effective_candidates - effective_candidates: list[_EffectiveRouteContext | _IncludedRouter] = [] - for route in self.original_router.routes: - if isinstance(route, _IncludedRouter): - child_context = self.include_context.combine(route.include_context) - child_branch = _IncludedRouter( - original_router=route.original_router, - include_context=child_context, - ) - effective_candidates.append(child_branch) - continue - route_context = self._build_effective_context(route) - if route_context is not None: - effective_candidates.append(route_context) - self._effective_candidates = effective_candidates - self._effective_candidates_version = routes_version - return effective_candidates - - def effective_low_priority_routes(self) -> list["_EffectiveRouteContext"]: - routes_version = self.original_router._get_routes_version() - if routes_version == self._effective_low_priority_routes_version: - return self._effective_low_priority_routes - with self._effective_routes_lock: - routes_version = self.original_router._get_routes_version() - if routes_version == self._effective_low_priority_routes_version: - return self._effective_low_priority_routes - effective_low_priority_routes: list[_EffectiveRouteContext] = [] - for route in self.original_router._low_priority_routes: - route_context = self._build_effective_context(route) - if route_context is not None: - effective_low_priority_routes.append(route_context) - for route in self.original_router.routes: - if isinstance(route, _IncludedRouter): - child_context = self.include_context.combine(route.include_context) - child_branch = _IncludedRouter( - original_router=route.original_router, - include_context=child_context, - ) - effective_low_priority_routes.extend( - child_branch.effective_low_priority_routes() - ) - self._effective_low_priority_routes = effective_low_priority_routes - self._effective_low_priority_routes_version = routes_version - return effective_low_priority_routes - - def _build_effective_context( - self, route: BaseRoute - ) -> _EffectiveRouteContext | None: - if isinstance(route, APIRoute): - return _EffectiveRouteContext.from_api_route( - original_route=route, - include_context=self.include_context, - ) - if isinstance(route, _FrontendRouteGroup): - return _EffectiveRouteContext.from_frontend_route_group( - original_route=route, - include_context=self.include_context, - ) - if isinstance(route, routing.Route): - starlette_route: BaseRoute = routing.Route( - self.include_context.path_for(route), - endpoint=route.endpoint, - methods=list(route.methods or []), - name=route.name, - include_in_schema=route.include_in_schema, - ) - return _EffectiveRouteContext( - original_route=route, - starlette_route=starlette_route, - ) - if isinstance(route, APIWebSocketRoute): - starlette_route = APIWebSocketRoute( - self.include_context.path_for(route), - endpoint=route.endpoint, - name=route.name, - dependencies=[*self.include_context.dependencies, *route.dependencies], - dependency_overrides_provider=( - self.include_context.dependency_overrides_provider - ), - ) - return _EffectiveRouteContext( - original_route=route, - starlette_route=starlette_route, - ) - if isinstance(route, routing.WebSocketRoute): - starlette_route = routing.WebSocketRoute( - self.include_context.path_for(route), route.endpoint, name=route.name - ) - return _EffectiveRouteContext( - original_route=route, - starlette_route=starlette_route, - ) - if isinstance(route, routing.Mount): - starlette_route = copy.copy(route) - starlette_route.path = self.include_context.path_for(route).rstrip("/") - ( - starlette_route.path_regex, - starlette_route.path_format, - starlette_route.param_convertors, - ) = compile_path(starlette_route.path + "/{path:path}") - return _EffectiveRouteContext( - original_route=route, - starlette_route=starlette_route, - ) - if isinstance(route, routing.Host): - if self.include_context.prefix: - prefixed_app: ASGIApp = routing.Router( - routes=[routing.Mount(self.include_context.prefix, app=route.app)] - ) - else: - prefixed_app = route.app - starlette_route = routing.Host( - route.host, app=prefixed_app, name=route.name - ) - return _EffectiveRouteContext( - original_route=route, - starlette_route=starlette_route, - ) - return None - - def _match( - self, scope: Scope - ) -> tuple[Match, Scope, BaseRoute | None, _EffectiveRouteContext | None]: - partial: tuple[Scope, BaseRoute, _EffectiveRouteContext | None] | None = None - for candidate in self.effective_candidates(): - if isinstance(candidate, _IncludedRouter): - match, child_scope = candidate.matches(scope) - route: BaseRoute = candidate - route_context = None - elif isinstance(candidate.original_route, APIRoute): - route_context = candidate - fastapi_scope = _get_fastapi_scope(scope) - previous_context = fastapi_scope.get( - _FASTAPI_EFFECTIVE_ROUTE_CONTEXT_KEY, _SCOPE_MISSING - ) - fastapi_scope[_FASTAPI_EFFECTIVE_ROUTE_CONTEXT_KEY] = route_context - try: - match, child_scope = candidate.original_route.matches(scope) - finally: - _restore_fastapi_scope_key( - scope, _FASTAPI_EFFECTIVE_ROUTE_CONTEXT_KEY, previous_context - ) - route = candidate.original_route - else: - route_context = candidate - match, child_scope = candidate.matches(scope) - route = candidate.starlette_route or candidate.original_route - if match == Match.FULL: - return match, child_scope, route, route_context - if match == Match.PARTIAL and partial is None: - partial = (child_scope, route, route_context) - if partial is not None: - child_scope, route, route_context = partial - return Match.PARTIAL, child_scope, route, route_context - return Match.NONE, {}, None, None - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - fastapi_scope = _get_fastapi_scope(scope) - previous_router = fastapi_scope.get( - _FASTAPI_INCLUDED_ROUTER_KEY, _SCOPE_MISSING - ) - fastapi_scope[_FASTAPI_INCLUDED_ROUTER_KEY] = self - try: - match, _ = self.original_router.matches(scope) - return match, {} - finally: - _restore_fastapi_scope_key( - scope, _FASTAPI_INCLUDED_ROUTER_KEY, previous_router - ) - - async def handle(self, scope: Scope, receive: Receive, send: Send) -> None: - _get_fastapi_scope(scope)[_FASTAPI_INCLUDED_ROUTER_KEY] = self - await self.original_router.handle(scope, receive, send) - - async def _handle_selected( - self, scope: Scope, receive: Receive, send: Send - ) -> None: - match, child_scope, route, effective_context = self._match(scope) - if match == Match.NONE or route is None: - await self.original_router.default(scope, receive, send) - return - scope.update(child_scope) - if isinstance(route, _IncludedRouter): - await route.handle(scope, receive, send) - return - if effective_context is not None: - _get_fastapi_scope(scope)[_FASTAPI_EFFECTIVE_ROUTE_CONTEXT_KEY] = ( - effective_context - ) - original_route = effective_context.original_route - if isinstance(original_route, APIRoute): - scope["route"] = original_route - await original_route.handle(scope, receive, send) - return - await route.handle(scope, receive, send) - - def effective_route_contexts(self) -> Iterator[_EffectiveRouteContext]: - for candidate in self.effective_candidates(): - if isinstance(candidate, _IncludedRouter): - yield from candidate.effective_route_contexts() - else: - yield candidate - - def url_path_for(self, name: str, /, **path_params: Any) -> Any: - for route_context in self.effective_route_contexts(): - try: - return route_context.url_path_for(name, **path_params) - except routing.NoMatchFound: - pass - raise routing.NoMatchFound(name, path_params) - - -def _iter_included_route_candidates(routes: Sequence[BaseRoute]) -> Iterator[BaseRoute]: - for route, route_context in _iter_routes_with_context(routes): - if route_context is not None and route_context.starlette_route is not None: - yield route_context.starlette_route - else: - yield route - - -def iter_route_contexts( - routes: Sequence[BaseRoute | RouteContext], -) -> Iterator[RouteContext]: - for route in routes: - if isinstance(route, RouteContext): - yield route - continue - for original_route, route_context in _iter_routes_with_context([route]): - if route_context is None: - yield RouteContext(original_route) - else: - yield RouteContext(original_route, route_context) - - -def _iter_routes_with_context( - routes: Sequence[BaseRoute], -) -> Iterator[tuple[BaseRoute, _EffectiveRouteContext | None]]: - for route in routes: - if isinstance(route, _IncludedRouter): - for route_context in route.effective_route_contexts(): - yield route_context.original_route, route_context - else: - yield route, None - - -def _normalize_frontend_path(path: str) -> str: - if not path: - raise AssertionError("A frontend path cannot be empty") - if not path.startswith("/"): - raise AssertionError("A frontend path must start with '/'") - if path != "/": - path = path.rstrip("/") - return path - - -def _join_frontend_paths(prefix: str, path: str) -> str: - if not prefix: - return path - if path == "/": - return prefix - return prefix + path - - -def _frontend_path_specificity(path: str) -> int: - if path == "/": - return 0 - return len(path) - - -def _get_resolved_absolute_path(path: str | os.PathLike[str]) -> str: - return os.path.realpath(os.fspath(path)) - - -def _resolve_frontend_check_dir( - *, - directory: str | os.PathLike[str], - check_dir: bool | Literal["auto"], -) -> bool: - if check_dir != "auto": - return check_dir - if os.environ.get("FASTAPI_ENV") != "development": - return True - if not os.path.isdir(directory): - warnings.warn( - f"Frontend directory '{directory}' does not exist. " - f"Resolved absolute path: '{_get_resolved_absolute_path(directory)}'", - stacklevel=3, - ) - return False - - -class _FrontendStaticFiles(StaticFiles): - def __init__( - self, - *, - directory: str | os.PathLike[str], - fallback: Literal["auto", "index.html", "404.html"] | None, - check_dir: bool, - ) -> None: - self.fallback = fallback - if check_dir and not os.path.isdir(directory): - raise RuntimeError( - f"Frontend directory '{directory}' does not exist. " - f"Resolved absolute path: '{_get_resolved_absolute_path(directory)}'" - ) - super().__init__( - directory=directory, - html=True, - check_dir=check_dir, - follow_symlink=False, - ) - if check_dir and fallback in {"index.html", "404.html"}: - self._check_fallback_file(fallback) - - def _check_fallback_file(self, fallback: str) -> None: - _, stat_result = self.lookup_path(fallback) - if stat_result is None or not stat.S_ISREG(stat_result.st_mode): - raise RuntimeError( - f"Frontend fallback file '{fallback}' does not exist in " - f"directory '{self.directory}'. Resolved absolute directory: " - f"'{self._get_resolved_directory()}'" - ) - - def _get_resolved_directory(self) -> str: - assert self.directory is not None - return _get_resolved_absolute_path(self.directory) - - def get_path(self, scope: Scope) -> str: - path = _get_fastapi_scope(scope).get(_FASTAPI_FRONTEND_PATH_KEY, "") - assert isinstance(path, str) - return os.path.normpath(os.path.join(*path.split("/"))) - - async def get_response_for_scope(self, scope: Scope) -> Response: - if not self.config_checked: - await self.check_config() - self.config_checked = True - return await self.get_response(self.get_path(scope), scope) - - async def get_response(self, path: str, scope: Scope) -> Response: - if scope["method"] not in ("GET", "HEAD"): - if await self._lookup_static_resource(path) is not None: - raise HTTPException(status_code=405) - raise HTTPException(status_code=404) - - static_resource = await self._lookup_static_resource(path) - if static_resource is not None: - full_path, stat_result, is_directory_index = static_resource - if is_directory_index and not scope["path"].endswith("/"): - url = URL(scope=scope) - url = url.replace(path=url.path + "/") - return RedirectResponse(url=url) - return self.file_response(full_path, stat_result, scope) - - if self.fallback == "404.html" or ( - self.fallback == "auto" and self._fallback_file_exists("404.html") - ): - return await self._fallback_response("404.html", scope, status_code=404) - - if ( - self.fallback == "index.html" - or (self.fallback == "auto" and self._fallback_file_exists("index.html")) - ) and _is_frontend_navigation_request(scope): - return await self._fallback_response("index.html", scope, status_code=200) - - raise HTTPException(status_code=404) - - async def _lookup_path(self, path: str) -> tuple[str, os.stat_result | None]: - try: - return await run_in_threadpool(self.lookup_path, path) - except PermissionError: - raise HTTPException(status_code=401) from None - except OSError as exc: - if exc.errno == errno.ENAMETOOLONG: - raise HTTPException(status_code=404) from None - raise exc - except ValueError: - raise HTTPException(status_code=404) from None - - async def _lookup_static_resource( - self, path: str - ) -> tuple[str, os.stat_result, bool] | None: - full_path, stat_result = await self._lookup_path(path) - if stat_result is None: - return None - if stat.S_ISREG(stat_result.st_mode): - return full_path, stat_result, False - if stat.S_ISDIR(stat_result.st_mode): - index_path = os.path.join(path, "index.html") - full_path, stat_result = await self._lookup_path(index_path) - if stat_result is not None and stat.S_ISREG(stat_result.st_mode): - return full_path, stat_result, True - return None - - def _fallback_file_exists(self, fallback: str) -> bool: - _, stat_result = self.lookup_path(fallback) - return stat_result is not None and stat.S_ISREG(stat_result.st_mode) - - async def _fallback_response( - self, fallback: str, scope: Scope, *, status_code: int - ) -> Response: - full_path, stat_result = await run_in_threadpool(self.lookup_path, fallback) - if stat_result is None or not stat.S_ISREG(stat_result.st_mode): - raise RuntimeError( - f"Frontend fallback file '{fallback}' does not exist in " - f"directory '{self.directory}'. Resolved absolute directory: " - f"'{self._get_resolved_directory()}'" - ) - return self.file_response( - full_path, stat_result, scope, status_code=status_code - ) - - -def _iter_accept_media_types(accept: str) -> Iterator[tuple[str, float]]: - for raw_value in accept.split(","): - message = email.message.Message() - message["content-type"] = raw_value.strip() - q = message.get_param("q") - quality = 1.0 - if isinstance(q, str): - try: - quality = float(q) - except ValueError: - pass - yield ( - f"{message.get_content_maintype()}/{message.get_content_subtype()}", - quality, - ) - - -def _is_frontend_navigation_request(scope: Scope) -> bool: - request = Request(scope) - for media_type, quality in _iter_accept_media_types( - request.headers.get("accept", "") - ): - if media_type in {"text/html", "application/xhtml+xml"} and quality != 0: - return True - return False - - -class _FrontendRoute(BaseRoute): - def __init__( - self, - path: str, - *, - directory: str | os.PathLike[str], - fallback: Literal["auto", "index.html", "404.html"] | None = "auto", - check_dir: bool, - ) -> None: - if fallback not in {"auto", "index.html", "404.html", None}: - raise AssertionError( - "fallback must be 'auto', 'index.html', '404.html', or None" - ) - self.path = _normalize_frontend_path(path) - self.methods = {"GET", "HEAD"} - self.app = _FrontendStaticFiles( - directory=directory, fallback=fallback, check_dir=check_dir - ) - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - return self.matches_with_path(scope, self.path) - - def matches_with_path(self, scope: Scope, path: str) -> tuple[Match, Scope]: - if scope["type"] != "http": - return Match.NONE, {} - frontend_path = self._get_frontend_path(path, get_route_path(scope)) - if frontend_path is None: - return Match.NONE, {} - child_scope = { - _FASTAPI_SCOPE_KEY: { - _FASTAPI_FRONTEND_PATH_KEY: frontend_path, - _FASTAPI_FRONTEND_SPECIFICITY_KEY: _frontend_path_specificity(path), - } - } - if scope["method"] not in self.methods: - return Match.PARTIAL, child_scope - return Match.FULL, child_scope - - def _get_frontend_path(self, path: str, route_path: str) -> str | None: - if path == "/": - return route_path.lstrip("/") - if route_path == path: - return "" - prefix = path + "/" - if route_path.startswith(prefix): - return route_path[len(prefix) :] - return None - - async def handle(self, scope: Scope, receive: Receive, send: Send) -> None: - response = await self.app.get_response_for_scope(scope) - await response(scope, receive, send) - - def url_path_for(self, name: str, /, **path_params: Any) -> URLPath: - raise NoMatchFound(name, path_params) - - -class _FrontendRouteGroup(BaseRoute): - def __init__( - self, - *, - dependencies: Sequence[params.Depends] | None = None, - dependency_overrides_provider: Any | None = None, - ) -> None: - self.routes: list[_FrontendRoute] = [] - self.dependencies = list(dependencies or []) - self.dependency_overrides_provider = dependency_overrides_provider - ( - self.dependant, - _, - self._embed_body_fields, - ) = _build_dependant_with_parameterless_dependencies( - path="", - call=_frontend_dependency_endpoint, - dependencies=self.dependencies, - ) - - def add_frontend_route( - self, - path: str, - *, - directory: str | os.PathLike[str], - fallback: Literal["auto", "index.html", "404.html"] | None = "auto", - check_dir: bool, - ) -> None: - self.routes.append( - _FrontendRoute( - path, - directory=directory, - fallback=fallback, - check_dir=check_dir, - ) - ) - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - match, child_scope, _ = self._match(scope, prefix="") - return match, child_scope - - def matches_with_prefix(self, scope: Scope, prefix: str) -> tuple[Match, Scope]: - match, child_scope, _ = self._match(scope, prefix=prefix) - return match, child_scope - - def _match( - self, scope: Scope, *, prefix: str - ) -> tuple[Match, Scope, _FrontendRoute | None]: - full: tuple[Scope, _FrontendRoute, int] | None = None - partial: tuple[Scope, _FrontendRoute, int] | None = None - for route in self.routes: - path = _join_frontend_paths(prefix, route.path) - match, child_scope = route.matches_with_path(scope, path) - specificity = _frontend_path_specificity(path) - if match == Match.FULL: - if full is None or specificity > full[2]: - full = (child_scope, route, specificity) - elif match == Match.PARTIAL: - if partial is None or specificity > partial[2]: - partial = (child_scope, route, specificity) - if full is not None: - child_scope, route, _ = full - return Match.FULL, child_scope, route - if partial is not None: - child_scope, route, _ = partial - return Match.PARTIAL, child_scope, route - return Match.NONE, {}, None - - async def handle(self, scope: Scope, receive: Receive, send: Send) -> None: - effective_context = _get_scope_effective_route_context(scope) - if ( - isinstance(effective_context, _EffectiveRouteContext) - and effective_context.original_route is self - ): - prefix = effective_context.frontend_prefix - dependant = effective_context.dependant - dependency_overrides_provider = ( - effective_context.dependency_overrides_provider - ) - embed_body_fields = effective_context._embed_body_fields - else: - prefix = "" - dependant = self.dependant - dependency_overrides_provider = self.dependency_overrides_provider - embed_body_fields = self._embed_body_fields - match, child_scope, route = self._match(scope, prefix=prefix) - if match == Match.NONE or route is None: - raise HTTPException(status_code=404) - _update_scope(scope, child_scope) - if match == Match.FULL and dependant and dependant.dependencies: - async with self._solve_dependencies( - scope, - receive, - send, - dependant=dependant, - dependency_overrides_provider=dependency_overrides_provider, - embed_body_fields=embed_body_fields, - ) as solved_result: - response = await route.app.get_response_for_scope(scope) - if response.background is None: - response.background = solved_result.background_tasks - response.headers.raw.extend(solved_result.response.headers.raw) - await response(scope, receive, send) - return - await route.handle(scope, receive, send) - - def url_path_for(self, name: str, /, **path_params: Any) -> URLPath: - raise NoMatchFound(name, path_params) - - # TODO: probably move this out of the Route / Route Group, same in APIRoute - # this should probably be top level FastAPI logic, not part of APIRoute and - # duplicated here - @asynccontextmanager - async def _solve_dependencies( - self, - scope: Scope, - receive: Receive, - send: Send, - *, - dependant: Dependant, - dependency_overrides_provider: Any | None, - embed_body_fields: bool, - ) -> AsyncIterator[SolvedDependency]: - request = Request(scope, receive, send) - previous_inner_astack = scope.get("fastapi_inner_astack", _SCOPE_MISSING) - previous_function_astack = scope.get("fastapi_function_astack", _SCOPE_MISSING) - try: - async with AsyncExitStack() as request_stack: - scope["fastapi_inner_astack"] = request_stack - async with AsyncExitStack() as function_stack: - scope["fastapi_function_astack"] = function_stack - solved_result = await solve_dependencies( - request=request, - dependant=dependant, - dependency_overrides_provider=dependency_overrides_provider, - async_exit_stack=request_stack, - embed_body_fields=embed_body_fields, - ) - if solved_result.errors: - raise RequestValidationError(solved_result.errors) - yield solved_result - finally: - if previous_inner_astack is _SCOPE_MISSING: - scope.pop("fastapi_inner_astack", None) - else: - scope["fastapi_inner_astack"] = previous_inner_astack - if previous_function_astack is _SCOPE_MISSING: - scope.pop("fastapi_function_astack", None) - else: - scope["fastapi_function_astack"] = previous_function_astack - - -class APIRouter(routing.Router): - """ - `APIRouter` class, used to group *path operations*, for example to structure - an app in multiple files. It would then be included in the `FastAPI` app, or - in another `APIRouter` (ultimately included in the app). - - Read more about it in the - [FastAPI docs for Bigger Applications - Multiple Files](https://fastapi.tiangolo.com/tutorial/bigger-applications/). - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - - app = FastAPI() - router = APIRouter() - - - @router.get("/users/", tags=["users"]) - async def read_users(): - return [{"username": "Rick"}, {"username": "Morty"}] - - - app.include_router(router) - ``` - """ - - def __init__( - self, - *, - prefix: Annotated[str, Doc("An optional path prefix for the router.")] = "", - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to all the *path operations* in this - router. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to all the - *path operations* in this router. - - Read more about it in the - [FastAPI docs for Bigger Applications - Multiple Files](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - default_response_class: Annotated[ - type[Response], - Doc( - """ - The default response class to be used. - - Read more in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#default-response-class). - """ - ), - ] = Default(JSONResponse), - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses to be shown in OpenAPI. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Additional Responses in OpenAPI](https://fastapi.tiangolo.com/advanced/additional-responses/). - - And in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - OpenAPI callbacks that should apply to all *path operations* in this - router. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - routes: Annotated[ - list[BaseRoute] | None, - Doc( - """ - **Note**: you probably shouldn't use this parameter, it is inherited - from Starlette and supported for compatibility. - - --- - - A list of routes to serve incoming HTTP and WebSocket requests. - """ - ), - deprecated( - """ - You normally wouldn't use this parameter with FastAPI, it is inherited - from Starlette and supported for compatibility. - - In FastAPI, you normally would use the *path operation methods*, - like `router.get()`, `router.post()`, etc. - """ - ), - ] = None, - redirect_slashes: Annotated[ - bool, - Doc( - """ - Whether to detect and redirect slashes in URLs when the client doesn't - use the same format. - """ - ), - ] = True, - default: Annotated[ - ASGIApp | None, - Doc( - """ - Default function handler for this router. Used to handle - 404 Not Found errors. - """ - ), - ] = None, - dependency_overrides_provider: Annotated[ - Any | None, - Doc( - """ - Only used internally by FastAPI to handle dependency overrides. - - You shouldn't need to use it. It normally points to the `FastAPI` app - object. - """ - ), - ] = None, - route_class: Annotated[ - type[APIRoute], - Doc( - """ - Custom route (*path operation*) class to be used by this router. - - Read more about it in the - [FastAPI docs for Custom Request and APIRoute class](https://fastapi.tiangolo.com/how-to/custom-request-and-route/#custom-apiroute-class-in-a-router). - """ - ), - ] = APIRoute, - on_startup: Annotated[ - Sequence[Callable[[], Any]] | None, - Doc( - """ - A list of startup event handler functions. - - You should instead use the `lifespan` handlers. - - Read more in the [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - on_shutdown: Annotated[ - Sequence[Callable[[], Any]] | None, - Doc( - """ - A list of shutdown event handler functions. - - You should instead use the `lifespan` handlers. - - Read more in the - [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - # the generic to Lifespan[AppType] is the type of the top level application - # which the router cannot know statically, so we use typing.Any - lifespan: Annotated[ - Lifespan[Any] | None, - Doc( - """ - A `Lifespan` context manager handler. This replaces `startup` and - `shutdown` functions with a single context manager. - - Read more in the - [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark all *path operations* in this router as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) all the *path operations* in this router in the - generated OpenAPI. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - strict_content_type: Annotated[ - bool, - Doc( - """ - Enable strict checking for request Content-Type headers. - - When `True` (the default), requests with a body that do not include - a `Content-Type` header will **not** be parsed as JSON. - - This prevents potential cross-site request forgery (CSRF) attacks - that exploit the browser's ability to send requests without a - Content-Type header, bypassing CORS preflight checks. In particular - applicable for apps that need to be run locally (in localhost). - - When `False`, requests without a `Content-Type` header will have - their body parsed as JSON, which maintains compatibility with - certain clients that don't send `Content-Type` headers. - - Read more about it in the - [FastAPI docs for Strict Content-Type](https://fastapi.tiangolo.com/advanced/strict-content-type/). - """ - ), - ] = Default(True), - ) -> None: - # Determine the lifespan context to use - if lifespan is None: - # Use the default lifespan that runs on_startup/on_shutdown handlers - lifespan_context: Lifespan[Any] = _DefaultLifespan(self) - elif inspect.isasyncgenfunction(lifespan): - lifespan_context = asynccontextmanager(lifespan) - elif inspect.isgeneratorfunction(lifespan): - lifespan_context = _wrap_gen_lifespan_context(lifespan) - else: - lifespan_context = lifespan - self.lifespan_context = lifespan_context - - super().__init__( - routes=routes, - redirect_slashes=redirect_slashes, - default=default, - lifespan=lifespan_context, - ) - if prefix: - assert prefix.startswith("/"), "A path prefix must start with '/'" - assert not prefix.endswith("/"), ( - "A path prefix must not end with '/', as the routes will start with '/'" - ) - - # Handle on_startup/on_shutdown locally since Starlette removed support - # Ref: https://github.com/Kludex/starlette/pull/3117 - # TODO: deprecate this once the lifespan (or alternative) interface is improved - self.on_startup: list[Callable[[], Any]] = ( - [] if on_startup is None else list(on_startup) - ) - self.on_shutdown: list[Callable[[], Any]] = ( - [] if on_shutdown is None else list(on_shutdown) - ) - - self.prefix = prefix - self.tags: list[str | Enum] = tags or [] - self.dependencies = list(dependencies or []) - self.deprecated = deprecated - self.include_in_schema = include_in_schema - self.responses = responses or {} - self.callbacks = callbacks or [] - self.dependency_overrides_provider = dependency_overrides_provider - self.route_class = route_class - self.default_response_class = default_response_class - self.generate_unique_id_function = generate_unique_id_function - self.strict_content_type = strict_content_type - self._routes_version = 0 - self._low_priority_routes: list[BaseRoute] = [] - self._frontend_routes: _FrontendRouteGroup | None = None - - def _mark_routes_changed(self) -> None: - self._routes_version += 1 - - def _get_routes_version(self, seen: set[int] | None = None) -> int: - if seen is None: - seen = set() - router_id = id(self) - if router_id in seen: - return self._routes_version - seen.add(router_id) - version = self._routes_version - for route in self.routes: - if isinstance(route, _IncludedRouter): - version += route.original_router._get_routes_version(seen) - return version - - def _contains_router( - self, router: "APIRouter", seen: set[int] | None = None - ) -> bool: - if seen is None: - seen = set() - router_id = id(self) - if router_id in seen: - return False - seen.add(router_id) - for route in self.routes: - if not isinstance(route, _IncludedRouter): - continue - if route.original_router is router: - return True - if route.original_router._contains_router(router, seen): - return True - return False - - def add_route( - self, - path: str, - endpoint: Callable[[Request], Awaitable[Response] | Response], - methods: Collection[str] | None = None, - name: str | None = None, - include_in_schema: bool = True, - ) -> None: - super().add_route( - path, - endpoint, - methods=methods, - name=name, - include_in_schema=include_in_schema, - ) - self._mark_routes_changed() - - def add_websocket_route( - self, - path: str, - endpoint: Callable[[WebSocket], Awaitable[None]], - name: str | None = None, - ) -> None: - super().add_websocket_route(path, endpoint, name=name) - self._mark_routes_changed() - - def frontend( - self, - path: Annotated[ - str, - Doc( - """ - The URL path prefix where the frontend build should be served. - """ - ), - ], - *, - directory: Annotated[ - str | os.PathLike[str], - Doc( - """ - The directory containing the static frontend build output. - """ - ), - ], - fallback: Annotated[ - Literal["auto", "index.html", "404.html"] | None, - Doc( - """ - The fallback file behavior for missing frontend paths. - """ - ), - ] = "auto", - check_dir: Annotated[ - bool | Literal["auto"], - Doc( - """ - Check that the frontend directory exists when the app is created. When - set to `"auto"`, skip the check with a warning when `FASTAPI_ENV` is - `"development"`, and check it otherwise. The `fastapi dev` command - sets `FASTAPI_ENV` to `"development"` if it is not already set. - """ - ), - ] = "auto", - ) -> None: - """ - Serve a static frontend build as low-priority routes. - - Use this for frontend tools that build static files into a directory, - such as `dist`. **FastAPI** path operations are checked first, and - the frontend files are checked only if no normal route matched. - - A typical project could look like this: - - ```text - . - ├── pyproject.toml - ├── app - │ ├── __init__.py - │ └── main.py - └── dist - ├── index.html - └── assets - └── app.js - ``` - - Then in `app/main.py`: - - ```python - from fastapi import APIRouter, FastAPI - - app = FastAPI() - router = APIRouter() - router.frontend("/", directory="dist") - app.include_router(router) - ``` - """ - check_dir = _resolve_frontend_check_dir( - directory=directory, check_dir=check_dir - ) - normalized_path = _normalize_frontend_path(path) - if self._frontend_routes is None: - self._frontend_routes = _FrontendRouteGroup( - dependencies=self.dependencies, - dependency_overrides_provider=self.dependency_overrides_provider, - ) - self._low_priority_routes.append(self._frontend_routes) - self._frontend_routes.add_frontend_route( - _join_frontend_paths(self.prefix, normalized_path), - directory=directory, - fallback=fallback, - check_dir=check_dir, - ) - self._mark_routes_changed() - - async def app(self, scope: Scope, receive: Receive, send: Send) -> None: - assert scope["type"] in ("http", "websocket", "lifespan") - - if "router" not in scope: - scope["router"] = self - - if scope["type"] == "lifespan": - await self.lifespan(scope, receive, send) - return - - partial: tuple[BaseRoute, Scope] | None = None - for route in self.routes: - match, child_scope = route.matches(scope) - if match == Match.FULL: - scope.update(child_scope) - await route.handle(scope, receive, send) - return - if match == Match.PARTIAL and partial is None: - partial = (route, child_scope) - - if partial is not None: - route, child_scope = partial - scope.update(child_scope) - await route.handle(scope, receive, send) - return - - route_path = get_route_path(scope) - if scope["type"] == "http" and self.redirect_slashes and route_path != "/": - redirect_scope = dict(scope) - if route_path.endswith("/"): - redirect_scope["path"] = redirect_scope["path"].rstrip("/") - else: - redirect_scope["path"] = redirect_scope["path"] + "/" - - for route in self.routes: - match, _ = route.matches(redirect_scope) - if match != Match.NONE: - redirect_url = URL(scope=redirect_scope) - response = RedirectResponse(url=str(redirect_url)) - await response(scope, receive, send) - return - - ( - low_priority_match, - low_priority_scope, - low_priority_route, - low_priority_context, - ) = self._match_low_priority(scope) - if low_priority_match != Match.NONE and low_priority_route is not None: - _update_scope(scope, low_priority_scope) - if low_priority_context is not None: - _get_fastapi_scope(scope)[_FASTAPI_EFFECTIVE_ROUTE_CONTEXT_KEY] = ( - low_priority_context - ) - original_route = low_priority_context.original_route - if isinstance(original_route, APIRoute): - scope["route"] = original_route - await original_route.handle(scope, receive, send) - return - await low_priority_route.handle(scope, receive, send) - return - - await self.default(scope, receive, send) - - async def handle(self, scope: Scope, receive: Receive, send: Send) -> None: - included_router = _get_scope_included_router(scope) - if ( - isinstance(included_router, _IncludedRouter) - and included_router.original_router is self - ): - await included_router._handle_selected(scope, receive, send) - return - await self.app(scope, receive, send) - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - included_router = _get_scope_included_router(scope) - if ( - isinstance(included_router, _IncludedRouter) - and included_router.original_router is self - ): - match, child_scope, _, _ = included_router._match(scope) - return match, child_scope - return Match.NONE, {} - - def _iter_low_priority_routes( - self, - ) -> Iterator[BaseRoute | _EffectiveRouteContext]: - yield from self._low_priority_routes - for route in self.routes: - if isinstance(route, _IncludedRouter): - yield from route.effective_low_priority_routes() - - def _match_low_priority( - self, scope: Scope - ) -> tuple[Match, Scope, BaseRoute | None, _EffectiveRouteContext | None]: - full: tuple[Scope, BaseRoute, _EffectiveRouteContext | None] | None = None - partial: tuple[Scope, BaseRoute, _EffectiveRouteContext | None] | None = None - for candidate in self._iter_low_priority_routes(): - route: BaseRoute - if isinstance(candidate, _EffectiveRouteContext): - route_context: _EffectiveRouteContext | None = candidate - original_route = candidate.original_route - if isinstance(original_route, APIRoute): - fastapi_scope = _get_fastapi_scope(scope) - previous_context = fastapi_scope.get( - _FASTAPI_EFFECTIVE_ROUTE_CONTEXT_KEY, _SCOPE_MISSING - ) - fastapi_scope[_FASTAPI_EFFECTIVE_ROUTE_CONTEXT_KEY] = route_context - try: - match, child_scope = original_route.matches(scope) - finally: - _restore_fastapi_scope_key( - scope, - _FASTAPI_EFFECTIVE_ROUTE_CONTEXT_KEY, - previous_context, - ) - route = original_route - else: - match, child_scope = candidate.matches(scope) - route = candidate.starlette_route or original_route - else: - route_context = None - match, child_scope = candidate.matches(scope) - route = candidate - if match == Match.FULL: - if full is None or self._frontend_match_is_more_specific( - child_scope, full[0] - ): - full = (child_scope, route, route_context) - elif match == Match.PARTIAL: - if partial is None or self._frontend_match_is_more_specific( - child_scope, partial[0] - ): - partial = (child_scope, route, route_context) - if full is not None: - child_scope, route, route_context = full - return Match.FULL, child_scope, route, route_context - if partial is not None: - child_scope, route, route_context = partial - return Match.PARTIAL, child_scope, route, route_context - return Match.NONE, {}, None, None - - def _frontend_match_is_more_specific( - self, child_scope: Scope, previous_child_scope: Scope - ) -> bool: - specificity = _frontend_scope_specificity(child_scope) - previous_specificity = _frontend_scope_specificity(previous_child_scope) - if specificity is None or previous_specificity is None: - return False - return specificity > previous_specificity - - def route( - self, - path: str, - methods: Collection[str] | None = None, - name: str | None = None, - include_in_schema: bool = True, - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_route( - path, - func, - methods=methods, - name=name, - include_in_schema=include_in_schema, - ) - return func - - return decorator - - def add_api_route( - self, - path: str, - endpoint: Callable[..., Any], - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[params.Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - methods: set[str] | list[str] | None = None, - operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] | DefaultPlaceholder = Default(JSONResponse), - name: str | None = None, - route_class_override: type[APIRoute] | None = None, - callbacks: list[BaseRoute] | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[[APIRoute], str] - | DefaultPlaceholder = Default(generate_unique_id), - strict_content_type: bool | DefaultPlaceholder = Default(True), - ) -> None: - route_class = route_class_override or self.route_class - responses = responses or {} - combined_responses = {**self.responses, **responses} - current_response_class = get_value_or_default( - response_class, self.default_response_class - ) - current_tags = self.tags.copy() - if tags: - current_tags.extend(tags) - current_dependencies = self.dependencies.copy() - if dependencies: - current_dependencies.extend(dependencies) - current_callbacks = self.callbacks.copy() - if callbacks: - current_callbacks.extend(callbacks) - current_generate_unique_id = get_value_or_default( - generate_unique_id_function, self.generate_unique_id_function - ) - route = route_class( - self.prefix + path, - endpoint=endpoint, - response_model=response_model, - status_code=status_code, - tags=current_tags, - dependencies=current_dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=combined_responses, - deprecated=deprecated or self.deprecated, - methods=methods, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema and self.include_in_schema, - response_class=current_response_class, - name=name, - dependency_overrides_provider=self.dependency_overrides_provider, - callbacks=current_callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=current_generate_unique_id, - strict_content_type=get_value_or_default( - strict_content_type, self.strict_content_type - ), - ) - self.routes.append(route) - self._mark_routes_changed() - - def api_route( - self, - path: str, - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[params.Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - methods: list[str] | None = None, - operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] = Default(JSONResponse), - name: str | None = None, - callbacks: list[BaseRoute] | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[[APIRoute], str] = Default( - generate_unique_id - ), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_api_route( - path, - func, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=methods, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - return func - - return decorator - - def add_api_websocket_route( - self, - path: str, - endpoint: Callable[..., Any], - name: str | None = None, - *, - dependencies: Sequence[params.Depends] | None = None, - ) -> None: - current_dependencies = self.dependencies.copy() - if dependencies: - current_dependencies.extend(dependencies) - - route = APIWebSocketRoute( - self.prefix + path, - endpoint=endpoint, - name=name, - dependencies=current_dependencies, - dependency_overrides_provider=self.dependency_overrides_provider, - ) - self.routes.append(route) - self._mark_routes_changed() - - def websocket( - self, - path: Annotated[ - str, - Doc( - """ - WebSocket path. - """ - ), - ], - name: Annotated[ - str | None, - Doc( - """ - A name for the WebSocket. Only used internally. - """ - ), - ] = None, - *, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be used for this - WebSocket. - - Read more about it in the - [FastAPI docs for WebSockets](https://fastapi.tiangolo.com/advanced/websockets/). - """ - ), - ] = None, - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Decorate a WebSocket function. - - Read more about it in the - [FastAPI docs for WebSockets](https://fastapi.tiangolo.com/advanced/websockets/). - - **Example** - - ## Example - - ```python - from fastapi import APIRouter, FastAPI, WebSocket - - app = FastAPI() - router = APIRouter() - - @router.websocket("/ws") - async def websocket_endpoint(websocket: WebSocket): - await websocket.accept() - while True: - data = await websocket.receive_text() - await websocket.send_text(f"Message text was: {data}") - - app.include_router(router) - ``` - """ - - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_api_websocket_route( - path, func, name=name, dependencies=dependencies - ) - return func - - return decorator - - def websocket_route( - self, path: str, name: str | None = None - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_websocket_route(path, func, name=name) - return func - - return decorator - - def include_router( - self, - router: Annotated["APIRouter", Doc("The `APIRouter` to include.")], - *, - prefix: Annotated[str, Doc("An optional path prefix for the router.")] = "", - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to all the *path operations* in this - router. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to all the - *path operations* in this router. - - Read more about it in the - [FastAPI docs for Bigger Applications - Multiple Files](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - default_response_class: Annotated[ - type[Response], - Doc( - """ - The default response class to be used. - - Read more in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#default-response-class). - """ - ), - ] = Default(JSONResponse), - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses to be shown in OpenAPI. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Additional Responses in OpenAPI](https://fastapi.tiangolo.com/advanced/additional-responses/). - - And in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - OpenAPI callbacks that should apply to all *path operations* in this - router. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark all *path operations* in this router as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include (or not) all the *path operations* in this router in the - generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> None: - """ - Include another `APIRouter` in the same current `APIRouter`. - - Read more about it in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/). - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - - app = FastAPI() - internal_router = APIRouter() - users_router = APIRouter() - - @users_router.get("/users/") - def read_users(): - return [{"name": "Rick"}, {"name": "Morty"}] - - internal_router.include_router(users_router) - app.include_router(internal_router) - ``` - """ - assert self is not router, ( - "Cannot include the same APIRouter instance into itself. " - "Did you mean to include a different router?" - ) - assert not router._contains_router(self), ( - "Cannot include an APIRouter instance that already includes this router. " - "Did you mean to include a different router?" - ) - if prefix: - assert prefix.startswith("/"), "A path prefix must start with '/'" - assert not prefix.endswith("/"), ( - "A path prefix must not end with '/', as the routes will start with '/'" - ) - else: - for route, route_context in _iter_routes_with_context(router.routes): - if route_context is None: - path = getattr(route, "path", None) - name = getattr(route, "name", "unknown") - elif route_context.starlette_route is not None: - path = getattr(route_context.starlette_route, "path", None) - name = getattr(route_context.starlette_route, "name", "unknown") - else: - path = route_context.path - name = route_context.name - if path is not None and not path: - raise FastAPIError( - f"Prefix and path cannot be both empty (path operation: {name})" - ) - include_context = _RouterIncludeContext.for_include( - parent_router=self, - included_router=router, - prefix=prefix, - tags=tags, - dependencies=dependencies, - default_response_class=default_response_class, - responses=responses, - callbacks=callbacks, - deprecated=deprecated, - include_in_schema=include_in_schema, - generate_unique_id_function=generate_unique_id_function, - ) - self.routes.append( - _IncludedRouter(original_router=router, include_context=include_context) - ) - self._mark_routes_changed() - for handler in router.on_startup: - self.add_event_handler("startup", handler) - for handler in router.on_shutdown: - self.add_event_handler("shutdown", handler) - self.lifespan_context = _merge_lifespan_context( - self.lifespan_context, - router.lifespan_context, - ) - - def get( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP GET operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - - app = FastAPI() - router = APIRouter() - - @router.get("/items/") - def read_items(): - return [{"name": "Empanada"}, {"name": "Arepa"}] - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["GET"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def put( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP PUT operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - router = APIRouter() - - @router.put("/items/{item_id}") - def replace_item(item_id: str, item: Item): - return {"message": "Item replaced", "id": item_id} - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["PUT"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def post( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP POST operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - router = APIRouter() - - @router.post("/items/") - def create_item(item: Item): - return {"message": "Item created"} - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["POST"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def delete( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP DELETE operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - - app = FastAPI() - router = APIRouter() - - @router.delete("/items/{item_id}") - def delete_item(item_id: str): - return {"message": "Item deleted"} - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["DELETE"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def options( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP OPTIONS operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - - app = FastAPI() - router = APIRouter() - - @router.options("/items/") - def get_item_options(): - return {"additions": ["Aji", "Guacamole"]} - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["OPTIONS"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def head( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP HEAD operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - router = APIRouter() - - @router.head("/items/", status_code=204) - def get_items_headers(response: Response): - response.headers["X-Cat-Dog"] = "Alone in the world" - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["HEAD"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def patch( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP PATCH operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - router = APIRouter() - - @router.patch("/items/") - def update_item(item: Item): - return {"message": "Item updated in place"} - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["PATCH"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def trace( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP TRACE operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - router = APIRouter() - - @router.trace("/items/{item_id}") - def trace_item(item_id: str): - return None - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["TRACE"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - # TODO: remove this once the lifespan (or alternative) interface is improved - async def _startup(self) -> None: - """ - Run any `.on_startup` event handlers. - - This method is kept for backward compatibility after Starlette removed - support for on_startup/on_shutdown handlers. - - Ref: https://github.com/Kludex/starlette/pull/3117 - """ - for handler in self.on_startup: - if is_async_callable(handler): - await handler() - else: - handler() - - # TODO: remove this once the lifespan (or alternative) interface is improved - async def _shutdown(self) -> None: - """ - Run any `.on_shutdown` event handlers. - - This method is kept for backward compatibility after Starlette removed - support for on_startup/on_shutdown handlers. - - Ref: https://github.com/Kludex/starlette/pull/3117 - """ - for handler in self.on_shutdown: - if is_async_callable(handler): - await handler() - else: - handler() - - # TODO: remove this once the lifespan (or alternative) interface is improved - def add_event_handler( - self, - event_type: str, - func: Callable[[], Any], - ) -> None: - """ - Add an event handler function for startup or shutdown. - - This method is kept for backward compatibility after Starlette removed - support for on_startup/on_shutdown handlers. - - Ref: https://github.com/Kludex/starlette/pull/3117 - """ - assert event_type in ("startup", "shutdown") - if event_type == "startup": - self.on_startup.append(func) - else: - self.on_shutdown.append(func) - - @deprecated( - """ - on_event is deprecated, use lifespan event handlers instead. - - Read more about it in the - [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/). - """ - ) - def on_event( - self, - event_type: Annotated[ - str, - Doc( - """ - The type of event. `startup` or `shutdown`. - """ - ), - ], - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add an event handler for the router. - - `on_event` is deprecated, use `lifespan` event handlers instead. - - Read more about it in the - [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/#alternative-events-deprecated). - """ - - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_event_handler(event_type, func) - return func - - return decorator diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/security/__init__.py b/bundle/python-cpu/Lib/site-packages/fastapi/security/__init__.py deleted file mode 100644 index 3aa6bf21e44f3069adb94242fbba5c8160532a1c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/security/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from .api_key import APIKeyCookie as APIKeyCookie -from .api_key import APIKeyHeader as APIKeyHeader -from .api_key import APIKeyQuery as APIKeyQuery -from .http import HTTPAuthorizationCredentials as HTTPAuthorizationCredentials -from .http import HTTPBasic as HTTPBasic -from .http import HTTPBasicCredentials as HTTPBasicCredentials -from .http import HTTPBearer as HTTPBearer -from .http import HTTPDigest as HTTPDigest -from .oauth2 import OAuth2 as OAuth2 -from .oauth2 import OAuth2AuthorizationCodeBearer as OAuth2AuthorizationCodeBearer -from .oauth2 import OAuth2PasswordBearer as OAuth2PasswordBearer -from .oauth2 import OAuth2PasswordRequestForm as OAuth2PasswordRequestForm -from .oauth2 import OAuth2PasswordRequestFormStrict as OAuth2PasswordRequestFormStrict -from .oauth2 import SecurityScopes as SecurityScopes -from .open_id_connect_url import OpenIdConnect as OpenIdConnect diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/security/api_key.py b/bundle/python-cpu/Lib/site-packages/fastapi/security/api_key.py deleted file mode 100644 index 83a4585a08cf43b367152dad06ce685f883ba940..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/security/api_key.py +++ /dev/null @@ -1,320 +0,0 @@ -from typing import Annotated - -from annotated_doc import Doc -from fastapi.openapi.models import APIKey, APIKeyIn -from fastapi.security.base import SecurityBase -from starlette.exceptions import HTTPException -from starlette.requests import Request -from starlette.status import HTTP_401_UNAUTHORIZED - - -class APIKeyBase(SecurityBase): - model: APIKey - - def __init__( - self, - location: APIKeyIn, - name: str, - description: str | None, - scheme_name: str | None, - auto_error: bool, - ): - self.auto_error = auto_error - - self.model: APIKey = APIKey( - **{"in": location}, # ty: ignore[invalid-argument-type] - name=name, - description=description, - ) - self.scheme_name = scheme_name or self.__class__.__name__ - - def make_not_authenticated_error(self) -> HTTPException: - """ - The WWW-Authenticate header is not standardized for API Key authentication but - the HTTP specification requires that an error of 401 "Unauthorized" must - include a WWW-Authenticate header. - - Ref: https://datatracker.ietf.org/doc/html/rfc9110#name-401-unauthorized - - For this, this method sends a custom challenge `APIKey`. - """ - return HTTPException( - status_code=HTTP_401_UNAUTHORIZED, - detail="Not authenticated", - headers={"WWW-Authenticate": "APIKey"}, - ) - - def check_api_key(self, api_key: str | None) -> str | None: - if not api_key: - if self.auto_error: - raise self.make_not_authenticated_error() - return None - return api_key - - -class APIKeyQuery(APIKeyBase): - """ - API key authentication using a query parameter. - - This defines the name of the query parameter that should be provided in the request - with the API key and integrates that into the OpenAPI documentation. It extracts - the key value sent in the query parameter automatically and provides it as the - dependency result. But it doesn't define how to send that API key to the client. - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be a string containing the key value. - - ## Example - - ```python - from fastapi import Depends, FastAPI - from fastapi.security import APIKeyQuery - - app = FastAPI() - - query_scheme = APIKeyQuery(name="api_key") - - - @app.get("/items/") - async def read_items(api_key: str = Depends(query_scheme)): - return {"api_key": api_key} - ``` - """ - - def __init__( - self, - *, - name: Annotated[ - str, - Doc("Query parameter name."), - ], - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the query parameter is not provided, `APIKeyQuery` will - automatically cancel the request and send the client an error. - - If `auto_error` is set to `False`, when the query parameter is not - available, instead of erroring out, the dependency result will be - `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in a query - parameter or in an HTTP Bearer token). - """ - ), - ] = True, - ): - super().__init__( - location=APIKeyIn.query, - name=name, - scheme_name=scheme_name, - description=description, - auto_error=auto_error, - ) - - async def __call__(self, request: Request) -> str | None: - api_key = request.query_params.get(self.model.name) - return self.check_api_key(api_key) - - -class APIKeyHeader(APIKeyBase): - """ - API key authentication using a header. - - This defines the name of the header that should be provided in the request with - the API key and integrates that into the OpenAPI documentation. It extracts - the key value sent in the header automatically and provides it as the dependency - result. But it doesn't define how to send that key to the client. - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be a string containing the key value. - - ## Example - - ```python - from fastapi import Depends, FastAPI - from fastapi.security import APIKeyHeader - - app = FastAPI() - - header_scheme = APIKeyHeader(name="x-key") - - - @app.get("/items/") - async def read_items(key: str = Depends(header_scheme)): - return {"key": key} - ``` - """ - - def __init__( - self, - *, - name: Annotated[str, Doc("Header name.")], - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the header is not provided, `APIKeyHeader` will - automatically cancel the request and send the client an error. - - If `auto_error` is set to `False`, when the header is not available, - instead of erroring out, the dependency result will be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in a header or - in an HTTP Bearer token). - """ - ), - ] = True, - ): - super().__init__( - location=APIKeyIn.header, - name=name, - scheme_name=scheme_name, - description=description, - auto_error=auto_error, - ) - - async def __call__(self, request: Request) -> str | None: - api_key = request.headers.get(self.model.name) - return self.check_api_key(api_key) - - -class APIKeyCookie(APIKeyBase): - """ - API key authentication using a cookie. - - This defines the name of the cookie that should be provided in the request with - the API key and integrates that into the OpenAPI documentation. It extracts - the key value sent in the cookie automatically and provides it as the dependency - result. But it doesn't define how to set that cookie. - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be a string containing the key value. - - ## Example - - ```python - from fastapi import Depends, FastAPI - from fastapi.security import APIKeyCookie - - app = FastAPI() - - cookie_scheme = APIKeyCookie(name="session") - - - @app.get("/items/") - async def read_items(session: str = Depends(cookie_scheme)): - return {"session": session} - ``` - """ - - def __init__( - self, - *, - name: Annotated[str, Doc("Cookie name.")], - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the cookie is not provided, `APIKeyCookie` will - automatically cancel the request and send the client an error. - - If `auto_error` is set to `False`, when the cookie is not available, - instead of erroring out, the dependency result will be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in a cookie or - in an HTTP Bearer token). - """ - ), - ] = True, - ): - super().__init__( - location=APIKeyIn.cookie, - name=name, - scheme_name=scheme_name, - description=description, - auto_error=auto_error, - ) - - async def __call__(self, request: Request) -> str | None: - api_key = request.cookies.get(self.model.name) - return self.check_api_key(api_key) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/security/base.py b/bundle/python-cpu/Lib/site-packages/fastapi/security/base.py deleted file mode 100644 index c43555deb8ea83b14241a5631c9ea451c96f6e7f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/security/base.py +++ /dev/null @@ -1,6 +0,0 @@ -from fastapi.openapi.models import SecurityBase as SecurityBaseModel - - -class SecurityBase: - model: SecurityBaseModel - scheme_name: str diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/security/http.py b/bundle/python-cpu/Lib/site-packages/fastapi/security/http.py deleted file mode 100644 index a32948ef0a0ec61d60d50229038d965ee815d60a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/security/http.py +++ /dev/null @@ -1,417 +0,0 @@ -import binascii -from base64 import b64decode -from typing import Annotated - -from annotated_doc import Doc -from fastapi.exceptions import HTTPException -from fastapi.openapi.models import HTTPBase as HTTPBaseModel -from fastapi.openapi.models import HTTPBearer as HTTPBearerModel -from fastapi.security.base import SecurityBase -from fastapi.security.utils import get_authorization_scheme_param -from pydantic import BaseModel -from starlette.requests import Request -from starlette.status import HTTP_401_UNAUTHORIZED - - -class HTTPBasicCredentials(BaseModel): - """ - The HTTP Basic credentials given as the result of using `HTTPBasic` in a - dependency. - - Read more about it in the - [FastAPI docs for HTTP Basic Auth](https://fastapi.tiangolo.com/advanced/security/http-basic-auth/). - """ - - username: Annotated[str, Doc("The HTTP Basic username.")] - password: Annotated[str, Doc("The HTTP Basic password.")] - - -class HTTPAuthorizationCredentials(BaseModel): - """ - The HTTP authorization credentials in the result of using `HTTPBearer` or - `HTTPDigest` in a dependency. - - The HTTP authorization header value is split by the first space. - - The first part is the `scheme`, the second part is the `credentials`. - - For example, in an HTTP Bearer token scheme, the client will send a header - like: - - ``` - Authorization: Bearer deadbeef12346 - ``` - - In this case: - - * `scheme` will have the value `"Bearer"` - * `credentials` will have the value `"deadbeef12346"` - """ - - scheme: Annotated[ - str, - Doc( - """ - The HTTP authorization scheme extracted from the header value. - """ - ), - ] - credentials: Annotated[ - str, - Doc( - """ - The HTTP authorization credentials extracted from the header value. - """ - ), - ] - - -class HTTPBase(SecurityBase): - model: HTTPBaseModel - - def __init__( - self, - *, - scheme: str, - scheme_name: str | None = None, - description: str | None = None, - auto_error: bool = True, - ): - self.model = HTTPBaseModel(scheme=scheme, description=description) - self.scheme_name = scheme_name or self.__class__.__name__ - self.auto_error = auto_error - - def make_authenticate_headers(self) -> dict[str, str]: - return {"WWW-Authenticate": f"{self.model.scheme.title()}"} - - def make_not_authenticated_error(self) -> HTTPException: - return HTTPException( - status_code=HTTP_401_UNAUTHORIZED, - detail="Not authenticated", - headers=self.make_authenticate_headers(), - ) - - async def __call__(self, request: Request) -> HTTPAuthorizationCredentials | None: - authorization = request.headers.get("Authorization") - scheme, credentials = get_authorization_scheme_param(authorization) - if not (authorization and scheme and credentials): - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return HTTPAuthorizationCredentials(scheme=scheme, credentials=credentials) - - -class HTTPBasic(HTTPBase): - """ - HTTP Basic authentication. - - Ref: https://datatracker.ietf.org/doc/html/rfc7617 - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be an `HTTPBasicCredentials` object containing the - `username` and the `password`. - - Read more about it in the - [FastAPI docs for HTTP Basic Auth](https://fastapi.tiangolo.com/advanced/security/http-basic-auth/). - - ## Example - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - from fastapi.security import HTTPBasic, HTTPBasicCredentials - - app = FastAPI() - - security = HTTPBasic() - - - @app.get("/users/me") - def read_current_user(credentials: Annotated[HTTPBasicCredentials, Depends(security)]): - return {"username": credentials.username, "password": credentials.password} - ``` - """ - - def __init__( - self, - *, - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - realm: Annotated[ - str | None, - Doc( - """ - HTTP Basic authentication realm. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the HTTP Basic authentication is not provided (a - header), `HTTPBasic` will automatically cancel the request and send the - client an error. - - If `auto_error` is set to `False`, when the HTTP Basic authentication - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in HTTP Basic - authentication or in an HTTP Bearer token). - """ - ), - ] = True, - ): - self.model = HTTPBaseModel(scheme="basic", description=description) - self.scheme_name = scheme_name or self.__class__.__name__ - self.realm = realm - self.auto_error = auto_error - - def make_authenticate_headers(self) -> dict[str, str]: - if self.realm: - return {"WWW-Authenticate": f'Basic realm="{self.realm}"'} - return {"WWW-Authenticate": "Basic"} - - async def __call__( # type: ignore - self, request: Request - ) -> HTTPBasicCredentials | None: - authorization = request.headers.get("Authorization") - scheme, param = get_authorization_scheme_param(authorization) - if not authorization or scheme.lower() != "basic": - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - try: - data = b64decode(param).decode("ascii") - except (ValueError, UnicodeDecodeError, binascii.Error) as e: - raise self.make_not_authenticated_error() from e - username, separator, password = data.partition(":") - if not separator: - raise self.make_not_authenticated_error() - return HTTPBasicCredentials(username=username, password=password) - - -class HTTPBearer(HTTPBase): - """ - HTTP Bearer token authentication. - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be an `HTTPAuthorizationCredentials` object containing - the `scheme` and the `credentials`. - - ## Example - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer - - app = FastAPI() - - security = HTTPBearer() - - - @app.get("/users/me") - def read_current_user( - credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)] - ): - return {"scheme": credentials.scheme, "credentials": credentials.credentials} - ``` - """ - - def __init__( - self, - *, - bearerFormat: Annotated[str | None, Doc("Bearer token format.")] = None, - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the HTTP Bearer token is not provided (in an - `Authorization` header), `HTTPBearer` will automatically cancel the - request and send the client an error. - - If `auto_error` is set to `False`, when the HTTP Bearer token - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in an HTTP - Bearer token or in a cookie). - """ - ), - ] = True, - ): - self.model = HTTPBearerModel(bearerFormat=bearerFormat, description=description) - self.scheme_name = scheme_name or self.__class__.__name__ - self.auto_error = auto_error - - async def __call__(self, request: Request) -> HTTPAuthorizationCredentials | None: - authorization = request.headers.get("Authorization") - scheme, credentials = get_authorization_scheme_param(authorization) - if not (authorization and scheme and credentials): - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - if scheme.lower() != "bearer": - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return HTTPAuthorizationCredentials(scheme=scheme, credentials=credentials) - - -class HTTPDigest(HTTPBase): - """ - HTTP Digest authentication. - - **Warning**: this is only a stub to connect the components with OpenAPI in FastAPI, - but it doesn't implement the full Digest scheme, you would need to subclass it - and implement it in your code. - - Ref: https://datatracker.ietf.org/doc/html/rfc7616 - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be an `HTTPAuthorizationCredentials` object containing - the `scheme` and the `credentials`. - - ## Example - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - from fastapi.security import HTTPAuthorizationCredentials, HTTPDigest - - app = FastAPI() - - security = HTTPDigest() - - - @app.get("/users/me") - def read_current_user( - credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)] - ): - return {"scheme": credentials.scheme, "credentials": credentials.credentials} - ``` - """ - - def __init__( - self, - *, - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the HTTP Digest is not provided, `HTTPDigest` will - automatically cancel the request and send the client an error. - - If `auto_error` is set to `False`, when the HTTP Digest is not - available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in HTTP - Digest or in a cookie). - """ - ), - ] = True, - ): - self.model = HTTPBaseModel(scheme="digest", description=description) - self.scheme_name = scheme_name or self.__class__.__name__ - self.auto_error = auto_error - - async def __call__(self, request: Request) -> HTTPAuthorizationCredentials | None: - authorization = request.headers.get("Authorization") - scheme, credentials = get_authorization_scheme_param(authorization) - if not (authorization and scheme and credentials): - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - if scheme.lower() != "digest": - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return HTTPAuthorizationCredentials(scheme=scheme, credentials=credentials) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/security/oauth2.py b/bundle/python-cpu/Lib/site-packages/fastapi/security/oauth2.py deleted file mode 100644 index 3fd9e41eb320b589ea680b313e1375d145b7085e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/security/oauth2.py +++ /dev/null @@ -1,693 +0,0 @@ -from typing import Annotated, Any, cast - -from annotated_doc import Doc -from fastapi.exceptions import HTTPException -from fastapi.openapi.models import OAuth2 as OAuth2Model -from fastapi.openapi.models import OAuthFlows as OAuthFlowsModel -from fastapi.param_functions import Form -from fastapi.security.base import SecurityBase -from fastapi.security.utils import get_authorization_scheme_param -from starlette.requests import Request -from starlette.status import HTTP_401_UNAUTHORIZED - - -class OAuth2PasswordRequestForm: - """ - This is a dependency class to collect the `username` and `password` as form data - for an OAuth2 password flow. - - The OAuth2 specification dictates that for a password flow the data should be - collected using form data (instead of JSON) and that it should have the specific - fields `username` and `password`. - - All the initialization parameters are extracted from the request. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - - ## Example - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - from fastapi.security import OAuth2PasswordRequestForm - - app = FastAPI() - - - @app.post("/login") - def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]): - data = {} - data["scopes"] = [] - for scope in form_data.scopes: - data["scopes"].append(scope) - if form_data.client_id: - data["client_id"] = form_data.client_id - if form_data.client_secret: - data["client_secret"] = form_data.client_secret - return data - ``` - - Note that for OAuth2 the scope `items:read` is a single scope in an opaque string. - You could have custom internal logic to separate it by colon characters (`:`) or - similar, and get the two parts `items` and `read`. Many applications do that to - group and organize permissions, you could do it as well in your application, just - know that it is application specific, it's not part of the specification. - """ - - def __init__( - self, - *, - grant_type: Annotated[ - str | None, - Form(pattern="^password$"), - Doc( - """ - The OAuth2 spec says it is required and MUST be the fixed string - "password". Nevertheless, this dependency class is permissive and - allows not passing it. If you want to enforce it, use instead the - `OAuth2PasswordRequestFormStrict` dependency. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ] = None, - username: Annotated[ - str, - Form(), - Doc( - """ - `username` string. The OAuth2 spec requires the exact field name - `username`. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - password: Annotated[ - str, - Form(json_schema_extra={"format": "password"}), - Doc( - """ - `password` string. The OAuth2 spec requires the exact field name - `password`. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - scope: Annotated[ - str, - Form(), - Doc( - """ - A single string with actually several scopes separated by spaces. Each - scope is also a string. - - For example, a single string with: - - ```python - "items:read items:write users:read profile openid" - ```` - - would represent the scopes: - - * `items:read` - * `items:write` - * `users:read` - * `profile` - * `openid` - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ] = "", - client_id: Annotated[ - str | None, - Form(), - Doc( - """ - If there's a `client_id`, it can be sent as part of the form fields. - But the OAuth2 specification recommends sending the `client_id` and - `client_secret` (if any) using HTTP Basic auth. - """ - ), - ] = None, - client_secret: Annotated[ - str | None, - Form(json_schema_extra={"format": "password"}), - Doc( - """ - If there's a `client_secret` (and a `client_id`), they can be sent - as part of the form fields. But the OAuth2 specification recommends - sending the `client_id` and `client_secret` (if any) using HTTP Basic - auth. - """ - ), - ] = None, - ): - self.grant_type = grant_type - self.username = username - self.password = password - self.scopes = scope.split() - self.client_id = client_id - self.client_secret = client_secret - - -class OAuth2PasswordRequestFormStrict(OAuth2PasswordRequestForm): - """ - This is a dependency class to collect the `username` and `password` as form data - for an OAuth2 password flow. - - The OAuth2 specification dictates that for a password flow the data should be - collected using form data (instead of JSON) and that it should have the specific - fields `username` and `password`. - - All the initialization parameters are extracted from the request. - - The only difference between `OAuth2PasswordRequestFormStrict` and - `OAuth2PasswordRequestForm` is that `OAuth2PasswordRequestFormStrict` requires the - client to send the form field `grant_type` with the value `"password"`, which - is required in the OAuth2 specification (it seems that for no particular reason), - while for `OAuth2PasswordRequestForm` `grant_type` is optional. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - - ## Example - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - from fastapi.security import OAuth2PasswordRequestForm - - app = FastAPI() - - - @app.post("/login") - def login(form_data: Annotated[OAuth2PasswordRequestFormStrict, Depends()]): - data = {} - data["scopes"] = [] - for scope in form_data.scopes: - data["scopes"].append(scope) - if form_data.client_id: - data["client_id"] = form_data.client_id - if form_data.client_secret: - data["client_secret"] = form_data.client_secret - return data - ``` - - Note that for OAuth2 the scope `items:read` is a single scope in an opaque string. - You could have custom internal logic to separate it by colon characters (`:`) or - similar, and get the two parts `items` and `read`. Many applications do that to - group and organize permissions, you could do it as well in your application, just - know that it is application specific, it's not part of the specification. - - - grant_type: the OAuth2 spec says it is required and MUST be the fixed string "password". - This dependency is strict about it. If you want to be permissive, use instead the - OAuth2PasswordRequestForm dependency class. - username: username string. The OAuth2 spec requires the exact field name "username". - password: password string. The OAuth2 spec requires the exact field name "password". - scope: Optional string. Several scopes (each one a string) separated by spaces. E.g. - "items:read items:write users:read profile openid" - client_id: optional string. OAuth2 recommends sending the client_id and client_secret (if any) - using HTTP Basic auth, as: client_id:client_secret - client_secret: optional string. OAuth2 recommends sending the client_id and client_secret (if any) - using HTTP Basic auth, as: client_id:client_secret - """ - - def __init__( - self, - grant_type: Annotated[ - str, - Form(pattern="^password$"), - Doc( - """ - The OAuth2 spec says it is required and MUST be the fixed string - "password". This dependency is strict about it. If you want to be - permissive, use instead the `OAuth2PasswordRequestForm` dependency - class. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - username: Annotated[ - str, - Form(), - Doc( - """ - `username` string. The OAuth2 spec requires the exact field name - `username`. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - password: Annotated[ - str, - Form(), - Doc( - """ - `password` string. The OAuth2 spec requires the exact field name - `password`. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - scope: Annotated[ - str, - Form(), - Doc( - """ - A single string with actually several scopes separated by spaces. Each - scope is also a string. - - For example, a single string with: - - ```python - "items:read items:write users:read profile openid" - ```` - - would represent the scopes: - - * `items:read` - * `items:write` - * `users:read` - * `profile` - * `openid` - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ] = "", - client_id: Annotated[ - str | None, - Form(), - Doc( - """ - If there's a `client_id`, it can be sent as part of the form fields. - But the OAuth2 specification recommends sending the `client_id` and - `client_secret` (if any) using HTTP Basic auth. - """ - ), - ] = None, - client_secret: Annotated[ - str | None, - Form(), - Doc( - """ - If there's a `client_secret` (and a `client_id`), they can be sent - as part of the form fields. But the OAuth2 specification recommends - sending the `client_id` and `client_secret` (if any) using HTTP Basic - auth. - """ - ), - ] = None, - ): - super().__init__( - grant_type=grant_type, - username=username, - password=password, - scope=scope, - client_id=client_id, - client_secret=client_secret, - ) - - -class OAuth2(SecurityBase): - """ - This is the base class for OAuth2 authentication, an instance of it would be used - as a dependency. All other OAuth2 classes inherit from it and customize it for - each OAuth2 flow. - - You normally would not create a new class inheriting from it but use one of the - existing subclasses, and maybe compose them if you want to support multiple flows. - - Read more about it in the - [FastAPI docs for Security](https://fastapi.tiangolo.com/tutorial/security/). - """ - - def __init__( - self, - *, - flows: Annotated[ - OAuthFlowsModel | dict[str, dict[str, Any]], - Doc( - """ - The dictionary of OAuth2 flows. - """ - ), - ] = OAuthFlowsModel(), - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if no HTTP Authorization header is provided, required for - OAuth2 authentication, it will automatically cancel the request and - send the client an error. - - If `auto_error` is set to `False`, when the HTTP Authorization header - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, with OAuth2 - or in a cookie). - """ - ), - ] = True, - ): - self.model = OAuth2Model( - flows=cast(OAuthFlowsModel, flows), description=description - ) - self.scheme_name = scheme_name or self.__class__.__name__ - self.auto_error = auto_error - - def make_not_authenticated_error(self) -> HTTPException: - """ - The OAuth 2 specification doesn't define the challenge that should be used, - because a `Bearer` token is not really the only option to authenticate. - - But declaring any other authentication challenge would be application-specific - as it's not defined in the specification. - - For practical reasons, this method uses the `Bearer` challenge by default, as - it's probably the most common one. - - If you are implementing an OAuth2 authentication scheme other than the provided - ones in FastAPI (based on bearer tokens), you might want to override this. - - Ref: https://datatracker.ietf.org/doc/html/rfc6749 - """ - return HTTPException( - status_code=HTTP_401_UNAUTHORIZED, - detail="Not authenticated", - headers={"WWW-Authenticate": "Bearer"}, - ) - - async def __call__(self, request: Request) -> str | None: - authorization = request.headers.get("Authorization") - if not authorization: - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return authorization - - -class OAuth2PasswordBearer(OAuth2): - """ - OAuth2 flow for authentication using a bearer token obtained with a password. - An instance of it would be used as a dependency. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - - def __init__( - self, - tokenUrl: Annotated[ - str, - Doc( - """ - The URL to obtain the OAuth2 token. This would be the *path operation* - that has `OAuth2PasswordRequestForm` as a dependency. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - scopes: Annotated[ - dict[str, str] | None, - Doc( - """ - The OAuth2 scopes that would be required by the *path operations* that - use this dependency. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if no HTTP Authorization header is provided, required for - OAuth2 authentication, it will automatically cancel the request and - send the client an error. - - If `auto_error` is set to `False`, when the HTTP Authorization header - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, with OAuth2 - or in a cookie). - """ - ), - ] = True, - refreshUrl: Annotated[ - str | None, - Doc( - """ - The URL to refresh the token and obtain a new one. - """ - ), - ] = None, - ): - if not scopes: - scopes = {} - flows = OAuthFlowsModel( - password=cast( - Any, - { - "tokenUrl": tokenUrl, - "refreshUrl": refreshUrl, - "scopes": scopes, - }, - ) - ) - super().__init__( - flows=flows, - scheme_name=scheme_name, - description=description, - auto_error=auto_error, - ) - - async def __call__(self, request: Request) -> str | None: - authorization = request.headers.get("Authorization") - scheme, param = get_authorization_scheme_param(authorization) - if not authorization or scheme.lower() != "bearer": - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return param - - -class OAuth2AuthorizationCodeBearer(OAuth2): - """ - OAuth2 flow for authentication using a bearer token obtained with an OAuth2 code - flow. An instance of it would be used as a dependency. - """ - - def __init__( - self, - authorizationUrl: str, - tokenUrl: Annotated[ - str, - Doc( - """ - The URL to obtain the OAuth2 token. - """ - ), - ], - refreshUrl: Annotated[ - str | None, - Doc( - """ - The URL to refresh the token and obtain a new one. - """ - ), - ] = None, - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - scopes: Annotated[ - dict[str, str] | None, - Doc( - """ - The OAuth2 scopes that would be required by the *path operations* that - use this dependency. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if no HTTP Authorization header is provided, required for - OAuth2 authentication, it will automatically cancel the request and - send the client an error. - - If `auto_error` is set to `False`, when the HTTP Authorization header - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, with OAuth2 - or in a cookie). - """ - ), - ] = True, - ): - if not scopes: - scopes = {} - flows = OAuthFlowsModel( - authorizationCode=cast( - Any, - { - "authorizationUrl": authorizationUrl, - "tokenUrl": tokenUrl, - "refreshUrl": refreshUrl, - "scopes": scopes, - }, - ) - ) - super().__init__( - flows=flows, - scheme_name=scheme_name, - description=description, - auto_error=auto_error, - ) - - async def __call__(self, request: Request) -> str | None: - authorization = request.headers.get("Authorization") - scheme, param = get_authorization_scheme_param(authorization) - if not authorization or scheme.lower() != "bearer": - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None # pragma: nocover - return param - - -class SecurityScopes: - """ - This is a special class that you can define in a parameter in a dependency to - obtain the OAuth2 scopes required by all the dependencies in the same chain. - - This way, multiple dependencies can have different scopes, even when used in the - same *path operation*. And with this, you can access all the scopes required in - all those dependencies in a single place. - - Read more about it in the - [FastAPI docs for OAuth2 scopes](https://fastapi.tiangolo.com/advanced/security/oauth2-scopes/). - """ - - def __init__( - self, - scopes: Annotated[ - list[str] | None, - Doc( - """ - This will be filled by FastAPI. - """ - ), - ] = None, - ): - self.scopes: Annotated[ - list[str], - Doc( - """ - The list of all the scopes required by dependencies. - """ - ), - ] = scopes or [] - self.scope_str: Annotated[ - str, - Doc( - """ - All the scopes required by all the dependencies in a single string - separated by spaces, as defined in the OAuth2 specification. - """ - ), - ] = " ".join(self.scopes) diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/security/open_id_connect_url.py b/bundle/python-cpu/Lib/site-packages/fastapi/security/open_id_connect_url.py deleted file mode 100644 index 125a81943183c818d3831f9bb2555a3881c903a1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/security/open_id_connect_url.py +++ /dev/null @@ -1,94 +0,0 @@ -from typing import Annotated - -from annotated_doc import Doc -from fastapi.openapi.models import OpenIdConnect as OpenIdConnectModel -from fastapi.security.base import SecurityBase -from starlette.exceptions import HTTPException -from starlette.requests import Request -from starlette.status import HTTP_401_UNAUTHORIZED - - -class OpenIdConnect(SecurityBase): - """ - OpenID Connect authentication class. An instance of it would be used as a - dependency. - - **Warning**: this is only a stub to connect the components with OpenAPI in FastAPI, - but it doesn't implement the full OpenIdConnect scheme, for example, it doesn't use - the OpenIDConnect URL. You would need to subclass it and implement it in your - code. - """ - - def __init__( - self, - *, - openIdConnectUrl: Annotated[ - str, - Doc( - """ - The OpenID Connect URL. - """ - ), - ], - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if no HTTP Authorization header is provided, required for - OpenID Connect authentication, it will automatically cancel the request - and send the client an error. - - If `auto_error` is set to `False`, when the HTTP Authorization header - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, with OpenID - Connect or in a cookie). - """ - ), - ] = True, - ): - self.model = OpenIdConnectModel( - openIdConnectUrl=openIdConnectUrl, description=description - ) - self.scheme_name = scheme_name or self.__class__.__name__ - self.auto_error = auto_error - - def make_not_authenticated_error(self) -> HTTPException: - return HTTPException( - status_code=HTTP_401_UNAUTHORIZED, - detail="Not authenticated", - headers={"WWW-Authenticate": "Bearer"}, - ) - - async def __call__(self, request: Request) -> str | None: - authorization = request.headers.get("Authorization") - if not authorization: - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return authorization diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/security/utils.py b/bundle/python-cpu/Lib/site-packages/fastapi/security/utils.py deleted file mode 100644 index 8ee66fd3812be2af9d9a3587feffb83cd8e512c3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/security/utils.py +++ /dev/null @@ -1,7 +0,0 @@ -def get_authorization_scheme_param( - authorization_header_value: str | None, -) -> tuple[str, str]: - if not authorization_header_value: - return "", "" - scheme, _, param = authorization_header_value.partition(" ") - return scheme, param.strip() diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/sse.py b/bundle/python-cpu/Lib/site-packages/fastapi/sse.py deleted file mode 100644 index c31334835032570d8244526a623ac249ffc77284..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/sse.py +++ /dev/null @@ -1,241 +0,0 @@ -from typing import Annotated, Any - -from annotated_doc import Doc -from pydantic import AfterValidator, BaseModel, Field, model_validator -from starlette.responses import StreamingResponse - -# Canonical SSE event schema matching the OpenAPI 3.2 spec -# (Section 4.14.4 "Special Considerations for Server-Sent Events") -_SSE_EVENT_SCHEMA: dict[str, Any] = { - "type": "object", - "properties": { - "data": {"type": "string"}, - "event": {"type": "string"}, - "id": {"type": "string"}, - "retry": {"type": "integer", "minimum": 0}, - }, -} - - -class EventSourceResponse(StreamingResponse): - """Streaming response with `text/event-stream` media type. - - Use as `response_class=EventSourceResponse` on a *path operation* that uses `yield` - to enable Server Sent Events (SSE) responses. - - Works with **any HTTP method** (`GET`, `POST`, etc.), which makes it compatible - with protocols like MCP that stream SSE over `POST`. - - The actual encoding logic lives in the FastAPI routing layer. This class - serves mainly as a marker and sets the correct `Content-Type`. - """ - - media_type = "text/event-stream" - - -def _check_single_line(v: str | None, field_name: str) -> str | None: - if v is not None and ("\r" in v or "\n" in v): - raise ValueError(f"SSE '{field_name}' must be a single line") - return v - - -def _check_event_single_line(v: str | None) -> str | None: - return _check_single_line(v, "event") - - -def _check_id_valid(v: str | None) -> str | None: - if v is not None and "\0" in v: - raise ValueError("SSE 'id' must not contain null characters") - return _check_single_line(v, "id") - - -class ServerSentEvent(BaseModel): - """Represents a single Server-Sent Event. - - When `yield`ed from a *path operation function* that uses - `response_class=EventSourceResponse`, each `ServerSentEvent` is encoded - into the [SSE wire format](https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream) - (`text/event-stream`). - - If you yield a plain object (dict, Pydantic model, etc.) instead, it is - automatically JSON-encoded and sent as the `data:` field. - - All `data` values **including plain strings** are JSON-serialized. - - For example, `data="hello"` produces `data: "hello"` on the wire (with - quotes). - """ - - data: Annotated[ - Any, - Doc( - """ - The event payload. - - Can be any JSON-serializable value: a Pydantic model, dict, list, - string, number, etc. It is **always** serialized to JSON: strings - are quoted (`"hello"` becomes `data: "hello"` on the wire). - - Mutually exclusive with `raw_data`. - """ - ), - ] = None - raw_data: Annotated[ - str | None, - Doc( - """ - Raw string to send as the `data:` field **without** JSON encoding. - - Use this when you need to send pre-formatted text, HTML fragments, - CSV lines, or any non-JSON payload. The string is placed directly - into the `data:` field as-is. - - Mutually exclusive with `data`. - """ - ), - ] = None - event: Annotated[ - str | None, - AfterValidator(_check_event_single_line), - Doc( - """ - Optional event type name. - - Maps to `addEventListener(event, ...)` on the browser. When omitted, - the browser dispatches on the generic `message` event. Must be a - single line. - """ - ), - ] = None - id: Annotated[ - str | None, - AfterValidator(_check_id_valid), - Doc( - """ - Optional event ID. - - The browser sends this value back as the `Last-Event-ID` header on - automatic reconnection. **Must be a single line** and must not contain - null (`\\0`) characters. - """ - ), - ] = None - retry: Annotated[ - int | None, - Field(ge=0), - Doc( - """ - Optional reconnection time in **milliseconds**. - - Tells the browser how long to wait before reconnecting after the - connection is lost. Must be a non-negative integer. - """ - ), - ] = None - comment: Annotated[ - str | None, - Doc( - """ - Optional comment line(s). - - Comment lines start with `:` in the SSE wire format and are ignored by - `EventSource` clients. Useful for keep-alive pings to prevent - proxy/load-balancer timeouts. - """ - ), - ] = None - - @model_validator(mode="after") - def _check_data_exclusive(self) -> "ServerSentEvent": - if self.data is not None and self.raw_data is not None: - raise ValueError( - "Cannot set both 'data' and 'raw_data' on the same " - "ServerSentEvent. Use 'data' for JSON-serialized payloads " - "or 'raw_data' for pre-formatted strings." - ) - return self - - -def _split_sse_lines(value: str) -> list[str]: - # Split on SSE-spec line terminators only (\n, \r\n, \r), preserving - # trailing empty strings. - return value.replace("\r\n", "\n").replace("\r", "\n").split("\n") - - -def format_sse_event( - *, - data_str: Annotated[ - str | None, - Doc( - """ - Pre-serialized data string to use as the `data:` field. - """ - ), - ] = None, - event: Annotated[ - str | None, - Doc( - """ - Optional event type name (`event:` field). - """ - ), - ] = None, - id: Annotated[ - str | None, - Doc( - """ - Optional event ID (`id:` field). - """ - ), - ] = None, - retry: Annotated[ - int | None, - Doc( - """ - Optional reconnection time in milliseconds (`retry:` field). - """ - ), - ] = None, - comment: Annotated[ - str | None, - Doc( - """ - Optional comment line(s) (`:` prefix). - """ - ), - ] = None, -) -> bytes: - """Build SSE wire-format bytes from **pre-serialized** data. - - The result always ends with `\\n\\n` (the event terminator). - """ - lines: list[str] = [] - - if comment is not None: - for line in _split_sse_lines(comment): - lines.append(f": {line}") - - if event is not None: - lines.append(f"event: {event}") - - if data_str is not None: - for line in _split_sse_lines(data_str): - lines.append(f"data: {line}") - - if id is not None: - lines.append(f"id: {id}") - - if retry is not None: - lines.append(f"retry: {retry}") - - lines.append("") - lines.append("") - return "\n".join(lines).encode("utf-8") - - -# Keep-alive comment, per the SSE spec recommendation -KEEPALIVE_COMMENT = b": ping\n\n" - -# Seconds between keep-alive pings when a generator is idle. -# Private but importable so tests can monkeypatch it. -_PING_INTERVAL: float = 15.0 diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/staticfiles.py b/bundle/python-cpu/Lib/site-packages/fastapi/staticfiles.py deleted file mode 100644 index 299015d4fef268cde91273790251f35192e1c8a6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/staticfiles.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.staticfiles import StaticFiles as StaticFiles # noqa diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/templating.py b/bundle/python-cpu/Lib/site-packages/fastapi/templating.py deleted file mode 100644 index 0cb868486edd9dda38f90c65f314597813128cf8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/templating.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.templating import Jinja2Templates as Jinja2Templates # noqa diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/testclient.py b/bundle/python-cpu/Lib/site-packages/fastapi/testclient.py deleted file mode 100644 index 4012406aa76f743c5c5d1ab8ff56d6d67cfb6653..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/testclient.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.testclient import TestClient as TestClient # noqa diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/types.py b/bundle/python-cpu/Lib/site-packages/fastapi/types.py deleted file mode 100644 index 1fb86e13b126ff7d20432d414f00b6bc372554a6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/types.py +++ /dev/null @@ -1,12 +0,0 @@ -import types -from collections.abc import Callable -from enum import Enum -from typing import Any, TypeVar, Union - -from pydantic import BaseModel -from pydantic.main import IncEx as IncEx - -DecoratedCallable = TypeVar("DecoratedCallable", bound=Callable[..., Any]) -UnionType = getattr(types, "UnionType", Union) -ModelNameMap = dict[type[BaseModel] | type[Enum], str] -DependencyCacheKey = tuple[Callable[..., Any] | None, tuple[str, ...], str] diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/utils.py b/bundle/python-cpu/Lib/site-packages/fastapi/utils.py deleted file mode 100644 index 12eaa2bf089c827f20a72dd82cbdb75deaac970a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/utils.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import warnings -from typing import ( - TYPE_CHECKING, - Any, - Literal, -) - -import fastapi -from fastapi._compat import ( - ModelField, - PydanticSchemaGenerationError, - Undefined, - annotation_is_pydantic_v1, -) -from fastapi.datastructures import DefaultPlaceholder, DefaultType -from fastapi.exceptions import FastAPIDeprecationWarning, PydanticV1NotSupportedError -from pydantic.fields import FieldInfo - -from ._compat import v2 - -if TYPE_CHECKING: # pragma: nocover - from .routing import APIRoute - - -def is_body_allowed_for_status_code(status_code: int | str | None) -> bool: - if status_code is None: - return True - # Ref: https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.1.0.md#patterned-fields-1 - if status_code in { - "default", - "1XX", - "2XX", - "3XX", - "4XX", - "5XX", - }: - return True - current_status_code = int(status_code) - return not (current_status_code < 200 or current_status_code in {204, 205, 304}) - - -def get_path_param_names(path: str) -> set[str]: - return set(re.findall("{(.*?)}", path)) - - -_invalid_args_message = ( - "Invalid args for response field! Hint: " - "check that {type_} is a valid Pydantic field type. " - "If you are using a return type annotation that is not a valid Pydantic " - "field (e.g. Union[Response, dict, None]) you can disable generating the " - "response model from the type annotation with the path operation decorator " - "parameter response_model=None. Read more: " - "https://fastapi.tiangolo.com/tutorial/response-model/" -) - - -def create_model_field( - name: str, - type_: Any, - default: Any | None = Undefined, - field_info: FieldInfo | None = None, - alias: str | None = None, - mode: Literal["validation", "serialization"] = "validation", -) -> ModelField: - if annotation_is_pydantic_v1(type_): - raise PydanticV1NotSupportedError( - "pydantic.v1 models are no longer supported by FastAPI." - f" Please update the response model {type_!r}." - ) - field_info = field_info or FieldInfo(annotation=type_, default=default, alias=alias) - try: - return v2.ModelField(mode=mode, name=name, field_info=field_info) - except PydanticSchemaGenerationError: - raise fastapi.exceptions.FastAPIError( - _invalid_args_message.format(type_=type_) - ) from None - - -def generate_operation_id_for_path( - *, name: str, path: str, method: str -) -> str: # pragma: nocover - warnings.warn( - message="fastapi.utils.generate_operation_id_for_path() was deprecated, " - "it is not used internally, and will be removed soon", - category=FastAPIDeprecationWarning, - stacklevel=2, - ) - operation_id = f"{name}{path}" - operation_id = re.sub(r"\W", "_", operation_id) - operation_id = f"{operation_id}_{method.lower()}" - return operation_id - - -def generate_unique_id(route: "APIRoute") -> str: - operation_id = f"{route.name}{route.path_format}" - operation_id = re.sub(r"\W", "_", operation_id) - assert route.methods - operation_id = f"{operation_id}_{list(route.methods)[0].lower()}" - return operation_id - - -def deep_dict_update(main_dict: dict[Any, Any], update_dict: dict[Any, Any]) -> None: - for key, value in update_dict.items(): - if ( - key in main_dict - and isinstance(main_dict[key], dict) - and isinstance(value, dict) - ): - deep_dict_update(main_dict[key], value) - elif ( - key in main_dict - and isinstance(main_dict[key], list) - and isinstance(update_dict[key], list) - ): - main_dict[key] = main_dict[key] + update_dict[key] - else: - main_dict[key] = value - - -def get_value_or_default( - first_item: DefaultPlaceholder | DefaultType, - *extra_items: DefaultPlaceholder | DefaultType, -) -> DefaultPlaceholder | DefaultType: - """ - Pass items or `DefaultPlaceholder`s by descending priority. - - The first one to _not_ be a `DefaultPlaceholder` will be returned. - - Otherwise, the first item (a `DefaultPlaceholder`) will be returned. - """ - items = (first_item,) + extra_items - for item in items: - if not isinstance(item, DefaultPlaceholder): - return item - return first_item diff --git a/bundle/python-cpu/Lib/site-packages/fastapi/websockets.py b/bundle/python-cpu/Lib/site-packages/fastapi/websockets.py deleted file mode 100644 index 55a4ac4a1a918720bb3b94eaea6f8737b968216a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fastapi/websockets.py +++ /dev/null @@ -1,3 +0,0 @@ -from starlette.websockets import WebSocket as WebSocket # noqa -from starlette.websockets import WebSocketDisconnect as WebSocketDisconnect # noqa -from starlette.websockets import WebSocketState as WebSocketState # noqa diff --git a/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/INSTALLER deleted file mode 100644 index a1b589e38a32041e49332e5e81c2d363dc418d68..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/METADATA deleted file mode 100644 index 75966d754f22a572e62d00112744e8cdb6809e9a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/METADATA +++ /dev/null @@ -1,39 +0,0 @@ -Metadata-Version: 2.5 -Name: filelock -Version: 3.32.3 -Summary: A platform independent file lock. -Project-URL: Documentation, https://py-filelock.readthedocs.io -Project-URL: Homepage, https://github.com/tox-dev/py-filelock -Project-URL: Source, https://github.com/tox-dev/py-filelock -Project-URL: Tracker, https://github.com/tox-dev/py-filelock/issues -Maintainer-email: Bernát Gábor -License-Expression: MIT -License-File: LICENSE -Keywords: application,cache,directory,log,user -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Programming Language :: Python :: 3.15 -Classifier: Topic :: Internet -Classifier: Topic :: Software Development :: Libraries -Classifier: Topic :: System -Requires-Python: >=3.10 -Description-Content-Type: text/markdown - -# filelock - -[![PyPI](https://img.shields.io/pypi/v/filelock)](https://pypi.org/project/filelock/) -[![Supported Python versions](https://img.shields.io/pypi/pyversions/filelock.svg)](https://pypi.org/project/filelock/) -[![Documentation status](https://readthedocs.org/projects/py-filelock/badge/?version=latest)](https://py-filelock.readthedocs.io/en/latest/?badge=latest) -[![Downloads](https://static.pepy.tech/badge/filelock/month)](https://pepy.tech/project/filelock) -[![check](https://github.com/tox-dev/py-filelock/actions/workflows/check.yaml/badge.svg)](https://github.com/tox-dev/py-filelock/actions/workflows/check.yaml) - -For more information checkout the [official documentation](https://py-filelock.readthedocs.io/en/latest/index.html). diff --git a/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/RECORD deleted file mode 100644 index 6042eef6a1332faf420ff4dbbab67b762598dada..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/RECORD +++ /dev/null @@ -1,48 +0,0 @@ -filelock-3.32.3.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -filelock-3.32.3.dist-info/METADATA,sha256=LYD0O9kFaJFFwAsxM7yViV1lTO1wqWoYAukjfg1mWZ8,2028 -filelock-3.32.3.dist-info/RECORD,, -filelock-3.32.3.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87 -filelock-3.32.3.dist-info/licenses/LICENSE,sha256=YIyJ1QYK6ZIa3M8yNmlbxlSplG4SMj72wCHfoE4pTUg,1088 -filelock/__init__.py,sha256=ZEhX0CozvRxFQwsm8YYc6uiLvuIYe-aoD20ACfjyizg,3469 -filelock/__pycache__/__init__.cpython-310.pyc,, -filelock/__pycache__/_api.cpython-310.pyc,, -filelock/__pycache__/_async.cpython-310.pyc,, -filelock/__pycache__/_async_read_write.cpython-310.pyc,, -filelock/__pycache__/_descriptor.cpython-310.pyc,, -filelock/__pycache__/_error.cpython-310.pyc,, -filelock/__pycache__/_identity.cpython-310.pyc,, -filelock/__pycache__/_lease.cpython-310.pyc,, -filelock/__pycache__/_marker.cpython-310.pyc,, -filelock/__pycache__/_read_write.cpython-310.pyc,, -filelock/__pycache__/_soft.cpython-310.pyc,, -filelock/__pycache__/_soft_protocol.cpython-310.pyc,, -filelock/__pycache__/_strict.cpython-310.pyc,, -filelock/__pycache__/_unix.cpython-310.pyc,, -filelock/__pycache__/_util.cpython-310.pyc,, -filelock/__pycache__/_windows.cpython-310.pyc,, -filelock/__pycache__/asyncio.cpython-310.pyc,, -filelock/__pycache__/version.cpython-310.pyc,, -filelock/_api.py,sha256=VWiuW2oWHrM9Fy3yRlsxre-daXjpmzCbJO43iFuYkJE,82436 -filelock/_async.py,sha256=rO6gt1h6ZvT2gw_0P1Xmf1jZs82kEipZ9xX4_rJUm3k,6111 -filelock/_async_read_write.py,sha256=ZAJJ02CALgZ6uhhWew4DefTtnEU7lslSoG-VjmMpDBs,13325 -filelock/_descriptor.py,sha256=ss1FfyEAyrZ1QB3LDXUtJHk5ZFAfSAaCdaa5Yp1Vzk4,2981 -filelock/_error.py,sha256=SfdLwQeooh81jIM1S4d1010ctZzXZBYBo78bJA1lEZs,2699 -filelock/_identity.py,sha256=dY_ngMJQJHm0fLD5TaGef12oOGeBAl9h0_7Xv1d3GMM,8415 -filelock/_lease.py,sha256=nEi4I02Ry3t8QSLYcctr36XV4jV6eLyaTyHo6zBeSKE,15152 -filelock/_marker.py,sha256=7VJWyyrh8wW8VywSTLLIR7OC__aNUn4zFnel5JdQwWI,6026 -filelock/_read_write.py,sha256=PBlLg_fbkqDcpRyJ5NkZgoWLR9DEUHM-1sQzBnZsOD8,32881 -filelock/_soft.py,sha256=ydpW15r3gGW2kBaHhj8FvnATQw5-kfXnSWyYhbEQNKA,13014 -filelock/_soft_protocol.py,sha256=qEM2x9Nz9zguXwwWjZsCsPEfas5nqHlJ-_oHvAM0Uo4,187 -filelock/_soft_rw/__init__.py,sha256=_ktpGmVObzrvL0K1_EAV3LF7B9JarYxW-mx_eyGPhT0,370 -filelock/_soft_rw/__pycache__/__init__.cpython-310.pyc,, -filelock/_soft_rw/__pycache__/_async.cpython-310.pyc,, -filelock/_soft_rw/__pycache__/_sync.cpython-310.pyc,, -filelock/_soft_rw/_async.py,sha256=oKDt9jgsRSAvx2ls9cA0AN2Lk0WlHc7bdmKTTCNjl9g,11492 -filelock/_soft_rw/_sync.py,sha256=XMC0kMoRBMoCAJqxK9bU_UW5o0KVQ5UmcdYqpp5j8kM,43201 -filelock/_strict.py,sha256=N5037nWAz2AF59LlyPMsOg4IwGL13D6x64hhQEn3vQQ,38823 -filelock/_unix.py,sha256=W95gKLjYivNZYLT3634o7l0pQfvU8SdBW_2C3bo3lLw,8722 -filelock/_util.py,sha256=gY839OXIttp1JOr9-FuhLTYYOoBtExkB2Nn50t8oxYQ,6356 -filelock/_windows.py,sha256=sLAk--cnOK1f-9RTuAAAOSjh4XBcxa0R3jy2w3XgUZs,15988 -filelock/asyncio.py,sha256=iE593J-CgUl3yAAGsnzitId4MtHDs5ne--F8N8oS6ys,34599 -filelock/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -filelock/version.py,sha256=7xrb9cafe38S02w7EZcEpqTbiGOe0tDHHrLpKSH8XgM,522 diff --git a/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/WHEEL deleted file mode 100644 index 51abe1f5e66cc9d07fda22a3bbe1f89475795969..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: hatchling 1.32.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/licenses/LICENSE b/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/licenses/LICENSE deleted file mode 100644 index 291919c0b6f41d014767f6c877af9f7595fcff99..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock-3.32.3.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 Bernát Gábor and contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/bundle/python-cpu/Lib/site-packages/filelock/__init__.py b/bundle/python-cpu/Lib/site-packages/filelock/__init__.py deleted file mode 100644 index 03cb401d57b4f6bbd37a071a913cd9793bbd86a3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/__init__.py +++ /dev/null @@ -1,115 +0,0 @@ -""" -A platform independent file lock that supports the with-statement. - -.. autodata:: filelock.__version__ - :no-value: - -""" - -from __future__ import annotations - -import sys -import warnings -from typing import TYPE_CHECKING, Final - -from ._api import AcquireReturnProxy, BaseFileLock, CloseErrorPolicy, ContextErrorPolicy, LockOptions -from ._descriptor import lock_descriptor, unlock_descriptor -from ._error import LeaseSettingsMismatch, SoftFileLockLifetimeWarning, SoftFileLockProtocolError, Timeout -from ._lease import LeaseCompromise, SoftFileLease -from ._marker import MarkerSoftFileLock, OwnerRecord - -if TYPE_CHECKING: - from ._async_read_write import ( - AsyncAcquireReadWriteReturnProxy, - AsyncReadWriteLock, - ) - from ._read_write import ReadWriteLock -else: - try: - from ._async_read_write import AsyncAcquireReadWriteReturnProxy, AsyncReadWriteLock - from ._read_write import ReadWriteLock - except ImportError: # pragma: lacks sqlite3 - AsyncAcquireReadWriteReturnProxy = None - AsyncReadWriteLock = None - ReadWriteLock = None - -from ._soft import SoftFileLock -from ._soft_rw import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock, SoftReadWriteLock -from ._strict import StrictSoftFileClaim, StrictSoftFileClaimState, StrictSoftFileLock -from ._unix import UnixFileLock, has_fcntl -from ._windows import WindowsFileLock -from .asyncio import ( - AsyncAcquireReturnProxy, - AsyncSoftFileLease, - AsyncSoftFileLock, - AsyncStrictSoftFileLock, - AsyncUnixFileLock, - AsyncWindowsFileLock, - BaseAsyncFileLock, -) -from .version import version - -#: version of the project as a string -__version__: Final[str] = version - - -if sys.platform == "win32": # pragma: win32 cover - _FileLock: type[BaseFileLock] = WindowsFileLock - _AsyncFileLock: type[BaseAsyncFileLock] = AsyncWindowsFileLock -else: # pragma: win32 no cover # ruff:ignore[collapsible-else-if] # the else carries the win32 no-cover pragma - if has_fcntl: - _FileLock: type[BaseFileLock] = UnixFileLock - _AsyncFileLock: type[BaseAsyncFileLock] = AsyncUnixFileLock - else: - _FileLock = SoftFileLock - _AsyncFileLock = AsyncSoftFileLock - warnings.warn("only soft file lock is available", stacklevel=2) - -if TYPE_CHECKING: - FileLock = SoftFileLock - AsyncFileLock = AsyncSoftFileLock -else: - #: Alias for the lock, which should be used for the current platform. - FileLock = _FileLock - AsyncFileLock = _AsyncFileLock - - -__all__ = [ - "AcquireReturnProxy", - "AsyncAcquireReadWriteReturnProxy", - "AsyncAcquireReturnProxy", - "AsyncAcquireSoftReadWriteReturnProxy", - "AsyncFileLock", - "AsyncReadWriteLock", - "AsyncSoftFileLease", - "AsyncSoftFileLock", - "AsyncSoftReadWriteLock", - "AsyncStrictSoftFileLock", - "AsyncUnixFileLock", - "AsyncWindowsFileLock", - "BaseAsyncFileLock", - "BaseFileLock", - "CloseErrorPolicy", - "ContextErrorPolicy", - "FileLock", - "LeaseCompromise", - "LeaseSettingsMismatch", - "LockOptions", - "MarkerSoftFileLock", - "OwnerRecord", - "ReadWriteLock", - "SoftFileLease", - "SoftFileLock", - "SoftFileLockLifetimeWarning", - "SoftFileLockProtocolError", - "SoftReadWriteLock", - "StrictSoftFileClaim", - "StrictSoftFileClaimState", - "StrictSoftFileLock", - "Timeout", - "UnixFileLock", - "WindowsFileLock", - "__version__", - "lock_descriptor", - "unlock_descriptor", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_api.py b/bundle/python-cpu/Lib/site-packages/filelock/_api.py deleted file mode 100644 index 243f94a139d74befc06c3126f8c5eb05d4a0a1e4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_api.py +++ /dev/null @@ -1,1826 +0,0 @@ -from __future__ import annotations - -import contextlib -import inspect -import logging -import math -import os -import secrets -import sys -import time -import warnings -from abc import ABCMeta, abstractmethod -from collections.abc import Callable, Hashable -from contextlib import contextmanager -from dataclasses import dataclass -from itertools import count, starmap -from threading import Condition, RLock, get_ident, local -from typing import TYPE_CHECKING, Final, Literal, NoReturn, TypedDict, TypeVar, cast -from weakref import WeakKeyDictionary, WeakValueDictionary - -from ._error import SoftFileLockLifetimeWarning, Timeout -from ._util import break_lock_file - -#: No explicit file permission mode was passed. Lock files then open with 0o666 so umask and default ACLs pick -#: the final permissions, and fchmod is skipped to preserve POSIX default ACL inheritance. -_UNSET_FILE_MODE: Final[int] = -1 - -#: Ceiling on the retry counter used as a power of two, so a long contended wait cannot overflow the backoff multiply. -_MAX_BACKOFF_EXPONENT: Final[int] = 20 - -#: How a context manager reconciles a body failure with a release failure on exit (see the property of this name). -ContextErrorPolicy = Literal["chain", "group"] -_CONTEXT_ERROR_POLICIES: Final[frozenset[str]] = frozenset({"chain", "group"}) - -#: What a descriptor-owning backend does with an ``os.close`` failure after relinquishing ownership (see the property). -CloseErrorPolicy = Literal["default", "raise", "suppress"] -_CLOSE_ERROR_POLICIES: Final[frozenset[str]] = frozenset({"default", "raise", "suppress"}) - -if TYPE_CHECKING: - from collections.abc import Generator - from types import TracebackType - from typing import Protocol - - from _typeshed import Unused - - from ._read_write import ReadWriteLock - from ._soft_rw import SoftReadWriteLock - - class _ForkResettable(Protocol): - def _reset_after_fork_in_child(self) -> None: ... - - class _ForkDescriptorOwner(Protocol): - def _descriptors_for_fork(self) -> tuple[tuple[int, tuple[int, int] | None], ...]: ... - - # Matched against the class object itself rather than `type[...]` of it. A metaclass supplies this method to the - # class while leaving instances without it, so a `type[_ForkResettableClass]` bound rejects `ReadWriteLock`. - class _ForkResettableClass(Protocol): - def _reset_class_after_fork(self) -> None: ... - - class _RegisterAtFork(Protocol): - def __call__( - self, - *, - before: Callable[[], None] | None = None, - after_in_parent: Callable[[], None] | None = None, - after_in_child: Callable[[], None] | None = None, - ) -> None: ... - - if sys.version_info >= (3, 11): # pragma: no cover (py311+) - from typing import Self - else: # pragma: no cover ( type[BaseException]: - # BaseExceptionGroup is a builtin on 3.11+; on 3.10 it needs the exceptiongroup backport. filelock keeps zero - # runtime dependencies, so the backport is imported lazily rather than required, and only group mode needs it. - if sys.version_info >= (3, 11): # pragma: no cover (py311+) - return BaseExceptionGroup # ruff:ignore[undefined-name] # builtin on 3.11+ - # Alias the import so BaseExceptionGroup above stays the builtin rather than an unbound local of this function. - from exceptiongroup import ( # ruff:ignore[import-outside-top-level] # pragma: no cover ( NoReturn: - errors = (first_error, second_error, *additional_errors) - _detach_grouped_contexts(errors) - group = _exception_group_cls()(message, errors) - if marker is not None: - setattr(group, marker[0], marker[1]) - raise group from None - - -def _detach_grouped_contexts(errors: tuple[BaseException, ...]) -> None: - seen: set[int] = set() - pending = list(errors) - while pending: - error = pending.pop() - if id(error) in seen: - continue - seen.add(id(error)) - if (context := error.__context__) is not None and ( - context is error - or _same_exception_tree(error, context) - or any(context is root or _contains_exception(root, context) for root in errors) - ): - error.__context__ = None - elif context is not None: - pending.append(context) - if error.__cause__ is not None: - pending.append(error.__cause__) - if isinstance(error, _exception_group_cls()): - pending.extend(cast("_ExceptionGroupProtocol", error).exceptions) - - -def _same_exception_tree(first: BaseException, second: BaseException) -> bool: - pending = [(first, second)] - seen: set[tuple[int, int]] = set() - while pending: - first_error, second_error = pending.pop() - if first_error is second_error: - continue - if (pair := (id(first_error), id(second_error))) in seen: - continue - seen.add(pair) - if ( - type(first_error) is not type(second_error) - or not isinstance(first_error, _exception_group_cls()) - or not isinstance(second_error, _exception_group_cls()) - ): - return False - first_group = cast("_ExceptionGroupProtocol", first_error) - second_group = cast("_ExceptionGroupProtocol", second_error) - if first_group.message != second_group.message or len(first_group.exceptions) != len(second_group.exceptions): - return False - pending.extend(zip(first_group.exceptions, second_group.exceptions, strict=True)) - return True - - -def _contains_exception(error: BaseException, target: BaseException | None) -> bool: - if target is None or not isinstance(error, _exception_group_cls()): - return False - pending = list(cast("_ExceptionGroupProtocol", error).exceptions) - seen: set[int] = set() - while pending: - child = pending.pop() - if child is target: - return True - if id(child) in seen: - continue - seen.add(id(child)) - if isinstance(child, _exception_group_cls()): - pending.extend(cast("_ExceptionGroupProtocol", child).exceptions) - return False - - -def _append_exception_context(error: BaseException, context: BaseException) -> None: - if _exception_graph_contains(error, context) or _exception_graph_contains(context, error): - return - if error.__context__ is None: - error.__context__ = context - return - tail = error - seen: set[int] = set() - while id(tail) not in seen: - seen.add(id(tail)) - if (next_error := tail.__cause__ if tail.__cause__ is not None else tail.__context__) is None: - tail.__context__ = context - return - tail = next_error - - -def _exception_graph_contains(error: BaseException, target: BaseException) -> bool: - pending = [error] - seen: set[int] = set() - while pending: - current = pending.pop() - if current is target: - return True - if id(current) in seen: # pragma: no cover - arbitrary caller exceptions can contain cycles - continue - seen.add(id(current)) - if current.__cause__ is not None: - pending.append(current.__cause__) - if current.__context__ is not None: - pending.append(current.__context__) - if isinstance(current, _exception_group_cls()): - pending.extend(cast("_ExceptionGroupProtocol", current).exceptions) - return False - - -def _grouped_errors( - error: BaseException, message: str, marker: tuple[str, _MarkerValue] -) -> tuple[BaseException, ...] | None: - if not isinstance(error, _exception_group_cls()): - return None - group = cast("_ExceptionGroupProtocol", error) - return group.exceptions if group.message == message and getattr(group, marker[0], None) is marker[1] else None - - -if TYPE_CHECKING: - - class _ExceptionGroupProtocol(Protocol): - @property - def message(self) -> str: ... - - @property - def exceptions(self) -> tuple[BaseException, ...]: ... - - -def _raise_chained_errors(first_error: BaseException, second_error: BaseException | None = None) -> NoReturn: - if second_error is None: - first_context = first_error.__context__ - try: - raise first_error # ruff:ignore[raise-within-try] # the handler restores caller-supplied context before propagation - except BaseException: - first_error.__context__ = first_context - raise - if (second_context := second_error.__context__) is not None and second_context is not first_error: - _detach_exception_context(second_context, first_error) - _append_exception_context(first_error, second_context) - first_context = first_error.__context__ - try: - raise first_error # ruff:ignore[raise-within-try] # the second raise needs this error as implicit context - except BaseException: # ruff:ignore[blind-except] # first_error may be a control-flow exception - first_error.__context__ = first_context - try: - raise second_error # ruff:ignore[raise-within-try] # the handler makes the chain interpreter-independent - except BaseException: - second_error.__context__ = first_error - first_error.__context__ = first_context - raise - - -def _detach_exception_context(error: BaseException, target: BaseException) -> None: - pending = [error] - seen: set[int] = set() - while pending: - current = pending.pop() - if id(current) in seen: - continue - seen.add(id(current)) - if current.__context__ is target: - current.__context__ = None - elif current.__context__ is not None: - pending.append(current.__context__) - if current.__cause__ is not None: - pending.append(current.__cause__) - if isinstance(current, _exception_group_cls()): - pending.extend(cast("_ExceptionGroupProtocol", current).exceptions) - - -def _raise_body_and_release(body_error: BaseException, release_error: BaseException) -> NoReturn: - # Group mode: surface the body failure and the release failure as sibling leaves instead of letting one hide in the - # other's __context__. BaseExceptionGroup returns a plain ExceptionGroup when both leaves subclass Exception, so - # ``except*`` and ``except Exception`` still catch them; a BaseException leaf (KeyboardInterrupt, CancelledError) - # keeps the group outside ordinary handlers. ``from None`` stops the group itself gaining a redundant __context__. - _raise_grouped_errors("lock body and release both failed", body_error, release_error) - - -def _raise_cleanup_errors( - message: str, - primary_error: BaseException, - *cleanup_errors: BaseException | None, -) -> NoReturn: - _raise_grouped_errors( - message, - primary_error, - *(error for error in cleanup_errors if error is not None), - ) - - -# On Windows os.path.realpath calls CreateFileW with share_mode=0, which blocks concurrent DeleteFileW and causes -# livelocks under threaded contention with SoftFileLock. os.path.abspath is purely string-based and avoids this. -_resolve_dir: Final[Callable[[str], str]] = os.path.abspath if sys.platform == "win32" else os.path.realpath - - -def _canonical(path: str | os.PathLike[str]) -> str: - """ - Return one stable key for *path*, collapsing equivalent spellings without following a final symlink. - - Relative, absolute, and ``./`` spellings of one lock file must map to a single singleton instance, deadlock-registry - entry, and removal key. Resolving the whole path with ``realpath`` would follow a final symlink and alias a lock - target the backend deliberately rejects, so the registry identity would differ from the backend's. Resolving only - the parent directory and re-appending the literal final component collapses the equivalent spellings while keeping a - final symlink a distinct key. On Windows the parent is resolved with ``abspath`` so junctions and reparse points are - not followed either. - """ - parent, name = os.path.split(os.fspath(path)) - return os.path.join(_resolve_dir(parent or os.curdir), name) # ruff:ignore[os-path-join] # string join matches abspath/realpath - - -class _ThreadLocalRegistry(local): - def __init__(self) -> None: - super().__init__() - self.held: dict[Hashable, int] = {} - - -_registry: Final[_ThreadLocalRegistry] = _ThreadLocalRegistry() - - -_T = TypeVar("_T", bound="BaseFileLock") - - -class FileLockMeta(ABCMeta): - _instances: WeakValueDictionary[str, BaseFileLock] - _instances_lock: RLock - _instances_under_construction: set[str] - - def __call__( # ruff:ignore[too-many-arguments] # forwards the public constructor's documented parameters - cls: type[_T], - lock_file: str | os.PathLike[str], - timeout: float = -1, - mode: int = _UNSET_FILE_MODE, - thread_local: bool = True, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility - *, - blocking: bool = True, - is_singleton: bool = False, - poll_interval: float = 0.05, - lifetime: float | None = None, - context_error_policy: ContextErrorPolicy = "chain", - close_error_policy: CloseErrorPolicy = "default", - fallback_to_soft: bool = True, - preserve_lock_file: bool = False, - on_acquired: Callable[[int], None] | None = None, - **kwargs: _ExtraValue, - ) -> _T: - _ensure_current_process() - lifetime = _resolve_lifetime(lifetime, cls, stacklevel=cls._constructor_lifetime_warning_stacklevel) - # Validate before building the instance: a raise inside __init__ would leave a half-constructed object whose - # __del__ then trips over the missing context. - context_error_policy = _resolve_context_error_policy(context_error_policy) - close_error_policy = _resolve_close_error_policy(close_error_policy) - preserve_lock_file = _resolve_preserve_lock_file( - preserve=preserve_lock_file, supported=cls._preserve_lock_file_supported, cls_name=cls.__name__ - ) - on_acquired = _resolve_on_acquired(on_acquired, supported=cls._on_acquired_supported, cls_name=cls.__name__) - params: dict[str, _LockInitValue | _ExtraValue] = { - "timeout": timeout, - "mode": mode, - "thread_local": thread_local, - "blocking": blocking, - "is_singleton": is_singleton, - "poll_interval": poll_interval, - "lifetime": lifetime, - "context_error_policy": context_error_policy, - "close_error_policy": close_error_policy, - "fallback_to_soft": fallback_to_soft, - "preserve_lock_file": preserve_lock_file, - "on_acquired": on_acquired, - **kwargs, - } - if not is_singleton: - return cls._create_instance(lock_file, params) - - # Look up, build and store under one lock. Without it two threads racing the first construction for a - # path both miss the cache and each build their own instance, so callers relying on is_singleton for - # reentrant locking across instances end up with two "singletons" and acquire()'s deadlock check then - # rejects a legitimate reentrant acquire; the unguarded writes to the WeakValueDictionary are a data - # race besides. ReadWriteLock and SoftReadWriteLock already guard their singleton caches this way. - # Key the cache on the canonical form so equivalent spellings of one path share a singleton, and it matches the - # deadlock-registry key acquire() uses. - singleton_key = _canonical(lock_file) - with cls._instances_lock: - if (instance := cls._instances.get(singleton_key)) is None: - if singleton_key in cls._instances_under_construction: # pragma: needs fork - msg = f"Singleton lock construction is already active for {lock_file!s}" - raise RuntimeError(msg) - construction_registry = cls._instances_under_construction - construction_pid = os.getpid() - construction_registry.add(singleton_key) - try: - instance = cls._create_instance(lock_file, params) - finally: - construction_registry.discard(singleton_key) - if os.getpid() != construction_pid: # pragma: needs fork - msg = "Lock construction cannot continue after fork; construct a new lock in the child" - raise RuntimeError(msg) - cls._instances[singleton_key] = instance - return instance - - params_to_check = { - "thread_local": (thread_local, instance.is_thread_local()), - "timeout": (timeout, instance.timeout), - "mode": (mode, instance._context.mode), # ruff:ignore[private-member-access] # compares against the managed instance's own context - "blocking": (blocking, instance.blocking), - "poll_interval": (poll_interval, instance.poll_interval), - "lifetime": (lifetime, instance.lifetime), - "context_error_policy": (context_error_policy, instance.context_error_policy), - "close_error_policy": (close_error_policy, instance.close_error_policy), - "fallback_to_soft": (fallback_to_soft, instance.fallback_to_soft), - "preserve_lock_file": (preserve_lock_file, instance.preserve_lock_file), - } - non_matching_params = { - name: (passed_param, set_param) - for name, (passed_param, set_param) in params_to_check.items() - if passed_param != set_param - } - # Callables compare by identity, not equality: two equal callables can close over different state, so a - # singleton must reject a different hook object even if it compares equal. Keep it out of the scalar dict above. - hook_mismatch = on_acquired is not instance.on_acquired - if not non_matching_params and not hook_mismatch: - return instance # ty: ignore[invalid-return-type] # https://github.com/astral-sh/ty/issues/3231 - - msg = "Singleton lock instances cannot be initialized with differing arguments" - msg += "\nNon-matching arguments: " - for param_name, (passed_param, set_param) in non_matching_params.items(): - msg += f"\n\t{param_name} (existing lock has {set_param} but {passed_param} was passed)" - if hook_mismatch: - msg += f"\n\ton_acquired (existing lock has {instance.on_acquired} but {on_acquired} was passed)" - raise ValueError(msg) - - def _create_instance( - cls: type[_T], lock_file: str | os.PathLike[str], params: dict[str, _LockInitValue | _ExtraValue] - ) -> _T: - model = _init_parameter_model(cls) - if model.accepts_kwargs: - return super().__call__(lock_file, **params) - - unsupported = sorted( - name - for name, value in params.items() - if name not in model.accepted_params - and ((parameter := model.default_params.get(name)) is None or value != parameter.default) - ) - if unsupported: - msg = f"{cls.__name__} does not support non-default lock options: {', '.join(unsupported)}" - raise TypeError(msg) - # virtualenv narrows a BaseFileLock descendant's signature; omit base defaults it does not accept (#340). - return super().__call__( - lock_file, - **{name: value for name, value in params.items() if name in model.accepted_params}, - ) - - -_INIT_PARAMETER_MODELS: Final[WeakKeyDictionary[type[BaseFileLock], _InitParameterModel]] = WeakKeyDictionary() - - -def _init_parameter_model(cls: type[BaseFileLock]) -> _InitParameterModel: - # A strong cache would keep dynamically created subclasses alive for the process lifetime. - with _fork_transition(), _FORK_STATE.parameter_models_lock: - if (model := _INIT_PARAMETER_MODELS.get(cls)) is None: - parameters = inspect.signature(cls.__init__).parameters.values() - model = _InitParameterModel( - accepted_params=frozenset( - parameter.name - for parameter in parameters - if parameter.kind in {inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY} - ), - accepts_kwargs=any(parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters), - default_params={ - name: parameter - for name, parameter in inspect.signature(type(cls).__call__).parameters.items() - if parameter.default is not inspect.Parameter.empty - }, - ) - _INIT_PARAMETER_MODELS[cls] = model - return model - - -@dataclass(frozen=True) -class _InitParameterModel: - accepted_params: frozenset[str] - accepts_kwargs: bool - default_params: dict[str, inspect.Parameter] - - -def _resolve_lifetime(lifetime: float | None, cls: type[BaseFileLock], *, stacklevel: int) -> float | None: - """ - Validate ``lifetime`` and drop a value the backend cannot honor. - - ``lifetime`` is a deliberate age-based lease: a lock file older than ``lifetime`` is broken even while its holder is - still alive. Existence locks (:class:`SoftFileLock`) implement that behavior by unlinking a reclaimable pathname, - which can overlap a live holder. A native OS lock lives on the inode, so unlinking the pathname by age cannot revoke - the kernel lock; a contender would lock a fresh inode and overlap the live holder (#590). Ignore the request with a - warning rather than accept a setting that breaks mutual exclusion. - """ - if lifetime is not None: - if isinstance(lifetime, bool) or not isinstance(lifetime, (int, float)): - msg = f"lifetime must be a finite non-negative number or None, not {type(lifetime).__name__}" - raise TypeError(msg) - if lifetime < 0 or (isinstance(lifetime, float) and not math.isfinite(lifetime)): - msg = f"lifetime must be finite and non-negative, not {lifetime!r}" - raise ValueError(msg) - if lifetime is not None and not cls._lifetime_supported: - warnings.warn( - f"lifetime is ignored for {cls.__name__}: {cls._lifetime_unsupported_reason}; " - f"only SoftFileLock supports lifetime-based expiry", - stacklevel=stacklevel, - ) - return None - if lifetime is not None and cls._lifetime_replacements is not None: - strict_lock, lease = cls._lifetime_replacements - warnings.warn( - f"{cls.__name__}(lifetime=...) uses age-based expiry and can overlap a live holder; " - f"use {lease} for expiry or {strict_lock} for fail-closed locking", - SoftFileLockLifetimeWarning, - stacklevel=stacklevel, - ) - return lifetime - - -def _resolve_context_error_policy(policy: str) -> ContextErrorPolicy: - if policy not in _CONTEXT_ERROR_POLICIES: - msg = f"context_error_policy must be 'chain' or 'group', got {policy!r}" - raise ValueError(msg) - if policy == "group": # fail fast at construction rather than only when a dual failure happens to occur - try: - _exception_group_cls() - except ImportError as exc: # pragma: no cover # only on 3.10 without the exceptiongroup backport - msg = "context_error_policy='group' requires Python 3.11+ or the 'exceptiongroup' backport installed" - raise ValueError(msg) from exc - return cast("ContextErrorPolicy", policy) - - -def _resolve_close_error_policy(policy: str) -> CloseErrorPolicy: - if policy not in _CLOSE_ERROR_POLICIES: - msg = f"close_error_policy must be 'default', 'raise', or 'suppress', got {policy!r}" - raise ValueError(msg) - return cast("CloseErrorPolicy", policy) - - -def _resolve_preserve_lock_file(*, preserve: bool, supported: bool, cls_name: str) -> bool: - # An existence lock unlinks its marker to release, so preserving the pathname would defeat unlocking. Reject the - # request rather than silently ignore it, since a caller asking for a stable identity must know it cannot be kept. - if preserve and not supported: - msg = f"preserve_lock_file=True is not supported by {cls_name}: unlinking its marker is how it releases" - raise ValueError(msg) - return preserve - - -def _resolve_on_acquired( - on_acquired: Callable[[int], None] | None, *, supported: bool, cls_name: str -) -> Callable[[int], None] | None: - if on_acquired is None: - return None - # An existence lock stores protocol state in its marker, so a caller writing through the descriptor would corrupt - # stale detection and ownership metadata; only native locks lend out the descriptor. - if not supported: - msg = f"on_acquired is not supported by {cls_name}: only native locks expose the lock descriptor" - raise ValueError(msg) - # A hook that fails and then also fails to release surfaces both errors as a BaseExceptionGroup. Require that class - # at construction rather than at the rare moment both fail, matching how context_error_policy='group' validates. - try: - _exception_group_cls() - except ImportError as exc: # pragma: no cover # only on 3.10 without the exceptiongroup backport - msg = "on_acquired requires Python 3.11+ or the 'exceptiongroup' backport for its rollback error path" - raise ValueError(msg) from exc - return on_acquired - - -class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta): # ruff:ignore[too-many-public-methods] # public config properties - """ - Abstract base class for a file lock object. - - Provides the common reentrant API and state management. Subclasses implement the locking mechanism - (:class:`UnixFileLock `, :class:`WindowsFileLock `, - :class:`SoftFileLock `). - - """ - - _instances: WeakValueDictionary[str, BaseFileLock] - _instances_lock: RLock - _instances_under_construction: set[str] - - #: How the cross-instance deadlock message names the conflicting holder; the async subclass says "task". - _deadlock_holder_desc: str = "FileLock instance in this thread" - - #: Whether an age-based :attr:`lifetime` lease may break this lock. Only existence locks set it (they reclaim by - #: unlinking a pathname); native OS locks leave it ``False`` since a kernel lock cannot be revoked by file age. - _lifetime_supported: bool = False - - #: Strict-lock and lease replacements for a backend with legacy age-based expiry. - _lifetime_replacements: tuple[str, str] | None = None - - #: Why a backend that refuses ``lifetime`` cannot honor it, named in the warning that drops the value. - _lifetime_unsupported_reason: str = "a native OS lock cannot be broken safely by file age" - - #: Async construction adds one metaclass frame before lifetime validation. - _constructor_lifetime_warning_stacklevel: int = 3 - - #: Whether :attr:`preserve_lock_file` may be ``True``. Native locks keep the pathname on release, so they support - #: it; existence locks unlink their marker to release and reject it. - _preserve_lock_file_supported: bool = True - - #: Whether an :attr:`on_acquired` hook may be set. Native locks lend the descriptor out; existence locks keep - #: protocol state in the marker and reject it. - _on_acquired_supported: bool = True - - #: Whether a shared instance serializes its physical acquire and release behind one gate. A backend that publishes - #: several files per owner needs it; a single-file backend is atomic and leaves it off to skip the gate entirely. - _serialize_transitions: bool = False - - #: Ceiling in seconds on the jittered backoff between contended acquisition retries. ``0`` keeps the fixed - #: poll cadence; a multi-file backend sets it so contending processes desynchronize instead of livelocking. - _poll_backoff_cap: float = 0.0 - - def __init_subclass__(cls, **kwargs: _SubclassValue) -> None: - """Give each lock subclass its own singleton registry and lock.""" - super().__init_subclass__(**kwargs) - cls._instances = WeakValueDictionary() - cls._instances_lock = RLock() - cls._instances_under_construction = set() - _register_fork_class(cls) - - @classmethod - def _reset_class_after_fork(cls) -> None: # pragma: forked child - cls._instances = WeakValueDictionary() - cls._instances_lock = RLock() - cls._instances_under_construction = set() - - def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option - self, - lock_file: str | os.PathLike[str], - timeout: float = -1, - mode: int = _UNSET_FILE_MODE, - thread_local: bool = True, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility - *, - blocking: bool = True, - is_singleton: bool = False, - poll_interval: float = 0.05, - lifetime: float | None = None, - context_error_policy: ContextErrorPolicy = "chain", - close_error_policy: CloseErrorPolicy = "default", - fallback_to_soft: bool = True, - preserve_lock_file: bool = False, - on_acquired: Callable[[int], None] | None = None, - ) -> None: - """ - Create a new lock object. - - :param lock_file: path to the file - :param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the - acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a - negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock. - :param mode: file permissions for the lockfile. When not specified, the OS controls permissions via umask and - default ACLs, preserving POSIX default ACL inheritance in shared directories. - :param thread_local: Whether this object's internal context should be thread local or not. If this is set to - ``False`` then the lock will be reentrant across threads. When ``True`` (the default), **all fields of the - lock's internal context are per-thread**, including the configuration values ``poll_interval``, ``timeout``, - ``blocking``, ``mode``, and ``lifetime``. Setting one of these properties from one thread does not change - the value seen by another thread; threads that did not perform the write continue to see the value supplied - at construction time. If you need configuration values to be visible across threads, construct the lock - with ``thread_local=False``. - :param blocking: whether the lock should be blocking or not - :param is_singleton: If this is set to ``True`` then only one instance of this class will be created per lock - file. This is useful if you want to use the lock object for reentrant locking without needing to pass the - same object around. - :param poll_interval: default interval for polling the lock file, in seconds. It will be used as fallback value - in the acquire method, if no poll_interval value (``None``) is given. - :param lifetime: for :class:`SoftFileLock`, the age in seconds after which a waiting process may delete the - marker, even while its holder remains alive. This legacy expiry mode does not provide strict mutual - exclusion. ``None`` (the default) disables age-based expiry. Native OS locks (:class:`FileLock`) cannot be - revoked by file age and ignore a non-``None`` ``lifetime`` with a warning. - :param context_error_policy: how a context manager reconciles a failure in its body with a failure while - releasing on exit. ``"chain"`` (the default) keeps Python's behavior: the release error propagates with the - body error in its ``__context__``. ``"group"`` raises a :class:`BaseExceptionGroup` holding the body error - first and the release error second, so neither hides the other. - :param close_error_policy: what to do with an ``os.close`` failure after relinquishing descriptor ownership. - ``"default"`` keeps each backend's historical behavior (Unix native locks drop a FUSE/Docker ``EIO``; - Windows native locks and :class:`SoftFileLock` propagate); ``"raise"`` always propagates the ``OSError``; - ``"suppress"`` always ignores it. Held state is released either way. It does not affect unlock failures or - lock-file deletion. - :param fallback_to_soft: for :class:`UnixFileLock`, whether to switch to :class:`SoftFileLock` when the - filesystem's ``flock`` returns ``ENOSYS``. ``True`` (the default) keeps the historical fallback; - ``False`` fails closed, letting the ``ENOSYS`` propagate so a caller that needs kernel-enforced - locking is never silently downgraded. It has no effect on Windows or :class:`SoftFileLock`. - :param preserve_lock_file: for native locks (:class:`FileLock`), whether filelock promises not to unlink the - lock pathname on release. ``False`` (the default) keeps each backend's cleanup: Windows removes the lock - file, Unix already leaves it. ``True`` keeps a stable file identity for ACLs, auditing, and holder metadata: - Windows skips its post-release unlink and Unix refuses to enter the ``ENOSYS`` soft fallback (which releases - by unlinking). :class:`SoftFileLock` rejects ``True``. The promise covers filelock's own release path only; - it cannot stop another process or the filesystem from removing the pathname. - :param on_acquired: for native locks (:class:`FileLock`), a callable invoked with the borrowed lock descriptor - once per physical acquisition, after filelock holds the native lock and finished backend initialization but - before :meth:`~BaseFileLock.acquire` returns. Recursive acquisitions do not call it again. The callback may - read, write, seek, truncate, or set metadata through ``os`` on the descriptor, but must not close, unlock, - or take ownership of it, and filelock does not fsync its writes. If it raises, filelock releases the lock - and re-raises. :class:`SoftFileLock` rejects the hook. - - """ - self._creator_pid = os.getpid() - self._transition_lock = RLock() - self._is_thread_local = thread_local - self._is_singleton = is_singleton - self._context_error_policy = context_error_policy # already validated by the metaclass - self._close_error_policy = close_error_policy # already validated by the metaclass - self._fallback_to_soft = fallback_to_soft - self._preserve_lock_file = preserve_lock_file # already validated by the metaclass - self._on_acquired = on_acquired # already validated by the metaclass - - self._context: FileLockContext = (ThreadLocalFileContext if thread_local else FileLockContext)( - lock_file=os.fspath(lock_file), - timeout=timeout, - mode=mode, - blocking=blocking, - poll_interval=poll_interval, - lifetime=lifetime, - ) - _register_fork_object(self) - - def is_thread_local(self) -> bool: - """:returns: a flag indicating if this lock is thread local or not""" - return self._is_thread_local - - @property - def is_singleton(self) -> bool: - """ - A flag indicating if this lock is singleton or not. - - .. versionadded:: 3.13.0 - - """ - return self._is_singleton - - @property - def context_error_policy(self) -> ContextErrorPolicy: - """ - How a context manager reconciles a body failure with a release failure on exit. - - .. versionadded:: 3.30.0 - - """ - return self._context_error_policy - - @property - def close_error_policy(self) -> CloseErrorPolicy: - """ - What a lock does with an ``os.close`` failure after relinquishing descriptor ownership. - - .. versionadded:: 3.30.0 - - """ - return self._close_error_policy - - def _close_released_fd(self, fd: int, *, default_suppresses: bool) -> None: - # CPython never retries close() after EINTR because the descriptor number may already be reused, so neither does - # this. close_error_policy decides the error's fate after the backend relinquishes descriptor ownership. - try: - os.close(fd) - except OSError: - if self._close_error_policy == "suppress" or (self._close_error_policy == "default" and default_suppresses): - return - raise - - @property - def fallback_to_soft(self) -> bool: - """ - Whether a :class:`FileLock` falls back to :class:`SoftFileLock` when the filesystem lacks ``flock``. - - Only :class:`UnixFileLock` acts on it: when ``False`` an ``ENOSYS`` from ``flock`` propagates instead of - switching to existence-lock semantics. - - .. versionadded:: 3.30.0 - - """ - return self._fallback_to_soft - - @property - def preserve_lock_file(self) -> bool: - """ - Whether filelock promises not to unlink the lock pathname on release. - - When ``True``, Windows skips its post-release unlink and Unix refuses the ``ENOSYS`` soft fallback. - :class:`SoftFileLock` rejects ``True`` because unlinking its marker is how it releases. - - .. versionadded:: 3.30.0 - - """ - return self._preserve_lock_file - - @property - def on_acquired(self) -> Callable[[int], None] | None: - """ - The callback run with the borrowed lock descriptor once per physical acquisition, or ``None``. - - Native locks only. It runs after the native lock is held and backend initialization finished, before - :meth:`~BaseFileLock.acquire` returns; a raise rolls back the acquisition. :class:`SoftFileLock` rejects it. - - .. versionadded:: 3.30.0 - - """ - return self._on_acquired - - @property - def lock_file(self) -> str: - """Path to the lock file.""" - return self._context.lock_file - - @property - def timeout(self) -> float: - """ - The default timeout value, in seconds. - - .. versionadded:: 2.0.0 - - """ - return self._context.timeout - - @timeout.setter - def timeout(self, value: float | str) -> None: - """ - Change the default timeout value. - - :param value: the new value, in seconds - - """ - self._context.timeout = float(value) - - @property - def blocking(self) -> bool: - """ - Whether the locking is blocking or not. - - .. versionadded:: 3.14.0 - - """ - return self._context.blocking - - @blocking.setter - def blocking(self, value: bool) -> None: - """ - Change the default blocking value. - - :param value: the new value as bool - - """ - self._context.blocking = value - - @property - def poll_interval(self) -> float: - """ - The default polling interval, in seconds. - - .. versionadded:: 3.24.0 - - """ - return self._context.poll_interval - - @poll_interval.setter - def poll_interval(self, value: float) -> None: - """ - Change the default polling interval. - - :param value: the new value, in seconds - - """ - self._context.poll_interval = value - - @property - def lifetime(self) -> float | None: - """ - The soft marker age in seconds that permits expiry, or ``None`` to disable age-based expiry. - - A non-``None`` value permits a waiter to enter while the previous holder remains active, so it does not provide - strict mutual exclusion. Native locks ignore the value with a warning. - - .. versionadded:: 3.24.0 - - """ - return self._context.lifetime - - @lifetime.setter - def lifetime(self, value: float | None) -> None: - """ - Change the legacy age-based expiry threshold. - - :param value: the new value in seconds, or ``None`` to disable expiration - - :raises ValueError: if *value* is negative or not finite - :raises TypeError: if *value* is not ``None`` and not a real number - - """ - self._context.lifetime = _resolve_lifetime(value, type(self), stacklevel=3) - - @property - def mode(self) -> int: - """The file permissions for the lockfile.""" - return 0o644 if self._context.mode == _UNSET_FILE_MODE else self._context.mode - - @property - def has_explicit_mode(self) -> bool: - """Whether the file permissions were explicitly set.""" - return self._context.mode != _UNSET_FILE_MODE - - def _open_mode(self) -> int: - """Mode for ``os.open``: 0o666 when unset so umask and ACLs decide, otherwise the explicit mode.""" - return 0o666 if self._context.mode == _UNSET_FILE_MODE else self._context.mode - - @property - def is_locked(self) -> bool: - """ - A boolean indicating if the lock file is holding the lock currently. - - .. versionchanged:: 2.0.0 - - This was previously a method and is now a property. - - """ - _ensure_current_process() - return self._context.lock_file_fd is not None - - @property - def lock_counter(self) -> int: - """The number of times this lock has been acquired (but not yet released).""" - _ensure_current_process() - return self._context.lock_counter - - def __enter__(self) -> Self: - """ - Acquire the lock. - - :returns: the lock object - - """ - self.acquire() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - """Release the lock, reconciling a release failure with any body failure per :attr:`context_error_policy`.""" - self._release_in_context(exc_value) - - def _release_in_context(self, body_error: BaseException | None) -> None: - # Release from a context-manager exit. "chain" lets a release failure propagate with the body error already in - # its __context__ (Python's default); "group" raises both as sibling leaves so neither one hides the other. - try: - self.release() - except BaseException as release_error: - if body_error is None or self._context_error_policy == "chain": - raise - _raise_body_and_release(body_error, release_error) - - def __del__(self) -> None: - """Force-release so a dropped reference never leaks a held lock.""" - if vars(self).get("_creator_pid") != os.getpid(): - return # pragma: forked child - # A finalizer must not raise. A release error during garbage collection would otherwise surface as an - # unraisable-exception warning, attributed to whichever code triggered collection. The dropped lock still gets - # best-effort cleanup; an explicit release() reports the same error to a caller who can act on it. - with contextlib.suppress(Exception): - self.release(force=True) - - def acquire( - self, - timeout: float | None = None, - poll_interval: float | None = None, - *, - poll_intervall: float | None = None, - blocking: bool | None = None, - cancel_check: Callable[[], bool] | None = None, - ) -> AcquireReturnProxy: - """ - Try to acquire the file lock. - - :param timeout: maximum wait time for acquiring the lock, ``None`` means use the default :attr:`~timeout` is and - if ``timeout < 0``, there is no timeout and this method will block until the lock could be acquired - :param poll_interval: interval of trying to acquire the lock file, ``None`` means use the default - :attr:`~poll_interval` - :param poll_intervall: deprecated, kept for backwards compatibility, use ``poll_interval`` instead - :param blocking: defaults to True. If False, function will return immediately if it cannot obtain a lock on the - first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired. - :param cancel_check: a callable returning ``True`` when the acquisition should be canceled. Checked on each poll - iteration. When triggered, raises :class:`~Timeout` just like an expired timeout. - - :returns: a context object that will unlock the file when the context is exited - - :raises Timeout: if fails to acquire lock within the timeout period - - .. code-block:: python - - # You can use this method in the context manager (recommended) - with lock.acquire(): - pass - - # Or use an equivalent try-finally construct: - lock.acquire() - try: - pass - finally: - lock.release() - - .. versionchanged:: 2.0.0 - - This method returns now a *proxy* object instead of *self*, so that it can be used in a with statement - without side effects. - - """ - self._raise_if_inherited() - if timeout is None: - timeout = self._context.timeout - - if blocking is None: - blocking = self._context.blocking - - if poll_intervall is not None: - msg = "use poll_interval instead of poll_intervall" - warnings.warn(msg, DeprecationWarning, stacklevel=2) - poll_interval = poll_intervall - - poll_interval = poll_interval if poll_interval is not None else self._context.poll_interval - - start_time = time.perf_counter() - # Wait for admission before touching any state: a caller refused entry must leave the counter, the registry and - # the descriptor exactly as it found them. - with self._transition_admission( - blocking=blocking, - cancel_check=cancel_check, - timeout=timeout, - poll_interval=poll_interval, - start_time=start_time, - ): - # Bump the counter up front; _undo_acquire rolls it back if acquisition fails. - self._context.lock_counter += 1 - - canonical = _canonical(self.lock_file) - self._raise_if_would_deadlock(canonical, timeout=timeout, blocking=blocking) - - try: - self._poll_until_acquired( - blocking=blocking, - cancel_check=cancel_check, - timeout=timeout, - poll_interval=poll_interval, - start_time=start_time, - ) - except BaseException: - self._reconcile_failed_acquire(canonical) - raise - self._commit_acquire(canonical) - return AcquireReturnProxy(lock=self) - - @contextlib.contextmanager - def _transition_admission( - self, - *, - blocking: bool, - cancel_check: Callable[[], bool] | None, - timeout: float, - poll_interval: float, - start_time: float, - ) -> Generator[None]: - # One thread at a time drives the physical transition of a shared instance. A protocol that publishes several - # files per owner leaves a half-built claim visible otherwise, and a second thread would read it as a holder. - # Only such a backend opts in; a single-file backend and a thread-local context each need no gate. - if not self._serialize_transitions or self._is_thread_local: - yield - return - while not self._transition_lock.acquire(blocking=False): # pragma: needs hard-link - if not blocking or (cancel_check is not None and cancel_check()): - raise Timeout(self.lock_file) - if timeout >= 0 and time.perf_counter() - start_time >= timeout: - raise Timeout(self.lock_file) - time.sleep(poll_interval) - try: # pragma: needs hard-link - yield - finally: # pragma: needs hard-link - self._transition_lock.release() - - def release(self, force: bool = False) -> None: # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility - """ - Release the file lock. The lock is only completely released when the lock counter reaches 0. The lock file - itself may be deleted automatically, the behavior is platform-specific. - - :param force: If true, the lock counter is ignored and the lock is released in every case. - - """ - # A shared instance releases under the same gate its acquisition ran through, so a thread entering the lock - # never observes a partially torn-down owner. - serialize = self._serialize_transitions and not self._is_thread_local - with self._transition_lock if serialize else contextlib.nullcontext(): - if self._creator_pid != os.getpid() or not self.is_locked: - return - if not force and self._context.lock_counter > 1: - self._context.lock_counter -= 1 - return - - lock_id, lock_filename = id(self), self.lock_file - _LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename) - try: - self._release_with_fork_tracking() - except BaseException: - # A failure after the OS unlock (during close or unlink) still released the lock: the backend cleared - # its descriptor, so commit the counter and registry to released even as the cleanup error propagates. - # A failure that left the lock held keeps the counter so a later release can retry the OS unlock. - if not self.is_locked: - self._commit_release() - raise - self._commit_release() - _LOGGER.debug("Lock %s released on %s", lock_id, lock_filename) - - def _raise_if_inherited(self) -> None: - if self._creator_pid != os.getpid(): # pragma: forked child - msg = f"{type(self).__name__} on {self.lock_file} was inherited across fork; construct a new instance" - raise RuntimeError(msg) - - def _mark_descriptor_owned(self, fd: int, identity: tuple[int, int] | None = None) -> None: - self._context.pending_lock_file_fd = None - self._context.pending_lock_file_fd_identity = None - self._context.lock_file_fd = fd - self._context.lock_file_fd_identity = identity - - def _mark_descriptor_pending(self, fd: int, identity: tuple[int, int] | None = None) -> None: - self._context.pending_lock_file_fd = fd - self._context.pending_lock_file_fd_identity = identity - - def _mark_descriptor_released(self) -> None: - self._context.pending_lock_file_fd = None - self._context.pending_lock_file_fd_identity = None - self._context.lock_file_fd = None - self._context.lock_file_fd_identity = None - - def _reset_after_fork_in_child(self) -> None: # pragma: forked child - # fork copies the lock in whatever state the parent's threads left it, so give the child an unheld one. - self._transition_lock = RLock() - self._context.owner_claim_paths = () - self._context.claim_root = None - self._context.lock_file_fd = None - self._context.lock_file_fd_token = None - self._context.lock_file_fd_identity = None - self._context.pending_lock_file_fd = None - self._context.pending_lock_file_fd_identity = None - self._context.lock_counter = 0 - self._context.lock_file_key = None - - def _descriptors_for_fork(self) -> tuple[tuple[int, tuple[int, int] | None], ...]: # pragma: needs fork - descriptors: list[tuple[int, tuple[int, int] | None]] = [] - if self._context.lock_file_fd is not None and self._context.lock_file_fd_token is None: - descriptors.append((self._context.lock_file_fd, self._context.lock_file_fd_identity)) - if self._context.pending_lock_file_fd is not None: - descriptors.append((self._context.pending_lock_file_fd, self._context.pending_lock_file_fd_identity)) - return tuple(descriptors) - - def _raise_if_would_deadlock(self, canonical: str, *, timeout: float, blocking: bool) -> None: - """ - Fail fast when a *different* live instance already holds this path in the current deadlock scope. - - Only the first, indefinitely-blocking acquire can self-deadlock this way: waiting in the OS primitive would - block on a lock this flow already owns. A finite timeout or ``blocking=False`` keeps the normal Timeout path. - """ - would_block = self._context.lock_counter == 1 and not self.is_locked and timeout < 0 and blocking - if would_block and _registry.held.get(self._registry_key(canonical)) not in {None, id(self)}: - self._context.lock_counter -= 1 - msg = ( - f"Deadlock: lock '{self.lock_file}' is already held by a different {self._deadlock_holder_desc}. " - f"Use is_singleton=True to enable reentrant locking across instances." - ) - raise RuntimeError(msg) - - def _registry_key(self, canonical: str) -> Hashable: - return canonical if (scope := self._deadlock_scope()) is None else (scope, canonical) - - @staticmethod - def _deadlock_scope() -> Hashable | None: - """ - Execution unit whose own hold would deadlock a new acquire. - - ``None`` scopes holders to the thread, which the thread-local registry already separates. Async locks - override it with the running task, since one event loop thread runs many tasks and only a reacquire from - the *same* task can self-deadlock. - """ - return None - - def _poll_until_acquired( - self, - *, - blocking: bool, - cancel_check: Callable[[], bool] | None, - timeout: float, - poll_interval: float, - start_time: float, - ) -> None: - lock_id = id(self) - lock_filename = self.lock_file - attempt = 0 - while True: - self._raise_if_inherited() - if not self.is_locked: - self._try_break_expired_lock() - _LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename) - self._acquire_with_fork_tracking() - self._raise_if_inherited() - if self.is_locked: - _LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename) - return - if self._check_give_up( - blocking=blocking, - cancel_check=cancel_check, - timeout=timeout, - start_time=start_time, - ): - raise Timeout(lock_filename) - attempt += 1 - delay = self._poll_delay(poll_interval, attempt) - msg = "Lock %s not acquired on %s, waiting %s seconds ..." - _LOGGER.debug(msg, lock_id, lock_filename, delay) - time.sleep(delay) - - def _poll_delay(self, poll_interval: float, attempt: int) -> float: - # A single-file lock retries on a fixed cadence. A backend that publishes several files per acquisition sets a - # cap, and then contending processes back off across a jittered, exponentially widening window instead of - # colliding on every poll; poll_interval stays the floor so a lone waiter is still responsive. - if not self._poll_backoff_cap: - return poll_interval - # Cap the exponent before doubling: under heavy contention attempt reaches the thousands, and 2**attempt would - # overflow the float multiply long before the window itself stops growing past the cap. - window = min( - self._poll_backoff_cap, poll_interval * 2 ** min(attempt, _MAX_BACKOFF_EXPONENT) - ) # pragma: needs hard-link - return max(poll_interval, secrets.randbelow(int(window * 1_000_000) + 1) / 1_000_000) # pragma: needs hard-link - - def _reconcile_failed_acquire(self, canonical: str) -> None: - # An acquire that raised while still holding the native lock (a hook that failed and whose rollback could not - # release) must keep the registry entry so a later release can retry the OS unlock; otherwise roll the counter - # back. is_locked was already reconciled by whichever release ran. - if self.is_locked: - self._commit_acquire(canonical) - else: - self._undo_acquire() - - def _invoke_on_acquired(self) -> None: - # The wrapper runs in the backend executor for async locks, preserving the callback's documented thread. - if self._on_acquired is None or self._context.lock_counter != 1: - return - try: - self._on_acquired(cast("int", self._context.lock_file_fd)) - except BaseException as callback_error: # arbitrary caller code; roll back on any failure - callback_context = callback_error.__context__ - try: - self._release_with_fork_tracking() - except BaseException as release_error: # ruff:ignore[blind-except] # both errors surface via the group below - _raise_body_and_release(callback_error, release_error) - callback_error.__context__ = callback_context - raise - - def _acquire_with_fork_tracking(self) -> None: - with _fork_transition(self): - try: - self._acquire() - except BaseException as acquisition_error: - self._rollback_failed_acquire(acquisition_error) - raise - try: - self._register_context_descriptor() - except BaseException as registration_error: # pragma: needs fork - self._rollback_failed_registration(registration_error) - raise - if self.is_locked: - self._invoke_on_acquired() - - def _rollback_failed_acquire(self, acquisition_error: BaseException) -> None: - if not self.is_locked: - return - registration_error: BaseException | None = None - tracking_error: BaseException | None = None - try: - self._register_context_descriptor() - except BaseException as error: # ruff:ignore[blind-except] # preserve registration and acquisition failures - registration_error = error - try: - # Rollback may fail too; retain the fd so a child can close it without another identity probe. - self._register_unverified_context_descriptor() - except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback - tracking_error = error - try: - self._release_with_fork_tracking() - except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and acquisition failures - _raise_cleanup_errors( - "lock acquisition cleanup failed", - acquisition_error, - registration_error, - tracking_error, - rollback_error, - ) - if registration_error is not None: # pragma: needs fork - _raise_cleanup_errors( - "lock acquisition cleanup failed", acquisition_error, registration_error, tracking_error - ) - - def _rollback_failed_registration(self, registration_error: BaseException) -> None: # pragma: needs fork - tracking_error: BaseException | None = None - try: - # Rollback may fail too; retain the fd so a child can close it without another identity probe. - self._register_unverified_context_descriptor() - except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback - tracking_error = error - try: - self._release_with_fork_tracking() - except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and registration failures - _raise_cleanup_errors( - "descriptor registration cleanup failed", registration_error, tracking_error, rollback_error - ) - if tracking_error is not None: # pragma: no cover - requires failed in-memory fallback - _raise_cleanup_errors("descriptor registration cleanup failed", registration_error, tracking_error) - - def _release_with_fork_tracking(self) -> None: - with _fork_transition(self): - try: - self._release() - finally: - self._unregister_released_descriptor() - - def _register_context_descriptor(self) -> None: - if self._context.lock_file_fd is not None and self._context.lock_file_fd_token is None: - self._context.lock_file_fd_token = _register_owned_descriptor( - self._context.lock_file_fd, - self._context.lock_file_fd_identity, - ) - - def _register_unverified_context_descriptor(self) -> None: - # The rollback only reaches here still holding a descriptor it never managed to register. - if self._context.lock_file_fd is not None and self._context.lock_file_fd_token is None: # pragma: no branch - self._context.lock_file_fd_token = _register_unverified_owned_descriptor(self._context.lock_file_fd) - - def _unregister_released_descriptor(self) -> None: - if self._context.lock_file_fd is None: - if (token := self._context.lock_file_fd_token) is not None: # pragma: needs fork - _unregister_owned_descriptor(token) - self._context.lock_file_fd_token = None - self._context.lock_file_fd_identity = None - - def _undo_acquire(self) -> None: - """Roll back the counter after a failed acquire, dropping the registry entry once nothing holds the path.""" - self._context.lock_counter = max(0, self._context.lock_counter - 1) - if self._context.lock_counter == 0: - self._drop_registry_entry() - - def _commit_acquire(self, canonical: str) -> None: - """Record this instance as the holder once the first acquire succeeds, so peers can detect the deadlock.""" - if self._context.lock_counter == 1: - key = self._registry_key(canonical) - # The holder scope is resolved once at commit so a later release from another flow drops the right entry. - self._context.lock_file_key = key - _registry.held[key] = id(self) - - def _drop_registry_entry(self) -> None: - """Forget the key owned by this hold without resolving a mutable path again.""" - key = self._context.lock_file_key - self._context.lock_file_key = None - if key is not None: - _registry.held.pop(key, None) - - def _commit_release(self) -> None: - """Record the lock as fully released: reset the recursion counter and drop the deadlock-registry entry.""" - self._context.lock_counter = 0 - self._drop_registry_entry() - - def _try_break_expired_lock(self) -> None: - """Remove the lock file if its modification time exceeds the configured :attr:`lifetime`.""" - if (lifetime := self._context.lifetime) is None: - return - with contextlib.suppress(OSError): - # lstat, not stat: an attacker with write access to the lock directory can replace a held - # lock file with a symlink pointing at an old file, making stat() report the target's stale - # mtime so a waiter breaks a live lock and two processes hold it at once. lstat reads the - # symlink's own mtime, matching the O_NOFOLLOW reads elsewhere. - st = os.lstat(self.lock_file) - if time.time() - st.st_mtime < lifetime: - return - break_lock_file(self.lock_file, st.st_mtime, st.st_ino) - - def _check_give_up( - self, - *, - blocking: bool, - cancel_check: Callable[[], bool] | None, - timeout: float, - start_time: float, - ) -> bool: - lock_id, lock_filename = id(self), self.lock_file - if blocking is False: - _LOGGER.debug("Failed to immediately acquire lock %s on %s", lock_id, lock_filename) - return True - if cancel_check is not None and cancel_check(): - _LOGGER.debug("Cancellation requested for lock %s on %s", lock_id, lock_filename) - return True - if 0 <= timeout < time.perf_counter() - start_time: - _LOGGER.debug("Timeout on acquiring lock %s on %s", lock_id, lock_filename) - return True - return False - - @abstractmethod - def _acquire(self) -> None: - """If the file lock could be acquired, self._context.lock_file_fd holds the file descriptor of the lock file.""" - raise NotImplementedError - - @abstractmethod - def _release(self) -> None: - """Releases the lock and sets self._context.lock_file_fd to None.""" - raise NotImplementedError - - -# acquire() returns this wrapper instead of self so entering the with-statement does not call __enter__ a second -# time; returning self would re-acquire the lock in BaseFileLock.__enter__ without a matching release (issue #37). -class AcquireReturnProxy: - """A context-aware object that will release the lock file when exiting.""" - - def __init__(self, lock: BaseFileLock | ReadWriteLock | SoftReadWriteLock) -> None: - self.lock: BaseFileLock | ReadWriteLock | SoftReadWriteLock = lock - - def __enter__(self) -> BaseFileLock | ReadWriteLock | SoftReadWriteLock: - return self.lock - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - if isinstance(self.lock, BaseFileLock): - self.lock._release_in_context(exc_value) # ruff:ignore[private-member-access] # forwards __exit__ to the owned lock's context release - else: # a reader/writer lock does not carry a context_error_policy - self.lock.release() - - -@dataclass -class FileLockContext: - """Holds the context for a ``BaseFileLock`` object.""" - - # A separate class so ThreadLocalFileContext can make the whole context thread-local. - - lock_file: str - timeout: float - mode: int - blocking: bool - poll_interval: float - - #: The lock lifetime in seconds; ``None`` means the lock never expires. - lifetime: float | None = None - - #: File descriptor from os.open for the lock file; not None while the lock is held. - lock_file_fd: int | None = None - - #: Registry token for the descriptor owned by this thread's context. - lock_file_fd_token: int | None = None - - #: Identity captured by a backend that already inspected the descriptor. - lock_file_fd_identity: tuple[int, int] | None = None - - #: Descriptor opened by a backend but not yet committed as the held lock. - pending_lock_file_fd: int | None = None - - #: Identity captured for a descriptor whose acquisition has not committed. - pending_lock_file_fd_identity: tuple[int, int] | None = None - - #: Depth of nested acquisitions; the lock is released only when it returns to 0. - lock_counter: int = 0 - - #: Canonical registry key captured when the first physical acquisition commits. - lock_file_key: Hashable | None = None - - #: Claim pathnames this owner published, removed by name on release so no holder ever unlinks a peer's claim. - owner_claim_paths: tuple[str, ...] = () - - #: Canonical lock path resolved when an acquisition starts. A waiter polling a relative path must keep publishing - #: into the directory it started in, even when another thread changes the working directory mid-wait. - claim_root: str | None = None - - -class ThreadLocalFileContext(FileLockContext, local): - """A thread local version of the ``FileLockContext`` class.""" - - -@dataclass(frozen=True) -class _OwnedDescriptor: - fd: int - creator_pid: int - device: int | None - inode: int | None - - -class _ForkTransitionContext(local): - depth: int = 0 - - -class _ForkState: - def __init__(self) -> None: - self.gate = Condition(RLock()) - self.registry_lock = RLock() - self.parameter_models_lock = RLock() - self.transition_context = _ForkTransitionContext() - self.transitions: dict[int, dict[int, _ForkDescriptorOwner | None]] = {} - self.active_transitions = 0 - self.admission_closed = False - self.fork_owner_depths: dict[int, int] = {} - self.pinned_objects: dict[int, list[tuple[_ForkResettable, ...]]] = {} - self.pinned_classes: dict[int, list[tuple[_ForkResettableClass, ...]]] = {} - self.provisional_descriptor_tokens: dict[int, list[tuple[int, ...]]] = {} - self.pid = os.getpid() - - def reset_synchronization(self) -> None: # pragma: forked child - self.gate = Condition(RLock()) - self.registry_lock = RLock() - self.parameter_models_lock = RLock() - self.transition_context = _ForkTransitionContext() - self.transitions = {} - self.active_transitions = 0 - self.admission_closed = False - self.fork_owner_depths = {} - self.pinned_objects = {} - self.pinned_classes = {} - self.provisional_descriptor_tokens = {} - - -_FORK_OBJECTS: Final[WeakValueDictionary[int, _ForkResettable]] = WeakValueDictionary() -_FORK_CLASSES: Final[WeakValueDictionary[int, _ForkResettableClass]] = WeakValueDictionary() -_OWNED_DESCRIPTORS: Final[dict[int, _OwnedDescriptor]] = {} -_DESCRIPTOR_TOKENS: Final[count[int]] = count() -_TRANSITION_TOKENS: Final[count[int]] = count() -_FORK_STATE: Final = _ForkState() -_FORK_AUDIT_EVENTS: Final[frozenset[str]] = frozenset({"os.fork", "os.forkpty"}) - - -def _register_fork_hooks() -> None: - if _REGISTER_AT_FORK is None: - return # pragma: lacks fork - sys.addaudithook(_audit_fork_safety) # pragma: needs fork - _REGISTER_AT_FORK( # pragma: needs fork - before=_pin_fork_objects, - after_in_parent=_resume_parent_after_fork, - after_in_child=_reset_child_after_fork, - ) - - -@contextmanager -def _fork_transition(descriptor_owner: _ForkDescriptorOwner | None = None) -> Generator[None]: - if not _HAS_REGISTER_AT_FORK: - yield # pragma: lacks fork - return # pragma: lacks fork - _ensure_current_process() # pragma: needs fork - creator_pid = os.getpid() # pragma: needs fork - token = _enter_fork_transition(descriptor_owner) # pragma: needs fork - try: # pragma: needs fork - yield - finally: - if os.getpid() == creator_pid: # pragma: needs fork - _leave_fork_transition(token) - - -def _register_fork_object(instance: _ForkResettable) -> None: - if not _HAS_REGISTER_AT_FORK: - return # pragma: lacks fork - with _fork_transition(), _FORK_STATE.registry_lock: # pragma: needs fork - _FORK_OBJECTS[id(instance)] = instance - _refresh_owner_pins() - - -def _register_fork_class(cls: _ForkResettableClass) -> None: - if not _HAS_REGISTER_AT_FORK: - return # pragma: lacks fork - with _fork_transition(), _FORK_STATE.registry_lock: # pragma: needs fork - _FORK_CLASSES[id(cls)] = cls - _refresh_owner_pins() - - -def _register_owned_descriptor(fd: int, identity: tuple[int, int] | None = None) -> int | None: - if not _HAS_REGISTER_AT_FORK: - return None # pragma: lacks fork - with _fork_transition(): # pragma: needs fork - if identity is None: - stat_result = os.fstat(fd) - identity = stat_result.st_dev, stat_result.st_ino - return _record_owned_descriptor(fd, identity) - - -def _register_unverified_owned_descriptor(fd: int) -> int | None: - if not _HAS_REGISTER_AT_FORK: - return None # pragma: lacks fork - with _fork_transition(): # pragma: needs fork - return _record_owned_descriptor(fd, None) - - -def _record_owned_descriptor(fd: int, identity: tuple[int, int] | None) -> int: # pragma: needs fork - with _FORK_STATE.registry_lock: - token = next(_DESCRIPTOR_TOKENS) - _OWNED_DESCRIPTORS[token] = _OwnedDescriptor( - fd=fd, - creator_pid=os.getpid(), - device=None if identity is None else identity[0], - inode=None if identity is None else identity[1], - ) - return token - - -def _unregister_owned_descriptor(token: int) -> None: # pragma: needs fork - with _fork_transition(), _FORK_STATE.registry_lock: - _OWNED_DESCRIPTORS.pop(token, None) - - -def _pin_fork_objects() -> None: # pragma: needs fork - _ensure_current_process() - thread_id = get_ident() - with _FORK_STATE.gate: - _FORK_STATE.fork_owner_depths[thread_id] = _FORK_STATE.fork_owner_depths.get(thread_id, 0) + 1 - _FORK_STATE.admission_closed = True - while _FORK_STATE.active_transitions > len(_FORK_STATE.transitions.get(thread_id, ())): - _FORK_STATE.gate.wait() - transition_owners = tuple(_FORK_STATE.transitions.get(thread_id, {}).values()) - _verify_unverified_descriptors() - owners = {id(owner): owner for owner in transition_owners if owner is not None} - provisional_descriptor_tokens = tuple( - starmap( - _snapshot_descriptor_for_fork, - (descriptor for owner in owners.values() for descriptor in owner._descriptors_for_fork()), # ruff:ignore[private-member-access] # snapshots each owner's own fork descriptors - ) - ) - with _FORK_STATE.registry_lock: - _FORK_STATE.provisional_descriptor_tokens.setdefault(thread_id, []).append(provisional_descriptor_tokens) - _FORK_STATE.pinned_objects.setdefault(thread_id, []).append(tuple(_FORK_OBJECTS.values())) - _FORK_STATE.pinned_classes.setdefault(thread_id, []).append(tuple(_FORK_CLASSES.values())) - - -def _resume_parent_after_fork() -> None: # pragma: needs fork - thread_id = get_ident() - with _FORK_STATE.registry_lock: - for token in _FORK_STATE.provisional_descriptor_tokens[thread_id].pop(): - _OWNED_DESCRIPTORS.pop(token, None) - _FORK_STATE.pinned_objects[thread_id].pop() - _FORK_STATE.pinned_classes[thread_id].pop() - if not _FORK_STATE.provisional_descriptor_tokens[thread_id]: # pragma: needs fork - del _FORK_STATE.provisional_descriptor_tokens[thread_id] - del _FORK_STATE.pinned_objects[thread_id] - del _FORK_STATE.pinned_classes[thread_id] - with _FORK_STATE.gate: - if _FORK_STATE.fork_owner_depths[thread_id] == 1: - del _FORK_STATE.fork_owner_depths[thread_id] - else: # pragma: no cover - earlier at-fork callbacks may deadlock first - _FORK_STATE.fork_owner_depths[thread_id] -= 1 - _FORK_STATE.admission_closed = bool(_FORK_STATE.fork_owner_depths) - _FORK_STATE.gate.notify_all() - - -def _ensure_current_process() -> None: - _reset_child_after_fork() - - -def _reset_child_after_fork() -> None: # pragma: forked child - if (pid := os.getpid()) == _FORK_STATE.pid: - return - thread_id = get_ident() - pinned_objects = _FORK_STATE.pinned_objects.get(thread_id, [()])[-1] - pinned_classes = _FORK_STATE.pinned_classes.get(thread_id, [()])[-1] - descriptors: list[_OwnedDescriptor] = [] - for token, descriptor in tuple(_OWNED_DESCRIPTORS.items()): - if descriptor.creator_pid != pid: - descriptors.append(_OWNED_DESCRIPTORS.pop(token)) - _FORK_STATE.reset_synchronization() - _FORK_STATE.pid = pid - _INIT_PARAMETER_MODELS.clear() - _detach_child_state(descriptors, pinned_objects, pinned_classes) - - -def _enter_fork_transition(descriptor_owner: _ForkDescriptorOwner | None) -> int: # pragma: needs fork - thread_id = get_ident() - with _FORK_STATE.gate: - while ( - _FORK_STATE.admission_closed - and thread_id not in _FORK_STATE.fork_owner_depths - and thread_id not in _FORK_STATE.transitions - ): - _FORK_STATE.gate.wait() # pragma: no cover - exercised in isolated interpreter - token = next(_TRANSITION_TOKENS) - _FORK_STATE.transitions.setdefault(thread_id, {})[token] = descriptor_owner - _FORK_STATE.active_transitions += 1 - _FORK_STATE.transition_context.depth += 1 - return token - - -def _leave_fork_transition(token: int) -> None: # pragma: needs fork - _FORK_STATE.transition_context.depth -= 1 - with _FORK_STATE.gate: - thread_id = get_ident() - del _FORK_STATE.transitions[thread_id][token] - if not _FORK_STATE.transitions[thread_id]: - del _FORK_STATE.transitions[thread_id] - _FORK_STATE.active_transitions -= 1 - _FORK_STATE.gate.notify_all() - - -def _verify_unverified_descriptors() -> None: # pragma: needs fork - with _FORK_STATE.registry_lock: - for token, descriptor in tuple(_OWNED_DESCRIPTORS.items()): - if descriptor.creator_pid != os.getpid() or descriptor.device is not None: - continue - try: - stat_result = os.fstat(descriptor.fd) - except OSError: - continue - _OWNED_DESCRIPTORS[token] = _OwnedDescriptor( - fd=descriptor.fd, - creator_pid=descriptor.creator_pid, - device=stat_result.st_dev, - inode=stat_result.st_ino, - ) - - -def _snapshot_descriptor_for_fork(fd: int, identity: tuple[int, int] | None) -> int: # pragma: needs fork - if identity is None: # pragma: needs fork - try: - stat_result = os.fstat(fd) - except OSError: - pass - else: - identity = stat_result.st_dev, stat_result.st_ino - return _record_owned_descriptor(fd, identity) - - -def _detach_child_state( # pragma: forked child - descriptors: list[_OwnedDescriptor], - pinned_objects: tuple[_ForkResettable, ...], - pinned_classes: tuple[_ForkResettableClass, ...], -) -> None: - for descriptor in descriptors: - if descriptor.device is None: - continue - try: - stat_result = os.fstat(descriptor.fd) - except OSError: - continue - if (stat_result.st_dev, stat_result.st_ino) != (descriptor.device, descriptor.inode): - continue - with contextlib.suppress(OSError): - os.close(descriptor.fd) - for instance in pinned_objects: - instance._reset_after_fork_in_child() # ruff:ignore[private-member-access] # resets each pinned instance in the fork child - for cls in pinned_classes: - cls._reset_class_after_fork() - _registry.held.clear() - - -def _refresh_owner_pins() -> None: # pragma: needs fork - with _FORK_STATE.gate: - fork_owner_thread_ids = tuple(_FORK_STATE.fork_owner_depths) - if get_ident() not in fork_owner_thread_ids: # pragma: no cover - at-fork callbacks disable tracing - return - objects = tuple(_FORK_OBJECTS.values()) - classes = tuple(_FORK_CLASSES.values()) - for thread_id in fork_owner_thread_ids: - if object_snapshots := _FORK_STATE.pinned_objects.get(thread_id): # pragma: needs fork - object_snapshots[:] = [objects] * len(object_snapshots) - _FORK_STATE.pinned_classes[thread_id][:] = [classes] * len(object_snapshots) - - -# The defaults capture the module globals: the hook outlives them at interpreter shutdown, where CPython wipes the -# module dict to None before the final audit events fire. -def _audit_fork_safety( # pragma: no cover - CPython disables tracing while Python audit hooks run - event: str, - _args: Unused, - *, - _fork_events: frozenset[str] = _FORK_AUDIT_EVENTS, - _state: _ForkState = _FORK_STATE, -) -> None: - if event in _fork_events: - if _state.transition_context.depth or _state.fork_owner_depths: - msg = f"{event} is unsafe while filelock is changing descriptor ownership" - raise RuntimeError(msg) - elif event == "_posixsubprocess.fork_exec" and _state.transition_context.depth: - msg = "fork_exec is unsafe while filelock is changing descriptor ownership" - raise RuntimeError(msg) - - -_register_fork_hooks() - - -__all__ = [ - "_UNSET_FILE_MODE", - "AcquireReturnProxy", - "BaseFileLock", - "CloseErrorPolicy", - "ContextErrorPolicy", - "FileLockContext", - "FileLockMeta", - "LockOptions", - "_append_exception_context", - "_canonical", - "_ensure_current_process", - "_fork_transition", - "_grouped_errors", - "_raise_body_and_release", - "_raise_chained_errors", - "_raise_cleanup_errors", - "_raise_grouped_errors", - "_register_fork_class", - "_register_fork_object", - "_register_owned_descriptor", - "_unregister_owned_descriptor", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_async.py b/bundle/python-cpu/Lib/site-packages/filelock/_async.py deleted file mode 100644 index 3848dbcc8f9623a129485e643e4c2173a96ff090..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_async.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Separate caller cancellation from backend task and executor-future results.""" - -from __future__ import annotations - -import asyncio -import contextlib -import time -from concurrent.futures import Future as ConcurrentFuture -from dataclasses import dataclass -from threading import Lock -from typing import TYPE_CHECKING, Final, Generic, NoReturn, TypeVar, cast - -from ._api import _append_exception_context, _raise_chained_errors - -if TYPE_CHECKING: - from collections.abc import AsyncIterator, Awaitable, Callable - -_T = TypeVar("_T") - - -class _AsyncTransitionUnavailableError(Exception): - pass - - -@dataclass(frozen=True) -class _BackendOutcome(Generic[_T]): - value: _T | None = None - error: BaseException | None = None - - -class _AsyncTransitionGate: - def __init__(self) -> None: - self._tail_lock: Final[Lock] = Lock() - self._tail: ConcurrentFuture[None] | None = None - - @contextlib.asynccontextmanager - async def hold(self) -> AsyncIterator[None]: - ticket: ConcurrentFuture[None] = ConcurrentFuture() - with self._tail_lock: - predecessor = self._tail - self._tail = ticket - if predecessor is not None: - try: - await _wait_until_done(asyncio.wrap_future(predecessor)) - except asyncio.CancelledError: - predecessor.add_done_callback(lambda _predecessor: self._leave(ticket)) - raise - try: - yield - finally: - self._leave(ticket) - - @contextlib.asynccontextmanager - async def hold_for_acquire( - self, - *, - blocking: bool, - cancel_check: Callable[[], bool] | None, - deadline: float | None, - poll_interval: float, - ) -> AsyncIterator[None]: - ticket: ConcurrentFuture[None] = ConcurrentFuture() - with self._tail_lock: - predecessor = self._tail - self._tail = ticket - if predecessor is not None and not predecessor.done(): - try: - await self._wait_for_predecessor( - predecessor, - blocking=blocking, - cancel_check=cancel_check, - deadline=deadline, - poll_interval=poll_interval, - ) - except BaseException: - predecessor.add_done_callback(lambda _predecessor: self._leave(ticket)) - raise - try: - yield - finally: - self._leave(ticket) - - @staticmethod - async def _wait_for_predecessor( - predecessor: ConcurrentFuture[None], - *, - blocking: bool, - cancel_check: Callable[[], bool] | None, - deadline: float | None, - poll_interval: float, - ) -> None: - if not blocking: - raise _AsyncTransitionUnavailableError - waiter = asyncio.wrap_future(predecessor) - while not predecessor.done(): - if cancel_check is not None and cancel_check(): - raise _AsyncTransitionUnavailableError - if deadline is not None: - if (remaining := deadline - time.perf_counter()) <= 0: - raise _AsyncTransitionUnavailableError - wait_interval = min(poll_interval, remaining) if cancel_check is not None else remaining - else: - wait_interval = poll_interval if cancel_check is not None else None - await asyncio.wait((waiter,), timeout=wait_interval) - - def _leave(self, ticket: ConcurrentFuture[None]) -> None: - with self._tail_lock: - if self._tail is ticket: - self._tail = None - ticket.set_result(None) - - -async def _drain_future(future: asyncio.Future[_BackendOutcome[_T]]) -> _T: - while not future.done(): - with contextlib.suppress(asyncio.CancelledError): - await _wait_until_done(future) - return _future_result(future) - - -async def _wait_until_done(future: asyncio.Future[_T]) -> None: - if not future.done(): - await asyncio.wait((future,)) - - -def _future_result(future: asyncio.Future[_BackendOutcome[_T]]) -> _T: - outcome = future.result() - if (error := outcome.error) is None: - return cast("_T", outcome.value) - context = error.__context__ - try: - raise error # ruff:ignore[raise-within-try] # the handler restores context changed across the async boundary - except BaseException: - error.__context__ = context - raise - - -def _capture_call(func: Callable[[], _T]) -> _BackendOutcome[_T]: - try: - return _BackendOutcome(value=func()) - except BaseException as error: # ruff:ignore[blind-except] # backend control-flow exceptions are operation results - return _BackendOutcome(error=error) - - -def _raise_cancelled_error(cancellation: asyncio.CancelledError, error: BaseException) -> NoReturn: - # A reconciliation step failed while unwinding a cancellation, so keep both exception chains. Splice the error's - # existing context onto the cancellation, then make the cancellation the error's context, so both the failure and - # the cancellation that triggered it survive. Shared by the async wrappers so cancellations report the same way. - if (context := error.__context__) is not None and context is not cancellation: - if (cancellation_context := cancellation.__context__) is not None: - _append_exception_context(context, cancellation_context) - cancellation.__context__ = context - error.__context__ = cancellation - _raise_chained_errors(error) - - -async def _capture_awaitable(awaitable: Awaitable[_T]) -> _BackendOutcome[_T]: - try: - return _BackendOutcome(value=await awaitable) - except BaseException as error: # ruff:ignore[blind-except] # backend cancellation must remain distinct from caller cancellation - return _BackendOutcome(error=error) - - -__all__ = [ - "_AsyncTransitionGate", - "_AsyncTransitionUnavailableError", - "_BackendOutcome", - "_capture_awaitable", - "_capture_call", - "_drain_future", - "_future_result", - "_raise_cancelled_error", - "_wait_until_done", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_async_read_write.py b/bundle/python-cpu/Lib/site-packages/filelock/_async_read_write.py deleted file mode 100644 index 58ad40cb2a2ed24d1dd15545e12f0489bc6dfca0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_async_read_write.py +++ /dev/null @@ -1,338 +0,0 @@ -"""Async wrapper around :class:`ReadWriteLock` for use with ``asyncio``.""" - -from __future__ import annotations - -import asyncio -import functools -import os -import sqlite3 -from concurrent.futures import ThreadPoolExecutor -from contextlib import asynccontextmanager -from typing import TYPE_CHECKING, ParamSpec, TypeVar - -from ._api import ( - _append_exception_context, - _ensure_current_process, - _fork_transition, - _register_fork_object, -) -from ._async import ( - _BackendOutcome, - _capture_call, - _drain_future, - _future_result, - _raise_cancelled_error, - _wait_until_done, -) -from ._read_write import ReadWriteLock - -if TYPE_CHECKING: - from collections.abc import AsyncGenerator, Callable - from concurrent import futures - from types import TracebackType - - from ._api import AcquireReturnProxy - -_P = ParamSpec("_P") -_R = TypeVar("_R") - - -class AsyncReadWriteLock: - """ - Async wrapper around :class:`ReadWriteLock` for use in ``asyncio`` applications. - - This wrapper dispatches every blocking SQLite operation to a thread pool via ``loop.run_in_executor()`` because - Python's :mod:`sqlite3` module has no async API. It delegates reentrancy, upgrade/downgrade rules, and singleton - behavior to the underlying :class:`ReadWriteLock`. - - :param lock_file: path to the SQLite database file used as the lock - :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable - :param is_singleton: if ``True``, reuse existing :class:`ReadWriteLock` instances for the same resolved path - :param loop: event loop for ``run_in_executor``; ``None`` uses the running loop - :param executor: executor for ``run_in_executor``. When ``None`` this lock creates and owns a dedicated - single-thread executor so every operation runs on the same thread (SQLite affinity requires this) and shuts it - down in :meth:`close`. This lock uses a caller-supplied executor as-is and never shuts it down, so after passing - no executor call :meth:`close` to release the owned one. - - .. versionadded:: 3.21.0 - - """ - - def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option - self, - lock_file: str | os.PathLike[str], - timeout: float = -1, - *, - blocking: bool = True, - is_singleton: bool = True, - loop: asyncio.AbstractEventLoop | None = None, - executor: futures.Executor | None = None, - ) -> None: - creator_pid = os.getpid() - self._creator_pid = creator_pid - self._fork_invalidated = False - self._closed = False - _register_fork_object(self) - with _fork_transition(): - self._lock = ReadWriteLock(lock_file, timeout, blocking=blocking, is_singleton=is_singleton) - self._loop = loop - self._owns_executor = executor is None - self._executor = executor or ThreadPoolExecutor(max_workers=1) - if os.getpid() != creator_pid: # pragma: forked child - msg = "AsyncReadWriteLock construction cannot continue after fork" - raise RuntimeError(msg) - - @property - def lock_file(self) -> str: - """The path to the lock file.""" - return self._lock.lock_file - - @property - def timeout(self) -> float: - """The default timeout.""" - return self._lock.timeout - - @property - def blocking(self) -> bool: - """Whether blocking is enabled by default.""" - return self._lock.blocking - - @property - def loop(self) -> asyncio.AbstractEventLoop | None: - """The event loop (or ``None`` for the running loop).""" - return self._loop - - @property - def executor(self) -> futures.Executor: - """The executor used for ``run_in_executor`` (a dedicated single-thread one if none was supplied).""" - return self._executor - - @asynccontextmanager - async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]: - """ - Async context manager that acquires and releases a shared read lock. - - Falls back to instance defaults for *timeout* and *blocking* when ``None``. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default - - """ - if timeout is None: - timeout = self._lock.timeout - if blocking is None: - blocking = self._lock.blocking - await self.acquire_read(timeout, blocking=blocking) - body_error: BaseException | None = None - try: - yield - except BaseException as error: - body_error = error - raise - finally: - await self._release_in_context(body_error) - - @asynccontextmanager - async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]: - """ - Async context manager that acquires and releases an exclusive write lock. - - Falls back to instance defaults for *timeout* and *blocking* when ``None``. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default - - """ - if timeout is None: - timeout = self._lock.timeout - if blocking is None: - blocking = self._lock.blocking - await self.acquire_write(timeout, blocking=blocking) - body_error: BaseException | None = None - try: - yield - except BaseException as error: - body_error = error - raise - finally: - await self._release_in_context(body_error) - - async def _release_in_context(self, body_error: BaseException | None) -> None: - try: - await self.release() - except BaseException as release_error: - if body_error is not None: - _append_exception_context(release_error, body_error) - raise - - async def acquire_read(self, timeout: float = -1, *, blocking: bool = True) -> AsyncAcquireReadWriteReturnProxy: - """ - Acquire a shared read lock. - - See :meth:`ReadWriteLock.acquire_read` for full semantics. - - :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable - - :returns: a proxy that can be used as an async context manager to release the lock - - :raises RuntimeError: if a write lock is already held on this instance - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - self._raise_if_unusable() - await self._run_acquire(functools.partial(self._lock.acquire_read, timeout, blocking=blocking)) - return AsyncAcquireReadWriteReturnProxy(lock=self) - - async def acquire_write(self, timeout: float = -1, *, blocking: bool = True) -> AsyncAcquireReadWriteReturnProxy: - """ - Acquire an exclusive write lock. - - See :meth:`ReadWriteLock.acquire_write` for full semantics. - - :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable - - :returns: a proxy that can be used as an async context manager to release the lock - - :raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - self._raise_if_unusable() - await self._run_acquire(functools.partial(self._lock.acquire_write, timeout, blocking=blocking)) - return AsyncAcquireReadWriteReturnProxy(lock=self) - - async def release(self, *, force: bool = False) -> None: - """ - Release one level of the current lock. - - See :meth:`ReadWriteLock.release` for full semantics. - - :param force: if ``True``, release the lock completely regardless of the current lock level - - :raises RuntimeError: if no lock is currently held and *force* is ``False`` - - """ - _ensure_current_process() - if self._inherited: # pragma: needs fork - return - await self._run(self._lock.release, force=force) - - async def close(self) -> None: - """ - Release the lock (if held) and close the underlying SQLite connection. - - After calling this method, the lock instance is no longer usable. - - """ - _ensure_current_process() - if self._inherited: # pragma: needs fork - return - if self._closed: - return - close_future = self._submit(self._lock.close) - try: - await _wait_until_done(close_future) - except asyncio.CancelledError as cancellation: - try: - await _drain_future(close_future) - except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below - _raise_cancelled_error(cancellation, error) - self._closed = True - self._shutdown_owned_executor() - raise - _future_result(close_future) - self._closed = True - # Wait for the worker to exit rather than letting it drain in the background: a caller that forks right - # after closing deserves a single-threaded process, and os.fork warns about any surviving thread. - if self._owns_executor: - await asyncio.to_thread(functools.partial(self._executor.shutdown, wait=True)) - - async def _run_acquire(self, acquire: Callable[[], AcquireReturnProxy]) -> None: - acquire_future = self._submit(acquire) - try: - await _wait_until_done(acquire_future) - except asyncio.CancelledError as cancellation: - try: - await _drain_future(acquire_future) - except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below - _raise_cancelled_error(cancellation, error) - try: - await _drain_future(self._submit(self._lock.release)) - except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below - _raise_cancelled_error(cancellation, error) - raise - _future_result(acquire_future) - - async def _run(self, func: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs) -> _R: - future = self._submit(func, *args, **kwargs) - try: - await _wait_until_done(future) - except asyncio.CancelledError as cancellation: - try: - await _drain_future(future) - except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below - _raise_cancelled_error(cancellation, error) - raise - return _future_result(future) - - def _submit( - self, func: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs - ) -> asyncio.Future[_BackendOutcome[_R]]: - return (self._loop or asyncio.get_running_loop()).run_in_executor( - self._executor, - _capture_call, - functools.partial(func, *args, **kwargs), - ) - - def _shutdown_owned_executor(self) -> None: - if self._owns_executor: - self._executor.shutdown(wait=False) - - @property - def _inherited(self) -> bool: - return self._fork_invalidated or os.getpid() != self._creator_pid - - def _raise_if_unusable(self) -> None: - _ensure_current_process() - if self._inherited: # pragma: needs fork - msg = f"AsyncReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance" - raise RuntimeError(msg) - if self._closed: - msg = "Cannot operate on a closed database." - raise sqlite3.ProgrammingError(msg) - - def _reset_after_fork_in_child(self) -> None: # pragma: forked child - self._fork_invalidated = True - - def __del__(self) -> None: - # Safety net when close() was never called: shut down the executor we own so its worker thread does not - # outlive the lock. shutdown(wait=False) never blocks. - if os.getpid() == getattr(self, "_creator_pid", None) and getattr(self, "_owns_executor", False): - self._executor.shutdown(wait=False) - - -class AsyncAcquireReadWriteReturnProxy: - """Context-aware object that releases the async read/write lock on exit.""" - - def __init__(self, lock: AsyncReadWriteLock) -> None: - self.lock = lock - - async def __aenter__(self) -> AsyncReadWriteLock: - return self.lock - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - await self.lock.release() - - -__all__ = [ - "AsyncAcquireReadWriteReturnProxy", - "AsyncReadWriteLock", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_descriptor.py b/bundle/python-cpu/Lib/site-packages/filelock/_descriptor.py deleted file mode 100644 index 7c9aa97101d7787cf8fc95df6160f8958fc76559..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_descriptor.py +++ /dev/null @@ -1,71 +0,0 @@ -"""A minimal native lock over a caller-owned file descriptor, contending with :class:`FileLock` on the same file.""" - -from __future__ import annotations - -import sys -import time -from math import isfinite -from typing import Final - -if sys.platform == "win32": # pragma: win32 cover - from ._windows import _lock_fd_nonblocking, _unlock_fd -else: # pragma: win32 no cover - from ._unix import _lock_fd_nonblocking, _unlock_fd - - -def lock_descriptor(fd: int, *, blocking: bool = True, poll_interval: float = 0.05) -> bool: - """ - Take the native OS lock on *fd*, a file descriptor the caller opened and owns. - - This is the same one-byte exclusive lock :class:`FileLock` uses, so a descriptor lock and a path lock on the same - file contend with each other. Unlike :class:`FileLock` it adds no path handling: it never opens, truncates, closes, - unlinks, chmods, canonicalizes, or falls back. The caller owns *fd* before, during, and after the call, and must - close it. On Windows *fd* must be a synchronous descriptor (its handle not opened with ``FILE_FLAG_OVERLAPPED``). - - For timeout, reentrancy, singleton, lifetime, or stale-break behavior, use :class:`FileLock`. There is no async - wrapper. Run this in an executor, or drive ``blocking=False`` from your own polling loop. - - :param fd: an open file descriptor the caller owns. - :param blocking: when ``True`` (default), retry the nonblocking attempt every *poll_interval* seconds until it - succeeds; when ``False``, make one attempt. - :param poll_interval: finite, positive seconds between attempts while blocking; ignored when *blocking* is - ``False``. - - :returns: ``True`` once the lock is held, or ``False`` on contention when ``blocking`` is ``False``. - - :raises OSError: for a permanent native failure, such as an invalid descriptor, or with ``errno.ENOSYS`` when the - Python build lacks the native locking primitive. The descriptor is left open. - :raises ValueError: if a blocking call receives a non-finite or non-positive *poll_interval*. - - .. versionadded:: 3.30.0 - - """ - if not blocking: - return _lock_fd_nonblocking(fd) - if not isfinite(poll_interval) or poll_interval <= 0: - msg: Final[str] = f"poll_interval must be finite and greater than 0, got {poll_interval}" - raise ValueError(msg) - while not _lock_fd_nonblocking(fd): - time.sleep(poll_interval) - return True - - -def unlock_descriptor(fd: int) -> None: - """ - Release the native OS lock on *fd* without touching the descriptor. - - :param fd: the descriptor a prior :func:`lock_descriptor` locked; the caller still owns and must close it. - - :raises OSError: if the native unlock fails, including ``errno.ENOSYS`` when the Python build lacks the native - locking primitive; the caller may retry on the same descriptor. - - .. versionadded:: 3.30.0 - - """ - _unlock_fd(fd) - - -__all__ = [ - "lock_descriptor", - "unlock_descriptor", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_error.py b/bundle/python-cpu/Lib/site-packages/filelock/_error.py deleted file mode 100644 index 6159b34e61a2932893acec0899173c2ee6f548da..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_error.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - - -class Timeout(TimeoutError): # ruff:ignore[error-suffix-on-exception-name] # public exception name; renaming breaks the API - """Raised when the lock could not be acquired in *timeout* seconds.""" - - def __init__(self, lock_file: str) -> None: - super().__init__() - self._lock_file = lock_file - - def __reduce__(self) -> tuple[type[Timeout], tuple[str]]: - # __init__ needs lock_file, so pickle must restore it as a constructor arg - return self.__class__, (self._lock_file,) - - def __str__(self) -> str: # pragma: needs hard-link - return f"The file lock '{self._lock_file}' could not be acquired." - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.lock_file!r})" - - @property - def lock_file(self) -> str: - """The path of the file lock.""" - return self._lock_file - - -class SoftFileLockLifetimeWarning(DeprecationWarning): - """The configured soft-lock lifetime permits overlapping live holders after expiry.""" - - -class LeaseSettingsMismatch(ValueError): # ruff:ignore[error-suffix-on-exception-name] # public exception name; renaming breaks the API - """A lease contender disagrees with the published claim about how long the lease lasts.""" - - -class SoftFileLockProtocolError(OSError): - """Raised when strict soft-lock state cannot be interpreted without risking overlap.""" - - def __init__(self, lock_file: str, claim_name: str | None, reason: str) -> None: - self._lock_file = lock_file - self._claim_name = claim_name - self._reason = reason - super().__init__(self.__str__()) - - def __reduce__( - self, - ) -> tuple[type[SoftFileLockProtocolError], tuple[str, str | None, str]]: # pragma: needs hard-link - return self.__class__, (self._lock_file, self._claim_name, self._reason) - - def __str__(self) -> str: - location = self._lock_file if self._claim_name is None else f"{self._lock_file}: claim {self._claim_name!r}" - return f"Invalid strict soft-lock state at {location}: {self._reason}" - - @property - def lock_file(self) -> str: # pragma: needs hard-link - """The requested lock path.""" - return self._lock_file - - @property - def claim_name(self) -> str | None: # pragma: needs hard-link - """The claim that caused the error, if scanning identified one.""" - return self._claim_name - - @property - def reason(self) -> str: # pragma: needs hard-link - """The protocol validation failure.""" - return self._reason - - -__all__ = [ - "LeaseSettingsMismatch", - "SoftFileLockLifetimeWarning", - "SoftFileLockProtocolError", - "Timeout", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_identity.py b/bundle/python-cpu/Lib/site-packages/filelock/_identity.py deleted file mode 100644 index fe1b03d425a7f1f6d1af736a96a6a493082a4d49..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_identity.py +++ /dev/null @@ -1,177 +0,0 @@ -from __future__ import annotations - -import os -import socket -import sys -from errno import EPERM, ESRCH -from pathlib import Path -from typing import Final - - -def host_name() -> str: - """The hostname recorded alongside an owner, so a marker written on another machine is never probed here.""" - return socket.gethostname() - - -def owner_is_stale(pid: int, hostname: str, start_token: int | None) -> bool: - """ - Whether the recorded owner is provably gone, so reclaiming its marker cannot detach a live holder. - - Fail closed: return ``True`` only when this process can prove the exact recorded owner is dead. A marker from - another host cannot be probed; a live PID whose start token still matches, or whose token cannot be read, is the - holder or is indistinguishable from it; a live PID whose start token differs is a recycled PID, so the process that - wrote the marker is gone. PostgreSQL, Qt ``QLockFile`` and Mercurial all break a stale lock only on proof of death - and treat an unreadable or foreign owner as still holding. - """ - if hostname != host_name(): - return False - if not process_alive(pid): - return True - if start_token is None: - return False - current = process_start_token(pid) - return current is not None and current != start_token - - -if sys.platform == "win32": # pragma: win32 cover - import ctypes - from ctypes import wintypes - - _KERNEL32: Final[ctypes.WinDLL] = ctypes.WinDLL("kernel32", use_last_error=True) - _KERNEL32.CloseHandle.argtypes = [wintypes.HANDLE] - _KERNEL32.CloseHandle.restype = wintypes.BOOL - _KERNEL32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] - _KERNEL32.OpenProcess.restype = wintypes.HANDLE - _KERNEL32.GetProcessTimes.argtypes = [ - wintypes.HANDLE, - ctypes.POINTER(wintypes.FILETIME), - ctypes.POINTER(wintypes.FILETIME), - ctypes.POINTER(wintypes.FILETIME), - ctypes.POINTER(wintypes.FILETIME), - ] - _KERNEL32.GetProcessTimes.restype = wintypes.BOOL - - _WIN_SYNCHRONIZE: Final[int] = 0x100000 - _WIN_PROCESS_QUERY_LIMITED_INFORMATION: Final[int] = 0x1000 - _WIN_ERROR_INVALID_PARAMETER: Final[int] = 87 - _WIN_INHERIT_HANDLE: Final[bool] = False - - def process_alive(pid: int) -> bool: - """Whether a process with this PID exists, treating an access denial as proof it does.""" - handle = _KERNEL32.OpenProcess(_WIN_SYNCHRONIZE, _WIN_INHERIT_HANDLE, pid) - if handle: - _KERNEL32.CloseHandle(handle) - return True - return ctypes.get_last_error() != _WIN_ERROR_INVALID_PARAMETER - - def process_start_token(pid: int) -> int | None: - """The process creation FILETIME as a 100ns tick count, or ``None`` when it cannot be read.""" - handle = _KERNEL32.OpenProcess(_WIN_PROCESS_QUERY_LIMITED_INFORMATION, _WIN_INHERIT_HANDLE, pid) - if not handle: - return None - creation, exit_time, kernel_time, user_time = (wintypes.FILETIME() for _ in range(4)) - try: - if not _KERNEL32.GetProcessTimes( - handle, - ctypes.byref(creation), - ctypes.byref(exit_time), - ctypes.byref(kernel_time), - ctypes.byref(user_time), - ): - return None # pragma: no cover # win32 GetProcessTimes failure path; not reproducible on a live handle - finally: - _KERNEL32.CloseHandle(handle) - return (creation.dwHighDateTime << 32) | creation.dwLowDateTime - -else: # pragma: win32 no cover - - def process_alive(pid: int) -> bool: - """Whether a process with this PID exists, treating an access denial (``EPERM``) as proof it does.""" - try: - os.kill(pid, 0) - except OSError as error: - if error.errno == ESRCH: - return False - if error.errno == EPERM: - return True - raise - return True - - if sys.platform in {"linux", "android"}: # pragma: linux cover - # Termux/Android reports sys.platform == "android" but runs the Linux kernel, so /proc//stat and the boot - # id are the same reliable start-time source; treat it exactly like Linux rather than the tokenless fallback. - # comm (field 2) is wrapped in parentheses and may itself contain spaces or a ')', so the fixed fields start - # after the final ')'. starttime is field 22 overall, the twentieth of those trailing fields (index 19). - _STARTTIME_INDEX: Final[int] = 19 - - def _read_boot_id() -> int: - # starttime is measured in clock ticks since boot, so on its own it repeats across a reboot. Folding the - # boot id into the high bits makes the Linux token reboot-safe like the absolute clocks macOS and Windows - # expose, while staying a single integer so a 3.29 reader still parses the third marker line. 0 when the - # kernel does not expose a boot id degrades to bare starttime, which stays safe (a reboot collision fails - # closed rather than reclaiming a live marker). - try: - boot_id = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii") - return int(boot_id.strip().replace("-", ""), 16) - except (OSError, ValueError): # pragma: no cover # the kernel always exposes boot_id as a UUID on Linux - return 0 - - _BOOT_ID: Final[int] = _read_boot_id() - - def process_start_token(pid: int) -> int | None: - """The ``/proc//stat`` ``starttime`` folded with the boot id, or ``None`` when the process is gone.""" - try: - data = Path(f"/proc/{pid}/stat").read_bytes() - except OSError: - return None - # psutil identifies a process by the same (pid, starttime) pair; the boot id extends that across reboots. - fields = data[data.rfind(b")") + 1 :].split() - if len(fields) <= _STARTTIME_INDEX: # pragma: no cover # a truncated /proc read, never seen in practice - return None - try: - starttime = int(fields[_STARTTIME_INDEX]) - except ValueError: # pragma: no cover # /proc always renders starttime as an integer - return None - return (_BOOT_ID << 64) | starttime - - elif sys.platform == "darwin": # pragma: darwin cover - import ctypes - import struct - - _LIBC: Final[ctypes.CDLL] = ctypes.CDLL(None, use_errno=True) - _CTL_KERN: Final[int] = 1 - _KERN_PROC: Final[int] = 14 - _KERN_PROC_PID: Final[int] = 1 - # kinfo_proc opens with kp_proc.p_starttime (a struct timeval) at offset 0: int64 seconds, int32 microseconds. - # The offset is fixed by the struct chain kinfo_proc -> extern_proc -> p_un, so a read at 0 is not a guess. - _TIMEVAL_AT_ZERO: Final[str] = " int | None: - """The process start time in microseconds from ``sysctl(KERN_PROC_PID)``, or ``None`` when it is gone.""" - mib = (ctypes.c_int * 4)(_CTL_KERN, _KERN_PROC, _KERN_PROC_PID, pid) - length = ctypes.c_size_t(0) - # The size probe reports the kinfo_proc size for any PID, so the read below, not the probe, tells a live - # process from a gone one: a gone PID leaves the fetch a zero-length success, so re-check the length after. - if _LIBC.sysctl(mib, 4, None, ctypes.byref(length), None, 0) != 0: - return None - buffer = (ctypes.c_char * length.value)() - if _LIBC.sysctl(mib, 4, buffer, ctypes.byref(length), None, 0) != 0 or length.value < _TIMEVAL_SIZE: - return None - seconds, microseconds = struct.unpack_from(_TIMEVAL_AT_ZERO, buffer.raw, 0) - return seconds * 1_000_000 + microseconds - - else: # pragma: no cover # a POSIX platform without a proven start-time source falls back to fail-closed liveness - - def process_start_token(pid: int) -> int | None: - """No proven start-time source, so the owner carries no token and liveness rests on the PID alone.""" - del pid - return None - - -__all__ = [ - "host_name", - "owner_is_stale", - "process_alive", - "process_start_token", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_lease.py b/bundle/python-cpu/Lib/site-packages/filelock/_lease.py deleted file mode 100644 index 16e15749416ccb544df3c0101555d68c0e64a967..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_lease.py +++ /dev/null @@ -1,334 +0,0 @@ -from __future__ import annotations - -import os -import secrets -import time -from contextlib import suppress -from dataclasses import dataclass -from threading import Event, Thread, current_thread, local -from typing import TYPE_CHECKING, Literal - -from ._error import LeaseSettingsMismatch -from ._identity import owner_is_stale -from ._marker import MarkerSoftFileLock, OwnerMode, OwnerRecord, parse_marker -from ._soft import _read_lock_file -from ._util import break_lock_file, touch - -if TYPE_CHECKING: - import sys - from collections.abc import Callable - - from ._api import LockOptions - - if sys.version_info >= (3, 11): # pragma: no cover (py311+) - from typing import Unpack - else: # pragma: no cover ( None: - self.claim = _LeaseClaim() - - -class _ThreadLocalLeaseClaimHolder(_LeaseClaimHolder, local): - """A thread local version of the ``_LeaseClaimHolder`` class.""" - - -class SoftFileLease(MarkerSoftFileLock): - """ - Existence lock whose claim expires, so a peer may take it while the previous holder still runs. - - A lease trades mutual exclusion for progress. The holder publishes a claim and refreshes it every - ``heartbeat_interval`` seconds; a contender takes the marker once it is ``lease_duration`` seconds stale. Nothing - stops the expired holder: it keeps running, and it keeps using whatever the lock protects. Treat the lease as a hint - about who *should* be working, not as a guarantee that only one worker is. - - To make a protected resource reject a superseded holder, that resource must be linearizable and must fence on a - monotonic generation it controls. :attr:`token` names a claim; it does not fence one. Where overlap is unacceptable, - use :class:`StrictSoftFileLock ` instead. - - Every contender for a path must agree on ``lease_duration``. A contender that finds a claim published under a - different duration raises :class:`LeaseSettingsMismatch ` rather than apply its own - expiry to a peer that never agreed to it. - - Expiry reclaims less on Windows, which refuses to rename or delete a file another process holds open. A peer there - takes an expired claim only once the previous holder's process exits and its handle closes; a holder that lives on - but stops refreshing keeps the marker. Unix reclaims the marker either way. - - ``on_compromise`` fires from the heartbeat thread when a refresh fails, or when the marker vanishes or names another - owner. The holder should stop touching the protected resource when it runs. Because it runs on that thread, a - ``release()`` inside it only takes effect when the lease was built with ``thread_local=False``; the default - thread-local context hides the claim from every thread but the one that acquired it, so the release does nothing. - Signal the acquiring thread instead when the context stays thread-local. - - .. versionadded:: 3.30.0 - - """ - - _owner_mode: OwnerMode = "lease" - - #: lease_duration replaces the legacy age-based lifetime, so accepting both would give one lock two expiry clocks. - _lifetime_supported: bool = False - _lifetime_unsupported_reason: str = "lease_duration sets when a lease expires" - - def __init__( - self, - lock_file: str | os.PathLike[str], - *, - lease_duration: float = 30.0, - heartbeat_interval: float | None = None, - on_compromise: Callable[[LeaseCompromise], None] | None = None, - **kwargs: Unpack[LockOptions], - ) -> None: - """ - Create a lease. - - :param lease_duration: seconds of marker staleness after which a contender may take the claim. Every contender - for the path must pass the same value. - :param heartbeat_interval: seconds between refreshes. Defaults to a third of ``lease_duration``, leaving room - for two missed refreshes before a peer may take the claim. Must be shorter than ``lease_duration``. - :param on_compromise: called from the heartbeat thread with a :class:`LeaseCompromise` when the claim is lost. - :param kwargs: every other :class:`BaseFileLock ` option, ``timeout`` and ``mode`` among - them. The metaclass passes them all by keyword, and taking them here lets - :class:`AsyncSoftFileLease ` add the async plumbing a fixed signature would - hide. - - """ - if lease_duration <= 0: - msg = f"lease_duration must be positive, got {lease_duration!r}" - raise ValueError(msg) - if heartbeat_interval is None: - heartbeat_interval = lease_duration / 3 - if not 0 < heartbeat_interval < lease_duration: - msg = f"heartbeat_interval must be positive and below lease_duration, got {heartbeat_interval!r}" - raise ValueError(msg) - super().__init__(lock_file, **kwargs) - self._lease_duration = lease_duration - self._heartbeat_interval = heartbeat_interval - self._on_compromise = on_compromise - # Sharing one claim across a thread-local lock lets a second thread's failed acquisition stop the heartbeat of - # the thread holding the lease, leaving its marker unrefreshed until a peer reclaims it. - self._claims: _LeaseClaimHolder = ( - _ThreadLocalLeaseClaimHolder if self.is_thread_local() else _LeaseClaimHolder - )() - - @property - def _claim(self) -> _LeaseClaim: - return self._claims.claim - - @property - def lease_duration(self) -> float: - """The staleness in seconds after which a contender may take this claim.""" - return self._lease_duration - - @property - def token(self) -> str | None: - """ - The token naming the claim this process published. - - :returns: the token while the lease is held, ``None`` otherwise. It identifies a claim; it does not fence one. - - """ - return self._claim.token - - @property - def compromise(self) -> LeaseCompromise | None: - """ - The loss of claim the heartbeat observed. - - :returns: the :class:`LeaseCompromise`, or ``None`` while the claim still holds - - """ - return self._claim.compromise - - def _acquire(self) -> None: - claim = self._claim - self._stop_heartbeat() # no earlier claim's heartbeat outlives the acquisition of the next one - claim.token = token = secrets.token_hex(16) - claim.compromise = None - super()._acquire() - # The context is thread-local by default, so the heartbeat thread cannot read the descriptor this one just - # published, nor the claim this one owns. Hand it the fd, the inode it verified and the claim instead. - if (fd := self._context.lock_file_fd) is not None and ( - identity := self._context.lock_file_fd_identity - ) is not None: - self._start_heartbeat(claim, fd, identity, token) - - def _release(self) -> None: - self._stop_heartbeat() - self._claim.token = None - super()._release() - - def _published_record(self) -> OwnerRecord: - return super()._published_record()._replace(token=self._claim.token, lease_duration=self._lease_duration) - - def _try_break_stale_lock(self) -> None: - if (peer := self._read_peer()) is None: - # Not a readable protocol 2 lease record: a partial write, a foreign or legacy protocol 1 marker, or the - # strict sentinel. The base self-heal evicts a genuinely malformed marker once it ages past the grace - # window and leaves a legitimate legacy or strict holder in place, so a corrupt marker no longer wedges - # every lease contender until its own timeout. - super()._try_break_stale_lock() - return - owner, mtime, ino = peer - # Only a peer that published a lease agreed to be superseded by one, so a record stating any other contract is - # never reclaimed by age. Raise the mismatch outside the read so the suppression cannot swallow it. - if owner.mode != "lease": - return - if owner.lease_duration != self._lease_duration: - msg = ( - f"{self.lock_file} holds a lease of {owner.lease_duration!r}s but this contender configured " - f"{self._lease_duration!r}s; every contender for a path must agree on lease_duration" - ) - raise LeaseSettingsMismatch(msg) - # A break can fail for reasons a contender must ride out rather than raise on: a peer broke the marker first, - # or Windows refuses to rename a file whose holder still has it open. Poll again instead. - with suppress(OSError): - # A dead or recycled owner is reclaimed at once; a live owner past its lease duration is superseded on the - # schedule every contender agreed to. - if owner_is_stale(owner.pid, owner.hostname, owner.start): - break_lock_file(self.lock_file, mtime, ino) - return - if time.time() - mtime >= self._lease_duration: - break_lock_file(self.lock_file, mtime, ino) - - def _read_peer(self) -> tuple[OwnerRecord, float, int] | None: - with suppress(OSError, ValueError): - content, mtime, ino = _read_lock_file(self.lock_file) - if (owner := parse_marker(content)) is not None: - return owner, mtime, ino - return None - - def _start_heartbeat(self, claim: _LeaseClaim, fd: int, identity: tuple[int, int], token: str) -> None: - # The thread watches the event it was handed rather than whatever the claim names later: a heartbeat that - # outlives its join timeout would otherwise adopt the next acquisition's event and never stop. - stop = Event() - thread = Thread( - target=self._refresh_until_stopped, - args=(claim, fd, identity, token, stop), - name=f"filelock-lease-{os.getpid()}", - daemon=True, - ) - # Record the heartbeat before starting the thread so a release racing this acquire on a shared, - # non-thread-local claim always sees it and sets the stop event; the thread then exits at its first wait - # instead of outliving the release. A start that raises leaves the unstarted thread for _stop_heartbeat. - claim.heartbeat = _Heartbeat(thread, stop) - thread.start() - - def _stop_heartbeat(self) -> None: - claim = self._claim - if (heartbeat := claim.heartbeat) is None: - return - heartbeat.stop.set() - claim.heartbeat = None - # thread.ident is None until start() runs: a heartbeat recorded before its thread started (a start that - # raised, or a release racing acquire on a shared claim) has nothing to join, and the stop above makes it - # exit at once. on_compromise runs on the heartbeat thread and may release the lease, landing back here. - if heartbeat.thread.ident is not None and heartbeat.thread is not current_thread(): - heartbeat.thread.join(timeout=self._heartbeat_interval) - - def _refresh_until_stopped( - self, - claim: _LeaseClaim, - fd: int, - identity: tuple[int, int], - token: str, - stop: Event, - ) -> None: - # The loop ends at the first loss of the claim, so the holder hears about it once. A transient filesystem - # error (ESTALE / EIO on the NFS-style filesystems a lease targets) is not a loss: retry rather than raise a - # false compromise. Report the claim unrefreshable only once failures have run long enough that a contender - # could take it before the next success would land, a margin before the marker actually ages out, the way - # restic declares a lock unrefreshable ahead of its stale time. - last_success = time.monotonic() - while not stop.wait(self._heartbeat_interval): - outcome, error = self._refresh_claim(claim, fd, identity, token) - if outcome == "lost": - return - if outcome == "ok": - last_success = time.monotonic() - elif time.monotonic() - last_success >= self._lease_duration - self._heartbeat_interval: - self._report_compromise(claim, "refresh-failed", error, token) - return - - def _refresh_claim( - self, - claim: _LeaseClaim, - fd: int, - identity: tuple[int, int], - token: str, - ) -> tuple[_RefreshOutcome, OSError | None]: - try: - st = os.lstat(self.lock_file) - except FileNotFoundError as error: - self._report_compromise(claim, "marker-missing", error, token) - return "lost", None - except OSError as error: - return "transient", error - # A peer that took the expired claim replaced the marker, so the pathname now names its inode, not ours. - if (st.st_dev, st.st_ino) != identity: - self._report_compromise(claim, "owner-changed", None, token) - return "lost", None - try: - touch(self.lock_file, fd=fd) - except OSError as error: - return "transient", error - return "ok", None - - def _report_compromise( - self, - claim: _LeaseClaim, - reason: CompromiseReason, - error: OSError | None, - token: str, - ) -> None: - # Record it on the claim this heartbeat serves, not on self._claim: a thread-local claim read from the - # heartbeat thread is a different, empty one, so the holder would never see the loss it is being told about. - # The token is the one this thread published, not claim.token, which a release may already have cleared. - claim.compromise = LeaseCompromise(lock_file=self.lock_file, token=token, reason=reason, error=error) - if self._on_compromise is not None: - self._on_compromise(claim.compromise) - - -__all__ = [ - "CompromiseReason", - "LeaseCompromise", - "SoftFileLease", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_marker.py b/bundle/python-cpu/Lib/site-packages/filelock/_marker.py deleted file mode 100644 index f8e7fa6f8aaeb726cd59970663b22340212109f2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_marker.py +++ /dev/null @@ -1,157 +0,0 @@ -from __future__ import annotations - -import math -import os -from contextlib import suppress -from typing import Final, Literal, NamedTuple - -from ._identity import host_name, process_start_token -from ._soft import SoftFileLock, _read_lock_file -from ._util import write_all - -#: Protocol 1 is the legacy ``\n\n[\n]`` marker that :class:`SoftFileLock` still writes. -#: Protocol 2 carries the owner mode and the lease claim. A protocol 1 reader treats a protocol 2 marker as malformed -#: and evicts it after its grace period, so the two never guarantee mutual exclusion against each other. -_PROTOCOL: Final[str] = "filelock/2" - -_MAX_PID: Final[int] = 2**31 - 1 - -#: ``unknown`` is never published: it names a mode some other filelock wrote that this version cannot interpret. Such a -#: record still identifies a live owner, so it is parsed rather than read as malformed and aged out. -OwnerMode = Literal["lease", "unknown"] - - -class OwnerRecord(NamedTuple): - """The owner published in a protocol 2 marker.""" - - pid: int - hostname: str - mode: OwnerMode - token: str | None = None - lease_duration: float | None = None - start: int | None = None - - -class MarkerSoftFileLock(SoftFileLock): - """An existence lock whose marker carries a protocol 2 owner record.""" - - #: Filled in by each mode so the published record states the contract its holder acquired under. - _owner_mode: OwnerMode - - @property - def owner(self) -> OwnerRecord | None: - """ - The owner named by the marker on disk. - - :returns: the published record, or ``None`` when no marker exists or its record is malformed or protocol 1 - - """ - return self._read_owner() - - @property - def pid(self) -> int | None: - """ - The PID of the process holding this lock, read from the marker. - - :returns: the PID, or ``None`` when no marker exists or its record is unreadable - - """ - return None if (owner := self._read_owner()) is None else owner.pid - - @property - def is_lock_held_by_us(self) -> bool: - """ - Whether the marker on disk names this process. - - :returns: ``True`` when the marker's PID and hostname match this process - - """ - owner = self._read_owner() - return owner is not None and owner.pid == os.getpid() and owner.hostname == host_name() - - def force_break(self) -> None: - """ - Remove the marker whoever holds it, so a later contender can acquire. - - Forced breaking voids mutual exclusion: the previous holder keeps running and keeps using whatever the lock - protects. Reserve it for an operator clearing a marker whose holder is known to be gone. - """ - self.break_lock() - - def _read_owner(self) -> OwnerRecord | None: - with suppress(OSError, ValueError): - return parse_marker(_read_lock_file(self.lock_file)[0]) - return None - - def _write_lock_info(self, fd: int) -> None: - write_all(fd, encode_marker(self._published_record())) - - def _published_record(self) -> OwnerRecord: - return OwnerRecord( - pid=os.getpid(), - hostname=host_name(), - mode=self._owner_mode, - start=process_start_token(os.getpid()), - ) - - -def encode_marker(record: OwnerRecord) -> bytes: - """Render an owner record as the bytes a protocol 2 marker holds.""" - lines = [_PROTOCOL, f"pid={record.pid}", f"host={record.hostname}", f"mode={record.mode}"] - if record.token is not None: - lines.append(f"token={record.token}") - if record.lease_duration is not None: - lines.append(f"duration={record.lease_duration!r}") - if record.start is not None: - lines.append(f"start={record.start}") - return "".join(f"{line}\n" for line in lines).encode() - - -def parse_marker(content: str | None) -> OwnerRecord | None: - """Return the owner a protocol 2 marker names, or ``None`` when the record is malformed or protocol 1.""" - if not content or not (lines := content.strip().splitlines()) or lines[0] != _PROTOCOL: - return None - fields: dict[str, str] = {} - for line in lines[1:]: - key, separator, value = line.partition("=") - if not separator: - return None - fields[key] = value - return _build_record(fields) - - -def _build_record(fields: dict[str, str]) -> OwnerRecord | None: - # An unknown key is a field a newer filelock published, so ignore it rather than read the record as malformed. An - # unrecognized mode is the same story one level up: a contract this version does not implement. Reading it as - # malformed would age the marker out of a live owner's hands, so keep it and let the caller refuse to reclaim it. - # A record naming no mode at all states no contract and stays malformed. - if (published := fields.get("mode")) is None: - return None - mode: OwnerMode = "lease" if published == "lease" else "unknown" - hostname = fields.get("host") - if not hostname or "pid" not in fields: - return None - try: - pid = int(fields["pid"]) - duration = float(fields["duration"]) if "duration" in fields else None - start = int(fields["start"]) if "start" in fields else None - except ValueError: - return None - if not 1 <= pid <= _MAX_PID: - return None - token = fields.get("token") - # float() accepts "nan" and "inf", and neither is non-positive, so a duration <= 0 guard alone would read such a - # marker as a valid lease. A nan duration mismatches every configured duration and so wedges reclaim, where a - # malformed marker ages out through the grace window. - if mode == "lease" and (token is None or duration is None or not (math.isfinite(duration) and duration > 0)): - return None - return OwnerRecord(pid=pid, hostname=hostname, mode=mode, token=token, lease_duration=duration, start=start) - - -__all__ = [ - "MarkerSoftFileLock", - "OwnerMode", - "OwnerRecord", - "encode_marker", - "parse_marker", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_read_write.py b/bundle/python-cpu/Lib/site-packages/filelock/_read_write.py deleted file mode 100644 index c043f7e67a9cc2136c0cc28ea27dc8660b1454cd..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_read_write.py +++ /dev/null @@ -1,783 +0,0 @@ -from __future__ import annotations - -import logging -import os -import pathlib -import sqlite3 -import sys -import threading -import time -from contextlib import contextmanager, suppress -from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, cast -from weakref import WeakValueDictionary - -from ._api import ( - AcquireReturnProxy, - _ensure_current_process, - _fork_transition, - _raise_chained_errors, - _register_fork_class, - _register_fork_object, -) -from ._error import Timeout - -if TYPE_CHECKING: - from collections.abc import Callable, Generator - - from _typeshed import Unused - - if sys.version_info >= (3, 11): - from typing import Self - else: - from typing_extensions import Self - -_LOGGER: Final[logging.Logger] = logging.getLogger("filelock") -_GETPID: Final[Callable[[], int]] = os.getpid -_IS_PYPY: Final[bool] = sys.implementation.name == "pypy" -_NEEDS_CONNECTION_ESCROW: Final[bool] = ( - hasattr(os, "register_at_fork") and sys.implementation.name == "cpython" and sys.version_info < (3, 12) -) -_ConnectionParameter: TypeAlias = ( - str | bytes | os.PathLike[str] | os.PathLike[bytes] | float | int | type[sqlite3.Connection] | None -) -_DatabaseIdentity: TypeAlias = tuple[int, int] - -# sqlite3_busy_timeout() accepts a C int, max 2_147_483_647 on 32-bit. Use a lower value to be safe (~23 days). -_MAX_SQLITE_TIMEOUT_MS: Final[int] = 2_000_000_000 - 1 -_UNSAFE_FORK_EXIT_STATUS: Final[int] = 70 - - -class _SQLiteTransitionContext(threading.local): - depth: int = 0 - - -_SQLITE_TRANSITION_CONTEXT: Final = _SQLiteTransitionContext() - - -class _ConnectionEscrow: - def __init__(self) -> None: - self._lock = threading.RLock() - self._functions: tuple[Callable[[sqlite3.Connection], None], Callable[[sqlite3.Connection], None]] | None = None - - def functions( - self, - ) -> tuple[Callable[[sqlite3.Connection], None], Callable[[sqlite3.Connection], None]] | None: - if not _NEEDS_CONNECTION_ESCROW: - return None # pragma: >=3.12 cover - with self._lock: # pragma: <3.12 cover # pragma: needs fork - if self._functions is None: - import ctypes # ruff:ignore[import-outside-top-level] # keep optional ctypes and its audited dlsym out of ordinary imports - - function_type = ctypes.PYFUNCTYPE(None, ctypes.py_object) - increment_address = ctypes.cast(ctypes.pythonapi.Py_IncRef, ctypes.c_void_p).value - decrement_address = ctypes.cast(ctypes.pythonapi.Py_DecRef, ctypes.c_void_p).value - if increment_address is None or decrement_address is None: # pragma: no cover - resolved CPython API - msg = "CPython reference functions have no address" - raise RuntimeError(msg) - self._functions = ( - cast("Callable[[sqlite3.Connection], None]", function_type(increment_address)), - cast("Callable[[sqlite3.Connection], None]", function_type(decrement_address)), - ) - return self._functions - - def _reset_after_fork_in_child(self) -> None: # pragma: forked child - self._lock = threading.RLock() - - -_CONNECTION_ESCROW: Final = _ConnectionEscrow() - - -class _ForkedDatabaseRegistry: - def __init__(self) -> None: - self._lock = threading.RLock() - self._paths: set[pathlib.Path] = set() - self._identities: set[_DatabaseIdentity] = set() - self._sqlite_used = False - self._all_paths_poisoned = False - - def raise_if_poisoned(self, path: pathlib.Path) -> None: - identity = self.identity(path) - with self._lock: - all_paths_poisoned = self._all_paths_poisoned - poisoned = ( - all_paths_poisoned or path in self._paths or (identity is not None and identity in self._identities) - ) - if poisoned: # pragma: needs fork - msg = ( - "ReadWriteLock is unavailable in a PyPy fork child; exec or exit before using it" - if all_paths_poisoned - else f"SQLite database {path!s} was active across fork(); exec or exit before using it in the child" - ) - raise RuntimeError(msg) - - def poison_after_fork(self, path: pathlib.Path, identity: _DatabaseIdentity | None) -> None: - self._paths.add(path) - if identity is not None: - self._identities.add(identity) - - def note_sqlite_use(self) -> None: - if _IS_PYPY: - with self._lock: - self._sqlite_used = True - - def _reset_after_fork_in_child(self) -> None: # pragma: forked child - self._lock = threading.RLock() - self._all_paths_poisoned = self._all_paths_poisoned or (_IS_PYPY and self._sqlite_used) - self._sqlite_used = False - - @staticmethod - def identity(path: pathlib.Path) -> _DatabaseIdentity | None: - try: - stat_result = path.stat() - except OSError: - return None - return stat_result.st_dev, stat_result.st_ino - - -_FORKED_DATABASES: Final = _ForkedDatabaseRegistry() - - -class _ForkSafeConnection(sqlite3.Connection): - _creator_pid: int - _decrement_escrow: Callable[[sqlite3.Connection], None] | None - - def __new__( - cls, - *_args: _ConnectionParameter, - **_kwargs: _ConnectionParameter, - ) -> Self: - connection = super().__new__(cls) - connection._creator_pid = _GETPID() - connection._decrement_escrow = None - return connection - - def close(self) -> None: - with _sqlite_transition(): - if _GETPID() != self._creator_pid: # pragma: needs fork - return - with _fork_transition(): - sqlite3.Connection.close(self) - if (decrement := self._decrement_escrow) is not None: # pragma: <3.12 cover # pragma: needs fork - self._decrement_escrow = None - decrement(self) - - def acquire_escrow( # pragma: <3.12 cover # pragma: needs fork - self, - functions: tuple[Callable[[sqlite3.Connection], None], Callable[[sqlite3.Connection], None]] | None, - ) -> None: - # The caller only reaches here holding the escrow functions; it skips the call entirely without them. - if functions is not None: # pragma: no branch - increment, decrement = functions - increment(self) - self._decrement_escrow = decrement - - def __del__(self) -> None: - with suppress(sqlite3.Error, RuntimeError): - self.close() - - -class _ReadWriteLockMeta(type): - """ - Resolve singleton instances for ``is_singleton=True`` construction. - - This logic lives here rather than in ReadWriteLock.get_lock so ``ReadWriteLock(path)`` returns cached instances - without a 2-arg ``super()`` call that type checkers cannot verify. - - """ - - _instances: WeakValueDictionary[pathlib.Path, ReadWriteLock] - _instances_lock: threading.RLock - _instances_pid: int - _instances_under_construction: set[pathlib.Path] - - def __call__( - cls, - lock_file: str | os.PathLike[str], - timeout: float = -1, - *, - blocking: bool = True, - is_singleton: bool = True, - ) -> ReadWriteLock: - _ensure_current_process() - if cls._instances_pid != _GETPID(): - cls._reset_class_after_fork() - construction_pid = _GETPID() - if not is_singleton: - instance = super().__call__(lock_file, timeout, blocking=blocking, is_singleton=is_singleton) - if _GETPID() != construction_pid: # pragma: forked child - msg = "ReadWriteLock construction cannot continue after fork" - raise RuntimeError(msg) - return instance - - normalized = pathlib.Path(lock_file).resolve() - with cls._instances_lock: - if normalized not in cls._instances: - if normalized in cls._instances_under_construction: # pragma: no cover - exercised in an audit callback - msg = f"Singleton lock construction is already active for {lock_file!s}" - raise RuntimeError(msg) - construction_registry = cls._instances_under_construction - construction_registry.add(normalized) - try: - instance = super().__call__(lock_file, timeout, blocking=blocking, is_singleton=is_singleton) - finally: - if _GETPID() == construction_pid: - construction_registry.discard(normalized) - if _GETPID() != construction_pid: - msg = "ReadWriteLock construction cannot continue after fork" - raise RuntimeError(msg) - cls._instances[normalized] = instance - else: - instance = cls._instances[normalized] - - if instance.timeout != timeout or instance.blocking != blocking: - msg = ( - f"Singleton lock created with timeout={instance.timeout}, blocking={instance.blocking}," - f" cannot be changed to timeout={timeout}, blocking={blocking}" - ) - raise ValueError(msg) - return instance - - def _reset_class_after_fork(cls) -> None: # pragma: forked child - cls._instances = WeakValueDictionary() - cls._instances_lock = threading.RLock() - cls._instances_pid = _GETPID() - cls._instances_under_construction = set() - - -class ReadWriteLock(metaclass=_ReadWriteLockMeta): - """ - Cross-process read-write lock backed by SQLite. - - Allows concurrent shared readers or a single exclusive writer. The lock is reentrant within the same mode (multiple - ``acquire_read`` calls nest, as do multiple ``acquire_write`` calls from the same thread), but upgrading from read - to write or downgrading from write to read raises :class:`RuntimeError`. Write locks are pinned to the thread that - acquired them. - - By default, ``is_singleton=True``: calling ``ReadWriteLock(path)`` with the same resolved path returns the same - instance. The path is handed to :func:`sqlite3.connect` as given, so a ``.db`` extension is a convention rather - than a requirement; the filesystem must be one the active SQLite VFS supports. - - :param lock_file: path to the SQLite database file used as the lock - :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable - :param is_singleton: if ``True``, reuse existing instances for the same resolved path - - .. versionadded:: 3.21.0 - - """ - - _instances: WeakValueDictionary[pathlib.Path, ReadWriteLock] = WeakValueDictionary() - _instances_lock = threading.RLock() - _instances_pid = _GETPID() - _instances_under_construction: ClassVar[set[pathlib.Path]] = set() - - def __init_subclass__(cls) -> None: - super().__init_subclass__() - cls._instances = WeakValueDictionary() - cls._instances_lock = threading.RLock() - cls._instances_pid = _GETPID() - cls._instances_under_construction = set() - _register_fork_class(cls) - - @classmethod - def get_lock( - cls, lock_file: str | os.PathLike[str], timeout: float = -1, *, blocking: bool = True - ) -> ReadWriteLock: - """ - Return the singleton :class:`ReadWriteLock` for *lock_file*. - - :param lock_file: path to the SQLite database file used as the lock - :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable - - :returns: the singleton lock instance - - :raises ValueError: if an instance already exists for this path with different *timeout* or *blocking* values - - """ - return cls(lock_file, timeout, blocking=blocking) - - def __init__( - self, - lock_file: str | os.PathLike[str], - timeout: float = -1, - *, - blocking: bool = True, - is_singleton: bool = True, # ruff:ignore[unused-method-argument] # consumed by _ReadWriteLockMeta.__call__ - ) -> None: - self.lock_file = os.fspath(lock_file) - self._canonical_path = pathlib.Path(lock_file).resolve() - _FORKED_DATABASES.raise_if_poisoned(self._canonical_path) - self.timeout = timeout - self.blocking = blocking - self._transaction_lock = threading.Lock() # serializes the (possibly blocking) SQLite transaction work - self._internal_lock = threading.Lock() # protects _lock_level / _current_mode updates and rollback - self._lock_level = 0 - self._current_mode: Literal["read", "write"] | None = None - self._write_thread_id: int | None = None - self._acquisition_thread_ids: set[int] = set() - self._con: _ForkSafeConnection | None = None - self._connection_transaction_released = True - self._connection_identity: _DatabaseIdentity | None = None - self._closed = False - self._creator_pid = _GETPID() - self._fork_invalidated = False - _register_fork_object(self) - with _fork_transition(), _sqlite_transition(): - validation_connection = self._open_connection(sqlite_timeout=5.0) - validation_connection.close() - - def acquire_read(self, timeout: float = -1, *, blocking: bool = True) -> AcquireReturnProxy: - """ - Acquire a shared read lock. - - If this instance already holds a read lock, the lock level is incremented (reentrant). Attempting to acquire a - read lock while holding a write lock raises :class:`RuntimeError` (downgrade not allowed). - - :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable - - :returns: a proxy that can be used as a context manager to release the lock - - :raises RuntimeError: if a write lock is already held on this instance - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - return self._acquire("read", timeout, blocking=blocking) - - def acquire_write(self, timeout: float = -1, *, blocking: bool = True) -> AcquireReturnProxy: - """ - Acquire an exclusive write lock. - - If this instance already holds a write lock from the same thread, the lock level is incremented (reentrant). - Attempting to acquire a write lock while holding a read lock raises :class:`RuntimeError` (upgrade not allowed). - Write locks are pinned to the acquiring thread: a different thread trying to re-enter also raises - :class:`RuntimeError`. - - :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable - - :returns: a proxy that can be used as a context manager to release the lock - - :raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - return self._acquire("write", timeout, blocking=blocking) - - @contextmanager - def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]: - """ - Context manager that acquires and releases a shared read lock. - - Falls back to instance defaults for *timeout* and *blocking* when ``None``. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default - - """ - if timeout is None: - timeout = self.timeout - if blocking is None: - blocking = self.blocking - self.acquire_read(timeout, blocking=blocking) - try: - yield - finally: - self.release() - - @contextmanager - def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]: - """ - Context manager that acquires and releases an exclusive write lock. - - Falls back to instance defaults for *timeout* and *blocking* when ``None``. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default - - """ - if timeout is None: - timeout = self.timeout - if blocking is None: - blocking = self.blocking - self.acquire_write(timeout, blocking=blocking) - try: - yield - finally: - self.release() - - def release(self, *, force: bool = False) -> None: - """ - Release one level of the current lock. - - When the lock level reaches zero the underlying SQLite transaction is rolled back, releasing the database lock. - - :param force: if ``True``, release the lock completely regardless of the current lock level - - :raises RuntimeError: if no lock is currently held and *force* is ``False`` - - """ - with _fork_transition(): - _ensure_current_process() - if self._inherited: # pragma: needs fork - return - self._raise_if_acquiring("release") - self._release(force=force, close=False) - - def close(self) -> None: - """ - Release the lock (if held) and close the underlying SQLite connection. - - After calling this method, the lock instance is no longer usable. - - """ - with _fork_transition(): - _ensure_current_process() - if self._inherited: # pragma: needs fork - return - self._raise_if_acquiring("close") - self._release(force=True, close=True) - - def _release(self, *, force: bool, close: bool) -> None: - with self._transaction_lock, self._internal_lock: - if self._lock_level == 0: - if force and self._con is None: - if close: - self._closed = True - return - if not force: - msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held" - raise RuntimeError(msg) - if not force and self._lock_level > 1: - self._lock_level -= 1 - return - try: - self._finish_connection() - except sqlite3.Error: - if self._connection_transaction_released: - self._clear_lock_state() - raise - self._clear_lock_state() - if close: - self._closed = True - - def _clear_lock_state(self) -> None: - self._lock_level = 0 - self._current_mode = None - self._write_thread_id = None - - def __del__(self) -> None: - if _GETPID() == getattr(self, "_creator_pid", None) and (connection := getattr(self, "_con", None)) is not None: - with suppress(sqlite3.Error, RuntimeError): - connection.close() - - def _reset_after_fork_in_child(self) -> None: # pragma: forked child - if self._con is not None: - _FORKED_DATABASES.poison_after_fork(self._canonical_path, self._connection_identity) - self._con = None - self._connection_transaction_released = True - self._connection_identity = None - self._transaction_lock = threading.Lock() - self._internal_lock = threading.Lock() - self._clear_lock_state() - self._acquisition_thread_ids = set() - self._fork_invalidated = True - - @property - def _inherited(self) -> bool: - return self._fork_invalidated or _GETPID() != self._creator_pid - - def _raise_if_unusable(self) -> None: - _ensure_current_process() - if self._inherited: # pragma: needs fork - msg = f"ReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance" - raise RuntimeError(msg) - if self._closed: - msg = "Cannot operate on a closed database." - raise sqlite3.ProgrammingError(msg) - - def _acquire(self, mode: Literal["read", "write"], timeout: float, *, blocking: bool) -> AcquireReturnProxy: - with _fork_transition(): - self._raise_if_unusable() - operation_pid = _GETPID() - thread_id = threading.get_ident() - with self._internal_lock: - if self._lock_level > 0: - return self._validate_reentrant(mode) - if thread_id in self._acquisition_thread_ids: # pragma: no cover - exercised in an audit callback - msg = f"Cannot acquire ReadWriteLock on {self.lock_file} while acquisition is active in this thread" - raise RuntimeError(msg) - self._acquisition_thread_ids.add(thread_id) - try: - start_time = time.perf_counter() - self._acquire_transaction_lock(blocking=blocking, timeout=timeout) - try: - self._raise_if_unusable() - return self._do_acquire_inner( - mode, - timeout, - blocking=blocking, - operation_pid=operation_pid, - start_time=start_time, - ) - finally: - self._transaction_lock.release() - finally: - with self._internal_lock: - self._acquisition_thread_ids.discard(thread_id) - - def _do_acquire_inner( - self, - mode: Literal["read", "write"], - timeout: float, - *, - blocking: bool, - operation_pid: int, - start_time: float, - ) -> AcquireReturnProxy: - # Double-check: another thread may have acquired the lock while we waited on _transaction_lock. - with self._internal_lock: - if self._lock_level > 0: - return self._validate_reentrant(mode) - if self._con is not None: - self._finish_connection() - try: - self._open_for_acquisition( - timeout, - blocking=blocking, - operation_pid=operation_pid, - start_time=start_time, - ) - self._configure_and_begin( - mode, - timeout, - blocking=blocking, - operation=(operation_pid, start_time), - ) - self._raise_if_process_changed(operation_pid) - except BaseException as error: - acquisition_error: BaseException - if isinstance(error, sqlite3.OperationalError) and "database is locked" in str(error): - acquisition_error = Timeout(self.lock_file) - else: - acquisition_error = error - try: - self._finish_connection() - except sqlite3.Error as cleanup_error: - _raise_chained_errors(acquisition_error, cleanup_error) - if acquisition_error is not error: - raise acquisition_error from None - raise - with self._internal_lock: - self._raise_if_process_changed(operation_pid) - self._current_mode = mode - self._lock_level = 1 - if mode == "write": - self._write_thread_id = threading.get_ident() - return AcquireReturnProxy(lock=self) - - def _open_for_acquisition(self, timeout: float, *, blocking: bool, operation_pid: int, start_time: float) -> None: - with _sqlite_transition(): - sqlite_timeout = ( - timeout_for_sqlite( - timeout, - blocking=blocking, - already_waited=time.perf_counter() - start_time, - ) - / 1000 - ) - connection = self._open_connection(sqlite_timeout=sqlite_timeout) - self._con, self._connection_transaction_released, self._connection_identity = ( - connection, - False, - _FORKED_DATABASES.identity(self._canonical_path), - ) - self._raise_if_process_changed(operation_pid) - - def _configure_and_begin( - self, - mode: Literal["read", "write"], - timeout: float, - *, - blocking: bool, - operation: tuple[int, float], - ) -> None: - with _sqlite_transition(): - operation_pid, start_time = operation - connection = cast("_ForkSafeConnection", self._con) - waited = time.perf_counter() - start_time - timeout_ms = timeout_for_sqlite(timeout, blocking=blocking, already_waited=waited) - self._raise_if_process_changed(operation_pid) - connection.executescript(f"PRAGMA busy_timeout={timeout_ms}; PRAGMA journal_mode=MEMORY;").close() - # Use legacy journal mode (not WAL) because WAL does not block readers while a concurrent EXCLUSIVE - # write transaction is active, which makes read-write locking impossible without modifying table data. - # MEMORY is safe here since no writes happen, so a crash cannot corrupt the DB. - # See https://sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions - # - # Recompute the remaining timeout after the blocking journal_mode pragma. - waited = time.perf_counter() - start_time - recomputed = timeout_for_sqlite(timeout, blocking=blocking, already_waited=waited) - self._raise_if_process_changed(operation_pid) - statements = f"PRAGMA busy_timeout={recomputed}; " if recomputed != timeout_ms else "" - statements += "BEGIN EXCLUSIVE TRANSACTION;" if mode == "write" else "BEGIN TRANSACTION;" - if mode == "read": - # SQLite takes the SHARED lock only when a statement reads; BEGIN alone stays deferred. - # https://www.sqlite.org/lockingv3.html#transaction_control - statements += " SELECT name FROM sqlite_schema LIMIT 1;" - connection.executescript(statements).close() - - def _open_connection(self, *, sqlite_timeout: float) -> _ForkSafeConnection: - with _sqlite_transition(): - creator_pid = _GETPID() - functions = _CONNECTION_ESCROW.functions() - if _GETPID() != creator_pid: # pragma: forked child - msg = "SQLite connection construction cannot continue after fork" - raise RuntimeError(msg) - connection = _connect( - os.fspath(self._canonical_path), - factory=_ForkSafeConnection, - timeout=sqlite_timeout, - ) - if functions is not None: # pragma: <3.12 cover # pragma: needs fork - connection.acquire_escrow(functions) - if _GETPID() != creator_pid: # pragma: forked child - _FORKED_DATABASES.poison_after_fork( - self._canonical_path, - _FORKED_DATABASES.identity(self._canonical_path), - ) - msg = "SQLite connection construction cannot continue after fork" - raise RuntimeError(msg) - return connection - - def _finish_connection(self) -> None: - with _sqlite_transition(): - if (connection := self._con) is None: - return - rollback_error: sqlite3.Error | None = None - if not self._connection_transaction_released: - if connection.in_transaction: - try: - connection.rollback() - except sqlite3.Error as error: - if connection.in_transaction: - raise - self._connection_transaction_released = True - rollback_error = error - else: - self._connection_transaction_released = True - else: - self._connection_transaction_released = True - try: - connection.close() - except sqlite3.Error as close_error: - if rollback_error is not None: - _raise_chained_errors(rollback_error, close_error) - raise - self._con = None - self._connection_transaction_released = True - self._connection_identity = None - if rollback_error is not None: - raise rollback_error - - def _validate_reentrant(self, mode: Literal["read", "write"]) -> AcquireReturnProxy: - if self._current_mode != mode: - opposite = "write" if mode == "read" else "read" - direction = "downgrade" if mode == "read" else "upgrade" - msg = ( - f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): " - f"already holding a {opposite} lock ({direction} not allowed)" - ) - raise RuntimeError(msg) - if mode == "write" and (cur := threading.get_ident()) != self._write_thread_id: - msg = ( - f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) " - f"from thread {cur} while it is held by thread {self._write_thread_id}" - ) - raise RuntimeError(msg) - self._lock_level += 1 - return AcquireReturnProxy(lock=self) - - def _acquire_transaction_lock(self, *, blocking: bool, timeout: float) -> None: - if not blocking: - acquired = self._transaction_lock.acquire(blocking=False) - elif timeout == -1: - acquired = self._transaction_lock.acquire(blocking=True) - else: - acquired = self._transaction_lock.acquire(blocking=True, timeout=timeout) - if not acquired: - raise Timeout(self.lock_file) from None - - def _raise_if_acquiring(self, operation: Literal["acquire", "close", "release"]) -> None: - with self._internal_lock: - active_in_current_thread = threading.get_ident() in self._acquisition_thread_ids - if active_in_current_thread: # pragma: no cover - exercised in an audit callback - msg = f"Cannot {operation} ReadWriteLock on {self.lock_file} while acquisition is active in this thread" - raise RuntimeError(msg) - - def _raise_if_process_changed(self, operation_pid: int) -> None: - if _GETPID() != operation_pid or self._inherited: # pragma: forked child - msg = f"ReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance" - raise RuntimeError(msg) - - -def _connect(database: str, *, factory: type[_ForkSafeConnection], timeout: float) -> _ForkSafeConnection: - _FORKED_DATABASES.note_sqlite_use() - return sqlite3.connect( - database, - check_same_thread=False, - factory=factory, - cached_statements=0, - timeout=timeout, - ) - - -@contextmanager -def _sqlite_transition() -> Generator[None]: - _SQLITE_TRANSITION_CONTEXT.depth += 1 - try: - yield - finally: - _SQLITE_TRANSITION_CONTEXT.depth -= 1 - - -def _abort_forked_sqlite_transition() -> None: # pragma: forked child - if _SQLITE_TRANSITION_CONTEXT.depth: - os._exit(_UNSAFE_FORK_EXIT_STATUS) # inherited SQLite handles cannot be used or closed safely - - -def _track_sqlite_use(event: str, _args: Unused) -> None: - if event == "sqlite3.connect": - _FORKED_DATABASES.note_sqlite_use() - - -def timeout_for_sqlite(timeout: float, *, blocking: bool, already_waited: float) -> int: - if blocking is False: - return 0 - - if timeout == -1: - return _MAX_SQLITE_TIMEOUT_MS - - if timeout < 0: - msg = "timeout must be a non-negative number or -1" - raise ValueError(msg) - - timeout_ms = int((max(timeout - already_waited, 0) if timeout > 0 else timeout) * 1000) - if timeout_ms > _MAX_SQLITE_TIMEOUT_MS or timeout_ms < 0: - _LOGGER.warning("timeout %s is too large for SQLite, using %s ms instead", timeout, _MAX_SQLITE_TIMEOUT_MS) - return _MAX_SQLITE_TIMEOUT_MS - return timeout_ms - - -_register_fork_object(_CONNECTION_ESCROW) -_register_fork_object(_FORKED_DATABASES) -_register_fork_class(ReadWriteLock) -if _IS_PYPY: - sys.addaudithook(_track_sqlite_use) # pragma: pypy cover -if hasattr(os, "register_at_fork"): # pragma: needs fork - os.register_at_fork(after_in_child=_abort_forked_sqlite_transition) - -__all__ = [ - "ReadWriteLock", - "timeout_for_sqlite", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_soft.py b/bundle/python-cpu/Lib/site-packages/filelock/_soft.py deleted file mode 100644 index ccb06e85a3d5c631ab1123373cdafcf3f99a5a91..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_soft.py +++ /dev/null @@ -1,260 +0,0 @@ -from __future__ import annotations - -import os -import stat -import sys -import time -from contextlib import suppress -from errno import EACCES, EEXIST, EPERM -from pathlib import Path -from typing import Final - -from ._api import BaseFileLock, _raise_grouped_errors -from ._identity import host_name, owner_is_stale, process_start_token -from ._soft_protocol import STRICT_SOFT_SENTINEL_RECORD -from ._util import break_lock_file, ensure_directory_exists, raise_on_not_writable_file, write_all - -_MALFORMED_LOCK_AGE_THRESHOLD: Final[float] = 2.0 -_MAX_LOCK_FILE_SIZE: Final[int] = 1024 -_UNLINK_MAX_RETRIES: Final[int] = 10 -_MARKER_WITH_START_TOKEN_LINE_COUNT: Final[int] = 3 - - -class SoftFileLock(BaseFileLock): - """ - Cooperative file lock based on a shared existence marker. - - Unlike :class:`UnixFileLock ` and :class:`WindowsFileLock `, this - lock does not use OS-level locking primitives. Instead, it creates the lock file with ``O_CREAT | O_EXCL`` and - treats its existence as the lock indicator. The filesystem must provide coherent exclusive creation and directory - updates to each participating process. A crash can leave the marker behind. - - The marker contains the holder's PID and hostname. A contender may remove it when it can no longer find a same-host - process with that PID. A configured :attr:`~filelock.BaseFileLock.lifetime` also permits removal based on marker - age, including while the holder remains alive. Age-based expiry can overlap protected operations and does not - provide strict mutual exclusion. - - """ - - #: Existence locks reclaim by unlinking a pathname, so an age-based lease may break one; a native inode lock cannot. - _lifetime_supported: bool = True - - #: Age-based expiry preserves historical behavior but does not provide strict mutual exclusion. - _lifetime_replacements: tuple[str, str] | None = ("StrictSoftFileLock", "SoftFileLease") - - #: An existence lock unlinks its marker to release, so it cannot promise to keep the pathname. - _preserve_lock_file_supported: bool = False - - #: An existence lock keeps protocol state in its marker, so it cannot lend the descriptor to an on_acquired hook. - _on_acquired_supported: bool = False - - def _acquire(self) -> None: - raise_on_not_writable_file(self.lock_file) - ensure_directory_exists(self.lock_file) - # O_CREAT | O_EXCL makes the create fail with EEXIST when the file already exists, so a successful open - # means this process now holds the lock. - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC - if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None: # pragma: needs o-nofollow - flags |= o_nofollow - try: - fd = os.open(self.lock_file, flags, self._open_mode()) - except OSError as exception: - if not ( - exception.errno == EEXIST or (exception.errno == EACCES and sys.platform == "win32") - ): # pragma: win32 no cover - raise - self._try_break_stale_lock() - return - self._mark_descriptor_pending(fd) - self._publish_held_marker(fd) - - def _publish_held_marker(self, fd: int) -> None: - # Publish held state only once the record is fully on disk. On any failure, including cancellation, close the - # descriptor and unlink the path only while it still names the file we opened, so a rollback never deletes a - # successor's marker that replaced ours at the same path after our lease expired. - identity: tuple[int, int] | None = None - try: - identity = _file_identity(os.fstat(fd)) - self._write_lock_info(fd) - except BaseException: - self._mark_descriptor_released() - os.close(fd) - with suppress(OSError): - if identity is not None and _file_identity(os.lstat(self.lock_file)) == identity: - Path(self.lock_file).unlink() - raise - self._mark_descriptor_owned(fd, identity) - - def _try_break_stale_lock(self) -> None: - with suppress(OSError, ValueError): - content, mtime, ino = _read_lock_file(self.lock_file) - if content == STRICT_SOFT_SENTINEL_RECORD: # pragma: needs hard-link - return - holder = _parse_lock_holder(content) - - if holder is None: - # Unparsable: wrong line count, a non-integer PID or start token, empty, oversized or not UTF-8. - # Self-heal only once the file is clearly not a half-written fresh lock (a peer between O_EXCL and - # _write_lock_info), so the brief create-then-write window is never mistaken for a stale lock. - if time.time() - mtime >= _MALFORMED_LOCK_AGE_THRESHOLD: - break_lock_file(self.lock_file, mtime, ino) - return - - if owner_is_stale(*holder): - break_lock_file(self.lock_file, mtime, ino) - - @staticmethod - def _write_lock_info(fd: int) -> None: - # No suppression: a write failure must reach the acquisition rollback so it never publishes a half-written - # marker as held state. The optional third line is this process's start token, absent when the platform - # exposes no proven start time, in which case a reader falls back to PID-only liveness. - info = f"{os.getpid()}\n{host_name()}\n" - if (token := process_start_token(os.getpid())) is not None: - info += f"{token}\n" - write_all(fd, info.encode()) - - @property - def pid(self) -> int | None: - """ - The PID of the process holding this lock, read from the lock file. - - :returns: the PID as an integer, or ``None`` if the lock file does not exist or cannot be parsed - - """ - with suppress(OSError, ValueError): - holder = _parse_lock_holder(_read_lock_file(self.lock_file)[0]) - if holder is not None: - return holder[0] - return None - - @property - def is_lock_held_by_us(self) -> bool: - """ - Whether this lock is held by the current process. - - :returns: ``True`` if the lock file exists and names the current process's PID and hostname - - """ - with suppress(OSError, ValueError): - holder = _parse_lock_holder(_read_lock_file(self.lock_file)[0]) - if holder is not None: - pid, hostname, _ = holder - return pid == os.getpid() and hostname == host_name() - return False - - def break_lock(self) -> None: - """Forcibly break the lock by removing the lock file, regardless of who holds it.""" - with suppress(OSError): - Path(self.lock_file).unlink() - - def _release(self) -> None: - fd = self._context.lock_file_fd - assert fd is not None # ruff:ignore[assert] # _release runs only while held, so the descriptor is set - # Capture the held file's identity before closing so cleanup can refuse to unlink a successor's marker. A - # supported lifetime lease lets a peer break our expired marker and create its own at this path before we - # release; unlinking by path alone would then delete the successor's lock. - identity: tuple[int, int] | None = None - with suppress(OSError): - identity = _file_identity(os.fstat(fd)) - # A failed close may already have released and recycled the descriptor number. Relinquish it before the one - # close attempt so no later release can close an unrelated descriptor that reused the same integer. - self._mark_descriptor_released() - try: - self._close_released_fd(fd, default_suppresses=False) - # Marker cleanup must also run for control-flow exceptions, and both failures must remain observable. - except BaseException as close_error: - try: - self._unlink_held_marker(identity) - except BaseException as cleanup_error: # ruff:ignore[blind-except] # preserve control-flow cleanup failures - _raise_grouped_errors( - "lock descriptor close and marker cleanup both failed", - close_error, - cleanup_error, - ) - raise - self._unlink_held_marker(identity) - - def _unlink_held_marker(self, identity: tuple[int, int] | None) -> None: - if identity is None: - return - if sys.platform == "win32": # pragma: win32 cover - self._windows_unlink_if_ours(identity) - else: # pragma: win32 no cover - with suppress(OSError): - if _file_identity(os.lstat(self.lock_file)) == identity: - Path(self.lock_file).unlink() - - def _windows_unlink_if_ours(self, identity: tuple[int, int]) -> None: # pragma: win32 cover - retry_delay = 0.001 - for attempt in range(_UNLINK_MAX_RETRIES): - # Windows doesn't immediately release file handles after close, causing EACCES/EPERM on unlink. Recheck - # identity each attempt: a failed unlink leaves a window for a successor to replace the marker at this path. - try: - if _file_identity(os.lstat(self.lock_file)) != identity: - return - Path(self.lock_file).unlink() - except OSError as exc: # ruff:ignore[try-except-in-loop] # each attempt's errno drives the retry choice - if exc.errno not in {EACCES, EPERM}: - return - if attempt < _UNLINK_MAX_RETRIES - 1: - time.sleep(retry_delay) - retry_delay *= 2 - else: - return - - -def _file_identity(st: os.stat_result) -> tuple[int, int]: - # (st_dev, st_ino) names the concrete inode behind a path, so a marker recreated at the same pathname after an - # expired lease reads as a different file. CPython populates both on Windows from the volume serial and file index. - return st.st_dev, st.st_ino - - -def _read_lock_file(path: str) -> tuple[str | None, float, int]: - # A legitimate lock file is always a regular file. Classify the path with lstat first, so any other node (symlink, - # FIFO, socket, device) is reported as a malformed lock the caller can evict, without an os.open that would follow - # a symlink, stall on a FIFO, or fail on a socket and leave acquisition wedged. The mtime and inode still flow back - # for the identity-checked stale break. lstat, not stat, so a hostile symlink is never followed onto its target. - st = os.lstat(path) - if not stat.S_ISREG(st.st_mode): # pragma: needs fifo - return None, st.st_mtime, st.st_ino - # Re-check on the opened handle: O_NOFOLLOW refuses a symlink swapped in after the lstat, O_NONBLOCK stops a FIFO - # swapped in from stalling the open, and the fstat catches any other non-regular replacement race before we read. - # The capped read stops a huge regular file (e.g. one filled from /dev/zero) from exhausting memory. - fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) - try: - st = os.fstat(fd) - if not stat.S_ISREG(st.st_mode): # pragma: no cover # only a non-regular node swapped in after the lstat - return None, st.st_mtime, st.st_ino - data = os.read(fd, _MAX_LOCK_FILE_SIZE + 1) - finally: - os.close(fd) - if len(data) <= _MAX_LOCK_FILE_SIZE: - with suppress(UnicodeDecodeError): - return data.decode("utf-8"), st.st_mtime, st.st_ino - return None, st.st_mtime, st.st_ino - - -def _parse_lock_holder(content: str | None) -> tuple[int, str, int | None] | None: - # A well-formed lock file is "\n\n" with an optional "\n" third line naming the - # holder's process start instant (a filelock 3.29 marker wrote this only on Windows; every platform writes it now). - # Anything else (wrong line count, a non-integer PID or start token, empty or unreadable content) is unparsable; - # returning None lets the caller treat it as a malformed lock to self-heal rather than a holder. - if not content or len(lines := content.strip().splitlines()) not in {2, 3}: - return None - try: - pid = int(lines[0]) - start_token = int(lines[2]) if len(lines) == _MARKER_WITH_START_TOKEN_LINE_COUNT else None - except ValueError: - return None - # A pid outside the valid range is a malformed lock, not a holder. Without this, a non-positive pid - # reaches os.kill() where 0 / -1 mean "the caller's own process group / every process" so a dead - # holder reads as alive and the lock is never reclaimed, while an oversized pid raises OverflowError - # (not OSError/ValueError) out of the self-heal path. _parse_marker_bytes already enforces this range. - if not 1 <= pid <= 2**31 - 1: - return None - return pid, lines[1], start_token - - -__all__ = [ - "SoftFileLock", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_soft_protocol.py b/bundle/python-cpu/Lib/site-packages/filelock/_soft_protocol.py deleted file mode 100644 index 490dd037ea225acd111a4826ba64acc90db9c671..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_soft_protocol.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import annotations - -from typing import Final - -STRICT_SOFT_SENTINEL_RECORD: Final[str] = "1\nfilelock-strict-v1\x00\n0\n" - -__all__ = [ - "STRICT_SOFT_SENTINEL_RECORD", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_soft_rw/__init__.py b/bundle/python-cpu/Lib/site-packages/filelock/_soft_rw/__init__.py deleted file mode 100644 index 0b737bcee9d257307ed31865e43023b967b116ab..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_soft_rw/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Cross-process and cross-host reader/writer lock on :class:`~filelock.SoftFileLock` primitives.""" - -from __future__ import annotations - -from ._async import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock -from ._sync import SoftReadWriteLock - -__all__ = [ - "AsyncAcquireSoftReadWriteReturnProxy", - "AsyncSoftReadWriteLock", - "SoftReadWriteLock", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_soft_rw/_async.py b/bundle/python-cpu/Lib/site-packages/filelock/_soft_rw/_async.py deleted file mode 100644 index 875c92541b1a36736943d0fdac40a1b74d8f4618..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_soft_rw/_async.py +++ /dev/null @@ -1,272 +0,0 @@ -"""Async wrapper around :class:`SoftReadWriteLock` for use with ``asyncio``.""" - -from __future__ import annotations - -import asyncio -import functools -import os -from contextlib import asynccontextmanager -from typing import TYPE_CHECKING, ParamSpec, TypeVar - -from filelock._async import ( - _BackendOutcome, - _capture_call, - _drain_future, - _future_result, - _raise_cancelled_error, - _wait_until_done, -) - -from ._sync import SoftReadWriteLock - -if TYPE_CHECKING: - from collections.abc import AsyncGenerator, Callable - from concurrent import futures - from types import TracebackType - - from filelock._api import AcquireReturnProxy - -_P = ParamSpec("_P") -_R = TypeVar("_R") - - -class AsyncSoftReadWriteLock: - """ - Async wrapper around :class:`SoftReadWriteLock` for ``asyncio`` applications. - - The sync class's blocking filesystem operations run on a thread pool via ``loop.run_in_executor()``. The - underlying :class:`SoftReadWriteLock` handles reentrancy, upgrade/downgrade rules, fork handling, heartbeat and - TTL stale detection, and singleton behavior. - - :param lock_file: path to the lock file; sidecar state/write/readers live next to it - :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention - :param is_singleton: if ``True``, reuse existing :class:`SoftReadWriteLock` instances per resolved path - :param heartbeat_interval: seconds between heartbeat refreshes; default 30 s - :param stale_threshold: seconds of mtime inactivity before a marker is stale; defaults to ``3 * heartbeat_interval`` - :param poll_interval: seconds between acquire retries under contention; default 0.25 s - :param loop: event loop for ``run_in_executor``; ``None`` uses the running loop - :param executor: executor for ``run_in_executor``; ``None`` uses the default executor - - .. versionadded:: 3.27.0 - - """ - - def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option - self, - lock_file: str | os.PathLike[str], - timeout: float = -1, - *, - blocking: bool = True, - is_singleton: bool = True, - heartbeat_interval: float = 30.0, - stale_threshold: float | None = None, - poll_interval: float = 0.25, - loop: asyncio.AbstractEventLoop | None = None, - executor: futures.Executor | None = None, - ) -> None: - self._creator_pid = os.getpid() - self._lock = SoftReadWriteLock( - lock_file, - timeout, - blocking=blocking, - is_singleton=is_singleton, - heartbeat_interval=heartbeat_interval, - stale_threshold=stale_threshold, - poll_interval=poll_interval, - ) - self._loop = loop - self._executor = executor - - @property - def lock_file(self) -> str: - """The path to the lock file passed to the constructor.""" - return self._lock.lock_file - - @property - def timeout(self) -> float: - """The default timeout applied when ``acquire_read`` / ``acquire_write`` is called without one.""" - return self._lock.timeout - - @property - def blocking(self) -> bool: - """Whether ``acquire_*`` defaults to blocking; ``False`` makes contention raise immediately.""" - return self._lock.blocking - - @property - def loop(self) -> asyncio.AbstractEventLoop | None: - """The event loop used for ``run_in_executor``, or ``None`` for the running loop.""" - return self._loop - - @property - def executor(self) -> futures.Executor | None: - """The executor used for ``run_in_executor``, or ``None`` for the default executor.""" - return self._executor - - @asynccontextmanager - async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]: - """ - Async context manager that acquires and releases a shared read lock. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default - - :raises RuntimeError: if a write lock is already held on this instance - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - await self.acquire_read(timeout, blocking=blocking) - try: - yield - finally: - await self.release() - - @asynccontextmanager - async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]: - """ - Async context manager that acquires and releases an exclusive write lock. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default - - :raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - await self.acquire_write(timeout, blocking=blocking) - try: - yield - finally: - await self.release() - - async def acquire_read( - self, timeout: float | None = None, *, blocking: bool | None = None - ) -> AsyncAcquireSoftReadWriteReturnProxy: - """ - Acquire a shared read lock. - - See :meth:`SoftReadWriteLock.acquire_read` for reentrancy / upgrade / fork semantics. The blocking work runs - inside ``run_in_executor`` so other coroutines on the same loop keep progressing while this call waits. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default - - :returns: a proxy usable as an async context manager to release the lock - - :raises RuntimeError: if a write lock is already held, if this instance was invalidated by - :func:`os.fork`, or if :meth:`close` was called - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - self._raise_if_inherited() - await self._run_acquire(functools.partial(self._lock.acquire_read, timeout, blocking=blocking)) - return AsyncAcquireSoftReadWriteReturnProxy(lock=self) - - async def acquire_write( - self, timeout: float | None = None, *, blocking: bool | None = None - ) -> AsyncAcquireSoftReadWriteReturnProxy: - """ - Acquire an exclusive write lock. - - See :meth:`SoftReadWriteLock.acquire_write` for the two-phase writer-preferring semantics. The blocking work - runs inside ``run_in_executor``. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default - - :returns: a proxy usable as an async context manager to release the lock - - :raises RuntimeError: if a read lock is already held, if a write lock is held by a different thread, if - this instance was invalidated by :func:`os.fork`, or if :meth:`close` was called - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - self._raise_if_inherited() - await self._run_acquire(functools.partial(self._lock.acquire_write, timeout, blocking=blocking)) - return AsyncAcquireSoftReadWriteReturnProxy(lock=self) - - async def release(self, *, force: bool = False) -> None: - """ - Release one level of the current lock. - - :param force: if ``True``, release the lock completely regardless of the current lock level - - :raises RuntimeError: if no lock is currently held and *force* is ``False`` - - """ - if self._creator_pid == os.getpid(): - await self._run(self._lock.release, force=force) - - async def close(self) -> None: - """Release any held lock and release the underlying filesystem resources. Idempotent.""" - if self._creator_pid == os.getpid(): - await self._run(self._lock.close) - - def _raise_if_inherited(self) -> None: - if self._creator_pid != os.getpid(): # pragma: forked child - msg = f"AsyncSoftReadWriteLock on {self.lock_file} was inherited across fork; construct a new instance" - raise RuntimeError(msg) - - async def _run_acquire(self, acquire: Callable[[], AcquireReturnProxy]) -> None: - # run_in_executor cannot recall work the pool already started, so canceling the caller does not stop the sync - # acquire: it still creates its marker, sets the hold, and starts the heartbeat, which keeps the marker fresh - # forever so no peer on any host can evict it as stale. Wait the submitted call out and hand the claim back, - # the way AsyncReadWriteLock does. - acquire_future = self._submit(acquire) - try: - await _wait_until_done(acquire_future) - except asyncio.CancelledError as cancellation: - try: - await _drain_future(acquire_future) - except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below - _raise_cancelled_error(cancellation, error) - try: - await _drain_future(self._submit(self._lock.release)) - except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below - _raise_cancelled_error(cancellation, error) - raise - _future_result(acquire_future) - - async def _run(self, func: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs) -> _R: - # A canceled release or close is already running on the pool thread; drain it so its outcome is observed - # instead of finishing unwatched, then let the cancellation through. - future = self._submit(func, *args, **kwargs) - try: - await _wait_until_done(future) - except asyncio.CancelledError as cancellation: - try: - await _drain_future(future) - except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below - _raise_cancelled_error(cancellation, error) - raise - return _future_result(future) - - def _submit( - self, func: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs - ) -> asyncio.Future[_BackendOutcome[_R]]: - loop = self._loop or asyncio.get_running_loop() - return loop.run_in_executor(self._executor, _capture_call, functools.partial(func, *args, **kwargs)) - - -class AsyncAcquireSoftReadWriteReturnProxy: - """Async context-aware object that releases an :class:`AsyncSoftReadWriteLock` on exit.""" - - def __init__(self, lock: AsyncSoftReadWriteLock) -> None: - self.lock = lock - - async def __aenter__(self) -> AsyncSoftReadWriteLock: - return self.lock - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - await self.lock.release() - - -__all__ = [ - "AsyncAcquireSoftReadWriteReturnProxy", - "AsyncSoftReadWriteLock", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_soft_rw/_sync.py b/bundle/python-cpu/Lib/site-packages/filelock/_soft_rw/_sync.py deleted file mode 100644 index 136bd614eb238e9e4bf1033181d2f83507b7267d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_soft_rw/_sync.py +++ /dev/null @@ -1,985 +0,0 @@ -"""Cross-process and cross-host reader/writer lock built on :class:`SoftFileLock` primitives.""" - -from __future__ import annotations - -import atexit -import hmac -import os -import re -import secrets -import socket -import stat -import sys -import threading -import time -import uuid -from contextlib import closing, contextmanager, suppress -from dataclasses import dataclass -from pathlib import Path -from typing import TYPE_CHECKING, Final, Literal -from weakref import WeakValueDictionary - -from filelock._api import ( - AcquireReturnProxy, - _ensure_current_process, - _fork_transition, - _raise_grouped_errors, - _register_fork_class, - _register_fork_object, - _register_owned_descriptor, - _unregister_owned_descriptor, -) -from filelock._error import Timeout -from filelock._soft import SoftFileLock -from filelock._util import ensure_directory_exists, touch, write_all - -if TYPE_CHECKING: - from collections.abc import Callable, Generator - - -_Mode = Literal["read", "write"] -_BREAK_SUFFIX: Final[str] = ".break" -_MAX_MARKER_SIZE: Final[int] = 1024 -_O_NOFOLLOW: Final[int] = getattr(os, "O_NOFOLLOW", 0) -_O_NONBLOCK: Final[int] = getattr(os, "O_NONBLOCK", 0) -# dirfd-relative I/O is a Unix-only optimization; Windows cannot ``os.open()`` a directory at all, and -# its ``os`` module skips dir_fd support entirely. When disabled, callers fall back to full-path ops. -_SUPPORTS_DIR_FD: Final[bool] = sys.platform != "win32" and os.open in os.supports_dir_fd - -_ALL_INSTANCES: Final[WeakValueDictionary[int, SoftReadWriteLock]] = WeakValueDictionary() -_ALL_INSTANCES_LOCK: threading.Lock = threading.Lock() -_SINGLETONS_UNDER_CONSTRUCTION: Final[set[Path]] = set() - - -class _SoftRWMeta(type): - _instances: WeakValueDictionary[Path, SoftReadWriteLock] - _instances_lock: threading.RLock - - def __call__( # ruff:ignore[too-many-arguments] # forwards the public constructor's documented parameters - cls, - lock_file: str | os.PathLike[str], - timeout: float = -1, - *, - blocking: bool = True, - is_singleton: bool = True, - heartbeat_interval: float = 30.0, - stale_threshold: float | None = None, - poll_interval: float = 0.25, - ) -> SoftReadWriteLock: - _ensure_current_process() - if not is_singleton: - return super().__call__( - lock_file, - timeout, - blocking=blocking, - is_singleton=is_singleton, - heartbeat_interval=heartbeat_interval, - stale_threshold=stale_threshold, - poll_interval=poll_interval, - ) - - normalized = Path(lock_file).resolve() - with cls._instances_lock: - instance = cls._instances.get(normalized) - if instance is None: - if normalized in _SINGLETONS_UNDER_CONSTRUCTION: # pragma: needs fork - msg = f"Singleton lock construction is already active for {lock_file!s}" - raise RuntimeError(msg) - construction_pid = os.getpid() - _SINGLETONS_UNDER_CONSTRUCTION.add(normalized) - try: - instance = super().__call__( - lock_file, - timeout, - blocking=blocking, - is_singleton=is_singleton, - heartbeat_interval=heartbeat_interval, - stale_threshold=stale_threshold, - poll_interval=poll_interval, - ) - finally: - _SINGLETONS_UNDER_CONSTRUCTION.discard(normalized) - if os.getpid() != construction_pid: # pragma: needs fork - msg = "Lock construction cannot continue after fork; construct a new lock in the child" - raise RuntimeError(msg) - cls._instances[normalized] = instance - elif instance.timeout != timeout or instance.blocking != blocking: - msg = ( - f"Singleton lock created with timeout={instance.timeout}, blocking={instance.blocking}," - f" cannot be changed to timeout={timeout}, blocking={blocking}" - ) - raise ValueError(msg) - return instance - - -class SoftReadWriteLock(metaclass=_SoftRWMeta): - """ - Cross-process and cross-host reader/writer lock built on :class:`SoftFileLock` primitives. - - Use this class instead of :class:`~filelock.ReadWriteLock` when the lock file lives on a network - filesystem (NFS, Lustre with ``-o flock``, HPC cluster shared storage). ``ReadWriteLock`` is backed - by SQLite and cannot run on NFS because SQLite's ``fcntl`` locking is unreliable there. - - Layout on disk for a lock at ``foo.lock``: - - - ``foo.lock.state`` — a :class:`SoftFileLock` taken only during state transitions (microseconds). - - ``foo.lock.write`` — writer marker; its presence means a writer is claiming or holding the lock. - - ``foo.lock.readers/..`` — one file per reader. - - Each marker stores a random token (``secrets.token_hex(16)``), the holder's pid, and the holder's - hostname. A daemon heartbeat thread refreshes ``mtime`` on every held marker. A marker whose mtime - has not advanced in ``stale_threshold`` seconds may be evicted by any process on any host, giving - correct behavior when a compute node crashes with a lock held. - - Writer acquire is two-phase and writer-preferring: phase 1 claims ``.write`` (blocking any new - reader), phase 2 waits for existing readers to drain. Writer starvation is impossible. - - Reentrancy, upgrade/downgrade rules, thread pinning, and singleton caching by resolved path match - :class:`~filelock.ReadWriteLock`. - - Forking invalidates the inherited instance in the child so the child cannot double-own the lock with its parent; - ``release()`` on that instance is a no-op, and the child must construct a new instance if it needs a lock. - - Trust boundary: protects against same-UID non-cooperating processes (one host or cross-host) and - same-host different-UID users via ``0o600`` / ``0o700`` permissions. Does not protect against root - compromise, NTP tampering on same-UID cross-host nodes, or multi-tenant mounts where hostile - co-tenants share the UID. - - :param lock_file: path to the lock file; sidecar state/write/readers live next to it - :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention - :param is_singleton: if ``True``, reuse existing instances for the same resolved path - :param heartbeat_interval: seconds between heartbeat refreshes; default 30 s - :param stale_threshold: seconds of ``mtime`` inactivity before a marker is stale; defaults to - ``3 * heartbeat_interval``, matching etcd's ``LeaseKeepAlive`` convention - :param poll_interval: seconds between acquire retries under contention; default 0.25 s - - .. versionadded:: 3.27.0 - - """ - - _instances: WeakValueDictionary[Path, SoftReadWriteLock] = WeakValueDictionary() - _instances_lock = threading.RLock() - - def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option - self, - lock_file: str | os.PathLike[str], - timeout: float = -1, - *, - blocking: bool = True, - is_singleton: bool = True, # ruff:ignore[unused-method-argument] # consumed by _SoftRWMeta.__call__ - heartbeat_interval: float = 30.0, - stale_threshold: float | None = None, - poll_interval: float = 0.25, - ) -> None: - self._creator_pid = os.getpid() - if heartbeat_interval <= 0: - msg = f"heartbeat_interval must be positive, got {heartbeat_interval}" - raise ValueError(msg) - if stale_threshold is None: - stale_threshold = heartbeat_interval * 3 - if stale_threshold <= heartbeat_interval: - msg = f"stale_threshold must exceed heartbeat_interval ({stale_threshold} <= {heartbeat_interval})" - raise ValueError(msg) - if poll_interval <= 0: - msg = f"poll_interval must be positive, got {poll_interval}" - raise ValueError(msg) - - self.lock_file: str = os.fspath(lock_file) - self.timeout: float = timeout - self.blocking: bool = blocking - self.heartbeat_interval: float = heartbeat_interval - self.stale_threshold: float = stale_threshold - self.poll_interval: float = poll_interval - - self._paths = _Paths( - state=f"{self.lock_file}.state", - write=f"{self.lock_file}.write", - readers=f"{self.lock_file}.readers", - ) - ensure_directory_exists(self.lock_file) - self._locks = _Locks( - internal=threading.Lock(), - transaction=threading.Lock(), - state=SoftFileLock(self._paths.state, timeout=-1), - ) - self._readers_dir_fd: int | None = None - self._readers_dir_fd_token: int | None = None - self._hold: _Hold | None = None - self._closed: bool = False - - with _ALL_INSTANCES_LOCK: - _ALL_INSTANCES[id(self)] = self - _register_fork_object(self) - - @classmethod - def _reset_class_after_fork(cls) -> None: # pragma: forked child - global _ALL_INSTANCES_LOCK # ruff:ignore[global-statement] # rebinds the module lock to a fresh one in the fork child - _ALL_INSTANCES_LOCK = threading.Lock() - cls._instances = WeakValueDictionary() - cls._instances_lock = threading.RLock() - _SINGLETONS_UNDER_CONSTRUCTION.clear() - - @contextmanager - def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]: - """ - Context manager that acquires and releases a shared read lock. - - Falls back to instance defaults for *timeout* and *blocking* when ``None``. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default - - :raises RuntimeError: if a write lock is already held on this instance - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - self.acquire_read(timeout, blocking=blocking) - try: - yield - finally: - self.release() - - @contextmanager - def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]: - """ - Context manager that acquires and releases an exclusive write lock. - - Falls back to instance defaults for *timeout* and *blocking* when ``None``. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default - - :raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - self.acquire_write(timeout, blocking=blocking) - try: - yield - finally: - self.release() - - def acquire_read(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy: - """ - Acquire a shared read lock. - - If this instance already holds a read lock, the lock level is incremented (reentrant). Attempting to acquire a - read lock while holding a write lock raises :class:`RuntimeError` (downgrade not allowed). On the 0→1 - transition a daemon heartbeat thread is started that refreshes the reader marker's ``mtime`` every - ``heartbeat_interval`` seconds so peers on other hosts do not evict the marker as stale. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block - indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable; - ``None`` uses the instance default - - :returns: a proxy that can be used as a context manager to release the lock - - :raises RuntimeError: if a write lock is already held on this instance, if this instance was invalidated by - :func:`os.fork`, or if :meth:`close` was called - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - return self._acquire("read", timeout, blocking=blocking) - - def acquire_write(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy: - """ - Acquire an exclusive write lock. - - If this instance already holds a write lock from the same thread, the lock level is incremented (reentrant). - Attempting to acquire a write lock while holding a read lock raises :class:`RuntimeError` (upgrade not - allowed). Write locks are pinned to the acquiring thread: a different thread trying to re-enter also raises - :class:`RuntimeError`. - - Writer acquisition runs in two phases. Phase 1 atomically claims ``.write`` via ``O_CREAT | O_EXCL``, - which immediately blocks any new reader on any host. Phase 2 waits for existing readers to drain. Writer - starvation is impossible: new readers see ``.write`` during phase 2 and wait behind the pending writer. - - :param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block - indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable; - ``None`` uses the instance default - - :returns: a proxy that can be used as a context manager to release the lock - - :raises RuntimeError: if a read lock is already held, if a write lock is held by a different thread, if this - instance was invalidated by :func:`os.fork`, or if :meth:`close` was called - :raises Timeout: if the lock cannot be acquired within *timeout* seconds - - """ - return self._acquire("write", timeout, blocking=blocking) - - @classmethod - def get_lock( - cls, - lock_file: str | os.PathLike[str], - timeout: float = -1, - *, - blocking: bool = True, - ) -> SoftReadWriteLock: - """ - Return the singleton :class:`SoftReadWriteLock` for *lock_file*. - - :param lock_file: path to the lock file; sidecar state/write/readers live next to it - :param timeout: maximum wait time in seconds; ``-1`` means block indefinitely - :param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable - - :returns: the singleton lock instance - - :raises ValueError: if an instance already exists for this path with different *timeout* or *blocking* values - - """ - return cls(lock_file, timeout, blocking=blocking) - - def close(self) -> None: - """ - Release any held lock and release internal filesystem resources. - - Idempotent. After calling this method the instance can no longer acquire locks — subsequent acquires raise - :class:`RuntimeError`. A fork-invalidated instance is closed without raising. - """ - if self._creator_pid != os.getpid(): # pragma: forked child - return - self.release(force=True) - with self._locks.internal: - if self._closed: - return - self._closed = True - if self._readers_dir_fd is not None: # pragma: needs dir-fd - with _fork_transition(): - if self._readers_dir_fd_token is not None: # pragma: needs dir-fd - _unregister_owned_descriptor(self._readers_dir_fd_token) - self._readers_dir_fd_token = None - fd, self._readers_dir_fd = self._readers_dir_fd, None - with suppress(OSError): # pragma: needs dir-fd - os.close(fd) - - def release(self, *, force: bool = False) -> None: - """ - Release one level of the current lock. - - When the lock level reaches zero the heartbeat thread is stopped and the held marker file is unlinked. On a - fork-invalidated instance (that is, the child of a :func:`os.fork` call made while the parent held a lock) - this method is a no-op so inherited ``with`` blocks can unwind cleanly in the child. - - :param force: if ``True``, release the lock completely regardless of the current lock level - - :raises RuntimeError: if no lock is currently held and *force* is ``False`` - - """ - if self._creator_pid != os.getpid(): # pragma: forked child - return - with self._locks.internal: - hold = self._hold - if hold is None: - if force: - return - msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held" - raise RuntimeError(msg) - if force: - hold.level = 0 - else: - hold.level -= 1 - if hold.level > 0: - return - self._hold = None - - # Order matters: signal → join → unlink. A late tick on a deleted marker is harmless and the - # heartbeat's token check would catch a re-acquisition race, but joining first removes that race. - hold.heartbeat_stop.set() - hold.heartbeat_thread.join(timeout=self.heartbeat_interval + 1.0) - if hold.is_reader: - _unlink(hold.marker_name, dir_fd=self._readers_dir_fd) - else: - self._unlink_writer_marker_if_ours(hold.token) - - def _unlink_writer_marker_if_ours(self, token: str) -> None: - # Remove the writer marker only while it still carries our token. If this holder was paused long - # enough (a stop-the-world GC pause, SIGSTOP, a suspended VM) for a peer to evict the marker as - # stale and claim the writer slot itself, the file now at .write is the peer's live marker; - # unlinking it by path would let a second writer through and break mutual exclusion. The state lock - # serializes this against a concurrent break/claim, and the heartbeat is already stopped, so the - # token we read is authoritative. Mirrors the token re-check the stale-break path already does. - with self._locks.state: - if (read := _read_marker(self._paths.write)) is None: - return - info, _ = read - if info is None or not hmac.compare_digest(info.token, token): - return - _unlink(self._paths.write) - - def _acquire( - self, - mode: _Mode, - timeout: float | None, - *, - blocking: bool | None, - ) -> AcquireReturnProxy: - if self._creator_pid != os.getpid(): # pragma: forked child - msg = f"SoftReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance" - raise RuntimeError(msg) - timeout = self.timeout if timeout is None else timeout - blocking = self.blocking if blocking is None else blocking - - with self._locks.internal: - if self._closed: - msg = f"SoftReadWriteLock on {self.lock_file} has been closed" - raise RuntimeError(msg) - if self._hold is not None: - return self._validate_reentrant(mode) - - start = time.perf_counter() - if not blocking: - acquired = self._locks.transaction.acquire(blocking=False) - elif timeout == -1: - acquired = self._locks.transaction.acquire(blocking=True) - else: - acquired = self._locks.transaction.acquire(blocking=True, timeout=timeout) - if not acquired: - raise Timeout(self.lock_file) from None - try: - return self._do_acquire_inner(mode, timeout, start, blocking=blocking) - finally: - self._locks.transaction.release() - - def _do_acquire_inner( - self, - mode: _Mode, - effective_timeout: float, - start: float, - *, - blocking: bool, - ) -> AcquireReturnProxy: - with self._locks.internal: - if self._hold is not None: - return self._validate_reentrant(mode) - deadline = None if effective_timeout == -1 else start + effective_timeout - token = secrets.token_hex(16) - if mode == "write": - marker_name, is_reader = self._acquire_writer_slot(token, deadline=deadline, blocking=blocking) - else: - marker_name, is_reader = self._acquire_reader_slot(token, deadline=deadline, blocking=blocking) - stop_event = threading.Event() - heartbeat = _HeartbeatThread( - refresh=self._refresh_marker, - interval=self.heartbeat_interval, - stop_event=stop_event, - name=f"filelock-heartbeat-{id(self):x}", - ) - # Publish the hold and start its heartbeat under one internal-lock section, so a concurrent release() never - # observes a hold whose thread has not started and joins it. If the OS refuses the thread, clear the hold and - # unlink the marker we claimed: left in place, a peer evicts it as stale and acquires while this instance still - # believes it holds the lock. - start_error: BaseException | None = None - with self._locks.internal: - self._hold = _Hold( - level=1, - mode=mode, - write_thread_id=threading.get_ident() if mode == "write" else None, - marker_name=marker_name, - is_reader=is_reader, - token=token, - heartbeat_thread=heartbeat, - heartbeat_stop=stop_event, - ) - try: - heartbeat.start() - except BaseException as error: # ruff:ignore[blind-except] # clear the slot below and re-raise - self._hold = None - start_error = error - if start_error is not None: - if is_reader: - _unlink(marker_name, dir_fd=self._readers_dir_fd) - else: - self._unlink_writer_marker_if_ours(token) - raise start_error - return AcquireReturnProxy(lock=self) - - def _validate_reentrant(self, mode: _Mode) -> AcquireReturnProxy: - hold = self._hold - assert hold is not None # ruff:ignore[assert] # callers dispatch here only inside the self._hold is not None branch - if hold.mode != mode: - opposite = "write" if mode == "read" else "read" - direction = "downgrade" if mode == "read" else "upgrade" - msg = ( - f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): " - f"already holding a {opposite} lock ({direction} not allowed)" - ) - raise RuntimeError(msg) - if mode == "write" and (cur := threading.get_ident()) != hold.write_thread_id: - msg = ( - f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) " - f"from thread {cur} while it is held by thread {hold.write_thread_id}" - ) - raise RuntimeError(msg) - hold.level += 1 - return AcquireReturnProxy(lock=self) - - def _acquire_writer_slot( - self, - token: str, - *, - deadline: float | None, - blocking: bool, - ) -> tuple[str, bool]: - # Phase 2 scans readers/ via dirfd (where supported), so we need it open even though writers never - # create files inside. - self._open_readers_dir() - - def try_claim_writer() -> bool: - with self._locks.state: - return self._claim_writer_marker(token) - - def readers_drained_touching() -> bool: - with self._locks.state: - # A peer may replace an expired marker while this process pauses. Refresh only our token; touching a - # successor's marker would let this acquisition proceed without owning the writer slot. - if not self._touch_writer_marker_if_ours(token) and not self._claim_writer_marker(token): - return False - self._break_stale_readers(time.time()) - return not self._any_readers() - - self._wait_for(try_claim_writer, deadline=deadline, blocking=blocking) - try: - self._wait_for(readers_drained_touching, deadline=deadline, blocking=blocking) - except Timeout: - # Give up our writer claim so readers can make progress again, but only while the marker is - # still ours: a peer may have evicted it as stale and claimed the slot while phase 2 waited. - self._unlink_writer_marker_if_ours(token) - raise - return self._paths.write, False - - def _claim_writer_marker(self, token: str) -> bool: - # Claim the writer slot for ``token``. Must be called holding ``self._locks.state``. Evicts a - # stale marker first, then refuses to claim while a live ``.write`` exists so a peer holding the - # slot is waited out instead of overwritten. - _break_stale_marker(self._paths.write, stale_threshold=self.stale_threshold, now=time.time()) - if _file_exists(self._paths.write): - return False - try: - _atomic_create_marker(self._paths.write, token) - except FileExistsError: - return False - return True - - def _touch_writer_marker_if_ours(self, token: str) -> bool: - # Refresh the writer marker through a single O_NOFOLLOW fd, but only while it still carries our - # token. Returns False when the marker is gone or now belongs to a peer that reclaimed the slot, - # so the caller can re-claim rather than keep a stranger's marker alive. Mirrors _refresh_marker. - fd = _open_marker(self._paths.write) - if fd is None: - return False - try: - try: - data = os.read(fd, _MAX_MARKER_SIZE + 1) - except OSError: # pragma: no cover - e.g. EAGAIN from a hostile FIFO that has a writer attached - return False - info = _parse_marker_bytes(data) - if info is None or not hmac.compare_digest(info.token, token): - return False - with suppress(OSError): - touch(self._paths.write, fd=fd) - return True - finally: - os.close(fd) - - def _acquire_reader_slot( - self, - token: str, - *, - deadline: float | None, - blocking: bool, - ) -> tuple[str, bool]: - self._open_readers_dir() - reader_name = f"{uuid.uuid4().hex}.{os.getpid()}" - dir_fd = self._readers_dir_fd - full_reader_path = str(Path(self._paths.readers) / reader_name) - - def try_claim_reader() -> bool: - with self._locks.state: - _break_stale_marker(self._paths.write, stale_threshold=self.stale_threshold, now=time.time()) - if _file_exists(self._paths.write): - return False - if dir_fd is not None: # pragma: needs dir-fd - _atomic_create_marker(reader_name, token, dir_fd=dir_fd) - else: # pragma: win32 cover - _atomic_create_marker(full_reader_path, token) - return True - - self._wait_for(try_claim_reader, deadline=deadline, blocking=blocking) - return (reader_name if dir_fd is not None else full_reader_path), True - - def _wait_for( - self, - predicate: Callable[[], bool], - *, - deadline: float | None, - blocking: bool, - ) -> None: - while True: - if predicate(): - return - now = time.perf_counter() - if not blocking: - raise Timeout(self.lock_file) - if deadline is not None and now >= deadline: - raise Timeout(self.lock_file) - sleep_for = self.poll_interval - if deadline is not None: - sleep_for = min(sleep_for, max(deadline - now, 0.0)) - time.sleep(sleep_for) - - def _open_readers_dir(self) -> None: - readers_path = Path(self._paths.readers) - with suppress(FileExistsError): - readers_path.mkdir(mode=0o700) - # mkdir has no O_NOFOLLOW, so verify via lstat that we did not land on an attacker-placed symlink - # or a regular file before we open or scan inside. - st = os.lstat(self._paths.readers) - if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode): - msg = f"{self._paths.readers} exists but is not a directory or is a symlink; refusing to use it" - raise RuntimeError(msg) - if self._readers_dir_fd is None and _SUPPORTS_DIR_FD: # pragma: needs dir-fd - with _fork_transition(): - fd = os.open(self._paths.readers, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | _O_NOFOLLOW) - try: - token = _register_owned_descriptor(fd) - except BaseException as registration_error: - try: - os.close(fd) - except BaseException as close_error: # ruff:ignore[blind-except] # both errors surface via the group below - _raise_grouped_errors( - "reader directory registration and descriptor close both failed", - registration_error, - close_error, - ) - raise - self._readers_dir_fd = fd - self._readers_dir_fd_token = token - - def _any_readers(self) -> bool: - with closing(self._iter_reader_entries()) as entries: - for _ in entries: - return True - return False - - def _iter_reader_entries(self) -> Generator[tuple[str, bool]]: - """ - Yield ``(name, dirfd_relative)`` pairs for every live reader marker. - - ``dirfd_relative`` is ``True`` when *name* should be passed to ``dir_fd=``-aware syscalls; ``False`` - when *name* is a full path because dirfd-relative I/O is unavailable on this platform. - - A consumer that stops early must close this generator: while suspended it holds the ``scandir`` handle open, - and leaving that to the collector surfaces as an unraisable exception inside whatever runs next. - """ - if self._readers_dir_fd is not None: # pragma: needs dir-fd - with os.scandir(self._readers_dir_fd) as it: - for entry in it: - if not _is_housekeeping_name(entry.name): - yield entry.name, True - return - readers_path = Path(self._paths.readers) # pragma: win32 cover - with os.scandir(readers_path) as it: # pragma: win32 cover - for entry in it: # pragma: win32 cover - if not _is_housekeeping_name(entry.name): # pragma: win32 cover - yield str(readers_path / entry.name), False # pragma: win32 cover - - def _break_stale_readers(self, now: float) -> None: - names: list[tuple[str, int | None]] = [] - try: - with closing(self._iter_reader_entries()) as entries: - for name, dirfd_relative in entries: - names.append((name, self._readers_dir_fd if dirfd_relative else None)) - except OSError: # pragma: no cover - transient NFS scandir hiccup - return - for name, fd in names: - _break_stale_marker(name, stale_threshold=self.stale_threshold, now=now, dir_fd=fd) - - def _refresh_marker(self) -> bool: - with self._locks.internal: - hold = self._hold - if hold is None: # pragma: no cover - race between stop_event.set and join - return False - marker_name = hold.marker_name - token = hold.token - dir_fd = self._readers_dir_fd if hold.is_reader else None - - # Open once with O_NOFOLLOW and touch that exact descriptor. Refreshing through the verified fd - # (instead of re-opening by name) closes the window where a peer unlinks our marker and drops a symlink - # or a different file at the path between the read and the touch: utime then lands on the inode we - # verified, or nowhere. Only an unambiguous loss stops the heartbeat: the marker gone, or a peer's token - # in its place. A transient filesystem error (ESTALE / EIO on the NFS-style filesystems this lock targets) - # keeps the heartbeat alive to retry next tick, the way the touch below already does, so one blip does not - # silently drop a held lock. - try: - fd = _open_marker_fd(marker_name, dir_fd=dir_fd) - except FileNotFoundError: - return False - except OSError: - return True - try: - try: - data = _read_marker_fd(fd) - except OSError: # a transient read error or EAGAIN from a hostile FIFO; retry rather than drop the lock - return True - info = _parse_marker_bytes(data) - # Token mismatch means another process already evicted our marker and created its own; stop the - # thread so it does not keep a stranger's file alive. - if info is None or not hmac.compare_digest(info.token, token): - return False - # A transient touch failure (ESTALE / EIO on the NFS-style filesystems this lock targets) must not - # kill the heartbeat thread: the read above just confirmed the marker is still ours, so swallow the - # error and retry on the next tick rather than letting the lease lapse while we still hold the lock. - with suppress(OSError): - touch(marker_name, fd=fd) - return True - finally: - os.close(fd) - - def _reset_after_fork_in_child(self) -> None: # pragma: forked child - self._locks = _Locks( - internal=threading.Lock(), - transaction=threading.Lock(), - state=self._locks.state, - ) - self._hold = None - self._readers_dir_fd = None - self._readers_dir_fd_token = None - - -class _HeartbeatThread(threading.Thread): - def __init__( - self, - refresh: Callable[[], bool], - interval: float, - stop_event: threading.Event, - name: str, - ) -> None: - super().__init__(name=name, daemon=True) - self._refresh = refresh - self._interval = interval - self._stop_event = stop_event - - def run(self) -> None: - while not self._stop_event.wait(self._interval): - if not self._refresh(): - self._stop_event.set() - return - - -def _read_marker(name: str, *, dir_fd: int | None = None) -> tuple[_MarkerInfo | None, float] | None: - fd = _open_marker(name, dir_fd=dir_fd) - if fd is None: - return None - try: - st = os.fstat(fd) - # A legitimate marker is a regular file, so anything else at the path (a FIFO, say) is reported as a - # malformed marker (its mtime still drives stale eviction) without being read. Reading is where - # platforms diverge: an empty non-blocking read yields 0 bytes on Linux/macOS but EAGAIN on FreeBSD, - # and the EAGAIN used to abort the stale-break and wedge the acquire until timeout (#587). - if not stat.S_ISREG(st.st_mode): # pragma: needs fifo - return None, st.st_mtime - data = os.read(fd, _MAX_MARKER_SIZE + 1) - except OSError: # pragma: no cover - marker vanished or turned unreadable between open and read - return None - finally: - os.close(fd) - return _parse_marker_bytes(data), st.st_mtime - - -def _read_marker_fd(fd: int) -> bytes: - return os.read(fd, _MAX_MARKER_SIZE + 1) - - -def _open_marker_fd(name: str, *, dir_fd: int | None = None) -> int: - # The file is ours; these guard a hostile mid-flight swap. O_NOFOLLOW rejects a symlink; O_NONBLOCK keeps - # a real FIFO from blocking the open forever, so it reads as a malformed marker instead of wedging a peer - # that holds the state lock. - flags = os.O_RDONLY | _O_NOFOLLOW | _O_NONBLOCK - return os.open(name, flags, dir_fd=dir_fd) if _SUPPORTS_DIR_FD and dir_fd is not None else os.open(name, flags) - - -def _open_marker(name: str, *, dir_fd: int | None = None) -> int | None: - try: - return _open_marker_fd(name, dir_fd=dir_fd) - except OSError: - return None - - -def _parse_marker_bytes(data: bytes) -> _MarkerInfo | None: - # Trust nothing about attacker-controlled markers; any deviation returns None so callers fall through - # to stale cleanup. ``re.match`` caches compiled patterns internally, so the regex is built only once - # despite being defined inline. - if not data or len(data) > _MAX_MARKER_SIZE: - return None - try: - text = data.decode("ascii") - except UnicodeDecodeError: - return None - match = re.match( - r""" - \A # start of string - (?P [0-9a-f]{32} ) \n # 128-bit hex token - (?P [1-9][0-9]{0,9} ) \n # decimal pid: no leading zero, ≤ 10 digits - (?P [\x21-\x7e]{1,253}) # printable non-whitespace ASCII (RFC 1123 hostname limit) - \n* # tolerate sloppy writers that append extra newlines - \Z # end of string - """, - text, - re.VERBOSE, - ) - if match is None: - return None - pid = int(match["pid"], 10) - if pid > 2**31 - 1: - return None - return _MarkerInfo(token=match["token"], pid=pid, hostname=match["hostname"]) - - -def _unlink(name: str, *, dir_fd: int | None = None) -> None: - with suppress(FileNotFoundError): - if _SUPPORTS_DIR_FD and dir_fd is not None: # pragma: needs dir-fd - # Path.unlink has no dir_fd support, so we stay on os.unlink for the dirfd path. - os.unlink(name, dir_fd=dir_fd) - else: - Path(name).unlink() - - -def _break_stale_marker( # ruff:ignore[too-many-return-statements] # each return is a distinct abort/commit point in the break protocol - name: str, - *, - stale_threshold: float, - now: float, - dir_fd: int | None = None, -) -> bool: - # Atomic break pattern: read → rename to unique break-name → re-verify → unlink. The rename gives us a - # private name nobody else can touch; if the re-verify sees a newer mtime or a different token, the - # legitimate holder's heartbeat fired between read and rename and we must abort (leaving the .break.* - # file behind rather than rollback-renaming, because rollback is itself racy). - if (read_result := _read_marker(name, dir_fd=dir_fd)) is None: - return False - info_before, mtime_before = read_result - if now - mtime_before <= stale_threshold: - return False - if info_before is None: - _unlink(name, dir_fd=dir_fd) - return True - - break_name = f"{name}{_BREAK_SUFFIX}.{os.getpid()}.{secrets.token_hex(16)}" - try: - if _SUPPORTS_DIR_FD and dir_fd is not None: # pragma: needs dir-fd - os.rename(name, break_name, src_dir_fd=dir_fd, dst_dir_fd=dir_fd) - else: - Path(name).rename(break_name) - except OSError: # pragma: no cover - race where the marker vanishes between read and rename - return False - - read_after = _read_marker(break_name, dir_fd=dir_fd) - if read_after is None: # pragma: no cover - race where a peer unlinks the break-name file - return False - info_after, mtime_after = read_after - if info_after is None: # pragma: no cover - content replaced post-rename by a racing peer - _unlink(break_name, dir_fd=dir_fd) - return True - if not hmac.compare_digest(info_before.token, info_after.token): # pragma: no cover - race only - return False - if mtime_after > mtime_before: # pragma: no cover - heartbeat raced our rename - return False - _unlink(break_name, dir_fd=dir_fd) - return True - - -def _atomic_create_marker(name: str, token: str, *, dir_fd: int | None = None) -> None: - # O_NOFOLLOW blocks the symlink-overwrite attack where an attacker pre-creates the marker path as a - # symlink pointing at a victim file. Mode 0o600 keeps the token unreadable to other users. - flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY | _O_NOFOLLOW - if _SUPPORTS_DIR_FD and dir_fd is not None: # pragma: needs dir-fd - fd = os.open(name, flags, 0o600, dir_fd=dir_fd) - else: - fd = os.open(name, flags, 0o600) - # Write the whole record before the marker counts as created. On failure remove it only while the path still names - # the file we opened, so a rollback never deletes a marker a concurrent reader recreated at this name. - identity: tuple[int, int] | None = None - try: - st = os.fstat(fd) - identity = st.st_dev, st.st_ino - write_all(fd, f"{token}\n{os.getpid()}\n{socket.gethostname()}\n".encode("ascii")) - except BaseException: - os.close(fd) - if identity is not None and _same_file(name, identity, dir_fd=dir_fd): - _unlink(name, dir_fd=dir_fd) - raise - else: - os.close(fd) - - -def _same_file(name: str, identity: tuple[int, int], *, dir_fd: int | None) -> bool: - try: - st = os.lstat(name, dir_fd=dir_fd) if _SUPPORTS_DIR_FD and dir_fd is not None else os.lstat(name) - except OSError: - return False - return (st.st_dev, st.st_ino) == identity - - -def _file_exists(path: str) -> bool: - try: - st = os.lstat(path) - except FileNotFoundError: - return False - return stat.S_ISREG(st.st_mode) - - -def _is_housekeeping_name(name: str) -> bool: - return name.startswith(".") or _BREAK_SUFFIX in name - - -@dataclass(frozen=True) -class _Paths: - state: str - write: str - readers: str - - -@dataclass -class _Locks: - internal: threading.Lock - transaction: threading.Lock - state: SoftFileLock - - -@dataclass(frozen=True) -class _MarkerInfo: - token: str - pid: int - hostname: str - - -@dataclass -class _Hold: - """Everything that exists only while a lock is held; ``None`` when the instance has no lock.""" - - level: int - mode: _Mode - write_thread_id: int | None - marker_name: str - is_reader: bool - token: str - heartbeat_thread: _HeartbeatThread - heartbeat_stop: threading.Event - - -def _cleanup_all_instances() -> None: # pragma: no cover - runs from atexit at interpreter shutdown - for instance in list(_ALL_INSTANCES.values()): - with suppress(Exception): - instance.release(force=True) - - -atexit.register(_cleanup_all_instances) -_register_fork_class(SoftReadWriteLock) - - -__all__ = [ - "SoftReadWriteLock", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_strict.py b/bundle/python-cpu/Lib/site-packages/filelock/_strict.py deleted file mode 100644 index e71eb3350fd8525f00aa999516d49225136d4c9d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_strict.py +++ /dev/null @@ -1,874 +0,0 @@ -from __future__ import annotations - -import contextlib -import errno -import os -import secrets -import stat -import sys -import tempfile -import time -from dataclasses import dataclass -from errno import EACCES, EEXIST, ENOENT, ENOSYS, EPERM, ESTALE, EXDEV -from pathlib import Path -from typing import TYPE_CHECKING, Final, Literal, cast - -from ._api import BaseFileLock, _canonical, _raise_cleanup_errors -from ._error import SoftFileLockProtocolError -from ._identity import host_name, process_start_token -from ._soft_protocol import STRICT_SOFT_SENTINEL_RECORD -from ._util import ensure_directory_exists, write_all - -if TYPE_CHECKING: - from collections.abc import Iterator - -StrictSoftFileClaimState = Literal["intent", "held"] - -_CLAIM_STATES: Final[frozenset[str]] = frozenset({"intent", "held"}) -_COORDINATION_SUFFIX: Final[str] = ".filelock" -_CLAIM_DIRECTORY_NAME: Final[str] = "claims" -_CLAIM_MAGIC: Final[str] = "filelock-strict-v1" -_CLAIM_RECORD_LIMIT: Final[int] = 1024 -_CLAIM_NAME_PART_COUNT: Final[int] = 3 -_TOKEN_HEX_LENGTH: Final[int] = 32 -_PRIVATE_RECORD_MARKER: Final[str] = ".private-v1-" -_PRIVATE_RECORD_SUFFIX: Final[str] = ".tmp" -_PRIVATE_RECORD_RANDOM_HEX_LENGTH: Final[int] = 32 -_PRIVATE_RECORD_GRACE: Final[float] = 2.0 -_UNLINK_MAX_RETRIES: Final[int] = 10 -#: How long a scan waits out a claim held in Windows' delete-pending state before treating it as unreadable. -_CLAIM_READ_GRACE: Final[float] = 0.5 -_CLAIM_READ_RETRY: Final[float] = 0.002 -#: Windows opens descriptors in text mode by default, which rewrites newlines and truncates a record at a control byte. -#: The claim and sentinel records are exact binary, so every record descriptor must be binary; POSIX ignores the flag. -_O_BINARY: Final[int] = getattr(os, "O_BINARY", 0) -_LEGACY_SENTINEL: Final[bytes] = STRICT_SOFT_SENTINEL_RECORD.encode() -_WINDOWS_HARD_LINK_UNSUPPORTED: Final[frozenset[int]] = frozenset({1, 17, 50}) - -# Termux/Android CPython ships without os.link (bionic long had only linkat), so the strict backend's whole hard-link -# mechanism is absent there. Probe once and gate every os.link reference on it, so importing filelock still works and -# only an actual StrictSoftFileLock acquire reports the missing capability. -_HAS_LINK: Final[bool] = hasattr(os, "link") - -# Probe dir_fd capability once at import. A per-call ``os.unlink in os.supports_dir_fd`` check flips to False the moment -# a test mocks os.unlink, silently diverting the code to a different branch than the one under test. -_OPEN_SUPPORTS_DIR_FD: Final[bool] = os.open in os.supports_dir_fd -_UNLINK_SUPPORTS_DIR_FD: Final[bool] = os.unlink in os.supports_dir_fd -_STAT_SUPPORTS_DIR_FD: Final[bool] = os.stat in os.supports_dir_fd -_LINK_SUPPORTS_DIR_FD: Final[bool] = _HAS_LINK and os.link in os.supports_dir_fd - - -def _probe_link_follow_symlinks() -> bool: - # os.supports_follow_symlinks lists os.link on PyPy, but its linkat then rejects follow_symlinks=False with EINVAL, - # and Windows raises NotImplementedError for the option outright. Link a throwaway file for real so the answer - # reflects the runtime rather than its advertisement, and treat any failure as "not honored": the option only - # hardens a source this process created with O_EXCL, so skipping it is safe, and a real environment fault surfaces - # when the actual link runs. - if not _HAS_LINK: - return False - try: - with tempfile.TemporaryDirectory() as directory: - source = Path(directory, "probe-source") - source.touch() - os.link(source, Path(directory, "probe-link"), follow_symlinks=False) - except (OSError, NotImplementedError, ValueError): - return False - return True - - -_LINK_HONORS_FOLLOW_SYMLINKS: Final[bool] = _probe_link_follow_symlinks() - - -def _probe_hard_link_unsupported_errnos() -> frozenset[int]: - # GraalPy's errno omits ENOTSUP, so importing the name outright breaks every runtime that ships without it. ENOTSUP - # wins wherever it exists, leaving every runtime that names it unchanged. EOPNOTSUPP only stands in for the ones - # that do not, and it approximates rather than matches: the two codes agree on Linux but differ on macOS/BSD and on - # Windows. Where neither exists a runtime that cannot name "operation not supported" cannot raise it either, and - # ENOSYS/EXDEV still classify the link failures it can raise. - not_supported = getattr(errno, "ENOTSUP", getattr(errno, "EOPNOTSUPP", None)) - return frozenset({ENOSYS, EXDEV} if not_supported is None else {ENOSYS, EXDEV, not_supported}) - - -_HARD_LINK_UNSUPPORTED_ERRNOS: Final[frozenset[int]] = _probe_hard_link_unsupported_errnos() - - -class StrictSoftFileLock(BaseFileLock): - """Portable fail-closed lock based on immutable owner claims.""" - - _preserve_lock_file_supported: bool = True - _on_acquired_supported: bool = False - #: Age cannot clear a strict claim: expiring one on a clock is the overlap the fail-closed contract exists to rule - #: out, so only force_break() removes it. - _lifetime_supported: bool = False - _lifetime_unsupported_reason: str = "a strict claim is never broken by age, only by force_break()" - #: The claim doorway publishes an intent and a held record per owner, so a shared instance must serialize them. - _serialize_transitions: bool = True - #: Contending processes each publish and rescan several files, so back their retries off across a jittered window - #: rather than let them collide on every poll. Seconds; keeps a waiter responsive once it wins. - _poll_backoff_cap: float = 0.05 - - def _acquire(self) -> None: - # Resolve once per acquisition, not per poll: a waiter on a relative path must keep publishing into the - # directory it started waiting in even when another thread changes the working directory mid-wait. - if (claim_root := self._context.claim_root) is None: - claim_root = self._context.claim_root = _canonical(self.lock_file) - lock_path = Path(claim_root) - coordination_directory = Path(f"{lock_path}{_COORDINATION_SUFFIX}") - claim_directory = coordination_directory / _CLAIM_DIRECTORY_NAME - ensure_directory_exists(os.fspath(lock_path)) - _ensure_protocol_directory(self.lock_file, coordination_directory) - _ensure_protocol_directory(self.lock_file, claim_directory) - if (sentinel_fd := _open_or_create_sentinel(self.lock_file, lock_path, self._open_mode())) is None: - return - try: - sentinel_identity = _file_identity(os.fstat(sentinel_fd)) - except BaseException as inspection_error: # preserve inspection and descriptor cleanup errors - try: - os.close(sentinel_fd) - except BaseException as close_error: # ruff:ignore[blind-except] # preserve inspection and descriptor cleanup errors - _raise_cleanup_errors("strict sentinel inspection cleanup failed", inspection_error, close_error) - raise - self._mark_descriptor_pending(sentinel_fd, sentinel_identity) - try: - self._attempt_doorway(claim_directory, sentinel_fd, sentinel_identity) - except BaseException: - if self._context.pending_lock_file_fd == sentinel_fd: - self._discard_doorway(sentinel_fd, sentinel_identity) - raise - - def _attempt_doorway(self, claim_directory: Path, sentinel_fd: int, sentinel_identity: tuple[int, int]) -> None: - if _read_existing_claims(self.lock_file, claim_directory): - self._discard_doorway(sentinel_fd, sentinel_identity) - return - - token = secrets.token_hex(_TOKEN_HEX_LENGTH // 2) - intent_name = _claim_name("intent", token) - intent_path = str(claim_directory / intent_name) - try: - publication_cleanup_error = _publish_record(intent_path, _claim_record(token), self._open_mode()) - except _PrivateRecordReclaimedError: - self._discard_doorway(sentinel_fd, sentinel_identity) - return - except (NotImplementedError, OSError) as error: - _raise_if_hard_links_unsupported(self.lock_file, error) - if isinstance(error, OSError) and error.errno == EEXIST: - self._discard_doorway(sentinel_fd, sentinel_identity) - return - raise - if publication_cleanup_error is not None: # pragma: needs dir-fd - raise publication_cleanup_error - self._context.owner_claim_paths = (intent_path,) - - claims = _read_existing_claims(self.lock_file, claim_directory) - if ( - not claims - or any(claim.state == "held" for claim in claims) - or min(claim.name for claim in claims) != intent_name - ): - self._discard_doorway(sentinel_fd, sentinel_identity) - return - - held_name = _claim_name("held", token) - held_path = str(claim_directory / held_name) - try: - link_cleanup_error = _link_no_replace(claim_directory, intent_name, held_name) - except (NotImplementedError, OSError) as error: - _raise_if_hard_links_unsupported(self.lock_file, error) - raise - self._context.owner_claim_paths = (held_path, intent_path) - if link_cleanup_error is not None: # pragma: needs dir-fd - self._context.owner_claim_paths = () - raise link_cleanup_error - - claims = _read_existing_claims(self.lock_file, claim_directory) - if ( - not {intent_name, held_name}.issubset(claim.name for claim in claims) - or min(_claim_token_key(claim.name) for claim in claims) != f"v1-{token}.claim" - ): - self._discard_doorway(sentinel_fd, sentinel_identity) - return - # Keep the intent claim for the whole hold rather than unlinking it now. The intent has existed, unchanged, - # since this owner published it, so a contender's os.scandir is guaranteed to return it (POSIX only leaves the - # visibility of entries created or removed *during* a scan unspecified). The freshly linked held claim carries - # no such guarantee: a scan that races its creation can miss it. Were the intent removed here, that scan could - # observe neither claim and let a larger-token contender win over this owner. The stable intent is the witness - # that keeps the phase-five min-token decision computed over the true set. Release unlinks both. - self._mark_descriptor_owned(sentinel_fd, sentinel_identity) - - @property - def claims(self) -> tuple[StrictSoftFileClaim, ...]: - """Published claims that block acquisition.""" - return _read_existing_claims(self.lock_file, self._claim_directory) - - def force_break(self, claim_name: str) -> None: - """Remove one named claim, allowing overlap if its owner still holds the protected resource.""" - _validate_force_break_name(claim_name) - _require_exact_name(self._claim_directory, claim_name) - if ( - cleanup_error := _unlink_in_directory(self._claim_directory, claim_name) - ) is not None: # pragma: needs dir-fd - raise cleanup_error - - def _rollback_failed_acquire(self, acquisition_error: BaseException) -> None: - # _acquire already reconciles a failed doorway through _discard_doorway: it either closes the pending - # descriptor or, when a held claim cannot be removed, commits it as owned so a later release retries and - # raises the cleanup errors. A base rollback would release that owned descriptor again and report each - # failure a second time, so leave the reconciled state alone. - if self.is_locked: - return - super()._rollback_failed_acquire(acquisition_error) - - def _reconcile_failed_acquire(self, canonical: str) -> None: - # The acquisition is over, so the next one resolves the working directory again rather than reuse this one's. - if not self.is_locked: - self._context.claim_root = None - super()._reconcile_failed_acquire(canonical) - - def _release(self) -> None: - fd = cast("int", self._context.lock_file_fd) - self._context.claim_root = None - remaining, errors = _unlink_owner_paths(self._context.owner_claim_paths) - self._context.owner_claim_paths = tuple(remaining) - if remaining: - _raise_recorded_errors("strict claim release failed", errors) - self._mark_descriptor_released() - try: - self._close_released_fd(fd, default_suppresses=False) - except BaseException as close_error: # ruff:ignore[blind-except] # preserve claim and sentinel cleanup errors - errors.append(close_error) - if errors: - _raise_recorded_errors("strict release cleanup failed", errors) - - def _discard_doorway(self, fd: int, identity: tuple[int, int]) -> None: - remaining, errors = _unlink_owner_paths(self._context.owner_claim_paths) - self._context.owner_claim_paths = tuple(remaining) - if remaining: - self._mark_descriptor_owned(fd, identity) - _raise_recorded_errors("strict doorway claim cleanup failed", errors) - self._mark_descriptor_released() - try: - self._close_released_fd(fd, default_suppresses=False) - except BaseException as close_error: # ruff:ignore[blind-except] # preserve claim and sentinel cleanup errors - errors.append(close_error) - if errors: - _raise_recorded_errors("strict doorway cleanup failed", errors) - - @property - def _claim_directory(self) -> Path: - if self._context.owner_claim_paths: - return Path(self._context.owner_claim_paths[0]).parent - return Path(f"{_canonical(self.lock_file)}{_COORDINATION_SUFFIX}") / _CLAIM_DIRECTORY_NAME - - -@dataclass(frozen=True) -class StrictSoftFileClaim: - """One parsed strict soft-lock claim.""" - - name: str - state: StrictSoftFileClaimState - token: str - pid: int - hostname: str - #: The owner's process start token, or ``None`` when the platform exposes no proven start time. A strict lock never - #: reclaims a claim on its own, so this identifies the owner for tooling rather than driving any automatic break. - start: int | None = None - - -class _PrivateRecordReclaimedError(Exception): - pass - - -def _open_or_create_sentinel(lock_file: str, path: Path, mode: int) -> int | None: - try: - return _open_sentinel(path) - except FileNotFoundError: - pass - except OSError: - return None - - _reclaim_sentinel_private_records(path, time.time()) - try: - publication_cleanup_error = _publish_record(os.fspath(path), _LEGACY_SENTINEL, mode) - except _PrivateRecordReclaimedError: - return None - except (NotImplementedError, OSError) as error: - _raise_if_hard_links_unsupported(lock_file, error) - if not isinstance(error, OSError) or error.errno != EEXIST: - raise - else: - if publication_cleanup_error is not None: # pragma: needs dir-fd - raise publication_cleanup_error - try: - return _open_sentinel(path) - except OSError: - return None - - -def _open_sentinel(path: Path) -> int | None: - fd, record = _open_record(path, len(_LEGACY_SENTINEL)) - if record == _LEGACY_SENTINEL: - return fd - os.close(fd) - return None - - -def _read_claims(lock_file: str, directory: Path) -> tuple[StrictSoftFileClaim, ...]: - try: - with os.scandir(directory) as entries: - names = _public_claim_names(directory, entries) - except OSError as error: - reason = f"cannot list claim directory: {error.strerror or type(error).__name__}" - raise SoftFileLockProtocolError(lock_file, None, reason) from error - - claims: list[StrictSoftFileClaim] = [] - for name in names: - if (name_parts := _parse_claim_name(name)) is None: - raise SoftFileLockProtocolError(lock_file, name, "unknown claim name or protocol version") - if (record := _read_claim_record(lock_file, directory, name)) is not None: - claims.append(_parse_claim(lock_file, name, name_parts, record)) - return tuple(claims) - - -def _read_claim_record(lock_file: str, directory: Path, name: str) -> bytes | None: - # A contended scan can list a claim that is not yet cleanly readable, in two ways that both resolve on a brief - # retry. Windows holds a claim mid-unlink in a delete-pending state that fails an open with EACCES until the unlink - # completes. On NFS, a peer that unlinks its own claim leaves this client's cached filehandle stale, so the next - # open returns ESTALE rather than a clean ENOENT until the lookup revalidates against the server. Retrying re-runs - # the path lookup, which turns the vanished claim into ENOENT (skip) or reads it if it still exists. A genuinely - # unreadable record (a locked-down file, an EIO fault) still fails closed. - deadline = time.monotonic() + _CLAIM_READ_GRACE - delaying = False - while True: - if delaying: - time.sleep(_CLAIM_READ_RETRY) - record, pending = _attempt_claim_read(lock_file, directory, name) - if pending is None: - return record - if time.monotonic() >= deadline: - if pending.errno == ESTALE: - # A stale handle that outlives revalidation is a claim the server no longer has (RFC 1813 - # NFS3ERR_STALE): skip it like ENOENT. Skipping a peer's vanished claim can only overcount - # contention, never free a held lock. - return None - reason = f"cannot read claim: {pending.strerror or str(pending) or type(pending).__name__}" - raise SoftFileLockProtocolError(lock_file, name, reason) from pending - delaying = True - - -def _attempt_claim_read(lock_file: str, directory: Path, name: str) -> tuple[bytes | None, OSError | None]: - # Return the record, or (None, None) when the claim has already gone, or (None, error) for an open the caller may - # retry: a Windows delete-pending EACCES that resolves to a clean removal, or an NFS ESTALE from a peer unlinking - # its own claim out from under this client's cached filehandle. Any other OSError is a real fault and fails closed. - try: - return _read_record(directory / name, _CLAIM_RECORD_LIMIT), None - except FileNotFoundError: - return None, None - except PermissionError as error: - return None, error - except OSError as error: - if error.errno == ESTALE: - return None, error - reason = f"cannot read claim: {error.strerror or str(error) or type(error).__name__}" - raise SoftFileLockProtocolError(lock_file, name, reason) from error - - -def _public_claim_names(directory: Path, entries: Iterator[os.DirEntry[str]]) -> list[str]: - names: list[str] = [] - now = time.time() - for entry in entries: - if not entry.name.startswith("."): - names.append(entry.name) - elif (public_name := _private_public_name(entry.name)) is not None and _parse_claim_name( - public_name - ) is not None: - _reclaim_private_record((os.fspath(directory), None), entry.name, now) - return sorted(names) - - -def _read_existing_claims(lock_file: str, directory: Path) -> tuple[StrictSoftFileClaim, ...]: - if not directory.exists(): - return () - return _read_claims(lock_file, directory) - - -def _parse_claim( - lock_file: str, - name: str, - name_parts: tuple[StrictSoftFileClaimState, str], - record: bytes, -) -> StrictSoftFileClaim: - try: - magic, token, pid_text, hostname_hex, start_text, trailing = record.decode("ascii").split("\n") - pid = int(pid_text) - start = int(start_text) if start_text else None - hostname = bytes.fromhex(hostname_hex).decode("utf-8") - except (UnicodeDecodeError, ValueError) as error: - raise SoftFileLockProtocolError(lock_file, name, "malformed claim record") from error - if not all(( - not trailing, - magic == _CLAIM_MAGIC, - token == name_parts[1], - 1 <= pid <= 2**31 - 1, - str(pid) == pid_text, - start is None or (start >= 0 and str(start) == start_text), - hostname.encode().hex() == hostname_hex - and hostname.isprintable() - and not any(character.isspace() for character in hostname), - )): - raise SoftFileLockProtocolError(lock_file, name, "malformed claim record") - return StrictSoftFileClaim(name=name, state=name_parts[0], token=token, pid=pid, hostname=hostname, start=start) - - -def _parse_claim_name(name: str) -> tuple[StrictSoftFileClaimState, str] | None: - if not name.endswith(".claim"): - return None - parts = name.removesuffix(".claim").split("-") - if len(parts) != _CLAIM_NAME_PART_COUNT or parts[0] not in _CLAIM_STATES or parts[1] != "v1": - return None - token = parts[2] - if len(token) != _TOKEN_HEX_LENGTH or any(character not in "0123456789abcdef" for character in token): - return None - return cast("StrictSoftFileClaimState", parts[0]), token - - -def _claim_name(state: StrictSoftFileClaimState, token: str) -> str: - return f"{state}-v1-{token}.claim" - - -def _claim_token_key(name: str) -> str: - return name.removeprefix("held-").removeprefix("intent-") - - -def _claim_record(token: str) -> bytes: - hostname_hex = host_name().encode().hex() - start = process_start_token(os.getpid()) - start_text = "" if start is None else str(start) - return f"{_CLAIM_MAGIC}\n{token}\n{os.getpid()}\n{hostname_hex}\n{start_text}\n".encode("ascii") - - -def _publish_record( - public_path: str, - record: bytes, - mode: int, -) -> BaseException | None: - directory, public_name = os.path.split(public_path) - directory = directory or os.curdir - private_name = _private_record_name(public_name) - directory_fd = _open_directory(directory) if _OPEN_SUPPORTS_DIR_FD else None - directory_ref = directory, directory_fd - try: - _publish_record_in_directory(directory_ref, (private_name, public_name), mode, record) - except BaseException as publication_error: # preserve publication and directory cleanup errors - try: - if directory_fd is not None: # pragma: needs dir-fd - os.close(directory_fd) - except BaseException as close_error: # ruff:ignore[blind-except] # pragma: needs dir-fd # preserve publication and directory cleanup errors - _raise_cleanup_errors("strict publication directory cleanup failed", publication_error, close_error) - raise - if directory_fd is not None: # pragma: needs dir-fd - try: - os.close(directory_fd) - except BaseException as close_error: # ruff:ignore[blind-except] # caller records the published path before raising - return close_error - return None - - -def _publish_record_in_directory( - directory_ref: tuple[str, int | None], - names: tuple[str, str], - mode: int, - record: bytes, -) -> None: - flags = os.O_RDWR | os.O_CREAT | os.O_EXCL | _O_BINARY - if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None: # pragma: needs o-nofollow - flags |= o_nofollow - private_fd = _open_relative(directory_ref, names[0], flags, mode) - private_identity: tuple[int, int] | None = None - try: - private_identity = _file_identity(os.fstat(private_fd)) - write_all(private_fd, record) - _link_private_record(directory_ref, names, private_identity) - except BaseException as publication_error: # preserve publication and cleanup errors - close_error, unlink_error = _close_and_unlink_private_record( - directory_ref, - names[0], - private_fd, - private_identity, - ) - if close_error is not None or unlink_error is not None: - _raise_cleanup_errors( - "strict record publication cleanup failed", - publication_error, - close_error, - unlink_error, - ) - raise - close_error, unlink_error = _close_and_unlink_private_record( - directory_ref, - names[0], - private_fd, - private_identity, - ) - if close_error is not None or unlink_error is not None: - _raise_record_finalization_errors(close_error, unlink_error) - - -def _close_and_unlink_private_record( - directory_ref: tuple[str, int | None], - private_name: str, - private_fd: int, - private_identity: tuple[int, int] | None, -) -> tuple[BaseException | None, BaseException | None]: - close_error: BaseException | None = None - try: - os.close(private_fd) - except BaseException as error: # ruff:ignore[blind-except] # returned for grouping with unlink failures - close_error = error - unlink_error: BaseException | None = None - try: - if private_identity is None: - _unlink_relative(directory_ref, private_name) - else: - _unlink_relative_if_identity(directory_ref, private_name, private_identity) - except FileNotFoundError: - pass - except BaseException as error: # ruff:ignore[blind-except] # returned for grouping with close failures - unlink_error = error - return close_error, unlink_error - - -def _link_private_record( - directory_ref: tuple[str, int | None], - names: tuple[str, str], - private_identity: tuple[int, int], -) -> None: - try: - _link_relative(directory_ref, *names) - except FileNotFoundError as error: - if _relative_identity(directory_ref, names[0]) is not None: - raise - msg = "private publication record was reclaimed" - raise _PrivateRecordReclaimedError(msg) from error - if _relative_identity(directory_ref, names[1]) == private_identity: - return - msg_0 = "private publication record was replaced" - raise _PrivateRecordReclaimedError(msg_0) - - -def _raise_record_finalization_errors( - close_error: BaseException | None, - unlink_error: BaseException | None, -) -> None: - errors = [error for error in (close_error, unlink_error) if error is not None] - if len(errors) > 1: - _raise_cleanup_errors("strict record finalization failed", errors[0], *errors[1:]) - raise errors[0] - - -def _private_record_name(public_name: str) -> str: - return f".{public_name}{_PRIVATE_RECORD_MARKER}{secrets.token_hex(_PRIVATE_RECORD_RANDOM_HEX_LENGTH // 2)}.tmp" - - -def _private_public_name(private_name: str) -> str | None: - if not private_name.startswith(".") or not private_name.endswith(_PRIVATE_RECORD_SUFFIX): - return None - public_name, marker, random_hex = private_name[1 : -len(_PRIVATE_RECORD_SUFFIX)].rpartition(_PRIVATE_RECORD_MARKER) - if ( - marker != _PRIVATE_RECORD_MARKER - or len(random_hex) != _PRIVATE_RECORD_RANDOM_HEX_LENGTH - or any(character not in "0123456789abcdef" for character in random_hex) - ): - return None - return public_name - - -def _reclaim_sentinel_private_records(path: Path, now: float) -> None: - directory_ref = os.fspath(path.parent), None - with os.scandir(path.parent) as entries: - for entry in entries: - if _private_public_name(entry.name) == path.name: - _reclaim_private_record(directory_ref, entry.name, now) - - -def _reclaim_private_record(directory_ref: tuple[str, int | None], private_name: str, now: float) -> None: - directory, directory_fd = directory_ref - try: - private_stat = ( - os.stat(private_name, dir_fd=directory_fd, follow_symlinks=False) - if directory_fd is not None and _STAT_SUPPORTS_DIR_FD - else Path(directory, private_name).lstat() - ) - except FileNotFoundError: - return - if not stat.S_ISREG(private_stat.st_mode): - msg = f"{Path(directory, private_name)} is not a regular private record" - raise OSError(msg) - if private_stat.st_nlink == 1 and now - private_stat.st_mtime < _PRIVATE_RECORD_GRACE: - return - try: - _unlink_private_record_once(directory_ref, private_name) - except FileNotFoundError: - pass - except OSError as error: - if error.errno not in {EACCES, EPERM}: - raise - - -def _unlink_private_record_once(directory_ref: tuple[str, int | None], private_name: str) -> None: - directory, directory_fd = directory_ref - if ( - directory_fd is not None and _UNLINK_SUPPORTS_DIR_FD - ): # pragma: no cover # callers always pass directory_fd=None - os.unlink(private_name, dir_fd=directory_fd) - else: - Path(directory, private_name).unlink() - - -def _relative_identity(directory_ref: tuple[str, int | None], name: str) -> tuple[int, int] | None: - directory, directory_fd = directory_ref - try: - if directory_fd is not None and _STAT_SUPPORTS_DIR_FD: # pragma: needs dir-fd - path_stat = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) - else: # pragma: win32 cover - path_stat = Path(directory, name).lstat() - except FileNotFoundError: - return None - return _file_identity(path_stat) - - -def _unlink_relative_if_identity( - directory_ref: tuple[str, int | None], - name: str, - identity: tuple[int, int], -) -> None: - if _relative_identity(directory_ref, name) == identity: - with contextlib.suppress(FileNotFoundError): - _unlink_relative(directory_ref, name) - - -def _open_relative(directory_ref: tuple[str, int | None], name: str, flags: int, mode: int) -> int: - directory, directory_fd = directory_ref - return ( - os.open(name, flags, mode, dir_fd=directory_fd) - if directory_fd is not None - else os.open(Path(directory, name), flags, mode) - ) - - -def _link_relative(directory_ref: tuple[str, int | None], source_name: str, destination_name: str) -> None: - directory, directory_fd = directory_ref - if directory_fd is not None and _LINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd - _link_no_follow(source_name, destination_name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd) - return - _link_no_follow(Path(directory, source_name), Path(directory, destination_name)) # pragma: win32 cover - - -def _link_no_follow( - source: str | Path, - destination: str | Path, - *, - src_dir_fd: int | None = None, - dst_dir_fd: int | None = None, -) -> None: - if not _HAS_LINK: - # No os.link at all (Termux/Android): report it as unsupported like a filesystem that refuses hard links, so - # the acquire path raises SoftFileLockProtocolError instead of a bare AttributeError. - msg = "os.link is unavailable on this platform" - raise NotImplementedError(msg) - # The source is a private record this process created with O_CREAT | O_EXCL, so follow_symlinks guards nothing an - # attacker can reach. Pass the option only when the runtime honors it: PyPy advertises it through - # os.supports_follow_symlinks yet its linkat rejects it with EINVAL, so probe once rather than trust the set. - if _LINK_HONORS_FOLLOW_SYMLINKS: - os.link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd, follow_symlinks=False) - else: # pragma: lacks link-follow-symlinks - os.link(source, destination, src_dir_fd=src_dir_fd, dst_dir_fd=dst_dir_fd) - - -def _unlink_relative(directory_ref: tuple[str, int | None], name: str) -> None: - directory, directory_fd = directory_ref - if directory_fd is not None and _UNLINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd - os.unlink(name, dir_fd=directory_fd) - elif sys.platform == "win32": # pragma: win32 cover - _unlink_in_directory(Path(directory), name) - else: - Path(directory, name).unlink() # pragma: win32 no cover - - -def _link_no_replace(directory: Path, source_name: str, destination_name: str) -> BaseException | None: - if _LINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd - directory_fd = _open_directory(str(directory)) - try: - _link_no_follow(source_name, destination_name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd) - except BaseException as link_error: # preserve link and directory cleanup errors - try: - os.close(directory_fd) - except BaseException as close_error: # ruff:ignore[blind-except] # preserve link and directory cleanup errors - _raise_cleanup_errors("strict link directory cleanup failed", link_error, close_error) - raise - try: - os.close(directory_fd) - except BaseException as close_error: # ruff:ignore[blind-except] # caller records the held path before raising - return close_error - return None - _link_no_follow(directory / source_name, directory / destination_name) # pragma: win32 cover - return None # pragma: win32 cover - - -def _unlink_owner_path(path: str) -> BaseException | None: - directory, name = os.path.split(path) - return _unlink_in_directory(Path(directory or os.curdir), name) - - -def _unlink_owner_paths(paths: tuple[str, ...]) -> tuple[list[str], list[BaseException]]: - results = [(path, _unlink_owner_path_result(path)) for path in paths] - return [path for path, (removed, _) in results if not removed], [ - error for _, (_, error) in results if error is not None - ] - - -def _unlink_owner_path_result(path: str) -> tuple[bool, BaseException | None]: - try: - cleanup_error = _unlink_owner_path(path) - except FileNotFoundError: - return True, None - except BaseException as error: # ruff:ignore[blind-except] # keep ownership when unlink did not commit - return False, error - return True, cleanup_error - - -def _raise_recorded_errors(message: str, errors: list[BaseException]) -> None: - if len(errors) > 1: - _raise_cleanup_errors(message, errors[0], *errors[1:]) - raise errors[0] - - -def _unlink_in_directory(directory: Path, name: str) -> BaseException | None: - if _UNLINK_SUPPORTS_DIR_FD: # pragma: needs dir-fd - directory_fd = _open_directory(str(directory)) - try: - os.unlink(name, dir_fd=directory_fd) - except BaseException as unlink_error: # preserve unlink and directory cleanup errors - try: - os.close(directory_fd) - except BaseException as close_error: # ruff:ignore[blind-except] # preserve unlink and directory cleanup errors - _raise_cleanup_errors("strict unlink directory cleanup failed", unlink_error, close_error) - raise - try: - os.close(directory_fd) - except BaseException as close_error: # ruff:ignore[blind-except] # caller commits the removed path before raising - return close_error - return None - if sys.platform != "win32": # pragma: win32 no cover - Path(directory / name).unlink() - return None - retry_delay = 0.001 # pragma: win32 cover - for attempt in range(_UNLINK_MAX_RETRIES): # pragma: win32 cover - if (error := _unlink_error(directory / name)) is None: - return None - if error.errno not in {EACCES, EPERM} or attempt == _UNLINK_MAX_RETRIES - 1: - raise error - time.sleep(retry_delay) - retry_delay *= 2 - return None # pragma: no cover # the final retry returns or raises - - -def _unlink_error(path: Path) -> OSError | None: # pragma: win32 cover - try: - path.unlink() - except OSError as error: - return error - return None - - -def _open_directory(directory: str) -> int: # pragma: needs dir-fd - return os.open( - directory, - os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), - ) - - -def _read_record(path: Path, limit: int) -> bytes: - fd, record = _open_record(path, limit) - os.close(fd) - return record - - -def _open_record(path: Path, limit: int) -> tuple[int, bytes]: - path_stat = path.lstat() - if not stat.S_ISREG(path_stat.st_mode): - msg = f"{path} is not a regular file" - raise OSError(msg) - flags = os.O_RDONLY | _O_BINARY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) - fd = os.open(path, flags) - try: - record = _read_opened_record(fd, path, path_stat, limit) - except BaseException as read_error: # preserve read and descriptor cleanup errors - try: - os.close(fd) - except BaseException as close_error: # ruff:ignore[blind-except] # preserve read and descriptor cleanup errors - _raise_cleanup_errors("strict record read cleanup failed", read_error, close_error) - raise - return fd, record - - -def _read_opened_record(fd: int, path: Path, path_stat: os.stat_result, limit: int) -> bytes: - opened_stat = os.fstat(fd) - if not stat.S_ISREG(opened_stat.st_mode): - msg = f"{path} is not a regular file" - raise OSError(msg) - if _file_identity(opened_stat) != _file_identity(path_stat): - msg = f"{path} changed while opening" - raise OSError(msg) - record = os.read(fd, limit + 1) - if len(record) > limit: - msg = f"{path} exceeds {limit} bytes" - raise OSError(msg) - return record - - -def _ensure_protocol_directory(lock_file: str, directory: Path) -> None: - try: - directory.mkdir() - except FileExistsError: - try: - mode = directory.lstat().st_mode - except OSError as error: - raise SoftFileLockProtocolError(lock_file, None, f"cannot inspect {directory}") from error - if stat.S_ISDIR(mode) and not stat.S_ISLNK(mode): - return - raise SoftFileLockProtocolError(lock_file, None, f"{directory} is not a real directory") from None - - -def _validate_force_break_name(name: str) -> None: - invalid_component = not name or name.startswith(".") or name in {os.curdir, os.pardir} - if invalid_component or any(separator in name for separator in ("/", "\\", "\x00")): - msg = "claim_name must be one public claim basename" - raise ValueError(msg) - - -def _require_exact_name(directory: Path, name: str) -> None: - with os.scandir(directory) as entries: - if not any(entry.name == name for entry in entries): - raise FileNotFoundError(ENOENT, os.strerror(ENOENT), name) - - -def _raise_if_hard_links_unsupported(lock_file: str, error: NotImplementedError | OSError) -> None: - unsupported = ( - isinstance(error, NotImplementedError) - or error.errno in _HARD_LINK_UNSUPPORTED_ERRNOS - or (getattr(error, "winerror", None) in _WINDOWS_HARD_LINK_UNSUPPORTED) - ) - if unsupported: - reason = "filesystem does not support atomic no-replace hard-link publication" - raise SoftFileLockProtocolError(lock_file, None, reason) from error - - -def _file_identity(st: os.stat_result) -> tuple[int, int]: - return st.st_dev, st.st_ino - - -__all__ = [ - "StrictSoftFileClaim", - "StrictSoftFileClaimState", - "StrictSoftFileLock", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_unix.py b/bundle/python-cpu/Lib/site-packages/filelock/_unix.py deleted file mode 100644 index 71e06deea70db6f0100b0158c669e652b4b8ad65..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_unix.py +++ /dev/null @@ -1,186 +0,0 @@ -from __future__ import annotations - -import os -import sys -import warnings -from contextlib import suppress -from errno import EACCES, EAGAIN, ENOSYS, EWOULDBLOCK -from pathlib import Path -from typing import Final, cast - -from ._api import BaseFileLock -from ._util import ensure_directory_exists - -has_fcntl = False -if sys.platform == "win32": # pragma: win32 cover - - class UnixFileLock(BaseFileLock): - """Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems.""" - - def _acquire(self) -> None: - raise NotImplementedError - - def _release(self) -> None: - raise NotImplementedError - -else: # pragma: win32 no cover - try: - import fcntl - - _ = (fcntl.flock, fcntl.LOCK_EX, fcntl.LOCK_NB, fcntl.LOCK_UN) - except (ImportError, AttributeError): - _FCNTL_UNAVAILABLE: Final[str] = "fcntl is unavailable" - - def _lock_fd_nonblocking(_fd: int) -> bool: - raise OSError(ENOSYS, _FCNTL_UNAVAILABLE) - - def _unlock_fd(_fd: int) -> None: - raise OSError(ENOSYS, _FCNTL_UNAVAILABLE) - - else: - has_fcntl = True - # Contention errnos for a nonblocking flock. EAGAIN/EWOULDBLOCK are the usual "held elsewhere" codes; some - # filesystems report EACCES instead, so treat it as contention too rather than a permanent error. - _CONTENTION_ERRNOS: Final[frozenset[int]] = frozenset({EACCES, EAGAIN, EWOULDBLOCK}) - - def _lock_fd_nonblocking(fd: int) -> bool: - # One nonblocking exclusive flock attempt shared by UnixFileLock and lock_descriptor, so both contend on - # the same lock and classify errors identically. The caller owns fd; this never closes it. - try: - fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as exception: - if exception.errno in _CONTENTION_ERRNOS: - return False - raise - return True - - def _unlock_fd(fd: int) -> None: - fcntl.flock(fd, fcntl.LOCK_UN) - - class UnixFileLock(BaseFileLock): - """ - Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems. - - We leave the lock file in place after release. Unlinking a locked file on Unix splits - waiters across inodes and breaks mutual exclusion for processes that coordinate via the - same path. - """ - - def _acquire(self) -> None: - missing_flock = self._acquire_native() - if missing_flock is not None: - self._switch_to_soft_lock(*missing_flock) - - def _acquire_native(self) -> tuple[int, OSError] | None: - ensure_directory_exists(self.lock_file) - # Open without O_TRUNC and defer truncation and fchmod until after flock succeeds: a contender that loses - # the lock must not truncate the holder's file (erasing caller diagnostics) or change its mode. The winner - # truncates and normalizes mode once it owns the lock (#591). - open_flags = os.O_RDWR - if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None: - open_flags |= o_nofollow - open_flags |= os.O_CREAT - open_mode = self._open_mode() - try: - fd = os.open(self.lock_file, open_flags, open_mode) - except FileNotFoundError: - # On FUSE/NFS, os.open(O_CREAT) is not atomic; a split LOOKUP + CREATE lets a concurrent unlink() - # delete the file between them. For a valid path, treat ENOENT as transient contention. For an - # invalid path (e.g. empty string), re-raise to avoid an infinite retry loop. - if self.lock_file and Path(self.lock_file).parent.exists(): - return None - raise - except PermissionError: - # Sticky-bit dirs (e.g. /tmp): O_CREAT fails if the file is owned by another user (#317). - # Fall back to opening the existing file without O_CREAT. - if not Path(self.lock_file).exists(): - raise - try: - fd = os.open(self.lock_file, open_flags & ~os.O_CREAT, open_mode) - except FileNotFoundError: - return None - self._mark_descriptor_pending(fd) - try: - locked = _lock_fd_nonblocking(fd) - except OSError as exception: - if exception.errno != ENOSYS: - self._mark_descriptor_released() - os.close(fd) - raise # contention returns False from _lock_fd_nonblocking, so any raise here is a real failure - return fd, exception - if locked: - self._finalize_locked_fd(fd) - else: - self._mark_descriptor_released() - os.close(fd) # contention; let the retry loop try again - return None - - def _switch_to_soft_lock(self, fd: int, missing_flock: OSError) -> None: - # The filesystem does not implement flock. Capture the opened file's identity before closing so the cleanup - # below removes only this attempt's placeholder, not a peer's replacement. - identity: tuple[int, int] | None = None - with suppress(OSError): - identity = (fstat := os.fstat(fd)).st_dev, fstat.st_ino - self._mark_descriptor_released() - os.close(fd) - if not self._fallback_to_soft or self._preserve_lock_file or self._on_acquired is not None: - # Fail closed: the caller opted out of existence-lock semantics (#603), asked to preserve the pathname - # (#605), or set an on_acquired hook (#607), none of which a soft lock can honor. - raise missing_flock - with suppress(OSError): - current = os.lstat(self.lock_file) - if identity == (current.st_dev, current.st_ino): - Path(self.lock_file).unlink() - self._fallback_to_soft_lock() - self._acquire() - - def _finalize_locked_fd(self, fd: int) -> None: - # Runs with the flock held. Truncate and normalize mode under a guard so any failure closes fd rather than - # leaking it and its lock. A concurrent _release() may have unlinked the inode between our open() and - # flock() (st_nlink 0), leaving a useless dead-inode lock; drop it and let the retry loop start fresh. - keep = False - try: - stat_result = os.fstat(fd) - if stat_result.st_nlink != 0: - os.ftruncate(fd, 0) - self._apply_explicit_mode(fd) - keep = True - except OSError: - self._mark_descriptor_released() - os.close(fd) - raise - if keep: - self._mark_descriptor_owned(fd, (stat_result.st_dev, stat_result.st_ino)) - else: - self._mark_descriptor_released() - os.close(fd) - - def _apply_explicit_mode(self, fd: int) -> None: - if self.has_explicit_mode: - with suppress(PermissionError): - os.fchmod(fd, self._context.mode) - - def _fallback_to_soft_lock(self) -> None: - # Import lazily: this runs only on the rare flock fallback, and asyncio imports _unix, so a - # module-level import of it here would cycle. - from ._soft import SoftFileLock # ruff:ignore[import-outside-top-level] - - warnings.warn("flock not supported on this filesystem, falling back to SoftFileLock", stacklevel=2) - from .asyncio import AsyncSoftFileLock, BaseAsyncFileLock # ruff:ignore[import-outside-top-level] - - self.__class__ = AsyncSoftFileLock if isinstance(self, BaseAsyncFileLock) else SoftFileLock - - def _release(self) -> None: - fd = cast("int", self._context.lock_file_fd) - # Retain the descriptor until flock succeeds: a failed unlock leaves the kernel lock held, so is_locked - # must keep reporting held for a retry. Once flock commits, clear held state and close as post-unlock - # cleanup; a close failure (EIO on FUSE/Docker bind mounts) does not make the kernel lock held again. - _unlock_fd(fd) - self._mark_descriptor_released() - self._close_released_fd(fd, default_suppresses=True) - - -if sys.platform == "win32": # pragma: win32 cover - __all__ = ["UnixFileLock", "has_fcntl"] -else: # pragma: win32 no cover - __all__ = ["UnixFileLock", "_lock_fd_nonblocking", "_unlock_fd", "has_fcntl"] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_util.py b/bundle/python-cpu/Lib/site-packages/filelock/_util.py deleted file mode 100644 index e3da8a09373fc7f0b28ca3dc28a8fb0113d9b2e0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_util.py +++ /dev/null @@ -1,137 +0,0 @@ -from __future__ import annotations - -import os -import secrets -import stat -import sys -from errno import EACCES, EIO, EISDIR -from pathlib import Path -from typing import Final - - -def write_all(fd: int, data: bytes) -> None: - """ - Write the whole buffer to *fd*, looping over the short writes ``os.write`` is allowed to make. - - A marker written with a bare ``os.write`` can land partially: a peer reading it mid-write parses a truncated record - as malformed or as a foreign holder. Looping until the buffer drains keeps the record atomic in the process and - kernel view. No ``fsync``: filelock needs a complete record, not crash-durable storage. - - :param fd: file descriptor open for writing. - :param data: bytes to write in full. - - :raises OSError: if a write reports zero progress before the record is complete. - - """ - remaining = memoryview(data) - while remaining: - if (written := os.write(fd, remaining)) == 0: - raise OSError(EIO, "os.write wrote 0 bytes before the record was complete") - remaining = remaining[written:] - - -def raise_on_not_writable_file(filename: str) -> None: - """ - Raise an exception if attempting to open the file for writing would fail. - - Separates files that can never be written from files that are writable but currently locked. - - :param filename: file to check - - :raises OSError: as if the file was opened for writing. - - """ - try: - # lstat, not stat: settles exists-and-writable in one syscall, and a hostile symlink at the lock path would - # make stat inspect the link target, letting an attacker turn a contended acquire into a misleading - # PermissionError / IsADirectoryError and probe that target's attributes. The real open passes O_NOFOLLOW and - # refuses the symlink anyway. - file_stat = os.lstat(filename) - except OSError: - return # does not exist, or an error the caller cannot act on - - # No mtime guard: the old `if st_mtime != 0` skip covered NFS/Linux quirks where os.lstat returned an all-zero - # struct, which it no longer does. Skipping on mtime 0 let a read-only file or a directory at the lock path pass - # as missing, so acquire() blocked forever on an open that cannot succeed. - if not (file_stat.st_mode & stat.S_IWUSR): - raise PermissionError(EACCES, "Permission denied", filename) - - if stat.S_ISDIR(file_stat.st_mode): - if sys.platform == "win32": # pragma: win32 cover - raise PermissionError(EACCES, "Permission denied", filename) - raise IsADirectoryError(EISDIR, "Is a directory", filename) # pragma: win32 no cover - - -def ensure_directory_exists(filename: Path | str) -> None: - """ - Ensure the directory containing the file exists (create it if necessary). - - :param filename: file. - - """ - Path(filename).parent.mkdir(parents=True, exist_ok=True) - - -def break_lock_file(lock_file: str, mtime_before: float, ino_before: int) -> None: - """ - Atomically break a stale lock file judged stale at modification time *mtime_before*. - - Rename the file to a process-private name before unlinking it, so two processes breaking the same lock cannot - delete each other's work: only one rename of a given inode wins, the loser gets ``OSError``. After the rename, - re-check the file. A newer modification time, or a different inode than *ino_before*, means a peer recreated the - lock between the stale decision and the rename, so we grabbed a live file and abort, leaving the renamed file in - place. A rollback rename is itself racy, the same trade-off as the soft read/write marker break. The inode check - matters because filesystems with coarse modification-time granularity (NFS, FAT) can give a same-second recreation - the old mtime, so mtime alone would miss it and unlink a live lock; the inode is the reliable identity, mirroring - the token re-check in the soft read/write marker break. ``lstat`` avoids following a hostile symlink swapped in - after the decision. - - The break name carries a random token so it is unguessable and unique per attempt. Without it two breakers in the - same process share ``.break.``, and a second break can rename a recreated live lock onto that path in - the window between the re-verify ``lstat`` above and the ``unlink`` below, deleting a live lock the inode check - just approved. A private name keeps anyone else from targeting our break path, matching the soft read/write marker - break. - - :param lock_file: path to the lock file to break. - :param mtime_before: modification time observed when the lock was judged stale. - :param ino_before: inode number observed when the lock was judged stale. - - :raises OSError: if the rename fails (e.g. the file vanished or is not owned in a sticky directory). - - """ - break_path = f"{lock_file}.break.{os.getpid()}.{secrets.token_hex(16)}" - Path(lock_file).rename(break_path) - try: - st_after = os.lstat(break_path) - except OSError: - return - if st_after.st_mtime > mtime_before or st_after.st_ino != ino_before: - return - Path(break_path).unlink() - - -def touch(name: str, *, fd: int | None = None) -> None: - # Prefer the already-open, already-verified fd so a peer that swaps a symlink or a different file in at the - # path after our O_NOFOLLOW read cannot redirect the touch: utime then targets the inode behind the fd. - # Where the platform cannot utime an fd, fall back to a path-based touch that still refuses to follow a - # symlink where supported, matching the O_NOFOLLOW reads used elsewhere here. - if fd is not None and _SUPPORTS_UTIME_FD: # pragma: needs utime-fd - os.utime(fd, None) - return - os.utime(name, None, follow_symlinks=not _SUPPORTS_UTIME_NOFOLLOW) - - -# Retargeting os.utime to an open fd lets a heartbeat refresh the exact inode it verified instead of whatever the -# pathname now names. -_SUPPORTS_UTIME_FD: Final[bool] = sys.platform != "win32" and os.utime in os.supports_fd -# os.utime follows symlinks unless told not to; not every platform can refuse the follow, so probe support. -_SUPPORTS_UTIME_NOFOLLOW: Final[bool] = os.utime in os.supports_follow_symlinks - - -__all__ = [ - "break_lock_file", - "ensure_directory_exists", - "raise_on_not_writable_file", - "touch", - "write_all", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/_windows.py b/bundle/python-cpu/Lib/site-packages/filelock/_windows.py deleted file mode 100644 index 3d72e5355429abf8f94aa7892737c46bb46ff9a3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/_windows.py +++ /dev/null @@ -1,323 +0,0 @@ -from __future__ import annotations - -import os -import sys -from contextlib import suppress -from pathlib import Path -from typing import Final, cast - -from ._api import BaseFileLock -from ._util import ensure_directory_exists, raise_on_not_writable_file - -if sys.platform == "win32": # pragma: win32 cover - import ctypes - import msvcrt - from ctypes import wintypes - - _GENERIC_READ: Final[int] = 0x80000000 - _GENERIC_WRITE: Final[int] = 0x40000000 - _SYNCHRONIZE: Final[int] = 0x00100000 - _DESIRED_ACCESS: Final[int] = _GENERIC_READ | _GENERIC_WRITE | _SYNCHRONIZE - _FILE_SHARE_READ_WRITE: Final[int] = ( - 0x00000001 | 0x00000002 - ) # read | write; matches os.open (_SH_DENYNO), no delete - _FILE_OPEN_IF: Final[int] = 3 # open the file if it exists, create it otherwise; the NtCreateFile OPEN_ALWAYS - _FILE_ATTRIBUTE_READONLY: Final[int] = 0x00000001 - _FILE_ATTRIBUTE_NORMAL: Final[int] = 0x00000080 - _FILE_ATTRIBUTE_REPARSE_POINT: Final[int] = 0x00000400 - # CreateOptions: keep the handle synchronous (the CRT and msvcrt.locking rely on a maintained file position), - # refuse a directory, and open a reparse point rather than following it so the check below acts on the link itself. - _FILE_SYNCHRONOUS_IO_NONALERT: Final[int] = 0x00000020 - _FILE_NON_DIRECTORY_FILE: Final[int] = 0x00000040 - _FILE_OPEN_REPARSE_POINT: Final[int] = 0x00200000 - _CREATE_OPTIONS: Final[int] = _FILE_SYNCHRONOUS_IO_NONALERT | _FILE_NON_DIRECTORY_FILE | _FILE_OPEN_REPARSE_POINT - _OBJ_CASE_INSENSITIVE: Final[int] = 0x00000040 # Win32 name lookups are case-insensitive - _OWNER_WRITE: Final[int] = 0o200 - - # LockFileEx locks a byte range at an offset carried in OVERLAPPED, independent of the descriptor's file position. - # msvcrt.locking starts at the current position instead, so a metadata write between lock and unlock could shift - # the byte a later unlock targets; the explicit offset removes that hazard for both the path lock and #608's - # descriptor lock. - _LOCKFILE_FAIL_IMMEDIATELY: Final[int] = 0x00000001 - _LOCKFILE_EXCLUSIVE_LOCK: Final[int] = 0x00000002 - _ERROR_LOCK_VIOLATION: Final[int] = 33 # another handle holds the byte range - - # NtCreateFile returns the raw NTSTATUS as its value, where CreateFileW collapses several of these into one - # ERROR_ACCESS_DENIED. Telling them apart is the point (#604): a name pending deletion or a share conflict is - # transient and worth a retry, a real access denial is not. - _STATUS_SUCCESS: Final[int] = 0x00000000 - _STATUS_ACCESS_DENIED: Final[int] = 0xC0000022 - _STATUS_SHARING_VIOLATION: Final[int] = 0xC0000043 - _STATUS_DELETE_PENDING: Final[int] = 0xC0000056 - - _ntdll: Final[ctypes.WinDLL] = ctypes.WinDLL("ntdll") - _kernel32: Final[ctypes.WinDLL] = ctypes.WinDLL("kernel32", use_last_error=True) - - class _UNICODE_STRING(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name - _fields_ = ( - ("Length", wintypes.USHORT), # byte length, not character count - ("MaximumLength", wintypes.USHORT), - ("Buffer", wintypes.LPWSTR), - ) - - class _OBJECT_ATTRIBUTES(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name - _fields_ = ( - ("Length", wintypes.ULONG), - ("RootDirectory", wintypes.HANDLE), - ("ObjectName", ctypes.POINTER(_UNICODE_STRING)), - ("Attributes", wintypes.ULONG), - ("SecurityDescriptor", ctypes.c_void_p), - ("SecurityQualityOfService", ctypes.c_void_p), - ) - - class _IO_STATUS_BLOCK(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name - _fields_ = ( - ("Status", ctypes.c_void_p), # a union of NTSTATUS and PVOID, so it is pointer-sized - ("Information", ctypes.c_void_p), - ) - - class _OVERLAPPED(ctypes.Structure): # mirrors the Win32 struct name - _fields_ = ( - ("Internal", ctypes.c_void_p), # ULONG_PTR: pointer-sized, not DWORD, or the x64 layout corrupts Offset - ("InternalHigh", ctypes.c_void_p), - ("Offset", wintypes.DWORD), # the DUMMYUNIONNAME struct, flattened: low 32 bits of the byte offset - ("OffsetHigh", wintypes.DWORD), - ("hEvent", wintypes.HANDLE), - ) - - class _BY_HANDLE_FILE_INFORMATION(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name - _fields_ = ( - ("dwFileAttributes", wintypes.DWORD), - ("ftCreationTime", wintypes.FILETIME), - ("ftLastAccessTime", wintypes.FILETIME), - ("ftLastWriteTime", wintypes.FILETIME), - ("dwVolumeSerialNumber", wintypes.DWORD), - ("nFileSizeHigh", wintypes.DWORD), - ("nFileSizeLow", wintypes.DWORD), - ("nNumberOfLinks", wintypes.DWORD), - ("nFileIndexHigh", wintypes.DWORD), - ("nFileIndexLow", wintypes.DWORD), - ) - - _ntdll.NtCreateFile.restype = wintypes.LONG # NTSTATUS - _ntdll.NtCreateFile.argtypes = [ - ctypes.POINTER(wintypes.HANDLE), - wintypes.DWORD, - ctypes.POINTER(_OBJECT_ATTRIBUTES), - ctypes.POINTER(_IO_STATUS_BLOCK), - ctypes.POINTER(ctypes.c_longlong), # PLARGE_INTEGER AllocationSize - wintypes.ULONG, - wintypes.ULONG, - wintypes.ULONG, - wintypes.ULONG, - ctypes.c_void_p, - wintypes.ULONG, - ] - _ntdll.RtlDosPathNameToNtPathName_U_WithStatus.restype = wintypes.LONG # NTSTATUS - _ntdll.RtlDosPathNameToNtPathName_U_WithStatus.argtypes = [ - wintypes.LPCWSTR, - ctypes.POINTER(_UNICODE_STRING), - ctypes.c_void_p, - ctypes.c_void_p, - ] - _ntdll.RtlFreeUnicodeString.restype = None - _ntdll.RtlFreeUnicodeString.argtypes = [ctypes.POINTER(_UNICODE_STRING)] - _ntdll.RtlNtStatusToDosError.restype = wintypes.ULONG - _ntdll.RtlNtStatusToDosError.argtypes = [wintypes.LONG] - - _kernel32.CloseHandle.argtypes = [wintypes.HANDLE] - _kernel32.CloseHandle.restype = wintypes.BOOL - _kernel32.GetFileInformationByHandle.argtypes = [wintypes.HANDLE, ctypes.POINTER(_BY_HANDLE_FILE_INFORMATION)] - _kernel32.GetFileInformationByHandle.restype = wintypes.BOOL - _kernel32.LockFileEx.argtypes = [ - wintypes.HANDLE, - wintypes.DWORD, - wintypes.DWORD, - wintypes.DWORD, - wintypes.DWORD, - ctypes.POINTER(_OVERLAPPED), - ] - _kernel32.LockFileEx.restype = wintypes.BOOL - _kernel32.UnlockFileEx.argtypes = [ - wintypes.HANDLE, - wintypes.DWORD, - wintypes.DWORD, - wintypes.DWORD, - ctypes.POINTER(_OVERLAPPED), - ] - _kernel32.UnlockFileEx.restype = wintypes.BOOL - - def _lock_fd_nonblocking(fd: int) -> bool: - # One nonblocking exclusive LockFileEx attempt shared by WindowsFileLock and lock_descriptor, over the one-byte - # range at offset 0. True on acquisition, False on contention, raise otherwise. The caller owns fd; the handle - # from get_osfhandle belongs to the CRT descriptor and must not be closed here. - overlapped = _OVERLAPPED() # zero-initialized, so Offset/OffsetHigh/hEvent are 0 - flags = _LOCKFILE_EXCLUSIVE_LOCK | _LOCKFILE_FAIL_IMMEDIATELY - if _kernel32.LockFileEx(msvcrt.get_osfhandle(fd), flags, 0, 1, 0, ctypes.byref(overlapped)): - return True - err = ctypes.get_last_error() - if err == _ERROR_LOCK_VIOLATION: - return False - # A non-contention LockFileEx failure is not reproducible in-process. - raise ctypes.WinError(err) # pragma: no cover - - def _unlock_fd(fd: int) -> None: - overlapped = _OVERLAPPED() # the same offset 0 and one-byte length the lock used - # Unlocking the exact range we hold does not fail. - if not _kernel32.UnlockFileEx(msvcrt.get_osfhandle(fd), 0, 1, 0, ctypes.byref(overlapped)): # pragma: no cover - raise ctypes.WinError(ctypes.get_last_error()) - - class WindowsFileLock(BaseFileLock): - """ - Uses ``LockFileEx`` to hard lock a byte range of the lock file on Windows systems. - - Lock file cleanup: Windows attempts to delete the lock file after release, but deletion is - not guaranteed in multi-threaded scenarios where another thread holds an open handle. The lock - file may persist on disk, which does not affect lock correctness. - """ - - def _acquire(self) -> None: - raise_on_not_writable_file(self.lock_file) - ensure_directory_exists(self.lock_file) - - # The reparse test is bound to the opened handle, so a symlink or junction swapped in cannot defeat it - # through a check-then-open TOCTOU race. - fd = _open_non_reparse_fd(self.lock_file, self._open_mode()) - if fd is None: - return # open contention (share conflict or a name pending deletion); let the retry loop try again - try: - locked = _lock_fd_nonblocking(fd) - if locked: - self._mark_descriptor_owned(fd) - except BaseException: # pragma: no cover # cleanup only if the lock attempt itself raises - os.close(fd) - raise - if not locked: - os.close(fd) # another holder owns the byte-range lock; let the retry loop try again - - def _release(self) -> None: - fd = cast("int", self._context.lock_file_fd) - # Retain the descriptor until the OS unlock succeeds: if UnlockFileEx raises, the byte-range lock is still - # held, so is_locked must keep reporting held rather than losing the fd. Only after the unlock commits do - # close and unlink run as post-unlock cleanup; their failure cannot make the lock held again. - _unlock_fd(fd) - self._mark_descriptor_released() - self._close_released_fd(fd, default_suppresses=False) - if not self._preserve_lock_file: # preserve_lock_file keeps a stable file identity for the caller (#605) - with suppress(OSError): - Path(self.lock_file).unlink() - - def _open_non_reparse_fd(path: str, mode: int) -> int | None: - """ - Open *path* for locking while refusing reparse points, bound to the handle actually locked. - - The file is opened through ``NtCreateFile`` with ``FILE_OPEN_REPARSE_POINT`` so a symlink or junction planted - at the path is not followed, and the reparse decision is read from *that* handle via - ``GetFileInformationByHandle`` rather than from a prior pathname query. Reading the held handle closes the - check-then-open race: an attacker cannot swap the path between validation and use because both act on the same - handle. Share mode omits delete so a peer cannot unlink or rename the file out from under a live holder, - matching ``os.open``'s ``_SH_DENYNO``. - - ``NtCreateFile`` is used instead of ``CreateFileW`` because its return value carries the raw ``NTSTATUS``. - Windows collapses a transient delete-pending name and a permanent access denial into the same Win32 - ``ERROR_ACCESS_DENIED``; the status keeps them apart, so a real denial fails fast instead of spinning until the - caller's timeout (#604). - - The reparse option only guards the final path component; Windows still follows reparse points in intermediate - directories. This assumes the lock file sits in a lock directory untrusted users cannot modify. A path with - attacker-controlled parent directories would need component-by-component handle validation. - - :param path: the lock file path. - :param mode: the permission mode; as ``os.open`` does on Windows, a cleared owner-write bit creates the file - read-only. The attribute only takes effect when the file is created, not when an existing one is opened. - - :returns: a file descriptor owning the opened handle, or ``None`` on a sharing violation or a delete-pending - name the caller should treat as contention and retry. - - :raises OSError: if the path resolves to a reparse point, or the open fails for any other reason, raised with - the Win32 error the status maps to. - - """ - # Emit the audit event os.open would, so consumers watching "open" still see the path-level open and can veto. - sys.audit("open", path, None, os.O_RDWR | os.O_CREAT) - handle, status = _nt_open(path, read_only=not mode & _OWNER_WRITE) - if status != _STATUS_SUCCESS: - if status in {_STATUS_SHARING_VIOLATION, _STATUS_DELETE_PENDING}: - return None - winerror = _ntdll.RtlNtStatusToDosError(status) - raise OSError(None, ctypes.FormatError(winerror).strip(), path, winerror) - - info = _BY_HANDLE_FILE_INFORMATION() - # Querying an open handle we just created does not fail. - if not _kernel32.GetFileInformationByHandle(handle, ctypes.byref(info)): # pragma: no cover - err = ctypes.get_last_error() - _kernel32.CloseHandle(handle) - raise ctypes.WinError(err) - if info.dwFileAttributes & _FILE_ATTRIBUTE_REPARSE_POINT: - _kernel32.CloseHandle(handle) - msg = f"Lock file is a reparse point (symlink/junction): {path}" - raise OSError(msg) - - try: - # O_NOINHERIT mirrors os.open on Windows: the lock fd must not leak into child processes. - return msvcrt.open_osfhandle(handle, os.O_RDWR | os.O_NOINHERIT) - except BaseException: # pragma: no cover # open_osfhandle audits too; a hook raising must not leak the handle - _kernel32.CloseHandle(handle) - raise - - def _nt_open(path: str, *, read_only: bool) -> tuple[int, int]: - """ - Open *path* through ``NtCreateFile`` and return ``(handle, status)``. - - ``RtlDosPathNameToNtPathName_U_WithStatus`` translates the Win32 path to the NT namespace, handling relative, - drive, UNC and extended-length path forms as Win32 itself would, and allocates a buffer that - ``RtlFreeUnicodeString`` releases. The handle is ``0`` unless the status is ``STATUS_SUCCESS``. - """ - nt_name = _UNICODE_STRING() - status = _ntdll.RtlDosPathNameToNtPathName_U_WithStatus(path, ctypes.byref(nt_name), None, None) & 0xFFFFFFFF - if status != _STATUS_SUCCESS: - return 0, status - try: - attributes = _OBJECT_ATTRIBUTES() - attributes.Length = ctypes.sizeof(_OBJECT_ATTRIBUTES) - attributes.ObjectName = ctypes.pointer(nt_name) - attributes.Attributes = _OBJ_CASE_INSENSITIVE - handle = wintypes.HANDLE() - io_status = _IO_STATUS_BLOCK() - status = ( - _ntdll.NtCreateFile( - ctypes.byref(handle), - _DESIRED_ACCESS, - ctypes.byref(attributes), - ctypes.byref(io_status), - None, - _FILE_ATTRIBUTE_READONLY if read_only else _FILE_ATTRIBUTE_NORMAL, - _FILE_SHARE_READ_WRITE, - _FILE_OPEN_IF, - _CREATE_OPTIONS, - None, - 0, - ) - & 0xFFFFFFFF - ) - finally: - _ntdll.RtlFreeUnicodeString(ctypes.byref(nt_name)) - if status != _STATUS_SUCCESS: - return 0, status - return handle.value or 0, status - -else: # pragma: win32 no cover - - class WindowsFileLock(BaseFileLock): - """Uses ``LockFileEx`` to hard lock a byte range of the lock file on Windows systems.""" - - def _acquire(self) -> None: - raise NotImplementedError - - def _release(self) -> None: - raise NotImplementedError - - -__all__ = [ - "WindowsFileLock", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/asyncio.py b/bundle/python-cpu/Lib/site-packages/filelock/asyncio.py deleted file mode 100644 index 9d97371e547f2156931fa9d79ce3c65295582fb2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/asyncio.py +++ /dev/null @@ -1,759 +0,0 @@ -"""An asyncio-based implementation of the file lock.""" - -from __future__ import annotations - -import asyncio -import contextlib -import logging -import os -import time -from dataclasses import dataclass -from inspect import iscoroutinefunction -from threading import local -from typing import TYPE_CHECKING, Final, NoReturn, TypeVar, cast - -from ._api import ( - _UNSET_FILE_MODE, - BaseFileLock, - CloseErrorPolicy, - ContextErrorPolicy, - FileLockContext, - FileLockMeta, - _append_exception_context, - _canonical, - _ExtraValue, - _fork_transition, - _grouped_errors, - _raise_body_and_release, - _raise_chained_errors, - _raise_cleanup_errors, - _raise_grouped_errors, - _register_fork_object, -) -from ._async import ( - _AsyncTransitionGate, - _AsyncTransitionUnavailableError, - _BackendOutcome, - _capture_awaitable, - _capture_call, - _drain_future, - _future_result, - _wait_until_done, -) -from ._error import Timeout -from ._lease import SoftFileLease -from ._soft import SoftFileLock -from ._strict import StrictSoftFileLock -from ._unix import UnixFileLock -from ._windows import WindowsFileLock - -if TYPE_CHECKING: - import sys - from collections.abc import Awaitable, Callable, Coroutine, Hashable - from concurrent import futures - from types import TracebackType - - if sys.version_info >= (3, 11): # pragma: no cover (py311+) - from typing import Self - else: # pragma: no cover ( _AT: - if thread_local and run_in_executor: - msg = "run_in_executor is not supported when thread_local is True" - raise ValueError(msg) - return super().__call__( # a subclass may add options of its own, as AsyncSoftFileLease does - **kwargs, - lock_file=lock_file, - timeout=timeout, - mode=mode, - thread_local=thread_local, - blocking=blocking, - is_singleton=is_singleton, - poll_interval=poll_interval, - lifetime=lifetime, - context_error_policy=context_error_policy, - close_error_policy=close_error_policy, - fallback_to_soft=fallback_to_soft, - preserve_lock_file=preserve_lock_file, - on_acquired=on_acquired, - loop=loop, - run_in_executor=run_in_executor, - executor=executor, - ) - - -class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta): - """ - Base class for asynchronous file locks. - - .. versionadded:: 3.15.0 - - """ - - _deadlock_holder_desc: str = "BaseAsyncFileLock instance in this task" - _constructor_lifetime_warning_stacklevel: int = 4 - - @staticmethod - def _deadlock_scope() -> Hashable | None: - # One event loop thread runs every task, so a thread-scoped registry cannot tell a same-task reacquire - # (a real deadlock: the polling task never reaches its own release) from another task queuing behind the - # holder (no deadlock: each poll yields, so the holder runs on and releases). Only the first may fail - # fast, so scope holders to the task. - return asyncio.current_task() - - def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option - self, - lock_file: str | os.PathLike[str], - timeout: float = -1, - mode: int = _UNSET_FILE_MODE, - thread_local: bool = False, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility - *, - blocking: bool = True, - is_singleton: bool = False, - poll_interval: float = 0.05, - lifetime: float | None = None, - context_error_policy: ContextErrorPolicy = "chain", - close_error_policy: CloseErrorPolicy = "default", - fallback_to_soft: bool = True, - preserve_lock_file: bool = False, - on_acquired: Callable[[int], None] | None = None, - loop: asyncio.AbstractEventLoop | None = None, - run_in_executor: bool = True, - executor: futures.Executor | None = None, - ) -> None: - """ - Create a new lock object. - - :param lock_file: path to the file - :param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the - acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a - negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock. - :param mode: file permissions for the lockfile. When not specified, the OS controls permissions via umask and - default ACLs, preserving POSIX default ACL inheritance in shared directories. - :param thread_local: Whether this object's internal context should be thread local or not. If this is set to - ``False`` then the lock will be reentrant across threads. When ``True`` (the default), **all fields of the - lock's internal context are per-thread**, including the configuration values ``poll_interval``, ``timeout``, - ``blocking``, ``mode``, and ``lifetime``. Setting one of these properties from one thread does not change - the value seen by another thread; threads that did not perform the write continue to see the value supplied - at construction time. If you need configuration values to be visible across threads, construct the lock - with ``thread_local=False``. - :param blocking: whether the lock should be blocking or not - :param is_singleton: If this is set to ``True`` then only one instance of this class will be created per lock - file. This is useful if you want to use the lock object for reentrant locking without needing to pass the - same object around. - :param poll_interval: default interval for polling the lock file, in seconds. It will be used as fallback value - in the acquire method, if no poll_interval value (``None``) is given. - :param lifetime: for :class:`AsyncSoftFileLock`, the age in seconds after which a waiting process may delete - the marker, even while its holder remains alive. This legacy expiry mode does not provide strict mutual - exclusion. ``None`` (the default) disables age-based expiry. Native OS locks (:class:`AsyncFileLock`) - cannot be revoked by file age and ignore a non-``None`` ``lifetime`` with a warning. - :param context_error_policy: how a context manager reconciles a failure in its body with a failure while - releasing on exit. ``"chain"`` (the default) keeps Python's behavior: the release error propagates with the - body error in its ``__context__``. ``"group"`` raises a :class:`BaseExceptionGroup` holding the body error - first and the release error second, so neither hides the other. - :param close_error_policy: for native locks (:class:`AsyncFileLock`), what to do with an ``os.close`` failure - after the OS unlock has already committed. ``"default"`` keeps each platform's historical behavior, - ``"raise"`` always propagates the ``OSError``, and ``"suppress"`` always ignores it. - :param fallback_to_soft: for :class:`AsyncFileLock`, whether to fall back to soft existence locking when - ``flock`` returns ``ENOSYS``. ``True`` (default) keeps the fallback; ``False`` propagates the error. - :param preserve_lock_file: for native locks (:class:`AsyncFileLock`), whether filelock promises not to unlink - the lock pathname on release. ``False`` (default) keeps each backend's cleanup; ``True`` keeps a stable file - identity (Windows skips its unlink, Unix refuses the ``ENOSYS`` soft fallback). :class:`AsyncSoftFileLock` - rejects ``True``. - :param on_acquired: for native locks (:class:`AsyncFileLock`), a callable invoked with the borrowed lock - descriptor once per physical acquisition, after the lock is held but before - :meth:`~BaseAsyncFileLock.acquire` returns. With ``run_in_executor=True`` (the default) it runs in the - backend executor. It must not close or unlock the descriptor; a raise rolls the acquisition back. - :class:`AsyncSoftFileLock` rejects it. - :param loop: The event loop to use. If not specified, the running event loop will be used. - :param run_in_executor: If this is set to ``True`` then the lock will be acquired in an executor. - :param executor: The executor to use. If not specified, the default executor will be used. - - """ - self._creator_pid = os.getpid() - self._is_thread_local = thread_local - self._is_singleton = is_singleton - self._context_error_policy = context_error_policy # already validated by the metaclass - self._close_error_policy = close_error_policy # already validated by the metaclass - self._fallback_to_soft = fallback_to_soft - self._preserve_lock_file = preserve_lock_file # already validated by the metaclass - self._on_acquired = on_acquired # already validated by the metaclass - self._transition_gate: Final[_AsyncTransitionGate] = _AsyncTransitionGate() - - self._context: AsyncFileLockContext = (AsyncThreadLocalFileContext if thread_local else AsyncFileLockContext)( - lock_file=os.fspath(lock_file), - timeout=timeout, - mode=mode, - blocking=blocking, - poll_interval=poll_interval, - lifetime=lifetime, - loop=loop, - run_in_executor=run_in_executor, - executor=executor, - ) - _register_fork_object(self) - - @property - def run_in_executor(self) -> bool: - """Whether run in executor.""" - return self._context.run_in_executor - - @property - def executor(self) -> futures.Executor | None: - """The executor.""" - return self._context.executor - - @executor.setter - def executor(self, value: futures.Executor | None) -> None: # pragma: no cover - """ - Change the executor. - - :param futures.Executor | None value: the new executor or ``None`` - - """ - self._context.executor = value - - @property - def loop(self) -> asyncio.AbstractEventLoop | None: - """The event loop.""" - return self._context.loop - - async def acquire( # ty: ignore[invalid-method-override] - self, - timeout: float | None = None, - poll_interval: float | None = None, - *, - blocking: bool | None = None, - cancel_check: Callable[[], bool] | None = None, - ) -> AsyncAcquireReturnProxy: - """ - Try to acquire the file lock. - - :param timeout: maximum wait time for acquiring the lock, ``None`` means use the default - :attr:`~BaseFileLock.timeout` is and if ``timeout < 0``, there is no timeout and this method will block - until the lock could be acquired - :param poll_interval: interval of trying to acquire the lock file, ``None`` means use the default - :attr:`~BaseFileLock.poll_interval` - :param blocking: defaults to True. If False, function will return immediately if it cannot obtain a lock on the - first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired. - :param cancel_check: a callable returning ``True`` when the acquisition should be canceled. Checked on each poll - iteration. When triggered, raises :class:`~Timeout` just like an expired timeout. - - :returns: a context object that will unlock the file when the context is exited - - :raises Timeout: if fails to acquire lock within the timeout period - - .. code-block:: python - - # You can use this method in the context manager (recommended) - with lock.acquire(): - pass - - # Or use an equivalent try-finally construct: - lock.acquire() - try: - pass - finally: - lock.release() - - """ - self._raise_if_inherited() - if timeout is None: - timeout = self._context.timeout - - if blocking is None: - blocking = self._context.blocking - - if poll_interval is None: - poll_interval = self._context.poll_interval - - start_time = time.perf_counter() - try: - return await self._acquire_with_admission( - blocking=blocking, - cancel_check=cancel_check, - timeout=timeout, - poll_interval=poll_interval, - start_time=start_time, - ) - except _AsyncTransitionUnavailableError: - raise Timeout(self.lock_file) from None - - async def _acquire_with_admission( - self, - *, - blocking: bool, - cancel_check: Callable[[], bool] | None, - timeout: float, - poll_interval: float, - start_time: float, - ) -> AsyncAcquireReturnProxy: - async with self._transition_gate.hold_for_acquire( - blocking=blocking, - cancel_check=cancel_check, - deadline=None if timeout < 0 else start_time + timeout, - poll_interval=poll_interval, - ): - # A canceled provisional acquire must finish rollback before another caller can claim its descriptor. - canonical = _canonical(self.lock_file) - self._context.lock_counter += 1 - self._raise_if_would_deadlock(canonical, timeout=timeout, blocking=blocking) - self._context.claim_root = canonical - try: - await self._async_poll_until_acquired( - blocking=blocking, - cancel_check=cancel_check, - timeout=timeout, - poll_interval=poll_interval, - start_time=start_time, - ) - except BaseException: - self._reconcile_failed_acquire(canonical) - raise - finally: - self._context.claim_root = None - self._commit_acquire(canonical) - return AsyncAcquireReturnProxy(lock=self) - - async def _async_poll_until_acquired( - self, - *, - blocking: bool, - cancel_check: Callable[[], bool] | None, - timeout: float, - poll_interval: float, - start_time: float, - ) -> None: - lock_id = id(self) - lock_filename = self.lock_file - while True: - self._raise_if_inherited() - if not self.is_locked: - self._try_break_expired_lock() - _LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename) - await self._run_acquire_attempt() - self._raise_if_inherited() - if self.is_locked: - _LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename) - return - if self._check_give_up( - blocking=blocking, - cancel_check=cancel_check, - timeout=timeout, - start_time=start_time, - ): - raise Timeout(lock_filename) - _LOGGER.debug("Lock %s not acquired on %s, waiting %s seconds ...", lock_id, lock_filename, poll_interval) - await asyncio.sleep(poll_interval) - - async def _run_acquire_attempt(self) -> None: - acquire_future = self._start_internal_method( - self._acquire_with_fork_tracking_async - if iscoroutinefunction(self._acquire) - else self._acquire_with_fork_tracking - ) - try: - await _wait_until_done(acquire_future) - except asyncio.CancelledError as cancellation: - acquire_error: BaseException | None = None - try: - await _drain_future(acquire_future) - except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below - acquire_error = error - - rollback_error: BaseException | None = None - if self.is_locked: - try: - await _drain_future(self._start_tracked_release()) - except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below - rollback_error = error - - if acquire_error is not None: - if rollback_error is not None: # pragma: needs fcntl - self._raise_cancelled_errors( - "lock acquisition cancellation, backend attempt, and rollback failed", - cancellation, - acquire_error, - rollback_error, - ) - self._raise_cancelled_errors( - "lock acquisition cancellation and backend attempt both failed", cancellation, acquire_error - ) - if rollback_error is not None: - self._raise_cancelled_errors( - "lock acquisition cancellation and rollback both failed", cancellation, rollback_error - ) - raise - try: - _future_result(acquire_future) - except asyncio.CancelledError as acquire_error: - await self._rollback_backend_cancelled_acquire(acquire_error) - - async def _rollback_backend_cancelled_acquire(self, acquire_error: asyncio.CancelledError) -> NoReturn: - if self.is_locked: - try: - await _drain_future(self._start_tracked_release()) - except BaseException as rollback_error: # ruff:ignore[blind-except] # both backend errors must surface - self._raise_acquire_rollback_errors(acquire_error, rollback_error) - raise acquire_error - - def _raise_acquire_rollback_errors(self, acquire_error: BaseException, rollback_error: BaseException) -> NoReturn: - if self._context_error_policy == "group": - _raise_grouped_errors("lock acquisition backend and rollback both failed", acquire_error, rollback_error) - _raise_chained_errors(acquire_error, rollback_error) - - def _raise_cancelled_errors( - self, - message: str, - cancellation: asyncio.CancelledError, - first_error: BaseException, - second_error: BaseException | None = None, - ) -> NoReturn: - if self._context_error_policy == "group": - marker = ( - (_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR, _ASYNC_RELEASE_CANCELLATION_MARKER) - if message == _ASYNC_RELEASE_CANCELLATION_ERRORS - else None - ) - if second_error is None: - _raise_grouped_errors(message, cancellation, first_error, marker=marker) - _raise_grouped_errors(message, cancellation, first_error, second_error, marker=marker) - if (context := first_error.__context__) is not None and context is not cancellation: - if (cancellation_context := cancellation.__context__) is not None: - _append_exception_context(context, cancellation_context) - cancellation.__context__ = context - first_error.__context__ = cancellation - _raise_chained_errors(first_error, second_error) - - async def release(self, force: bool = False) -> None: # ty: ignore[invalid-method-override] # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility - """ - Release the file lock. The lock is only completely released when the lock counter reaches 0. The lock file - itself may be deleted automatically, the behavior is platform-specific. - - :param force: If true, the lock counter is ignored and the lock is released in every case. - - """ - if self._creator_pid != os.getpid() or not self.is_locked: - return - async with self._transition_gate.hold(): - await self._release_serialized(force=force) - - async def _release_serialized(self, *, force: bool) -> None: - if not self.is_locked: - return - if not force and self._context.lock_counter > 1: - self._context.lock_counter -= 1 - return - - lock_id, lock_filename = id(self), self.lock_file - _LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename) - release_future = self._start_tracked_release() - try: - await _wait_until_done(release_future) - except asyncio.CancelledError as cancellation: - try: - await _drain_future(release_future) - except BaseException as release_error: # ruff:ignore[blind-except] # cancellation and backend failure must both surface - self._commit_release_if_released() - self._raise_cancelled_errors(_ASYNC_RELEASE_CANCELLATION_ERRORS, cancellation, release_error) - self._commit_release() - raise - try: - _future_result(release_future) - except BaseException: # state follows the backend for control-flow exceptions too - self._commit_release_if_released() - raise - self._commit_release() - _LOGGER.debug("Lock %s released on %s", lock_id, lock_filename) - - def _commit_release_if_released(self) -> None: - # Commit only when the backend actually unlocked (close or unlink failed after the OS unlock). If the lock is - # still held, keep the counter so a later release can retry. - if not self.is_locked: - self._commit_release() - - def _start_internal_method( - self, method: Callable[[], None] | Callable[[], Coroutine[None, None, None]] - ) -> asyncio.Future[_BackendOutcome[None]]: - if iscoroutinefunction(method): - return asyncio.create_task(_capture_awaitable(cast("Callable[[], Coroutine[None, None, None]]", method)())) - loop = asyncio.get_running_loop() - sync_method = cast("Callable[[], None]", method) - if self.run_in_executor: - return loop.run_in_executor(self.executor, _capture_call, sync_method) - future: asyncio.Future[_BackendOutcome[None]] = loop.create_future() - future.set_result(_capture_call(sync_method)) - return future - - def _start_tracked_release(self) -> asyncio.Future[_BackendOutcome[None]]: - return self._start_internal_method( - self._release_with_fork_tracking_async - if iscoroutinefunction(self._release) - else self._release_with_fork_tracking - ) - - async def _acquire_with_fork_tracking_async(self) -> None: - with _fork_transition(self): - try: - await cast("Callable[[], Awaitable[None]]", self._acquire)() - except BaseException as acquisition_error: - await self._rollback_failed_acquire_async(acquisition_error) - raise - try: - self._register_context_descriptor() - except BaseException as registration_error: # cancellation must roll back the descriptor - await self._rollback_failed_registration_async(registration_error) - raise - if self.is_locked: - await self._invoke_on_acquired_async() - - async def _rollback_failed_acquire_async(self, acquisition_error: BaseException) -> None: - if not self.is_locked: - return - registration_error: BaseException | None = None - tracking_error: BaseException | None = None - try: - self._register_context_descriptor() - except BaseException as error: # ruff:ignore[blind-except] # preserve registration and acquisition failures - registration_error = error - try: - # Rollback may fail too; retain the fd so a child can close it without another identity probe. - self._register_unverified_context_descriptor() - except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback - tracking_error = error - try: - await _drain_future(self._start_tracked_release()) - except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and acquisition failures - if registration_error is None and tracking_error is None: - self._raise_acquire_rollback_errors(acquisition_error, rollback_error) - _raise_cleanup_errors( - "lock acquisition cleanup failed", - acquisition_error, - registration_error, - tracking_error, - rollback_error, - ) - if registration_error is not None: - _raise_cleanup_errors( - "lock acquisition cleanup failed", acquisition_error, registration_error, tracking_error - ) - - async def _rollback_failed_registration_async(self, registration_error: BaseException) -> None: - tracking_error: BaseException | None = None - try: - # Rollback may fail too; retain the fd so a child can close it without another identity probe. - self._register_unverified_context_descriptor() - except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback - tracking_error = error - try: - await _drain_future(self._start_tracked_release()) - except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and registration failures - _raise_cleanup_errors( - "descriptor registration cleanup failed", registration_error, tracking_error, rollback_error - ) - if tracking_error is not None: # pragma: no cover - requires failed in-memory fallback - _raise_cleanup_errors("descriptor registration cleanup failed", registration_error, tracking_error) - - async def _invoke_on_acquired_async(self) -> None: - if self._on_acquired is None or self._context.lock_counter != 1: - return - try: - self._on_acquired(cast("int", self._context.lock_file_fd)) - except BaseException as callback_error: # caller control-flow errors must release the lock - callback_context = callback_error.__context__ - try: - await _drain_future(self._start_tracked_release()) - except BaseException as release_error: # ruff:ignore[blind-except] # both errors surface via the group below - _raise_body_and_release(callback_error, release_error) - callback_error.__context__ = callback_context - raise - - async def _release_with_fork_tracking_async(self) -> None: - with _fork_transition(self): - try: - await cast("Callable[[], Awaitable[None]]", self._release)() - finally: - self._unregister_released_descriptor() - - def __enter__(self) -> NoReturn: - """Sync context manager entry is not supported because lock acquisition is a coroutine.""" - msg = "Use `async with`: acquire/release are coroutines and cannot be awaited in a sync context manager." - raise NotImplementedError(msg) - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - """Sync context manager exit is not supported because lock release is a coroutine.""" - msg = "Use `async with`: acquire/release are coroutines and cannot be awaited in a sync context manager." - raise NotImplementedError(msg) - - async def __aenter__(self) -> Self: - """ - Acquire the lock. - - :returns: the lock object - - """ - await self.acquire() - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - """ - Release the lock, reconciling a release failure with any body failure per :attr:`context_error_policy`. - - :param exc_type: the exception type if raised - :param exc_value: the exception value if raised - :param traceback: the exception traceback if raised - - """ - await self._release_in_context(exc_value) - - async def _release_in_context( # ty: ignore[invalid-method-override] - self, body_error: BaseException | None - ) -> None: - # The async counterpart of BaseFileLock._release_in_context: await release, then apply the same policy. - try: - await self.release() - except BaseException as release_error: - if body_error is None: - raise - if self._context_error_policy == "chain": - _append_exception_context(release_error, body_error) - raise - if ( - errors := _grouped_errors( - release_error, - _ASYNC_RELEASE_CANCELLATION_ERRORS, - (_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR, _ASYNC_RELEASE_CANCELLATION_MARKER), - ) - ) is not None: - match errors: - # The marker is only ever set on a (CancelledError, backend_error) pair, so this always matches. - case (asyncio.CancelledError() as cancellation, backend_error): # pragma: no branch - _raise_grouped_errors(_ASYNC_CONTEXT_RELEASE_ERRORS, body_error, cancellation, backend_error) - _raise_body_and_release(body_error, release_error) - - def __del__(self) -> None: - """Release on deletion; safe to call during GC even when no event loop is running.""" - if vars(self).get("_creator_pid") != os.getpid(): - return # pragma: forked child - with contextlib.suppress(Exception): - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = self._context.loop if self._context.loop and not self._context.loop.is_closed() else None - if loop is None: - return - if not loop.is_running(): # pragma: no cover - loop.run_until_complete(self.release(force=True)) - else: - loop.create_task(self.release(force=True)) - - -@dataclass -class AsyncFileLockContext(FileLockContext): - """A dataclass which holds the context for a ``BaseAsyncFileLock`` object.""" - - #: Whether run in executor - run_in_executor: bool = True - - #: The executor - executor: futures.Executor | None = None - - #: The loop - loop: asyncio.AbstractEventLoop | None = None - - -class AsyncThreadLocalFileContext(AsyncFileLockContext, local): - """A thread local version of the ``FileLockContext`` class.""" - - -class AsyncAcquireReturnProxy: - """A context-aware object that will release the lock file when exiting.""" - - def __init__(self, lock: BaseAsyncFileLock) -> None: # ruff:ignore[undocumented-public-init] # trivial release-on-exit proxy - self.lock = lock - - async def __aenter__(self) -> BaseAsyncFileLock: # ruff:ignore[undocumented-magic-method] # returns the wrapped lock - return self.lock - - async def __aexit__( # ruff:ignore[undocumented-magic-method] # releases the wrapped lock on exit - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - await self.lock._release_in_context(exc_value) # ruff:ignore[private-member-access] # releases the wrapped lock's context - - -class AsyncSoftFileLock(SoftFileLock, BaseAsyncFileLock): - """Simply watches the existence of the lock file.""" - - _lifetime_replacements: tuple[str, str] | None = ("AsyncStrictSoftFileLock", "AsyncSoftFileLease") - - -class AsyncStrictSoftFileLock(StrictSoftFileLock, BaseAsyncFileLock): - """Run strict owner-claim locking without blocking the event loop.""" - - -class AsyncSoftFileLease(SoftFileLease, BaseAsyncFileLock): - """Existence lock whose claim expires, so a peer may take it while the previous holder still runs.""" - - -class AsyncUnixFileLock(UnixFileLock, BaseAsyncFileLock): - """Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems.""" - - -class AsyncWindowsFileLock(WindowsFileLock, BaseAsyncFileLock): - """Uses the :func:`msvcrt.locking` to hard lock the lock file on windows systems.""" - - -__all__ = [ - "AsyncAcquireReturnProxy", - "AsyncSoftFileLease", - "AsyncSoftFileLock", - "AsyncStrictSoftFileLock", - "AsyncUnixFileLock", - "AsyncWindowsFileLock", - "BaseAsyncFileLock", -] diff --git a/bundle/python-cpu/Lib/site-packages/filelock/py.typed b/bundle/python-cpu/Lib/site-packages/filelock/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/filelock/version.py b/bundle/python-cpu/Lib/site-packages/filelock/version.py deleted file mode 100644 index 91df61eeb6075a4f6c97a71caf096dbf1283cb90..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/filelock/version.py +++ /dev/null @@ -1,24 +0,0 @@ -# file generated by vcs-versioning -# don't change, don't track in version control -from __future__ import annotations - -__all__ = [ - "__version__", - "__version_tuple__", - "version", - "version_tuple", - "__commit_id__", - "commit_id", -] - -version: str -__version__: str -__version_tuple__: tuple[int | str, ...] -version_tuple: tuple[int | str, ...] -commit_id: str | None -__commit_id__: str | None - -__version__ = version = '3.32.3' -__version_tuple__ = version_tuple = (3, 32, 3) - -__commit_id__ = commit_id = None diff --git a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/METADATA deleted file mode 100644 index 9d8623c9aef83d5bc5eaeef3bb42591edb8e2cfb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/METADATA +++ /dev/null @@ -1,91 +0,0 @@ -Metadata-Version: 2.4 -Name: Flask -Version: 3.1.3 -Summary: A simple framework for building complex web applications. -Maintainer-email: Pallets -Requires-Python: >=3.9 -Description-Content-Type: text/markdown -License-Expression: BSD-3-Clause -Classifier: Development Status :: 5 - Production/Stable -Classifier: Environment :: Web Environment -Classifier: Framework :: Flask -Classifier: Intended Audience :: Developers -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content -Classifier: Topic :: Internet :: WWW/HTTP :: WSGI -Classifier: Topic :: Internet :: WWW/HTTP :: WSGI :: Application -Classifier: Topic :: Software Development :: Libraries :: Application Frameworks -Classifier: Typing :: Typed -License-File: LICENSE.txt -Requires-Dist: blinker>=1.9.0 -Requires-Dist: click>=8.1.3 -Requires-Dist: importlib-metadata>=3.6.0; python_version < '3.10' -Requires-Dist: itsdangerous>=2.2.0 -Requires-Dist: jinja2>=3.1.2 -Requires-Dist: markupsafe>=2.1.1 -Requires-Dist: werkzeug>=3.1.0 -Requires-Dist: asgiref>=3.2 ; extra == "async" -Requires-Dist: python-dotenv ; extra == "dotenv" -Project-URL: Changes, https://flask.palletsprojects.com/page/changes/ -Project-URL: Chat, https://discord.gg/pallets -Project-URL: Documentation, https://flask.palletsprojects.com/ -Project-URL: Donate, https://palletsprojects.com/donate -Project-URL: Source, https://github.com/pallets/flask/ -Provides-Extra: async -Provides-Extra: dotenv - -
- -# Flask - -Flask is a lightweight [WSGI] web application framework. It is designed -to make getting started quick and easy, with the ability to scale up to -complex applications. It began as a simple wrapper around [Werkzeug] -and [Jinja], and has become one of the most popular Python web -application frameworks. - -Flask offers suggestions, but doesn't enforce any dependencies or -project layout. It is up to the developer to choose the tools and -libraries they want to use. There are many extensions provided by the -community that make adding new functionality easy. - -[WSGI]: https://wsgi.readthedocs.io/ -[Werkzeug]: https://werkzeug.palletsprojects.com/ -[Jinja]: https://jinja.palletsprojects.com/ - -## A Simple Example - -```python -# save this as app.py -from flask import Flask - -app = Flask(__name__) - -@app.route("/") -def hello(): - return "Hello, World!" -``` - -``` -$ flask run - * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit) -``` - -## Donate - -The Pallets organization develops and supports Flask and the libraries -it uses. In order to grow the community of contributors and users, and -allow the maintainers to devote more time to the projects, [please -donate today]. - -[please donate today]: https://palletsprojects.com/donate - -## Contributing - -See our [detailed contributing documentation][contrib] for many ways to -contribute, including reporting issues, requesting features, asking or answering -questions, and making PRs. - -[contrib]: https://palletsprojects.com/contributing/ - diff --git a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/RECORD deleted file mode 100644 index 0a37350006bfe191dd7c05e274e3eb901e38b0d9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/RECORD +++ /dev/null @@ -1,34 +0,0 @@ -../../Scripts/flask.exe,sha256=kl49YVVKZ2UZnGnHTi8JK6o5jH3RVcxTbr5-FqUgr2o,46080 -flask-3.1.3.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -flask-3.1.3.dist-info/METADATA,sha256=qmdg7W9UVwRHTXBzPkpjp_FIHjdpc-3IlqE9AqciTHw,3167 -flask-3.1.3.dist-info/RECORD,, -flask-3.1.3.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -flask-3.1.3.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 -flask-3.1.3.dist-info/entry_points.txt,sha256=bBP7hTOS5fz9zLtC7sPofBZAlMkEvBxu7KqS6l5lvc4,40 -flask-3.1.3.dist-info/licenses/LICENSE.txt,sha256=SJqOEQhQntmKN7uYPhHg9-HTHwvY-Zp5yESOf_N9B-o,1475 -flask/__init__.py,sha256=mHvJN9Swtl1RDtjCqCIYyIniK_SZ_l_hqUynOzgpJ9o,2701 -flask/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30 -flask/app.py,sha256=k7tW8LHRSldUi6zKsFKK7Axa_WL4zu1e2wPNthIsu7o,61719 -flask/blueprints.py,sha256=p5QE2lY18GItbdr_RKRpZ8Do17g0PvQGIgZkSUDhX2k,4541 -flask/cli.py,sha256=Pfh72-BxlvoH0QHCDOc1HvXG7Kq5Xetf3zzNz2kNSHk,37184 -flask/config.py,sha256=PiqF0DPam6HW0FH4CH1hpXTBe30NSzjPEOwrz1b6kt0,13219 -flask/ctx.py,sha256=oMe0TRsScW0qdaIqavVsk8P9qiEvAY5VHn1FAgkX8nk,15521 -flask/debughelpers.py,sha256=PGIDhStW_efRjpaa3zHIpo-htStJOR41Ip3OJWPYBwo,6080 -flask/globals.py,sha256=XdQZmStBmPIs8t93tjx6pO7Bm3gobAaONWkFcUHaGas,1713 -flask/helpers.py,sha256=rJZge7_J288J1UQv5-kNf4oEaw332PP8NTW0QRIBbXE,23517 -flask/json/__init__.py,sha256=hLNR898paqoefdeAhraa5wyJy-bmRB2k2dV4EgVy2Z8,5602 -flask/json/provider.py,sha256=5imEzY5HjV2HoUVrQbJLqXCzMNpZXfD0Y1XqdLV2XBA,7672 -flask/json/tag.py,sha256=DhaNwuIOhdt2R74oOC9Y4Z8ZprxFYiRb5dUP5byyINw,9281 -flask/logging.py,sha256=8sM3WMTubi1cBb2c_lPkWpN0J8dMAqrgKRYLLi1dCVI,2377 -flask/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -flask/sansio/README.md,sha256=-0X1tECnilmz1cogx-YhNw5d7guK7GKrq_DEV2OzlU0,228 -flask/sansio/app.py,sha256=whGURQDkN0jmhS4CHO7DQ96GGlZS0kETkKkAkoRjl4U,38106 -flask/sansio/blueprints.py,sha256=Tqe-7EkZ-tbWchm8iDoCfD848f0_3nLv6NNjeIPvHwM,24637 -flask/sansio/scaffold.py,sha256=wSASXYdFRWJmqcL0Xq-T7N-PDVUSiFGvjO9kPZg58bk,30371 -flask/sessions.py,sha256=eywRqmytTmYnX_EC78-YBGJoTc5XD_lRphQG5LbN1d0,14969 -flask/signals.py,sha256=V7lMUww7CqgJ2ThUBn1PiatZtQanOyt7OZpu2GZI-34,750 -flask/templating.py,sha256=vbIkwYAxsSEfDxQID1gKRvBQQcGWEuWYCnH0XK3EqOI,7678 -flask/testing.py,sha256=zzC7XxhBWOP9H697IV_4SG7Lg3Lzb5PWiyEP93_KQXE,10117 -flask/typing.py,sha256=L-L5t2jKgS0aOmVhioQ_ylqcgiVFnA6yxO-RLNhq-GU,3293 -flask/views.py,sha256=xzJx6oJqGElThtEghZN7ZQGMw5TDFyuRxUkecwRuAoA,6962 -flask/wrappers.py,sha256=jUkv4mVek2Iq4hwxd4RvqrIMb69Bv0PElDgWLmd5ORo,9406 diff --git a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/WHEEL deleted file mode 100644 index d8b9936dad9ab2513fa6979f411560d3b6b57e37..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: flit 3.12.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/entry_points.txt b/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/entry_points.txt deleted file mode 100644 index eec6733e577feb9487435b9722713a820bd4ccc1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/entry_points.txt +++ /dev/null @@ -1,3 +0,0 @@ -[console_scripts] -flask=flask.cli:main - diff --git a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/licenses/LICENSE.txt b/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/licenses/LICENSE.txt deleted file mode 100644 index 9d227a0cc43c3268d15722b763bd94ad298645a1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask-3.1.3.dist-info/licenses/LICENSE.txt +++ /dev/null @@ -1,28 +0,0 @@ -Copyright 2010 Pallets - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bundle/python-cpu/Lib/site-packages/flask/__init__.py b/bundle/python-cpu/Lib/site-packages/flask/__init__.py deleted file mode 100644 index 1fdc50cea1352b2f1e789ff07c430376924b2f2f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/__init__.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -import typing as t - -from . import json as json -from .app import Flask as Flask -from .blueprints import Blueprint as Blueprint -from .config import Config as Config -from .ctx import after_this_request as after_this_request -from .ctx import copy_current_request_context as copy_current_request_context -from .ctx import has_app_context as has_app_context -from .ctx import has_request_context as has_request_context -from .globals import current_app as current_app -from .globals import g as g -from .globals import request as request -from .globals import session as session -from .helpers import abort as abort -from .helpers import flash as flash -from .helpers import get_flashed_messages as get_flashed_messages -from .helpers import get_template_attribute as get_template_attribute -from .helpers import make_response as make_response -from .helpers import redirect as redirect -from .helpers import send_file as send_file -from .helpers import send_from_directory as send_from_directory -from .helpers import stream_with_context as stream_with_context -from .helpers import url_for as url_for -from .json import jsonify as jsonify -from .signals import appcontext_popped as appcontext_popped -from .signals import appcontext_pushed as appcontext_pushed -from .signals import appcontext_tearing_down as appcontext_tearing_down -from .signals import before_render_template as before_render_template -from .signals import got_request_exception as got_request_exception -from .signals import message_flashed as message_flashed -from .signals import request_finished as request_finished -from .signals import request_started as request_started -from .signals import request_tearing_down as request_tearing_down -from .signals import template_rendered as template_rendered -from .templating import render_template as render_template -from .templating import render_template_string as render_template_string -from .templating import stream_template as stream_template -from .templating import stream_template_string as stream_template_string -from .wrappers import Request as Request -from .wrappers import Response as Response - -if not t.TYPE_CHECKING: - - def __getattr__(name: str) -> t.Any: - if name == "__version__": - import importlib.metadata - import warnings - - warnings.warn( - "The '__version__' attribute is deprecated and will be removed in" - " Flask 3.2. Use feature detection or" - " 'importlib.metadata.version(\"flask\")' instead.", - DeprecationWarning, - stacklevel=2, - ) - return importlib.metadata.version("flask") - - raise AttributeError(name) diff --git a/bundle/python-cpu/Lib/site-packages/flask/__main__.py b/bundle/python-cpu/Lib/site-packages/flask/__main__.py deleted file mode 100644 index 4e28416e104515e90fca4b69cc60d0c61fd15d61..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .cli import main - -main() diff --git a/bundle/python-cpu/Lib/site-packages/flask/app.py b/bundle/python-cpu/Lib/site-packages/flask/app.py deleted file mode 100644 index cc326dbe3c2a373171ecb372206b9e694e9c5ffe..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/app.py +++ /dev/null @@ -1,1536 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import os -import sys -import typing as t -import weakref -from datetime import timedelta -from inspect import iscoroutinefunction -from itertools import chain -from types import TracebackType -from urllib.parse import quote as _url_quote - -import click -from werkzeug.datastructures import Headers -from werkzeug.datastructures import ImmutableDict -from werkzeug.exceptions import BadRequestKeyError -from werkzeug.exceptions import HTTPException -from werkzeug.exceptions import InternalServerError -from werkzeug.routing import BuildError -from werkzeug.routing import MapAdapter -from werkzeug.routing import RequestRedirect -from werkzeug.routing import RoutingException -from werkzeug.routing import Rule -from werkzeug.serving import is_running_from_reloader -from werkzeug.wrappers import Response as BaseResponse -from werkzeug.wsgi import get_host - -from . import cli -from . import typing as ft -from .ctx import AppContext -from .ctx import RequestContext -from .globals import _cv_app -from .globals import _cv_request -from .globals import current_app -from .globals import g -from .globals import request -from .globals import request_ctx -from .globals import session -from .helpers import get_debug_flag -from .helpers import get_flashed_messages -from .helpers import get_load_dotenv -from .helpers import send_from_directory -from .sansio.app import App -from .sansio.scaffold import _sentinel -from .sessions import SecureCookieSessionInterface -from .sessions import SessionInterface -from .signals import appcontext_tearing_down -from .signals import got_request_exception -from .signals import request_finished -from .signals import request_started -from .signals import request_tearing_down -from .templating import Environment -from .wrappers import Request -from .wrappers import Response - -if t.TYPE_CHECKING: # pragma: no cover - from _typeshed.wsgi import StartResponse - from _typeshed.wsgi import WSGIEnvironment - - from .testing import FlaskClient - from .testing import FlaskCliRunner - from .typing import HeadersValue - -T_shell_context_processor = t.TypeVar( - "T_shell_context_processor", bound=ft.ShellContextProcessorCallable -) -T_teardown = t.TypeVar("T_teardown", bound=ft.TeardownCallable) -T_template_filter = t.TypeVar("T_template_filter", bound=ft.TemplateFilterCallable) -T_template_global = t.TypeVar("T_template_global", bound=ft.TemplateGlobalCallable) -T_template_test = t.TypeVar("T_template_test", bound=ft.TemplateTestCallable) - - -def _make_timedelta(value: timedelta | int | None) -> timedelta | None: - if value is None or isinstance(value, timedelta): - return value - - return timedelta(seconds=value) - - -class Flask(App): - """The flask object implements a WSGI application and acts as the central - object. It is passed the name of the module or package of the - application. Once it is created it will act as a central registry for - the view functions, the URL rules, template configuration and much more. - - The name of the package is used to resolve resources from inside the - package or the folder the module is contained in depending on if the - package parameter resolves to an actual python package (a folder with - an :file:`__init__.py` file inside) or a standard module (just a ``.py`` file). - - For more information about resource loading, see :func:`open_resource`. - - Usually you create a :class:`Flask` instance in your main module or - in the :file:`__init__.py` file of your package like this:: - - from flask import Flask - app = Flask(__name__) - - .. admonition:: About the First Parameter - - The idea of the first parameter is to give Flask an idea of what - belongs to your application. This name is used to find resources - on the filesystem, can be used by extensions to improve debugging - information and a lot more. - - So it's important what you provide there. If you are using a single - module, `__name__` is always the correct value. If you however are - using a package, it's usually recommended to hardcode the name of - your package there. - - For example if your application is defined in :file:`yourapplication/app.py` - you should create it with one of the two versions below:: - - app = Flask('yourapplication') - app = Flask(__name__.split('.')[0]) - - Why is that? The application will work even with `__name__`, thanks - to how resources are looked up. However it will make debugging more - painful. Certain extensions can make assumptions based on the - import name of your application. For example the Flask-SQLAlchemy - extension will look for the code in your application that triggered - an SQL query in debug mode. If the import name is not properly set - up, that debugging information is lost. (For example it would only - pick up SQL queries in `yourapplication.app` and not - `yourapplication.views.frontend`) - - .. versionadded:: 0.7 - The `static_url_path`, `static_folder`, and `template_folder` - parameters were added. - - .. versionadded:: 0.8 - The `instance_path` and `instance_relative_config` parameters were - added. - - .. versionadded:: 0.11 - The `root_path` parameter was added. - - .. versionadded:: 1.0 - The ``host_matching`` and ``static_host`` parameters were added. - - .. versionadded:: 1.0 - The ``subdomain_matching`` parameter was added. Subdomain - matching needs to be enabled manually now. Setting - :data:`SERVER_NAME` does not implicitly enable it. - - :param import_name: the name of the application package - :param static_url_path: can be used to specify a different path for the - static files on the web. Defaults to the name - of the `static_folder` folder. - :param static_folder: The folder with static files that is served at - ``static_url_path``. Relative to the application ``root_path`` - or an absolute path. Defaults to ``'static'``. - :param static_host: the host to use when adding the static route. - Defaults to None. Required when using ``host_matching=True`` - with a ``static_folder`` configured. - :param host_matching: set ``url_map.host_matching`` attribute. - Defaults to False. - :param subdomain_matching: consider the subdomain relative to - :data:`SERVER_NAME` when matching routes. Defaults to False. - :param template_folder: the folder that contains the templates that should - be used by the application. Defaults to - ``'templates'`` folder in the root path of the - application. - :param instance_path: An alternative instance path for the application. - By default the folder ``'instance'`` next to the - package or module is assumed to be the instance - path. - :param instance_relative_config: if set to ``True`` relative filenames - for loading the config are assumed to - be relative to the instance path instead - of the application root. - :param root_path: The path to the root of the application files. - This should only be set manually when it can't be detected - automatically, such as for namespace packages. - """ - - default_config = ImmutableDict( - { - "DEBUG": None, - "TESTING": False, - "PROPAGATE_EXCEPTIONS": None, - "SECRET_KEY": None, - "SECRET_KEY_FALLBACKS": None, - "PERMANENT_SESSION_LIFETIME": timedelta(days=31), - "USE_X_SENDFILE": False, - "TRUSTED_HOSTS": None, - "SERVER_NAME": None, - "APPLICATION_ROOT": "/", - "SESSION_COOKIE_NAME": "session", - "SESSION_COOKIE_DOMAIN": None, - "SESSION_COOKIE_PATH": None, - "SESSION_COOKIE_HTTPONLY": True, - "SESSION_COOKIE_SECURE": False, - "SESSION_COOKIE_PARTITIONED": False, - "SESSION_COOKIE_SAMESITE": None, - "SESSION_REFRESH_EACH_REQUEST": True, - "MAX_CONTENT_LENGTH": None, - "MAX_FORM_MEMORY_SIZE": 500_000, - "MAX_FORM_PARTS": 1_000, - "SEND_FILE_MAX_AGE_DEFAULT": None, - "TRAP_BAD_REQUEST_ERRORS": None, - "TRAP_HTTP_EXCEPTIONS": False, - "EXPLAIN_TEMPLATE_LOADING": False, - "PREFERRED_URL_SCHEME": "http", - "TEMPLATES_AUTO_RELOAD": None, - "MAX_COOKIE_SIZE": 4093, - "PROVIDE_AUTOMATIC_OPTIONS": True, - } - ) - - #: The class that is used for request objects. See :class:`~flask.Request` - #: for more information. - request_class: type[Request] = Request - - #: The class that is used for response objects. See - #: :class:`~flask.Response` for more information. - response_class: type[Response] = Response - - #: the session interface to use. By default an instance of - #: :class:`~flask.sessions.SecureCookieSessionInterface` is used here. - #: - #: .. versionadded:: 0.8 - session_interface: SessionInterface = SecureCookieSessionInterface() - - def __init__( - self, - import_name: str, - static_url_path: str | None = None, - static_folder: str | os.PathLike[str] | None = "static", - static_host: str | None = None, - host_matching: bool = False, - subdomain_matching: bool = False, - template_folder: str | os.PathLike[str] | None = "templates", - instance_path: str | None = None, - instance_relative_config: bool = False, - root_path: str | None = None, - ): - super().__init__( - import_name=import_name, - static_url_path=static_url_path, - static_folder=static_folder, - static_host=static_host, - host_matching=host_matching, - subdomain_matching=subdomain_matching, - template_folder=template_folder, - instance_path=instance_path, - instance_relative_config=instance_relative_config, - root_path=root_path, - ) - - #: The Click command group for registering CLI commands for this - #: object. The commands are available from the ``flask`` command - #: once the application has been discovered and blueprints have - #: been registered. - self.cli = cli.AppGroup() - - # Set the name of the Click group in case someone wants to add - # the app's commands to another CLI tool. - self.cli.name = self.name - - # Add a static route using the provided static_url_path, static_host, - # and static_folder if there is a configured static_folder. - # Note we do this without checking if static_folder exists. - # For one, it might be created while the server is running (e.g. during - # development). Also, Google App Engine stores static files somewhere - if self.has_static_folder: - assert bool(static_host) == host_matching, ( - "Invalid static_host/host_matching combination" - ) - # Use a weakref to avoid creating a reference cycle between the app - # and the view function (see #3761). - self_ref = weakref.ref(self) - self.add_url_rule( - f"{self.static_url_path}/", - endpoint="static", - host=static_host, - view_func=lambda **kw: self_ref().send_static_file(**kw), # type: ignore - ) - - def get_send_file_max_age(self, filename: str | None) -> int | None: - """Used by :func:`send_file` to determine the ``max_age`` cache - value for a given file path if it wasn't passed. - - By default, this returns :data:`SEND_FILE_MAX_AGE_DEFAULT` from - the configuration of :data:`~flask.current_app`. This defaults - to ``None``, which tells the browser to use conditional requests - instead of a timed cache, which is usually preferable. - - Note this is a duplicate of the same method in the Flask - class. - - .. versionchanged:: 2.0 - The default configuration is ``None`` instead of 12 hours. - - .. versionadded:: 0.9 - """ - value = current_app.config["SEND_FILE_MAX_AGE_DEFAULT"] - - if value is None: - return None - - if isinstance(value, timedelta): - return int(value.total_seconds()) - - return value # type: ignore[no-any-return] - - def send_static_file(self, filename: str) -> Response: - """The view function used to serve files from - :attr:`static_folder`. A route is automatically registered for - this view at :attr:`static_url_path` if :attr:`static_folder` is - set. - - Note this is a duplicate of the same method in the Flask - class. - - .. versionadded:: 0.5 - - """ - if not self.has_static_folder: - raise RuntimeError("'static_folder' must be set to serve static_files.") - - # send_file only knows to call get_send_file_max_age on the app, - # call it here so it works for blueprints too. - max_age = self.get_send_file_max_age(filename) - return send_from_directory( - t.cast(str, self.static_folder), filename, max_age=max_age - ) - - def open_resource( - self, resource: str, mode: str = "rb", encoding: str | None = None - ) -> t.IO[t.AnyStr]: - """Open a resource file relative to :attr:`root_path` for reading. - - For example, if the file ``schema.sql`` is next to the file - ``app.py`` where the ``Flask`` app is defined, it can be opened - with: - - .. code-block:: python - - with app.open_resource("schema.sql") as f: - conn.executescript(f.read()) - - :param resource: Path to the resource relative to :attr:`root_path`. - :param mode: Open the file in this mode. Only reading is supported, - valid values are ``"r"`` (or ``"rt"``) and ``"rb"``. - :param encoding: Open the file with this encoding when opening in text - mode. This is ignored when opening in binary mode. - - .. versionchanged:: 3.1 - Added the ``encoding`` parameter. - """ - if mode not in {"r", "rt", "rb"}: - raise ValueError("Resources can only be opened for reading.") - - path = os.path.join(self.root_path, resource) - - if mode == "rb": - return open(path, mode) # pyright: ignore - - return open(path, mode, encoding=encoding) - - def open_instance_resource( - self, resource: str, mode: str = "rb", encoding: str | None = "utf-8" - ) -> t.IO[t.AnyStr]: - """Open a resource file relative to the application's instance folder - :attr:`instance_path`. Unlike :meth:`open_resource`, files in the - instance folder can be opened for writing. - - :param resource: Path to the resource relative to :attr:`instance_path`. - :param mode: Open the file in this mode. - :param encoding: Open the file with this encoding when opening in text - mode. This is ignored when opening in binary mode. - - .. versionchanged:: 3.1 - Added the ``encoding`` parameter. - """ - path = os.path.join(self.instance_path, resource) - - if "b" in mode: - return open(path, mode) - - return open(path, mode, encoding=encoding) - - def create_jinja_environment(self) -> Environment: - """Create the Jinja environment based on :attr:`jinja_options` - and the various Jinja-related methods of the app. Changing - :attr:`jinja_options` after this will have no effect. Also adds - Flask-related globals and filters to the environment. - - .. versionchanged:: 0.11 - ``Environment.auto_reload`` set in accordance with - ``TEMPLATES_AUTO_RELOAD`` configuration option. - - .. versionadded:: 0.5 - """ - options = dict(self.jinja_options) - - if "autoescape" not in options: - options["autoescape"] = self.select_jinja_autoescape - - if "auto_reload" not in options: - auto_reload = self.config["TEMPLATES_AUTO_RELOAD"] - - if auto_reload is None: - auto_reload = self.debug - - options["auto_reload"] = auto_reload - - rv = self.jinja_environment(self, **options) - rv.globals.update( - url_for=self.url_for, - get_flashed_messages=get_flashed_messages, - config=self.config, - # request, session and g are normally added with the - # context processor for efficiency reasons but for imported - # templates we also want the proxies in there. - request=request, - session=session, - g=g, - ) - rv.policies["json.dumps_function"] = self.json.dumps - return rv - - def create_url_adapter(self, request: Request | None) -> MapAdapter | None: - """Creates a URL adapter for the given request. The URL adapter - is created at a point where the request context is not yet set - up so the request is passed explicitly. - - .. versionchanged:: 3.1 - If :data:`SERVER_NAME` is set, it does not restrict requests to - only that domain, for both ``subdomain_matching`` and - ``host_matching``. - - .. versionchanged:: 1.0 - :data:`SERVER_NAME` no longer implicitly enables subdomain - matching. Use :attr:`subdomain_matching` instead. - - .. versionchanged:: 0.9 - This can be called outside a request when the URL adapter is created - for an application context. - - .. versionadded:: 0.6 - """ - if request is not None: - if (trusted_hosts := self.config["TRUSTED_HOSTS"]) is not None: - request.trusted_hosts = trusted_hosts - - # Check trusted_hosts here until bind_to_environ does. - request.host = get_host(request.environ, request.trusted_hosts) # pyright: ignore - subdomain = None - server_name = self.config["SERVER_NAME"] - - if self.url_map.host_matching: - # Don't pass SERVER_NAME, otherwise it's used and the actual - # host is ignored, which breaks host matching. - server_name = None - elif not self.subdomain_matching: - # Werkzeug doesn't implement subdomain matching yet. Until then, - # disable it by forcing the current subdomain to the default, or - # the empty string. - subdomain = self.url_map.default_subdomain or "" - - return self.url_map.bind_to_environ( - request.environ, server_name=server_name, subdomain=subdomain - ) - - # Need at least SERVER_NAME to match/build outside a request. - if self.config["SERVER_NAME"] is not None: - return self.url_map.bind( - self.config["SERVER_NAME"], - script_name=self.config["APPLICATION_ROOT"], - url_scheme=self.config["PREFERRED_URL_SCHEME"], - ) - - return None - - def raise_routing_exception(self, request: Request) -> t.NoReturn: - """Intercept routing exceptions and possibly do something else. - - In debug mode, intercept a routing redirect and replace it with - an error if the body will be discarded. - - With modern Werkzeug this shouldn't occur, since it now uses a - 308 status which tells the browser to resend the method and - body. - - .. versionchanged:: 2.1 - Don't intercept 307 and 308 redirects. - - :meta private: - :internal: - """ - if ( - not self.debug - or not isinstance(request.routing_exception, RequestRedirect) - or request.routing_exception.code in {307, 308} - or request.method in {"GET", "HEAD", "OPTIONS"} - ): - raise request.routing_exception # type: ignore[misc] - - from .debughelpers import FormDataRoutingRedirect - - raise FormDataRoutingRedirect(request) - - def update_template_context(self, context: dict[str, t.Any]) -> None: - """Update the template context with some commonly used variables. - This injects request, session, config and g into the template - context as well as everything template context processors want - to inject. Note that the as of Flask 0.6, the original values - in the context will not be overridden if a context processor - decides to return a value with the same key. - - :param context: the context as a dictionary that is updated in place - to add extra variables. - """ - names: t.Iterable[str | None] = (None,) - - # A template may be rendered outside a request context. - if request: - names = chain(names, reversed(request.blueprints)) - - # The values passed to render_template take precedence. Keep a - # copy to re-apply after all context functions. - orig_ctx = context.copy() - - for name in names: - if name in self.template_context_processors: - for func in self.template_context_processors[name]: - context.update(self.ensure_sync(func)()) - - context.update(orig_ctx) - - def make_shell_context(self) -> dict[str, t.Any]: - """Returns the shell context for an interactive shell for this - application. This runs all the registered shell context - processors. - - .. versionadded:: 0.11 - """ - rv = {"app": self, "g": g} - for processor in self.shell_context_processors: - rv.update(processor()) - return rv - - def run( - self, - host: str | None = None, - port: int | None = None, - debug: bool | None = None, - load_dotenv: bool = True, - **options: t.Any, - ) -> None: - """Runs the application on a local development server. - - Do not use ``run()`` in a production setting. It is not intended to - meet security and performance requirements for a production server. - Instead, see :doc:`/deploying/index` for WSGI server recommendations. - - If the :attr:`debug` flag is set the server will automatically reload - for code changes and show a debugger in case an exception happened. - - If you want to run the application in debug mode, but disable the - code execution on the interactive debugger, you can pass - ``use_evalex=False`` as parameter. This will keep the debugger's - traceback screen active, but disable code execution. - - It is not recommended to use this function for development with - automatic reloading as this is badly supported. Instead you should - be using the :command:`flask` command line script's ``run`` support. - - .. admonition:: Keep in Mind - - Flask will suppress any server error with a generic error page - unless it is in debug mode. As such to enable just the - interactive debugger without the code reloading, you have to - invoke :meth:`run` with ``debug=True`` and ``use_reloader=False``. - Setting ``use_debugger`` to ``True`` without being in debug mode - won't catch any exceptions because there won't be any to - catch. - - :param host: the hostname to listen on. Set this to ``'0.0.0.0'`` to - have the server available externally as well. Defaults to - ``'127.0.0.1'`` or the host in the ``SERVER_NAME`` config variable - if present. - :param port: the port of the webserver. Defaults to ``5000`` or the - port defined in the ``SERVER_NAME`` config variable if present. - :param debug: if given, enable or disable debug mode. See - :attr:`debug`. - :param load_dotenv: Load the nearest :file:`.env` and :file:`.flaskenv` - files to set environment variables. Will also change the working - directory to the directory containing the first file found. - :param options: the options to be forwarded to the underlying Werkzeug - server. See :func:`werkzeug.serving.run_simple` for more - information. - - .. versionchanged:: 1.0 - If installed, python-dotenv will be used to load environment - variables from :file:`.env` and :file:`.flaskenv` files. - - The :envvar:`FLASK_DEBUG` environment variable will override :attr:`debug`. - - Threaded mode is enabled by default. - - .. versionchanged:: 0.10 - The default port is now picked from the ``SERVER_NAME`` - variable. - """ - # Ignore this call so that it doesn't start another server if - # the 'flask run' command is used. - if os.environ.get("FLASK_RUN_FROM_CLI") == "true": - if not is_running_from_reloader(): - click.secho( - " * Ignoring a call to 'app.run()' that would block" - " the current 'flask' CLI command.\n" - " Only call 'app.run()' in an 'if __name__ ==" - ' "__main__"\' guard.', - fg="red", - ) - - return - - if get_load_dotenv(load_dotenv): - cli.load_dotenv() - - # if set, env var overrides existing value - if "FLASK_DEBUG" in os.environ: - self.debug = get_debug_flag() - - # debug passed to method overrides all other sources - if debug is not None: - self.debug = bool(debug) - - server_name = self.config.get("SERVER_NAME") - sn_host = sn_port = None - - if server_name: - sn_host, _, sn_port = server_name.partition(":") - - if not host: - if sn_host: - host = sn_host - else: - host = "127.0.0.1" - - if port or port == 0: - port = int(port) - elif sn_port: - port = int(sn_port) - else: - port = 5000 - - options.setdefault("use_reloader", self.debug) - options.setdefault("use_debugger", self.debug) - options.setdefault("threaded", True) - - cli.show_server_banner(self.debug, self.name) - - from werkzeug.serving import run_simple - - try: - run_simple(t.cast(str, host), port, self, **options) - finally: - # reset the first request information if the development server - # reset normally. This makes it possible to restart the server - # without reloader and that stuff from an interactive shell. - self._got_first_request = False - - def test_client(self, use_cookies: bool = True, **kwargs: t.Any) -> FlaskClient: - """Creates a test client for this application. For information - about unit testing head over to :doc:`/testing`. - - Note that if you are testing for assertions or exceptions in your - application code, you must set ``app.testing = True`` in order for the - exceptions to propagate to the test client. Otherwise, the exception - will be handled by the application (not visible to the test client) and - the only indication of an AssertionError or other exception will be a - 500 status code response to the test client. See the :attr:`testing` - attribute. For example:: - - app.testing = True - client = app.test_client() - - The test client can be used in a ``with`` block to defer the closing down - of the context until the end of the ``with`` block. This is useful if - you want to access the context locals for testing:: - - with app.test_client() as c: - rv = c.get('/?vodka=42') - assert request.args['vodka'] == '42' - - Additionally, you may pass optional keyword arguments that will then - be passed to the application's :attr:`test_client_class` constructor. - For example:: - - from flask.testing import FlaskClient - - class CustomClient(FlaskClient): - def __init__(self, *args, **kwargs): - self._authentication = kwargs.pop("authentication") - super(CustomClient,self).__init__( *args, **kwargs) - - app.test_client_class = CustomClient - client = app.test_client(authentication='Basic ....') - - See :class:`~flask.testing.FlaskClient` for more information. - - .. versionchanged:: 0.4 - added support for ``with`` block usage for the client. - - .. versionadded:: 0.7 - The `use_cookies` parameter was added as well as the ability - to override the client to be used by setting the - :attr:`test_client_class` attribute. - - .. versionchanged:: 0.11 - Added `**kwargs` to support passing additional keyword arguments to - the constructor of :attr:`test_client_class`. - """ - cls = self.test_client_class - if cls is None: - from .testing import FlaskClient as cls - return cls( # type: ignore - self, self.response_class, use_cookies=use_cookies, **kwargs - ) - - def test_cli_runner(self, **kwargs: t.Any) -> FlaskCliRunner: - """Create a CLI runner for testing CLI commands. - See :ref:`testing-cli`. - - Returns an instance of :attr:`test_cli_runner_class`, by default - :class:`~flask.testing.FlaskCliRunner`. The Flask app object is - passed as the first argument. - - .. versionadded:: 1.0 - """ - cls = self.test_cli_runner_class - - if cls is None: - from .testing import FlaskCliRunner as cls - - return cls(self, **kwargs) # type: ignore - - def handle_http_exception( - self, e: HTTPException - ) -> HTTPException | ft.ResponseReturnValue: - """Handles an HTTP exception. By default this will invoke the - registered error handlers and fall back to returning the - exception as response. - - .. versionchanged:: 1.0.3 - ``RoutingException``, used internally for actions such as - slash redirects during routing, is not passed to error - handlers. - - .. versionchanged:: 1.0 - Exceptions are looked up by code *and* by MRO, so - ``HTTPException`` subclasses can be handled with a catch-all - handler for the base ``HTTPException``. - - .. versionadded:: 0.3 - """ - # Proxy exceptions don't have error codes. We want to always return - # those unchanged as errors - if e.code is None: - return e - - # RoutingExceptions are used internally to trigger routing - # actions, such as slash redirects raising RequestRedirect. They - # are not raised or handled in user code. - if isinstance(e, RoutingException): - return e - - handler = self._find_error_handler(e, request.blueprints) - if handler is None: - return e - return self.ensure_sync(handler)(e) # type: ignore[no-any-return] - - def handle_user_exception( - self, e: Exception - ) -> HTTPException | ft.ResponseReturnValue: - """This method is called whenever an exception occurs that - should be handled. A special case is :class:`~werkzeug - .exceptions.HTTPException` which is forwarded to the - :meth:`handle_http_exception` method. This function will either - return a response value or reraise the exception with the same - traceback. - - .. versionchanged:: 1.0 - Key errors raised from request data like ``form`` show the - bad key in debug mode rather than a generic bad request - message. - - .. versionadded:: 0.7 - """ - if isinstance(e, BadRequestKeyError) and ( - self.debug or self.config["TRAP_BAD_REQUEST_ERRORS"] - ): - e.show_exception = True - - if isinstance(e, HTTPException) and not self.trap_http_exception(e): - return self.handle_http_exception(e) - - handler = self._find_error_handler(e, request.blueprints) - - if handler is None: - raise - - return self.ensure_sync(handler)(e) # type: ignore[no-any-return] - - def handle_exception(self, e: Exception) -> Response: - """Handle an exception that did not have an error handler - associated with it, or that was raised from an error handler. - This always causes a 500 ``InternalServerError``. - - Always sends the :data:`got_request_exception` signal. - - If :data:`PROPAGATE_EXCEPTIONS` is ``True``, such as in debug - mode, the error will be re-raised so that the debugger can - display it. Otherwise, the original exception is logged, and - an :exc:`~werkzeug.exceptions.InternalServerError` is returned. - - If an error handler is registered for ``InternalServerError`` or - ``500``, it will be used. For consistency, the handler will - always receive the ``InternalServerError``. The original - unhandled exception is available as ``e.original_exception``. - - .. versionchanged:: 1.1.0 - Always passes the ``InternalServerError`` instance to the - handler, setting ``original_exception`` to the unhandled - error. - - .. versionchanged:: 1.1.0 - ``after_request`` functions and other finalization is done - even for the default 500 response when there is no handler. - - .. versionadded:: 0.3 - """ - exc_info = sys.exc_info() - got_request_exception.send(self, _async_wrapper=self.ensure_sync, exception=e) - propagate = self.config["PROPAGATE_EXCEPTIONS"] - - if propagate is None: - propagate = self.testing or self.debug - - if propagate: - # Re-raise if called with an active exception, otherwise - # raise the passed in exception. - if exc_info[1] is e: - raise - - raise e - - self.log_exception(exc_info) - server_error: InternalServerError | ft.ResponseReturnValue - server_error = InternalServerError(original_exception=e) - handler = self._find_error_handler(server_error, request.blueprints) - - if handler is not None: - server_error = self.ensure_sync(handler)(server_error) - - return self.finalize_request(server_error, from_error_handler=True) - - def log_exception( - self, - exc_info: (tuple[type, BaseException, TracebackType] | tuple[None, None, None]), - ) -> None: - """Logs an exception. This is called by :meth:`handle_exception` - if debugging is disabled and right before the handler is called. - The default implementation logs the exception as error on the - :attr:`logger`. - - .. versionadded:: 0.8 - """ - self.logger.error( - f"Exception on {request.path} [{request.method}]", exc_info=exc_info - ) - - def dispatch_request(self) -> ft.ResponseReturnValue: - """Does the request dispatching. Matches the URL and returns the - return value of the view or error handler. This does not have to - be a response object. In order to convert the return value to a - proper response object, call :func:`make_response`. - - .. versionchanged:: 0.7 - This no longer does the exception handling, this code was - moved to the new :meth:`full_dispatch_request`. - """ - req = request_ctx.request - if req.routing_exception is not None: - self.raise_routing_exception(req) - rule: Rule = req.url_rule # type: ignore[assignment] - # if we provide automatic options for this URL and the - # request came with the OPTIONS method, reply automatically - if ( - getattr(rule, "provide_automatic_options", False) - and req.method == "OPTIONS" - ): - return self.make_default_options_response() - # otherwise dispatch to the handler for that endpoint - view_args: dict[str, t.Any] = req.view_args # type: ignore[assignment] - return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args) # type: ignore[no-any-return] - - def full_dispatch_request(self) -> Response: - """Dispatches the request and on top of that performs request - pre and postprocessing as well as HTTP exception catching and - error handling. - - .. versionadded:: 0.7 - """ - self._got_first_request = True - - try: - request_started.send(self, _async_wrapper=self.ensure_sync) - rv = self.preprocess_request() - if rv is None: - rv = self.dispatch_request() - except Exception as e: - rv = self.handle_user_exception(e) - return self.finalize_request(rv) - - def finalize_request( - self, - rv: ft.ResponseReturnValue | HTTPException, - from_error_handler: bool = False, - ) -> Response: - """Given the return value from a view function this finalizes - the request by converting it into a response and invoking the - postprocessing functions. This is invoked for both normal - request dispatching as well as error handlers. - - Because this means that it might be called as a result of a - failure a special safe mode is available which can be enabled - with the `from_error_handler` flag. If enabled, failures in - response processing will be logged and otherwise ignored. - - :internal: - """ - response = self.make_response(rv) - try: - response = self.process_response(response) - request_finished.send( - self, _async_wrapper=self.ensure_sync, response=response - ) - except Exception: - if not from_error_handler: - raise - self.logger.exception( - "Request finalizing failed with an error while handling an error" - ) - return response - - def make_default_options_response(self) -> Response: - """This method is called to create the default ``OPTIONS`` response. - This can be changed through subclassing to change the default - behavior of ``OPTIONS`` responses. - - .. versionadded:: 0.7 - """ - adapter = request_ctx.url_adapter - methods = adapter.allowed_methods() # type: ignore[union-attr] - rv = self.response_class() - rv.allow.update(methods) - return rv - - def ensure_sync(self, func: t.Callable[..., t.Any]) -> t.Callable[..., t.Any]: - """Ensure that the function is synchronous for WSGI workers. - Plain ``def`` functions are returned as-is. ``async def`` - functions are wrapped to run and wait for the response. - - Override this method to change how the app runs async views. - - .. versionadded:: 2.0 - """ - if iscoroutinefunction(func): - return self.async_to_sync(func) - - return func - - def async_to_sync( - self, func: t.Callable[..., t.Coroutine[t.Any, t.Any, t.Any]] - ) -> t.Callable[..., t.Any]: - """Return a sync function that will run the coroutine function. - - .. code-block:: python - - result = app.async_to_sync(func)(*args, **kwargs) - - Override this method to change how the app converts async code - to be synchronously callable. - - .. versionadded:: 2.0 - """ - try: - from asgiref.sync import async_to_sync as asgiref_async_to_sync - except ImportError: - raise RuntimeError( - "Install Flask with the 'async' extra in order to use async views." - ) from None - - return asgiref_async_to_sync(func) - - def url_for( - self, - /, - endpoint: str, - *, - _anchor: str | None = None, - _method: str | None = None, - _scheme: str | None = None, - _external: bool | None = None, - **values: t.Any, - ) -> str: - """Generate a URL to the given endpoint with the given values. - - This is called by :func:`flask.url_for`, and can be called - directly as well. - - An *endpoint* is the name of a URL rule, usually added with - :meth:`@app.route() `, and usually the same name as the - view function. A route defined in a :class:`~flask.Blueprint` - will prepend the blueprint's name separated by a ``.`` to the - endpoint. - - In some cases, such as email messages, you want URLs to include - the scheme and domain, like ``https://example.com/hello``. When - not in an active request, URLs will be external by default, but - this requires setting :data:`SERVER_NAME` so Flask knows what - domain to use. :data:`APPLICATION_ROOT` and - :data:`PREFERRED_URL_SCHEME` should also be configured as - needed. This config is only used when not in an active request. - - Functions can be decorated with :meth:`url_defaults` to modify - keyword arguments before the URL is built. - - If building fails for some reason, such as an unknown endpoint - or incorrect values, the app's :meth:`handle_url_build_error` - method is called. If that returns a string, that is returned, - otherwise a :exc:`~werkzeug.routing.BuildError` is raised. - - :param endpoint: The endpoint name associated with the URL to - generate. If this starts with a ``.``, the current blueprint - name (if any) will be used. - :param _anchor: If given, append this as ``#anchor`` to the URL. - :param _method: If given, generate the URL associated with this - method for the endpoint. - :param _scheme: If given, the URL will have this scheme if it - is external. - :param _external: If given, prefer the URL to be internal - (False) or require it to be external (True). External URLs - include the scheme and domain. When not in an active - request, URLs are external by default. - :param values: Values to use for the variable parts of the URL - rule. Unknown keys are appended as query string arguments, - like ``?a=b&c=d``. - - .. versionadded:: 2.2 - Moved from ``flask.url_for``, which calls this method. - """ - req_ctx = _cv_request.get(None) - - if req_ctx is not None: - url_adapter = req_ctx.url_adapter - blueprint_name = req_ctx.request.blueprint - - # If the endpoint starts with "." and the request matches a - # blueprint, the endpoint is relative to the blueprint. - if endpoint[:1] == ".": - if blueprint_name is not None: - endpoint = f"{blueprint_name}{endpoint}" - else: - endpoint = endpoint[1:] - - # When in a request, generate a URL without scheme and - # domain by default, unless a scheme is given. - if _external is None: - _external = _scheme is not None - else: - app_ctx = _cv_app.get(None) - - # If called by helpers.url_for, an app context is active, - # use its url_adapter. Otherwise, app.url_for was called - # directly, build an adapter. - if app_ctx is not None: - url_adapter = app_ctx.url_adapter - else: - url_adapter = self.create_url_adapter(None) - - if url_adapter is None: - raise RuntimeError( - "Unable to build URLs outside an active request" - " without 'SERVER_NAME' configured. Also configure" - " 'APPLICATION_ROOT' and 'PREFERRED_URL_SCHEME' as" - " needed." - ) - - # When outside a request, generate a URL with scheme and - # domain by default. - if _external is None: - _external = True - - # It is an error to set _scheme when _external=False, in order - # to avoid accidental insecure URLs. - if _scheme is not None and not _external: - raise ValueError("When specifying '_scheme', '_external' must be True.") - - self.inject_url_defaults(endpoint, values) - - try: - rv = url_adapter.build( # type: ignore[union-attr] - endpoint, - values, - method=_method, - url_scheme=_scheme, - force_external=_external, - ) - except BuildError as error: - values.update( - _anchor=_anchor, _method=_method, _scheme=_scheme, _external=_external - ) - return self.handle_url_build_error(error, endpoint, values) - - if _anchor is not None: - _anchor = _url_quote(_anchor, safe="%!#$&'()*+,/:;=?@") - rv = f"{rv}#{_anchor}" - - return rv - - def make_response(self, rv: ft.ResponseReturnValue) -> Response: - """Convert the return value from a view function to an instance of - :attr:`response_class`. - - :param rv: the return value from the view function. The view function - must return a response. Returning ``None``, or the view ending - without returning, is not allowed. The following types are allowed - for ``view_rv``: - - ``str`` - A response object is created with the string encoded to UTF-8 - as the body. - - ``bytes`` - A response object is created with the bytes as the body. - - ``dict`` - A dictionary that will be jsonify'd before being returned. - - ``list`` - A list that will be jsonify'd before being returned. - - ``generator`` or ``iterator`` - A generator that returns ``str`` or ``bytes`` to be - streamed as the response. - - ``tuple`` - Either ``(body, status, headers)``, ``(body, status)``, or - ``(body, headers)``, where ``body`` is any of the other types - allowed here, ``status`` is a string or an integer, and - ``headers`` is a dictionary or a list of ``(key, value)`` - tuples. If ``body`` is a :attr:`response_class` instance, - ``status`` overwrites the exiting value and ``headers`` are - extended. - - :attr:`response_class` - The object is returned unchanged. - - other :class:`~werkzeug.wrappers.Response` class - The object is coerced to :attr:`response_class`. - - :func:`callable` - The function is called as a WSGI application. The result is - used to create a response object. - - .. versionchanged:: 2.2 - A generator will be converted to a streaming response. - A list will be converted to a JSON response. - - .. versionchanged:: 1.1 - A dict will be converted to a JSON response. - - .. versionchanged:: 0.9 - Previously a tuple was interpreted as the arguments for the - response object. - """ - - status: int | None = None - headers: HeadersValue | None = None - - # unpack tuple returns - if isinstance(rv, tuple): - len_rv = len(rv) - - # a 3-tuple is unpacked directly - if len_rv == 3: - rv, status, headers = rv # type: ignore[misc] - # decide if a 2-tuple has status or headers - elif len_rv == 2: - if isinstance(rv[1], (Headers, dict, tuple, list)): - rv, headers = rv # pyright: ignore - else: - rv, status = rv # type: ignore[assignment,misc] - # other sized tuples are not allowed - else: - raise TypeError( - "The view function did not return a valid response tuple." - " The tuple must have the form (body, status, headers)," - " (body, status), or (body, headers)." - ) - - # the body must not be None - if rv is None: - raise TypeError( - f"The view function for {request.endpoint!r} did not" - " return a valid response. The function either returned" - " None or ended without a return statement." - ) - - # make sure the body is an instance of the response class - if not isinstance(rv, self.response_class): - if isinstance(rv, (str, bytes, bytearray)) or isinstance(rv, cabc.Iterator): - # let the response class set the status and headers instead of - # waiting to do it manually, so that the class can handle any - # special logic - rv = self.response_class( - rv, # pyright: ignore - status=status, - headers=headers, # type: ignore[arg-type] - ) - status = headers = None - elif isinstance(rv, (dict, list)): - rv = self.json.response(rv) - elif isinstance(rv, BaseResponse) or callable(rv): - # evaluate a WSGI callable, or coerce a different response - # class to the correct type - try: - rv = self.response_class.force_type( - rv, # type: ignore[arg-type] - request.environ, - ) - except TypeError as e: - raise TypeError( - f"{e}\nThe view function did not return a valid" - " response. The return type must be a string," - " dict, list, tuple with headers or status," - " Response instance, or WSGI callable, but it" - f" was a {type(rv).__name__}." - ).with_traceback(sys.exc_info()[2]) from None - else: - raise TypeError( - "The view function did not return a valid" - " response. The return type must be a string," - " dict, list, tuple with headers or status," - " Response instance, or WSGI callable, but it was a" - f" {type(rv).__name__}." - ) - - rv = t.cast(Response, rv) - # prefer the status if it was provided - if status is not None: - if isinstance(status, (str, bytes, bytearray)): - rv.status = status - else: - rv.status_code = status - - # extend existing headers with provided headers - if headers: - rv.headers.update(headers) - - return rv - - def preprocess_request(self) -> ft.ResponseReturnValue | None: - """Called before the request is dispatched. Calls - :attr:`url_value_preprocessors` registered with the app and the - current blueprint (if any). Then calls :attr:`before_request_funcs` - registered with the app and the blueprint. - - If any :meth:`before_request` handler returns a non-None value, the - value is handled as if it was the return value from the view, and - further request handling is stopped. - """ - names = (None, *reversed(request.blueprints)) - - for name in names: - if name in self.url_value_preprocessors: - for url_func in self.url_value_preprocessors[name]: - url_func(request.endpoint, request.view_args) - - for name in names: - if name in self.before_request_funcs: - for before_func in self.before_request_funcs[name]: - rv = self.ensure_sync(before_func)() - - if rv is not None: - return rv # type: ignore[no-any-return] - - return None - - def process_response(self, response: Response) -> Response: - """Can be overridden in order to modify the response object - before it's sent to the WSGI server. By default this will - call all the :meth:`after_request` decorated functions. - - .. versionchanged:: 0.5 - As of Flask 0.5 the functions registered for after request - execution are called in reverse order of registration. - - :param response: a :attr:`response_class` object. - :return: a new response object or the same, has to be an - instance of :attr:`response_class`. - """ - ctx = request_ctx._get_current_object() # type: ignore[attr-defined] - - for func in ctx._after_request_functions: - response = self.ensure_sync(func)(response) - - for name in chain(request.blueprints, (None,)): - if name in self.after_request_funcs: - for func in reversed(self.after_request_funcs[name]): - response = self.ensure_sync(func)(response) - - if not self.session_interface.is_null_session(ctx._session): - self.session_interface.save_session(self, ctx._session, response) - - return response - - def do_teardown_request( - self, - exc: BaseException | None = _sentinel, # type: ignore[assignment] - ) -> None: - """Called after the request is dispatched and the response is - returned, right before the request context is popped. - - This calls all functions decorated with - :meth:`teardown_request`, and :meth:`Blueprint.teardown_request` - if a blueprint handled the request. Finally, the - :data:`request_tearing_down` signal is sent. - - This is called by - :meth:`RequestContext.pop() `, - which may be delayed during testing to maintain access to - resources. - - :param exc: An unhandled exception raised while dispatching the - request. Detected from the current exception information if - not passed. Passed to each teardown function. - - .. versionchanged:: 0.9 - Added the ``exc`` argument. - """ - if exc is _sentinel: - exc = sys.exc_info()[1] - - for name in chain(request.blueprints, (None,)): - if name in self.teardown_request_funcs: - for func in reversed(self.teardown_request_funcs[name]): - self.ensure_sync(func)(exc) - - request_tearing_down.send(self, _async_wrapper=self.ensure_sync, exc=exc) - - def do_teardown_appcontext( - self, - exc: BaseException | None = _sentinel, # type: ignore[assignment] - ) -> None: - """Called right before the application context is popped. - - When handling a request, the application context is popped - after the request context. See :meth:`do_teardown_request`. - - This calls all functions decorated with - :meth:`teardown_appcontext`. Then the - :data:`appcontext_tearing_down` signal is sent. - - This is called by - :meth:`AppContext.pop() `. - - .. versionadded:: 0.9 - """ - if exc is _sentinel: - exc = sys.exc_info()[1] - - for func in reversed(self.teardown_appcontext_funcs): - self.ensure_sync(func)(exc) - - appcontext_tearing_down.send(self, _async_wrapper=self.ensure_sync, exc=exc) - - def app_context(self) -> AppContext: - """Create an :class:`~flask.ctx.AppContext`. Use as a ``with`` - block to push the context, which will make :data:`current_app` - point at this application. - - An application context is automatically pushed by - :meth:`RequestContext.push() ` - when handling a request, and when running a CLI command. Use - this to manually create a context outside of these situations. - - :: - - with app.app_context(): - init_db() - - See :doc:`/appcontext`. - - .. versionadded:: 0.9 - """ - return AppContext(self) - - def request_context(self, environ: WSGIEnvironment) -> RequestContext: - """Create a :class:`~flask.ctx.RequestContext` representing a - WSGI environment. Use a ``with`` block to push the context, - which will make :data:`request` point at this request. - - See :doc:`/reqcontext`. - - Typically you should not call this from your own code. A request - context is automatically pushed by the :meth:`wsgi_app` when - handling a request. Use :meth:`test_request_context` to create - an environment and context instead of this method. - - :param environ: a WSGI environment - """ - return RequestContext(self, environ) - - def test_request_context(self, *args: t.Any, **kwargs: t.Any) -> RequestContext: - """Create a :class:`~flask.ctx.RequestContext` for a WSGI - environment created from the given values. This is mostly useful - during testing, where you may want to run a function that uses - request data without dispatching a full request. - - See :doc:`/reqcontext`. - - Use a ``with`` block to push the context, which will make - :data:`request` point at the request for the created - environment. :: - - with app.test_request_context(...): - generate_report() - - When using the shell, it may be easier to push and pop the - context manually to avoid indentation. :: - - ctx = app.test_request_context(...) - ctx.push() - ... - ctx.pop() - - Takes the same arguments as Werkzeug's - :class:`~werkzeug.test.EnvironBuilder`, with some defaults from - the application. See the linked Werkzeug docs for most of the - available arguments. Flask-specific behavior is listed here. - - :param path: URL path being requested. - :param base_url: Base URL where the app is being served, which - ``path`` is relative to. If not given, built from - :data:`PREFERRED_URL_SCHEME`, ``subdomain``, - :data:`SERVER_NAME`, and :data:`APPLICATION_ROOT`. - :param subdomain: Subdomain name to append to - :data:`SERVER_NAME`. - :param url_scheme: Scheme to use instead of - :data:`PREFERRED_URL_SCHEME`. - :param data: The request body, either as a string or a dict of - form keys and values. - :param json: If given, this is serialized as JSON and passed as - ``data``. Also defaults ``content_type`` to - ``application/json``. - :param args: other positional arguments passed to - :class:`~werkzeug.test.EnvironBuilder`. - :param kwargs: other keyword arguments passed to - :class:`~werkzeug.test.EnvironBuilder`. - """ - from .testing import EnvironBuilder - - builder = EnvironBuilder(self, *args, **kwargs) - - try: - return self.request_context(builder.get_environ()) - finally: - builder.close() - - def wsgi_app( - self, environ: WSGIEnvironment, start_response: StartResponse - ) -> cabc.Iterable[bytes]: - """The actual WSGI application. This is not implemented in - :meth:`__call__` so that middlewares can be applied without - losing a reference to the app object. Instead of doing this:: - - app = MyMiddleware(app) - - It's a better idea to do this instead:: - - app.wsgi_app = MyMiddleware(app.wsgi_app) - - Then you still have the original application object around and - can continue to call methods on it. - - .. versionchanged:: 0.7 - Teardown events for the request and app contexts are called - even if an unhandled error occurs. Other events may not be - called depending on when an error occurs during dispatch. - See :ref:`callbacks-and-errors`. - - :param environ: A WSGI environment. - :param start_response: A callable accepting a status code, - a list of headers, and an optional exception context to - start the response. - """ - ctx = self.request_context(environ) - error: BaseException | None = None - try: - try: - ctx.push() - response = self.full_dispatch_request() - except Exception as e: - error = e - response = self.handle_exception(e) - except: - error = sys.exc_info()[1] - raise - return response(environ, start_response) - finally: - if "werkzeug.debug.preserve_context" in environ: - environ["werkzeug.debug.preserve_context"](_cv_app.get()) - environ["werkzeug.debug.preserve_context"](_cv_request.get()) - - if error is not None and self.should_ignore_error(error): - error = None - - ctx.pop(error) - - def __call__( - self, environ: WSGIEnvironment, start_response: StartResponse - ) -> cabc.Iterable[bytes]: - """The WSGI server calls the Flask application object as the - WSGI application. This calls :meth:`wsgi_app`, which can be - wrapped to apply middleware. - """ - return self.wsgi_app(environ, start_response) diff --git a/bundle/python-cpu/Lib/site-packages/flask/blueprints.py b/bundle/python-cpu/Lib/site-packages/flask/blueprints.py deleted file mode 100644 index b6d4e43339f0d88fc551cb5211c62be8988d3666..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/blueprints.py +++ /dev/null @@ -1,128 +0,0 @@ -from __future__ import annotations - -import os -import typing as t -from datetime import timedelta - -from .cli import AppGroup -from .globals import current_app -from .helpers import send_from_directory -from .sansio.blueprints import Blueprint as SansioBlueprint -from .sansio.blueprints import BlueprintSetupState as BlueprintSetupState # noqa -from .sansio.scaffold import _sentinel - -if t.TYPE_CHECKING: # pragma: no cover - from .wrappers import Response - - -class Blueprint(SansioBlueprint): - def __init__( - self, - name: str, - import_name: str, - static_folder: str | os.PathLike[str] | None = None, - static_url_path: str | None = None, - template_folder: str | os.PathLike[str] | None = None, - url_prefix: str | None = None, - subdomain: str | None = None, - url_defaults: dict[str, t.Any] | None = None, - root_path: str | None = None, - cli_group: str | None = _sentinel, # type: ignore - ) -> None: - super().__init__( - name, - import_name, - static_folder, - static_url_path, - template_folder, - url_prefix, - subdomain, - url_defaults, - root_path, - cli_group, - ) - - #: The Click command group for registering CLI commands for this - #: object. The commands are available from the ``flask`` command - #: once the application has been discovered and blueprints have - #: been registered. - self.cli = AppGroup() - - # Set the name of the Click group in case someone wants to add - # the app's commands to another CLI tool. - self.cli.name = self.name - - def get_send_file_max_age(self, filename: str | None) -> int | None: - """Used by :func:`send_file` to determine the ``max_age`` cache - value for a given file path if it wasn't passed. - - By default, this returns :data:`SEND_FILE_MAX_AGE_DEFAULT` from - the configuration of :data:`~flask.current_app`. This defaults - to ``None``, which tells the browser to use conditional requests - instead of a timed cache, which is usually preferable. - - Note this is a duplicate of the same method in the Flask - class. - - .. versionchanged:: 2.0 - The default configuration is ``None`` instead of 12 hours. - - .. versionadded:: 0.9 - """ - value = current_app.config["SEND_FILE_MAX_AGE_DEFAULT"] - - if value is None: - return None - - if isinstance(value, timedelta): - return int(value.total_seconds()) - - return value # type: ignore[no-any-return] - - def send_static_file(self, filename: str) -> Response: - """The view function used to serve files from - :attr:`static_folder`. A route is automatically registered for - this view at :attr:`static_url_path` if :attr:`static_folder` is - set. - - Note this is a duplicate of the same method in the Flask - class. - - .. versionadded:: 0.5 - - """ - if not self.has_static_folder: - raise RuntimeError("'static_folder' must be set to serve static_files.") - - # send_file only knows to call get_send_file_max_age on the app, - # call it here so it works for blueprints too. - max_age = self.get_send_file_max_age(filename) - return send_from_directory( - t.cast(str, self.static_folder), filename, max_age=max_age - ) - - def open_resource( - self, resource: str, mode: str = "rb", encoding: str | None = "utf-8" - ) -> t.IO[t.AnyStr]: - """Open a resource file relative to :attr:`root_path` for reading. The - blueprint-relative equivalent of the app's :meth:`~.Flask.open_resource` - method. - - :param resource: Path to the resource relative to :attr:`root_path`. - :param mode: Open the file in this mode. Only reading is supported, - valid values are ``"r"`` (or ``"rt"``) and ``"rb"``. - :param encoding: Open the file with this encoding when opening in text - mode. This is ignored when opening in binary mode. - - .. versionchanged:: 3.1 - Added the ``encoding`` parameter. - """ - if mode not in {"r", "rt", "rb"}: - raise ValueError("Resources can only be opened for reading.") - - path = os.path.join(self.root_path, resource) - - if mode == "rb": - return open(path, mode) # pyright: ignore - - return open(path, mode, encoding=encoding) diff --git a/bundle/python-cpu/Lib/site-packages/flask/cli.py b/bundle/python-cpu/Lib/site-packages/flask/cli.py deleted file mode 100644 index ed11f256a153300887657e51b0b2f00543ca5229..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/cli.py +++ /dev/null @@ -1,1135 +0,0 @@ -from __future__ import annotations - -import ast -import collections.abc as cabc -import importlib.metadata -import inspect -import os -import platform -import re -import sys -import traceback -import typing as t -from functools import update_wrapper -from operator import itemgetter -from types import ModuleType - -import click -from click.core import ParameterSource -from werkzeug import run_simple -from werkzeug.serving import is_running_from_reloader -from werkzeug.utils import import_string - -from .globals import current_app -from .helpers import get_debug_flag -from .helpers import get_load_dotenv - -if t.TYPE_CHECKING: - import ssl - - from _typeshed.wsgi import StartResponse - from _typeshed.wsgi import WSGIApplication - from _typeshed.wsgi import WSGIEnvironment - - from .app import Flask - - -class NoAppException(click.UsageError): - """Raised if an application cannot be found or loaded.""" - - -def find_best_app(module: ModuleType) -> Flask: - """Given a module instance this tries to find the best possible - application in the module or raises an exception. - """ - from . import Flask - - # Search for the most common names first. - for attr_name in ("app", "application"): - app = getattr(module, attr_name, None) - - if isinstance(app, Flask): - return app - - # Otherwise find the only object that is a Flask instance. - matches = [v for v in module.__dict__.values() if isinstance(v, Flask)] - - if len(matches) == 1: - return matches[0] - elif len(matches) > 1: - raise NoAppException( - "Detected multiple Flask applications in module" - f" '{module.__name__}'. Use '{module.__name__}:name'" - " to specify the correct one." - ) - - # Search for app factory functions. - for attr_name in ("create_app", "make_app"): - app_factory = getattr(module, attr_name, None) - - if inspect.isfunction(app_factory): - try: - app = app_factory() - - if isinstance(app, Flask): - return app - except TypeError as e: - if not _called_with_wrong_args(app_factory): - raise - - raise NoAppException( - f"Detected factory '{attr_name}' in module '{module.__name__}'," - " but could not call it without arguments. Use" - f" '{module.__name__}:{attr_name}(args)'" - " to specify arguments." - ) from e - - raise NoAppException( - "Failed to find Flask application or factory in module" - f" '{module.__name__}'. Use '{module.__name__}:name'" - " to specify one." - ) - - -def _called_with_wrong_args(f: t.Callable[..., Flask]) -> bool: - """Check whether calling a function raised a ``TypeError`` because - the call failed or because something in the factory raised the - error. - - :param f: The function that was called. - :return: ``True`` if the call failed. - """ - tb = sys.exc_info()[2] - - try: - while tb is not None: - if tb.tb_frame.f_code is f.__code__: - # In the function, it was called successfully. - return False - - tb = tb.tb_next - - # Didn't reach the function. - return True - finally: - # Delete tb to break a circular reference. - # https://docs.python.org/2/library/sys.html#sys.exc_info - del tb - - -def find_app_by_string(module: ModuleType, app_name: str) -> Flask: - """Check if the given string is a variable name or a function. Call - a function to get the app instance, or return the variable directly. - """ - from . import Flask - - # Parse app_name as a single expression to determine if it's a valid - # attribute name or function call. - try: - expr = ast.parse(app_name.strip(), mode="eval").body - except SyntaxError: - raise NoAppException( - f"Failed to parse {app_name!r} as an attribute name or function call." - ) from None - - if isinstance(expr, ast.Name): - name = expr.id - args = [] - kwargs = {} - elif isinstance(expr, ast.Call): - # Ensure the function name is an attribute name only. - if not isinstance(expr.func, ast.Name): - raise NoAppException( - f"Function reference must be a simple name: {app_name!r}." - ) - - name = expr.func.id - - # Parse the positional and keyword arguments as literals. - try: - args = [ast.literal_eval(arg) for arg in expr.args] - kwargs = { - kw.arg: ast.literal_eval(kw.value) - for kw in expr.keywords - if kw.arg is not None - } - except ValueError: - # literal_eval gives cryptic error messages, show a generic - # message with the full expression instead. - raise NoAppException( - f"Failed to parse arguments as literal values: {app_name!r}." - ) from None - else: - raise NoAppException( - f"Failed to parse {app_name!r} as an attribute name or function call." - ) - - try: - attr = getattr(module, name) - except AttributeError as e: - raise NoAppException( - f"Failed to find attribute {name!r} in {module.__name__!r}." - ) from e - - # If the attribute is a function, call it with any args and kwargs - # to get the real application. - if inspect.isfunction(attr): - try: - app = attr(*args, **kwargs) - except TypeError as e: - if not _called_with_wrong_args(attr): - raise - - raise NoAppException( - f"The factory {app_name!r} in module" - f" {module.__name__!r} could not be called with the" - " specified arguments." - ) from e - else: - app = attr - - if isinstance(app, Flask): - return app - - raise NoAppException( - "A valid Flask application was not obtained from" - f" '{module.__name__}:{app_name}'." - ) - - -def prepare_import(path: str) -> str: - """Given a filename this will try to calculate the python path, add it - to the search path and return the actual module name that is expected. - """ - path = os.path.realpath(path) - - fname, ext = os.path.splitext(path) - if ext == ".py": - path = fname - - if os.path.basename(path) == "__init__": - path = os.path.dirname(path) - - module_name = [] - - # move up until outside package structure (no __init__.py) - while True: - path, name = os.path.split(path) - module_name.append(name) - - if not os.path.exists(os.path.join(path, "__init__.py")): - break - - if sys.path[0] != path: - sys.path.insert(0, path) - - return ".".join(module_name[::-1]) - - -@t.overload -def locate_app( - module_name: str, app_name: str | None, raise_if_not_found: t.Literal[True] = True -) -> Flask: ... - - -@t.overload -def locate_app( - module_name: str, app_name: str | None, raise_if_not_found: t.Literal[False] = ... -) -> Flask | None: ... - - -def locate_app( - module_name: str, app_name: str | None, raise_if_not_found: bool = True -) -> Flask | None: - try: - __import__(module_name) - except ImportError: - # Reraise the ImportError if it occurred within the imported module. - # Determine this by checking whether the trace has a depth > 1. - if sys.exc_info()[2].tb_next: # type: ignore[union-attr] - raise NoAppException( - f"While importing {module_name!r}, an ImportError was" - f" raised:\n\n{traceback.format_exc()}" - ) from None - elif raise_if_not_found: - raise NoAppException(f"Could not import {module_name!r}.") from None - else: - return None - - module = sys.modules[module_name] - - if app_name is None: - return find_best_app(module) - else: - return find_app_by_string(module, app_name) - - -def get_version(ctx: click.Context, param: click.Parameter, value: t.Any) -> None: - if not value or ctx.resilient_parsing: - return - - flask_version = importlib.metadata.version("flask") - werkzeug_version = importlib.metadata.version("werkzeug") - - click.echo( - f"Python {platform.python_version()}\n" - f"Flask {flask_version}\n" - f"Werkzeug {werkzeug_version}", - color=ctx.color, - ) - ctx.exit() - - -version_option = click.Option( - ["--version"], - help="Show the Flask version.", - expose_value=False, - callback=get_version, - is_flag=True, - is_eager=True, -) - - -class ScriptInfo: - """Helper object to deal with Flask applications. This is usually not - necessary to interface with as it's used internally in the dispatching - to click. In future versions of Flask this object will most likely play - a bigger role. Typically it's created automatically by the - :class:`FlaskGroup` but you can also manually create it and pass it - onwards as click object. - - .. versionchanged:: 3.1 - Added the ``load_dotenv_defaults`` parameter and attribute. - """ - - def __init__( - self, - app_import_path: str | None = None, - create_app: t.Callable[..., Flask] | None = None, - set_debug_flag: bool = True, - load_dotenv_defaults: bool = True, - ) -> None: - #: Optionally the import path for the Flask application. - self.app_import_path = app_import_path - #: Optionally a function that is passed the script info to create - #: the instance of the application. - self.create_app = create_app - #: A dictionary with arbitrary data that can be associated with - #: this script info. - self.data: dict[t.Any, t.Any] = {} - self.set_debug_flag = set_debug_flag - - self.load_dotenv_defaults = get_load_dotenv(load_dotenv_defaults) - """Whether default ``.flaskenv`` and ``.env`` files should be loaded. - - ``ScriptInfo`` doesn't load anything, this is for reference when doing - the load elsewhere during processing. - - .. versionadded:: 3.1 - """ - - self._loaded_app: Flask | None = None - - def load_app(self) -> Flask: - """Loads the Flask app (if not yet loaded) and returns it. Calling - this multiple times will just result in the already loaded app to - be returned. - """ - if self._loaded_app is not None: - return self._loaded_app - app: Flask | None = None - if self.create_app is not None: - app = self.create_app() - else: - if self.app_import_path: - path, name = ( - re.split(r":(?![\\/])", self.app_import_path, maxsplit=1) + [None] - )[:2] - import_name = prepare_import(path) - app = locate_app(import_name, name) - else: - for path in ("wsgi.py", "app.py"): - import_name = prepare_import(path) - app = locate_app(import_name, None, raise_if_not_found=False) - - if app is not None: - break - - if app is None: - raise NoAppException( - "Could not locate a Flask application. Use the" - " 'flask --app' option, 'FLASK_APP' environment" - " variable, or a 'wsgi.py' or 'app.py' file in the" - " current directory." - ) - - if self.set_debug_flag: - # Update the app's debug flag through the descriptor so that - # other values repopulate as well. - app.debug = get_debug_flag() - - self._loaded_app = app - return app - - -pass_script_info = click.make_pass_decorator(ScriptInfo, ensure=True) - -F = t.TypeVar("F", bound=t.Callable[..., t.Any]) - - -def with_appcontext(f: F) -> F: - """Wraps a callback so that it's guaranteed to be executed with the - script's application context. - - Custom commands (and their options) registered under ``app.cli`` or - ``blueprint.cli`` will always have an app context available, this - decorator is not required in that case. - - .. versionchanged:: 2.2 - The app context is active for subcommands as well as the - decorated callback. The app context is always available to - ``app.cli`` command and parameter callbacks. - """ - - @click.pass_context - def decorator(ctx: click.Context, /, *args: t.Any, **kwargs: t.Any) -> t.Any: - if not current_app: - app = ctx.ensure_object(ScriptInfo).load_app() - ctx.with_resource(app.app_context()) - - return ctx.invoke(f, *args, **kwargs) - - return update_wrapper(decorator, f) # type: ignore[return-value] - - -class AppGroup(click.Group): - """This works similar to a regular click :class:`~click.Group` but it - changes the behavior of the :meth:`command` decorator so that it - automatically wraps the functions in :func:`with_appcontext`. - - Not to be confused with :class:`FlaskGroup`. - """ - - def command( # type: ignore[override] - self, *args: t.Any, **kwargs: t.Any - ) -> t.Callable[[t.Callable[..., t.Any]], click.Command]: - """This works exactly like the method of the same name on a regular - :class:`click.Group` but it wraps callbacks in :func:`with_appcontext` - unless it's disabled by passing ``with_appcontext=False``. - """ - wrap_for_ctx = kwargs.pop("with_appcontext", True) - - def decorator(f: t.Callable[..., t.Any]) -> click.Command: - if wrap_for_ctx: - f = with_appcontext(f) - return super(AppGroup, self).command(*args, **kwargs)(f) # type: ignore[no-any-return] - - return decorator - - def group( # type: ignore[override] - self, *args: t.Any, **kwargs: t.Any - ) -> t.Callable[[t.Callable[..., t.Any]], click.Group]: - """This works exactly like the method of the same name on a regular - :class:`click.Group` but it defaults the group class to - :class:`AppGroup`. - """ - kwargs.setdefault("cls", AppGroup) - return super().group(*args, **kwargs) # type: ignore[no-any-return] - - -def _set_app(ctx: click.Context, param: click.Option, value: str | None) -> str | None: - if value is None: - return None - - info = ctx.ensure_object(ScriptInfo) - info.app_import_path = value - return value - - -# This option is eager so the app will be available if --help is given. -# --help is also eager, so --app must be before it in the param list. -# no_args_is_help bypasses eager processing, so this option must be -# processed manually in that case to ensure FLASK_APP gets picked up. -_app_option = click.Option( - ["-A", "--app"], - metavar="IMPORT", - help=( - "The Flask application or factory function to load, in the form 'module:name'." - " Module can be a dotted import or file path. Name is not required if it is" - " 'app', 'application', 'create_app', or 'make_app', and can be 'name(args)' to" - " pass arguments." - ), - is_eager=True, - expose_value=False, - callback=_set_app, -) - - -def _set_debug(ctx: click.Context, param: click.Option, value: bool) -> bool | None: - # If the flag isn't provided, it will default to False. Don't use - # that, let debug be set by env in that case. - source = ctx.get_parameter_source(param.name) # type: ignore[arg-type] - - if source is not None and source in ( - ParameterSource.DEFAULT, - ParameterSource.DEFAULT_MAP, - ): - return None - - # Set with env var instead of ScriptInfo.load so that it can be - # accessed early during a factory function. - os.environ["FLASK_DEBUG"] = "1" if value else "0" - return value - - -_debug_option = click.Option( - ["--debug/--no-debug"], - help="Set debug mode.", - expose_value=False, - callback=_set_debug, -) - - -def _env_file_callback( - ctx: click.Context, param: click.Option, value: str | None -) -> str | None: - try: - import dotenv # noqa: F401 - except ImportError: - # Only show an error if a value was passed, otherwise we still want to - # call load_dotenv and show a message without exiting. - if value is not None: - raise click.BadParameter( - "python-dotenv must be installed to load an env file.", - ctx=ctx, - param=param, - ) from None - - # Load if a value was passed, or we want to load default files, or both. - if value is not None or ctx.obj.load_dotenv_defaults: - load_dotenv(value, load_defaults=ctx.obj.load_dotenv_defaults) - - return value - - -# This option is eager so env vars are loaded as early as possible to be -# used by other options. -_env_file_option = click.Option( - ["-e", "--env-file"], - type=click.Path(exists=True, dir_okay=False), - help=( - "Load environment variables from this file, taking precedence over" - " those set by '.env' and '.flaskenv'. Variables set directly in the" - " environment take highest precedence. python-dotenv must be installed." - ), - is_eager=True, - expose_value=False, - callback=_env_file_callback, -) - - -class FlaskGroup(AppGroup): - """Special subclass of the :class:`AppGroup` group that supports - loading more commands from the configured Flask app. Normally a - developer does not have to interface with this class but there are - some very advanced use cases for which it makes sense to create an - instance of this. see :ref:`custom-scripts`. - - :param add_default_commands: if this is True then the default run and - shell commands will be added. - :param add_version_option: adds the ``--version`` option. - :param create_app: an optional callback that is passed the script info and - returns the loaded app. - :param load_dotenv: Load the nearest :file:`.env` and :file:`.flaskenv` - files to set environment variables. Will also change the working - directory to the directory containing the first file found. - :param set_debug_flag: Set the app's debug flag. - - .. versionchanged:: 3.1 - ``-e path`` takes precedence over default ``.env`` and ``.flaskenv`` files. - - .. versionchanged:: 2.2 - Added the ``-A/--app``, ``--debug/--no-debug``, ``-e/--env-file`` options. - - .. versionchanged:: 2.2 - An app context is pushed when running ``app.cli`` commands, so - ``@with_appcontext`` is no longer required for those commands. - - .. versionchanged:: 1.0 - If installed, python-dotenv will be used to load environment variables - from :file:`.env` and :file:`.flaskenv` files. - """ - - def __init__( - self, - add_default_commands: bool = True, - create_app: t.Callable[..., Flask] | None = None, - add_version_option: bool = True, - load_dotenv: bool = True, - set_debug_flag: bool = True, - **extra: t.Any, - ) -> None: - params: list[click.Parameter] = list(extra.pop("params", None) or ()) - # Processing is done with option callbacks instead of a group - # callback. This allows users to make a custom group callback - # without losing the behavior. --env-file must come first so - # that it is eagerly evaluated before --app. - params.extend((_env_file_option, _app_option, _debug_option)) - - if add_version_option: - params.append(version_option) - - if "context_settings" not in extra: - extra["context_settings"] = {} - - extra["context_settings"].setdefault("auto_envvar_prefix", "FLASK") - - super().__init__(params=params, **extra) - - self.create_app = create_app - self.load_dotenv = load_dotenv - self.set_debug_flag = set_debug_flag - - if add_default_commands: - self.add_command(run_command) - self.add_command(shell_command) - self.add_command(routes_command) - - self._loaded_plugin_commands = False - - def _load_plugin_commands(self) -> None: - if self._loaded_plugin_commands: - return - - if sys.version_info >= (3, 10): - from importlib import metadata - else: - # Use a backport on Python < 3.10. We technically have - # importlib.metadata on 3.8+, but the API changed in 3.10, - # so use the backport for consistency. - import importlib_metadata as metadata # pyright: ignore - - for ep in metadata.entry_points(group="flask.commands"): - self.add_command(ep.load(), ep.name) - - self._loaded_plugin_commands = True - - def get_command(self, ctx: click.Context, name: str) -> click.Command | None: - self._load_plugin_commands() - # Look up built-in and plugin commands, which should be - # available even if the app fails to load. - rv = super().get_command(ctx, name) - - if rv is not None: - return rv - - info = ctx.ensure_object(ScriptInfo) - - # Look up commands provided by the app, showing an error and - # continuing if the app couldn't be loaded. - try: - app = info.load_app() - except NoAppException as e: - click.secho(f"Error: {e.format_message()}\n", err=True, fg="red") - return None - - # Push an app context for the loaded app unless it is already - # active somehow. This makes the context available to parameter - # and command callbacks without needing @with_appcontext. - if not current_app or current_app._get_current_object() is not app: # type: ignore[attr-defined] - ctx.with_resource(app.app_context()) - - return app.cli.get_command(ctx, name) - - def list_commands(self, ctx: click.Context) -> list[str]: - self._load_plugin_commands() - # Start with the built-in and plugin commands. - rv = set(super().list_commands(ctx)) - info = ctx.ensure_object(ScriptInfo) - - # Add commands provided by the app, showing an error and - # continuing if the app couldn't be loaded. - try: - rv.update(info.load_app().cli.list_commands(ctx)) - except NoAppException as e: - # When an app couldn't be loaded, show the error message - # without the traceback. - click.secho(f"Error: {e.format_message()}\n", err=True, fg="red") - except Exception: - # When any other errors occurred during loading, show the - # full traceback. - click.secho(f"{traceback.format_exc()}\n", err=True, fg="red") - - return sorted(rv) - - def make_context( - self, - info_name: str | None, - args: list[str], - parent: click.Context | None = None, - **extra: t.Any, - ) -> click.Context: - # Set a flag to tell app.run to become a no-op. If app.run was - # not in a __name__ == __main__ guard, it would start the server - # when importing, blocking whatever command is being called. - os.environ["FLASK_RUN_FROM_CLI"] = "true" - - if "obj" not in extra and "obj" not in self.context_settings: - extra["obj"] = ScriptInfo( - create_app=self.create_app, - set_debug_flag=self.set_debug_flag, - load_dotenv_defaults=self.load_dotenv, - ) - - return super().make_context(info_name, args, parent=parent, **extra) - - def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: - if (not args and self.no_args_is_help) or ( - len(args) == 1 and args[0] in self.get_help_option_names(ctx) - ): - # Attempt to load --env-file and --app early in case they - # were given as env vars. Otherwise no_args_is_help will not - # see commands from app.cli. - _env_file_option.handle_parse_result(ctx, {}, []) - _app_option.handle_parse_result(ctx, {}, []) - - return super().parse_args(ctx, args) - - -def _path_is_ancestor(path: str, other: str) -> bool: - """Take ``other`` and remove the length of ``path`` from it. Then join it - to ``path``. If it is the original value, ``path`` is an ancestor of - ``other``.""" - return os.path.join(path, other[len(path) :].lstrip(os.sep)) == other - - -def load_dotenv( - path: str | os.PathLike[str] | None = None, load_defaults: bool = True -) -> bool: - """Load "dotenv" files to set environment variables. A given path takes - precedence over ``.env``, which takes precedence over ``.flaskenv``. After - loading and combining these files, values are only set if the key is not - already set in ``os.environ``. - - This is a no-op if `python-dotenv`_ is not installed. - - .. _python-dotenv: https://github.com/theskumar/python-dotenv#readme - - :param path: Load the file at this location. - :param load_defaults: Search for and load the default ``.flaskenv`` and - ``.env`` files. - :return: ``True`` if at least one env var was loaded. - - .. versionchanged:: 3.1 - Added the ``load_defaults`` parameter. A given path takes precedence - over default files. - - .. versionchanged:: 2.0 - The current directory is not changed to the location of the - loaded file. - - .. versionchanged:: 2.0 - When loading the env files, set the default encoding to UTF-8. - - .. versionchanged:: 1.1.0 - Returns ``False`` when python-dotenv is not installed, or when - the given path isn't a file. - - .. versionadded:: 1.0 - """ - try: - import dotenv - except ImportError: - if path or os.path.isfile(".env") or os.path.isfile(".flaskenv"): - click.secho( - " * Tip: There are .env files present. Install python-dotenv" - " to use them.", - fg="yellow", - err=True, - ) - - return False - - data: dict[str, str | None] = {} - - if load_defaults: - for default_name in (".flaskenv", ".env"): - if not (default_path := dotenv.find_dotenv(default_name, usecwd=True)): - continue - - data |= dotenv.dotenv_values(default_path, encoding="utf-8") - - if path is not None and os.path.isfile(path): - data |= dotenv.dotenv_values(path, encoding="utf-8") - - for key, value in data.items(): - if key in os.environ or value is None: - continue - - os.environ[key] = value - - return bool(data) # True if at least one env var was loaded. - - -def show_server_banner(debug: bool, app_import_path: str | None) -> None: - """Show extra startup messages the first time the server is run, - ignoring the reloader. - """ - if is_running_from_reloader(): - return - - if app_import_path is not None: - click.echo(f" * Serving Flask app '{app_import_path}'") - - if debug is not None: - click.echo(f" * Debug mode: {'on' if debug else 'off'}") - - -class CertParamType(click.ParamType): - """Click option type for the ``--cert`` option. Allows either an - existing file, the string ``'adhoc'``, or an import for a - :class:`~ssl.SSLContext` object. - """ - - name = "path" - - def __init__(self) -> None: - self.path_type = click.Path(exists=True, dir_okay=False, resolve_path=True) - - def convert( - self, value: t.Any, param: click.Parameter | None, ctx: click.Context | None - ) -> t.Any: - try: - import ssl - except ImportError: - raise click.BadParameter( - 'Using "--cert" requires Python to be compiled with SSL support.', - ctx, - param, - ) from None - - try: - return self.path_type(value, param, ctx) - except click.BadParameter: - value = click.STRING(value, param, ctx).lower() - - if value == "adhoc": - try: - import cryptography # noqa: F401 - except ImportError: - raise click.BadParameter( - "Using ad-hoc certificates requires the cryptography library.", - ctx, - param, - ) from None - - return value - - obj = import_string(value, silent=True) - - if isinstance(obj, ssl.SSLContext): - return obj - - raise - - -def _validate_key(ctx: click.Context, param: click.Parameter, value: t.Any) -> t.Any: - """The ``--key`` option must be specified when ``--cert`` is a file. - Modifies the ``cert`` param to be a ``(cert, key)`` pair if needed. - """ - cert = ctx.params.get("cert") - is_adhoc = cert == "adhoc" - - try: - import ssl - except ImportError: - is_context = False - else: - is_context = isinstance(cert, ssl.SSLContext) - - if value is not None: - if is_adhoc: - raise click.BadParameter( - 'When "--cert" is "adhoc", "--key" is not used.', ctx, param - ) - - if is_context: - raise click.BadParameter( - 'When "--cert" is an SSLContext object, "--key" is not used.', - ctx, - param, - ) - - if not cert: - raise click.BadParameter('"--cert" must also be specified.', ctx, param) - - ctx.params["cert"] = cert, value - - else: - if cert and not (is_adhoc or is_context): - raise click.BadParameter('Required when using "--cert".', ctx, param) - - return value - - -class SeparatedPathType(click.Path): - """Click option type that accepts a list of values separated by the - OS's path separator (``:``, ``;`` on Windows). Each value is - validated as a :class:`click.Path` type. - """ - - def convert( - self, value: t.Any, param: click.Parameter | None, ctx: click.Context | None - ) -> t.Any: - items = self.split_envvar_value(value) - # can't call no-arg super() inside list comprehension until Python 3.12 - super_convert = super().convert - return [super_convert(item, param, ctx) for item in items] - - -@click.command("run", short_help="Run a development server.") -@click.option("--host", "-h", default="127.0.0.1", help="The interface to bind to.") -@click.option("--port", "-p", default=5000, help="The port to bind to.") -@click.option( - "--cert", - type=CertParamType(), - help="Specify a certificate file to use HTTPS.", - is_eager=True, -) -@click.option( - "--key", - type=click.Path(exists=True, dir_okay=False, resolve_path=True), - callback=_validate_key, - expose_value=False, - help="The key file to use when specifying a certificate.", -) -@click.option( - "--reload/--no-reload", - default=None, - help="Enable or disable the reloader. By default the reloader " - "is active if debug is enabled.", -) -@click.option( - "--debugger/--no-debugger", - default=None, - help="Enable or disable the debugger. By default the debugger " - "is active if debug is enabled.", -) -@click.option( - "--with-threads/--without-threads", - default=True, - help="Enable or disable multithreading.", -) -@click.option( - "--extra-files", - default=None, - type=SeparatedPathType(), - help=( - "Extra files that trigger a reload on change. Multiple paths" - f" are separated by {os.path.pathsep!r}." - ), -) -@click.option( - "--exclude-patterns", - default=None, - type=SeparatedPathType(), - help=( - "Files matching these fnmatch patterns will not trigger a reload" - " on change. Multiple patterns are separated by" - f" {os.path.pathsep!r}." - ), -) -@pass_script_info -def run_command( - info: ScriptInfo, - host: str, - port: int, - reload: bool, - debugger: bool, - with_threads: bool, - cert: ssl.SSLContext | tuple[str, str | None] | t.Literal["adhoc"] | None, - extra_files: list[str] | None, - exclude_patterns: list[str] | None, -) -> None: - """Run a local development server. - - This server is for development purposes only. It does not provide - the stability, security, or performance of production WSGI servers. - - The reloader and debugger are enabled by default with the '--debug' - option. - """ - try: - app: WSGIApplication = info.load_app() # pyright: ignore - except Exception as e: - if is_running_from_reloader(): - # When reloading, print out the error immediately, but raise - # it later so the debugger or server can handle it. - traceback.print_exc() - err = e - - def app( - environ: WSGIEnvironment, start_response: StartResponse - ) -> cabc.Iterable[bytes]: - raise err from None - - else: - # When not reloading, raise the error immediately so the - # command fails. - raise e from None - - debug = get_debug_flag() - - if reload is None: - reload = debug - - if debugger is None: - debugger = debug - - show_server_banner(debug, info.app_import_path) - - run_simple( - host, - port, - app, - use_reloader=reload, - use_debugger=debugger, - threaded=with_threads, - ssl_context=cert, - extra_files=extra_files, - exclude_patterns=exclude_patterns, - ) - - -run_command.params.insert(0, _debug_option) - - -@click.command("shell", short_help="Run a shell in the app context.") -@with_appcontext -def shell_command() -> None: - """Run an interactive Python shell in the context of a given - Flask application. The application will populate the default - namespace of this shell according to its configuration. - - This is useful for executing small snippets of management code - without having to manually configure the application. - """ - import code - - banner = ( - f"Python {sys.version} on {sys.platform}\n" - f"App: {current_app.import_name}\n" - f"Instance: {current_app.instance_path}" - ) - ctx: dict[str, t.Any] = {} - - # Support the regular Python interpreter startup script if someone - # is using it. - startup = os.environ.get("PYTHONSTARTUP") - if startup and os.path.isfile(startup): - with open(startup) as f: - eval(compile(f.read(), startup, "exec"), ctx) - - ctx.update(current_app.make_shell_context()) - - # Site, customize, or startup script can set a hook to call when - # entering interactive mode. The default one sets up readline with - # tab and history completion. - interactive_hook = getattr(sys, "__interactivehook__", None) - - if interactive_hook is not None: - try: - import readline - from rlcompleter import Completer - except ImportError: - pass - else: - # rlcompleter uses __main__.__dict__ by default, which is - # flask.__main__. Use the shell context instead. - readline.set_completer(Completer(ctx).complete) - - interactive_hook() - - code.interact(banner=banner, local=ctx) - - -@click.command("routes", short_help="Show the routes for the app.") -@click.option( - "--sort", - "-s", - type=click.Choice(("endpoint", "methods", "domain", "rule", "match")), - default="endpoint", - help=( - "Method to sort routes by. 'match' is the order that Flask will match routes" - " when dispatching a request." - ), -) -@click.option("--all-methods", is_flag=True, help="Show HEAD and OPTIONS methods.") -@with_appcontext -def routes_command(sort: str, all_methods: bool) -> None: - """Show all registered routes with endpoints and methods.""" - rules = list(current_app.url_map.iter_rules()) - - if not rules: - click.echo("No routes were registered.") - return - - ignored_methods = set() if all_methods else {"HEAD", "OPTIONS"} - host_matching = current_app.url_map.host_matching - has_domain = any(rule.host if host_matching else rule.subdomain for rule in rules) - rows = [] - - for rule in rules: - row = [ - rule.endpoint, - ", ".join(sorted((rule.methods or set()) - ignored_methods)), - ] - - if has_domain: - row.append((rule.host if host_matching else rule.subdomain) or "") - - row.append(rule.rule) - rows.append(row) - - headers = ["Endpoint", "Methods"] - sorts = ["endpoint", "methods"] - - if has_domain: - headers.append("Host" if host_matching else "Subdomain") - sorts.append("domain") - - headers.append("Rule") - sorts.append("rule") - - try: - rows.sort(key=itemgetter(sorts.index(sort))) - except ValueError: - pass - - rows.insert(0, headers) - widths = [max(len(row[i]) for row in rows) for i in range(len(headers))] - rows.insert(1, ["-" * w for w in widths]) - template = " ".join(f"{{{i}:<{w}}}" for i, w in enumerate(widths)) - - for row in rows: - click.echo(template.format(*row)) - - -cli = FlaskGroup( - name="flask", - help="""\ -A general utility script for Flask applications. - -An application to load must be given with the '--app' option, -'FLASK_APP' environment variable, or with a 'wsgi.py' or 'app.py' file -in the current directory. -""", -) - - -def main() -> None: - cli.main() - - -if __name__ == "__main__": - main() diff --git a/bundle/python-cpu/Lib/site-packages/flask/config.py b/bundle/python-cpu/Lib/site-packages/flask/config.py deleted file mode 100644 index 34ef1a57217878e21daba0bc4f20ded293a33e7e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/config.py +++ /dev/null @@ -1,367 +0,0 @@ -from __future__ import annotations - -import errno -import json -import os -import types -import typing as t - -from werkzeug.utils import import_string - -if t.TYPE_CHECKING: - import typing_extensions as te - - from .sansio.app import App - - -T = t.TypeVar("T") - - -class ConfigAttribute(t.Generic[T]): - """Makes an attribute forward to the config""" - - def __init__( - self, name: str, get_converter: t.Callable[[t.Any], T] | None = None - ) -> None: - self.__name__ = name - self.get_converter = get_converter - - @t.overload - def __get__(self, obj: None, owner: None) -> te.Self: ... - - @t.overload - def __get__(self, obj: App, owner: type[App]) -> T: ... - - def __get__(self, obj: App | None, owner: type[App] | None = None) -> T | te.Self: - if obj is None: - return self - - rv = obj.config[self.__name__] - - if self.get_converter is not None: - rv = self.get_converter(rv) - - return rv # type: ignore[no-any-return] - - def __set__(self, obj: App, value: t.Any) -> None: - obj.config[self.__name__] = value - - -class Config(dict): # type: ignore[type-arg] - """Works exactly like a dict but provides ways to fill it from files - or special dictionaries. There are two common patterns to populate the - config. - - Either you can fill the config from a config file:: - - app.config.from_pyfile('yourconfig.cfg') - - Or alternatively you can define the configuration options in the - module that calls :meth:`from_object` or provide an import path to - a module that should be loaded. It is also possible to tell it to - use the same module and with that provide the configuration values - just before the call:: - - DEBUG = True - SECRET_KEY = 'development key' - app.config.from_object(__name__) - - In both cases (loading from any Python file or loading from modules), - only uppercase keys are added to the config. This makes it possible to use - lowercase values in the config file for temporary values that are not added - to the config or to define the config keys in the same file that implements - the application. - - Probably the most interesting way to load configurations is from an - environment variable pointing to a file:: - - app.config.from_envvar('YOURAPPLICATION_SETTINGS') - - In this case before launching the application you have to set this - environment variable to the file you want to use. On Linux and OS X - use the export statement:: - - export YOURAPPLICATION_SETTINGS='/path/to/config/file' - - On windows use `set` instead. - - :param root_path: path to which files are read relative from. When the - config object is created by the application, this is - the application's :attr:`~flask.Flask.root_path`. - :param defaults: an optional dictionary of default values - """ - - def __init__( - self, - root_path: str | os.PathLike[str], - defaults: dict[str, t.Any] | None = None, - ) -> None: - super().__init__(defaults or {}) - self.root_path = root_path - - def from_envvar(self, variable_name: str, silent: bool = False) -> bool: - """Loads a configuration from an environment variable pointing to - a configuration file. This is basically just a shortcut with nicer - error messages for this line of code:: - - app.config.from_pyfile(os.environ['YOURAPPLICATION_SETTINGS']) - - :param variable_name: name of the environment variable - :param silent: set to ``True`` if you want silent failure for missing - files. - :return: ``True`` if the file was loaded successfully. - """ - rv = os.environ.get(variable_name) - if not rv: - if silent: - return False - raise RuntimeError( - f"The environment variable {variable_name!r} is not set" - " and as such configuration could not be loaded. Set" - " this variable and make it point to a configuration" - " file" - ) - return self.from_pyfile(rv, silent=silent) - - def from_prefixed_env( - self, prefix: str = "FLASK", *, loads: t.Callable[[str], t.Any] = json.loads - ) -> bool: - """Load any environment variables that start with ``FLASK_``, - dropping the prefix from the env key for the config key. Values - are passed through a loading function to attempt to convert them - to more specific types than strings. - - Keys are loaded in :func:`sorted` order. - - The default loading function attempts to parse values as any - valid JSON type, including dicts and lists. - - Specific items in nested dicts can be set by separating the - keys with double underscores (``__``). If an intermediate key - doesn't exist, it will be initialized to an empty dict. - - :param prefix: Load env vars that start with this prefix, - separated with an underscore (``_``). - :param loads: Pass each string value to this function and use - the returned value as the config value. If any error is - raised it is ignored and the value remains a string. The - default is :func:`json.loads`. - - .. versionadded:: 2.1 - """ - prefix = f"{prefix}_" - - for key in sorted(os.environ): - if not key.startswith(prefix): - continue - - value = os.environ[key] - key = key.removeprefix(prefix) - - try: - value = loads(value) - except Exception: - # Keep the value as a string if loading failed. - pass - - if "__" not in key: - # A non-nested key, set directly. - self[key] = value - continue - - # Traverse nested dictionaries with keys separated by "__". - current = self - *parts, tail = key.split("__") - - for part in parts: - # If an intermediate dict does not exist, create it. - if part not in current: - current[part] = {} - - current = current[part] - - current[tail] = value - - return True - - def from_pyfile( - self, filename: str | os.PathLike[str], silent: bool = False - ) -> bool: - """Updates the values in the config from a Python file. This function - behaves as if the file was imported as module with the - :meth:`from_object` function. - - :param filename: the filename of the config. This can either be an - absolute filename or a filename relative to the - root path. - :param silent: set to ``True`` if you want silent failure for missing - files. - :return: ``True`` if the file was loaded successfully. - - .. versionadded:: 0.7 - `silent` parameter. - """ - filename = os.path.join(self.root_path, filename) - d = types.ModuleType("config") - d.__file__ = filename - try: - with open(filename, mode="rb") as config_file: - exec(compile(config_file.read(), filename, "exec"), d.__dict__) - except OSError as e: - if silent and e.errno in (errno.ENOENT, errno.EISDIR, errno.ENOTDIR): - return False - e.strerror = f"Unable to load configuration file ({e.strerror})" - raise - self.from_object(d) - return True - - def from_object(self, obj: object | str) -> None: - """Updates the values from the given object. An object can be of one - of the following two types: - - - a string: in this case the object with that name will be imported - - an actual object reference: that object is used directly - - Objects are usually either modules or classes. :meth:`from_object` - loads only the uppercase attributes of the module/class. A ``dict`` - object will not work with :meth:`from_object` because the keys of a - ``dict`` are not attributes of the ``dict`` class. - - Example of module-based configuration:: - - app.config.from_object('yourapplication.default_config') - from yourapplication import default_config - app.config.from_object(default_config) - - Nothing is done to the object before loading. If the object is a - class and has ``@property`` attributes, it needs to be - instantiated before being passed to this method. - - You should not use this function to load the actual configuration but - rather configuration defaults. The actual config should be loaded - with :meth:`from_pyfile` and ideally from a location not within the - package because the package might be installed system wide. - - See :ref:`config-dev-prod` for an example of class-based configuration - using :meth:`from_object`. - - :param obj: an import name or object - """ - if isinstance(obj, str): - obj = import_string(obj) - for key in dir(obj): - if key.isupper(): - self[key] = getattr(obj, key) - - def from_file( - self, - filename: str | os.PathLike[str], - load: t.Callable[[t.IO[t.Any]], t.Mapping[str, t.Any]], - silent: bool = False, - text: bool = True, - ) -> bool: - """Update the values in the config from a file that is loaded - using the ``load`` parameter. The loaded data is passed to the - :meth:`from_mapping` method. - - .. code-block:: python - - import json - app.config.from_file("config.json", load=json.load) - - import tomllib - app.config.from_file("config.toml", load=tomllib.load, text=False) - - :param filename: The path to the data file. This can be an - absolute path or relative to the config root path. - :param load: A callable that takes a file handle and returns a - mapping of loaded data from the file. - :type load: ``Callable[[Reader], Mapping]`` where ``Reader`` - implements a ``read`` method. - :param silent: Ignore the file if it doesn't exist. - :param text: Open the file in text or binary mode. - :return: ``True`` if the file was loaded successfully. - - .. versionchanged:: 2.3 - The ``text`` parameter was added. - - .. versionadded:: 2.0 - """ - filename = os.path.join(self.root_path, filename) - - try: - with open(filename, "r" if text else "rb") as f: - obj = load(f) - except OSError as e: - if silent and e.errno in (errno.ENOENT, errno.EISDIR): - return False - - e.strerror = f"Unable to load configuration file ({e.strerror})" - raise - - return self.from_mapping(obj) - - def from_mapping( - self, mapping: t.Mapping[str, t.Any] | None = None, **kwargs: t.Any - ) -> bool: - """Updates the config like :meth:`update` ignoring items with - non-upper keys. - - :return: Always returns ``True``. - - .. versionadded:: 0.11 - """ - mappings: dict[str, t.Any] = {} - if mapping is not None: - mappings.update(mapping) - mappings.update(kwargs) - for key, value in mappings.items(): - if key.isupper(): - self[key] = value - return True - - def get_namespace( - self, namespace: str, lowercase: bool = True, trim_namespace: bool = True - ) -> dict[str, t.Any]: - """Returns a dictionary containing a subset of configuration options - that match the specified namespace/prefix. Example usage:: - - app.config['IMAGE_STORE_TYPE'] = 'fs' - app.config['IMAGE_STORE_PATH'] = '/var/app/images' - app.config['IMAGE_STORE_BASE_URL'] = 'http://img.website.com' - image_store_config = app.config.get_namespace('IMAGE_STORE_') - - The resulting dictionary `image_store_config` would look like:: - - { - 'type': 'fs', - 'path': '/var/app/images', - 'base_url': 'http://img.website.com' - } - - This is often useful when configuration options map directly to - keyword arguments in functions or class constructors. - - :param namespace: a configuration namespace - :param lowercase: a flag indicating if the keys of the resulting - dictionary should be lowercase - :param trim_namespace: a flag indicating if the keys of the resulting - dictionary should not include the namespace - - .. versionadded:: 0.11 - """ - rv = {} - for k, v in self.items(): - if not k.startswith(namespace): - continue - if trim_namespace: - key = k[len(namespace) :] - else: - key = k - if lowercase: - key = key.lower() - rv[key] = v - return rv - - def __repr__(self) -> str: - return f"<{type(self).__name__} {dict.__repr__(self)}>" diff --git a/bundle/python-cpu/Lib/site-packages/flask/ctx.py b/bundle/python-cpu/Lib/site-packages/flask/ctx.py deleted file mode 100644 index 5f7b1f1db013739d1b3b24f5014af62cf2bd21ba..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/ctx.py +++ /dev/null @@ -1,459 +0,0 @@ -from __future__ import annotations - -import contextvars -import sys -import typing as t -from functools import update_wrapper -from types import TracebackType - -from werkzeug.exceptions import HTTPException - -from . import typing as ft -from .globals import _cv_app -from .globals import _cv_request -from .signals import appcontext_popped -from .signals import appcontext_pushed - -if t.TYPE_CHECKING: # pragma: no cover - from _typeshed.wsgi import WSGIEnvironment - - from .app import Flask - from .sessions import SessionMixin - from .wrappers import Request - - -# a singleton sentinel value for parameter defaults -_sentinel = object() - - -class _AppCtxGlobals: - """A plain object. Used as a namespace for storing data during an - application context. - - Creating an app context automatically creates this object, which is - made available as the :data:`g` proxy. - - .. describe:: 'key' in g - - Check whether an attribute is present. - - .. versionadded:: 0.10 - - .. describe:: iter(g) - - Return an iterator over the attribute names. - - .. versionadded:: 0.10 - """ - - # Define attr methods to let mypy know this is a namespace object - # that has arbitrary attributes. - - def __getattr__(self, name: str) -> t.Any: - try: - return self.__dict__[name] - except KeyError: - raise AttributeError(name) from None - - def __setattr__(self, name: str, value: t.Any) -> None: - self.__dict__[name] = value - - def __delattr__(self, name: str) -> None: - try: - del self.__dict__[name] - except KeyError: - raise AttributeError(name) from None - - def get(self, name: str, default: t.Any | None = None) -> t.Any: - """Get an attribute by name, or a default value. Like - :meth:`dict.get`. - - :param name: Name of attribute to get. - :param default: Value to return if the attribute is not present. - - .. versionadded:: 0.10 - """ - return self.__dict__.get(name, default) - - def pop(self, name: str, default: t.Any = _sentinel) -> t.Any: - """Get and remove an attribute by name. Like :meth:`dict.pop`. - - :param name: Name of attribute to pop. - :param default: Value to return if the attribute is not present, - instead of raising a ``KeyError``. - - .. versionadded:: 0.11 - """ - if default is _sentinel: - return self.__dict__.pop(name) - else: - return self.__dict__.pop(name, default) - - def setdefault(self, name: str, default: t.Any = None) -> t.Any: - """Get the value of an attribute if it is present, otherwise - set and return a default value. Like :meth:`dict.setdefault`. - - :param name: Name of attribute to get. - :param default: Value to set and return if the attribute is not - present. - - .. versionadded:: 0.11 - """ - return self.__dict__.setdefault(name, default) - - def __contains__(self, item: str) -> bool: - return item in self.__dict__ - - def __iter__(self) -> t.Iterator[str]: - return iter(self.__dict__) - - def __repr__(self) -> str: - ctx = _cv_app.get(None) - if ctx is not None: - return f"" - return object.__repr__(self) - - -def after_this_request( - f: ft.AfterRequestCallable[t.Any], -) -> ft.AfterRequestCallable[t.Any]: - """Executes a function after this request. This is useful to modify - response objects. The function is passed the response object and has - to return the same or a new one. - - Example:: - - @app.route('/') - def index(): - @after_this_request - def add_header(response): - response.headers['X-Foo'] = 'Parachute' - return response - return 'Hello World!' - - This is more useful if a function other than the view function wants to - modify a response. For instance think of a decorator that wants to add - some headers without converting the return value into a response object. - - .. versionadded:: 0.9 - """ - ctx = _cv_request.get(None) - - if ctx is None: - raise RuntimeError( - "'after_this_request' can only be used when a request" - " context is active, such as in a view function." - ) - - ctx._after_request_functions.append(f) - return f - - -F = t.TypeVar("F", bound=t.Callable[..., t.Any]) - - -def copy_current_request_context(f: F) -> F: - """A helper function that decorates a function to retain the current - request context. This is useful when working with greenlets. The moment - the function is decorated a copy of the request context is created and - then pushed when the function is called. The current session is also - included in the copied request context. - - Example:: - - import gevent - from flask import copy_current_request_context - - @app.route('/') - def index(): - @copy_current_request_context - def do_some_work(): - # do some work here, it can access flask.request or - # flask.session like you would otherwise in the view function. - ... - gevent.spawn(do_some_work) - return 'Regular response' - - .. versionadded:: 0.10 - """ - ctx = _cv_request.get(None) - - if ctx is None: - raise RuntimeError( - "'copy_current_request_context' can only be used when a" - " request context is active, such as in a view function." - ) - - ctx = ctx.copy() - - def wrapper(*args: t.Any, **kwargs: t.Any) -> t.Any: - with ctx: - return ctx.app.ensure_sync(f)(*args, **kwargs) - - return update_wrapper(wrapper, f) # type: ignore[return-value] - - -def has_request_context() -> bool: - """If you have code that wants to test if a request context is there or - not this function can be used. For instance, you may want to take advantage - of request information if the request object is available, but fail - silently if it is unavailable. - - :: - - class User(db.Model): - - def __init__(self, username, remote_addr=None): - self.username = username - if remote_addr is None and has_request_context(): - remote_addr = request.remote_addr - self.remote_addr = remote_addr - - Alternatively you can also just test any of the context bound objects - (such as :class:`request` or :class:`g`) for truthness:: - - class User(db.Model): - - def __init__(self, username, remote_addr=None): - self.username = username - if remote_addr is None and request: - remote_addr = request.remote_addr - self.remote_addr = remote_addr - - .. versionadded:: 0.7 - """ - return _cv_request.get(None) is not None - - -def has_app_context() -> bool: - """Works like :func:`has_request_context` but for the application - context. You can also just do a boolean check on the - :data:`current_app` object instead. - - .. versionadded:: 0.9 - """ - return _cv_app.get(None) is not None - - -class AppContext: - """The app context contains application-specific information. An app - context is created and pushed at the beginning of each request if - one is not already active. An app context is also pushed when - running CLI commands. - """ - - def __init__(self, app: Flask) -> None: - self.app = app - self.url_adapter = app.create_url_adapter(None) - self.g: _AppCtxGlobals = app.app_ctx_globals_class() - self._cv_tokens: list[contextvars.Token[AppContext]] = [] - - def push(self) -> None: - """Binds the app context to the current context.""" - self._cv_tokens.append(_cv_app.set(self)) - appcontext_pushed.send(self.app, _async_wrapper=self.app.ensure_sync) - - def pop(self, exc: BaseException | None = _sentinel) -> None: # type: ignore - """Pops the app context.""" - try: - if len(self._cv_tokens) == 1: - if exc is _sentinel: - exc = sys.exc_info()[1] - self.app.do_teardown_appcontext(exc) - finally: - ctx = _cv_app.get() - _cv_app.reset(self._cv_tokens.pop()) - - if ctx is not self: - raise AssertionError( - f"Popped wrong app context. ({ctx!r} instead of {self!r})" - ) - - appcontext_popped.send(self.app, _async_wrapper=self.app.ensure_sync) - - def __enter__(self) -> AppContext: - self.push() - return self - - def __exit__( - self, - exc_type: type | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> None: - self.pop(exc_value) - - -class RequestContext: - """The request context contains per-request information. The Flask - app creates and pushes it at the beginning of the request, then pops - it at the end of the request. It will create the URL adapter and - request object for the WSGI environment provided. - - Do not attempt to use this class directly, instead use - :meth:`~flask.Flask.test_request_context` and - :meth:`~flask.Flask.request_context` to create this object. - - When the request context is popped, it will evaluate all the - functions registered on the application for teardown execution - (:meth:`~flask.Flask.teardown_request`). - - The request context is automatically popped at the end of the - request. When using the interactive debugger, the context will be - restored so ``request`` is still accessible. Similarly, the test - client can preserve the context after the request ends. However, - teardown functions may already have closed some resources such as - database connections. - """ - - def __init__( - self, - app: Flask, - environ: WSGIEnvironment, - request: Request | None = None, - session: SessionMixin | None = None, - ) -> None: - self.app = app - if request is None: - request = app.request_class(environ) - request.json_module = app.json - self.request: Request = request - self.url_adapter = None - try: - self.url_adapter = app.create_url_adapter(self.request) - except HTTPException as e: - self.request.routing_exception = e - self.flashes: list[tuple[str, str]] | None = None - self._session: SessionMixin | None = session - # Functions that should be executed after the request on the response - # object. These will be called before the regular "after_request" - # functions. - self._after_request_functions: list[ft.AfterRequestCallable[t.Any]] = [] - - self._cv_tokens: list[ - tuple[contextvars.Token[RequestContext], AppContext | None] - ] = [] - - def copy(self) -> RequestContext: - """Creates a copy of this request context with the same request object. - This can be used to move a request context to a different greenlet. - Because the actual request object is the same this cannot be used to - move a request context to a different thread unless access to the - request object is locked. - - .. versionadded:: 0.10 - - .. versionchanged:: 1.1 - The current session object is used instead of reloading the original - data. This prevents `flask.session` pointing to an out-of-date object. - """ - return self.__class__( - self.app, - environ=self.request.environ, - request=self.request, - session=self._session, - ) - - def match_request(self) -> None: - """Can be overridden by a subclass to hook into the matching - of the request. - """ - try: - result = self.url_adapter.match(return_rule=True) # type: ignore - self.request.url_rule, self.request.view_args = result # type: ignore - except HTTPException as e: - self.request.routing_exception = e - - @property - def session(self) -> SessionMixin: - """The session data associated with this request. Not available until - this context has been pushed. Accessing this property, also accessed by - the :data:`~flask.session` proxy, sets :attr:`.SessionMixin.accessed`. - """ - assert self._session is not None, "The session has not yet been opened." - self._session.accessed = True - return self._session - - def push(self) -> None: - # Before we push the request context we have to ensure that there - # is an application context. - app_ctx = _cv_app.get(None) - - if app_ctx is None or app_ctx.app is not self.app: - app_ctx = self.app.app_context() - app_ctx.push() - else: - app_ctx = None - - self._cv_tokens.append((_cv_request.set(self), app_ctx)) - - # Open the session at the moment that the request context is available. - # This allows a custom open_session method to use the request context. - # Only open a new session if this is the first time the request was - # pushed, otherwise stream_with_context loses the session. - if self._session is None: - session_interface = self.app.session_interface - self._session = session_interface.open_session(self.app, self.request) - - if self._session is None: - self._session = session_interface.make_null_session(self.app) - - # Match the request URL after loading the session, so that the - # session is available in custom URL converters. - if self.url_adapter is not None: - self.match_request() - - def pop(self, exc: BaseException | None = _sentinel) -> None: # type: ignore - """Pops the request context and unbinds it by doing that. This will - also trigger the execution of functions registered by the - :meth:`~flask.Flask.teardown_request` decorator. - - .. versionchanged:: 0.9 - Added the `exc` argument. - """ - clear_request = len(self._cv_tokens) == 1 - - try: - if clear_request: - if exc is _sentinel: - exc = sys.exc_info()[1] - self.app.do_teardown_request(exc) - - request_close = getattr(self.request, "close", None) - if request_close is not None: - request_close() - finally: - ctx = _cv_request.get() - token, app_ctx = self._cv_tokens.pop() - _cv_request.reset(token) - - # get rid of circular dependencies at the end of the request - # so that we don't require the GC to be active. - if clear_request: - ctx.request.environ["werkzeug.request"] = None - - if app_ctx is not None: - app_ctx.pop(exc) - - if ctx is not self: - raise AssertionError( - f"Popped wrong request context. ({ctx!r} instead of {self!r})" - ) - - def __enter__(self) -> RequestContext: - self.push() - return self - - def __exit__( - self, - exc_type: type | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> None: - self.pop(exc_value) - - def __repr__(self) -> str: - return ( - f"<{type(self).__name__} {self.request.url!r}" - f" [{self.request.method}] of {self.app.name}>" - ) diff --git a/bundle/python-cpu/Lib/site-packages/flask/debughelpers.py b/bundle/python-cpu/Lib/site-packages/flask/debughelpers.py deleted file mode 100644 index 2c8c4c483677d2233f34baf82263d747f882d561..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/debughelpers.py +++ /dev/null @@ -1,178 +0,0 @@ -from __future__ import annotations - -import typing as t - -from jinja2.loaders import BaseLoader -from werkzeug.routing import RequestRedirect - -from .blueprints import Blueprint -from .globals import request_ctx -from .sansio.app import App - -if t.TYPE_CHECKING: - from .sansio.scaffold import Scaffold - from .wrappers import Request - - -class UnexpectedUnicodeError(AssertionError, UnicodeError): - """Raised in places where we want some better error reporting for - unexpected unicode or binary data. - """ - - -class DebugFilesKeyError(KeyError, AssertionError): - """Raised from request.files during debugging. The idea is that it can - provide a better error message than just a generic KeyError/BadRequest. - """ - - def __init__(self, request: Request, key: str) -> None: - form_matches = request.form.getlist(key) - buf = [ - f"You tried to access the file {key!r} in the request.files" - " dictionary but it does not exist. The mimetype for the" - f" request is {request.mimetype!r} instead of" - " 'multipart/form-data' which means that no file contents" - " were transmitted. To fix this error you should provide" - ' enctype="multipart/form-data" in your form.' - ] - if form_matches: - names = ", ".join(repr(x) for x in form_matches) - buf.append( - "\n\nThe browser instead transmitted some file names. " - f"This was submitted: {names}" - ) - self.msg = "".join(buf) - - def __str__(self) -> str: - return self.msg - - -class FormDataRoutingRedirect(AssertionError): - """This exception is raised in debug mode if a routing redirect - would cause the browser to drop the method or body. This happens - when method is not GET, HEAD or OPTIONS and the status code is not - 307 or 308. - """ - - def __init__(self, request: Request) -> None: - exc = request.routing_exception - assert isinstance(exc, RequestRedirect) - buf = [ - f"A request was sent to '{request.url}', but routing issued" - f" a redirect to the canonical URL '{exc.new_url}'." - ] - - if f"{request.base_url}/" == exc.new_url.partition("?")[0]: - buf.append( - " The URL was defined with a trailing slash. Flask" - " will redirect to the URL with a trailing slash if it" - " was accessed without one." - ) - - buf.append( - " Send requests to the canonical URL, or use 307 or 308 for" - " routing redirects. Otherwise, browsers will drop form" - " data.\n\n" - "This exception is only raised in debug mode." - ) - super().__init__("".join(buf)) - - -def attach_enctype_error_multidict(request: Request) -> None: - """Patch ``request.files.__getitem__`` to raise a descriptive error - about ``enctype=multipart/form-data``. - - :param request: The request to patch. - :meta private: - """ - oldcls = request.files.__class__ - - class newcls(oldcls): # type: ignore[valid-type, misc] - def __getitem__(self, key: str) -> t.Any: - try: - return super().__getitem__(key) - except KeyError as e: - if key not in request.form: - raise - - raise DebugFilesKeyError(request, key).with_traceback( - e.__traceback__ - ) from None - - newcls.__name__ = oldcls.__name__ - newcls.__module__ = oldcls.__module__ - request.files.__class__ = newcls - - -def _dump_loader_info(loader: BaseLoader) -> t.Iterator[str]: - yield f"class: {type(loader).__module__}.{type(loader).__name__}" - for key, value in sorted(loader.__dict__.items()): - if key.startswith("_"): - continue - if isinstance(value, (tuple, list)): - if not all(isinstance(x, str) for x in value): - continue - yield f"{key}:" - for item in value: - yield f" - {item}" - continue - elif not isinstance(value, (str, int, float, bool)): - continue - yield f"{key}: {value!r}" - - -def explain_template_loading_attempts( - app: App, - template: str, - attempts: list[ - tuple[ - BaseLoader, - Scaffold, - tuple[str, str | None, t.Callable[[], bool] | None] | None, - ] - ], -) -> None: - """This should help developers understand what failed""" - info = [f"Locating template {template!r}:"] - total_found = 0 - blueprint = None - if request_ctx and request_ctx.request.blueprint is not None: - blueprint = request_ctx.request.blueprint - - for idx, (loader, srcobj, triple) in enumerate(attempts): - if isinstance(srcobj, App): - src_info = f"application {srcobj.import_name!r}" - elif isinstance(srcobj, Blueprint): - src_info = f"blueprint {srcobj.name!r} ({srcobj.import_name})" - else: - src_info = repr(srcobj) - - info.append(f"{idx + 1:5}: trying loader of {src_info}") - - for line in _dump_loader_info(loader): - info.append(f" {line}") - - if triple is None: - detail = "no match" - else: - detail = f"found ({triple[1] or ''!r})" - total_found += 1 - info.append(f" -> {detail}") - - seems_fishy = False - if total_found == 0: - info.append("Error: the template could not be found.") - seems_fishy = True - elif total_found > 1: - info.append("Warning: multiple loaders returned a match for the template.") - seems_fishy = True - - if blueprint is not None and seems_fishy: - info.append( - " The template was looked up from an endpoint that belongs" - f" to the blueprint {blueprint!r}." - ) - info.append(" Maybe you did not place a template in the right folder?") - info.append(" See https://flask.palletsprojects.com/blueprints/#templates") - - app.logger.info("\n".join(info)) diff --git a/bundle/python-cpu/Lib/site-packages/flask/globals.py b/bundle/python-cpu/Lib/site-packages/flask/globals.py deleted file mode 100644 index e2c410cc5b6bf7e5facc4e604fffd2c620adfd58..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/globals.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -import typing as t -from contextvars import ContextVar - -from werkzeug.local import LocalProxy - -if t.TYPE_CHECKING: # pragma: no cover - from .app import Flask - from .ctx import _AppCtxGlobals - from .ctx import AppContext - from .ctx import RequestContext - from .sessions import SessionMixin - from .wrappers import Request - - -_no_app_msg = """\ -Working outside of application context. - -This typically means that you attempted to use functionality that needed -the current application. To solve this, set up an application context -with app.app_context(). See the documentation for more information.\ -""" -_cv_app: ContextVar[AppContext] = ContextVar("flask.app_ctx") -app_ctx: AppContext = LocalProxy( # type: ignore[assignment] - _cv_app, unbound_message=_no_app_msg -) -current_app: Flask = LocalProxy( # type: ignore[assignment] - _cv_app, "app", unbound_message=_no_app_msg -) -g: _AppCtxGlobals = LocalProxy( # type: ignore[assignment] - _cv_app, "g", unbound_message=_no_app_msg -) - -_no_req_msg = """\ -Working outside of request context. - -This typically means that you attempted to use functionality that needed -an active HTTP request. Consult the documentation on testing for -information about how to avoid this problem.\ -""" -_cv_request: ContextVar[RequestContext] = ContextVar("flask.request_ctx") -request_ctx: RequestContext = LocalProxy( # type: ignore[assignment] - _cv_request, unbound_message=_no_req_msg -) -request: Request = LocalProxy( # type: ignore[assignment] - _cv_request, "request", unbound_message=_no_req_msg -) -session: SessionMixin = LocalProxy( # type: ignore[assignment] - _cv_request, "session", unbound_message=_no_req_msg -) diff --git a/bundle/python-cpu/Lib/site-packages/flask/helpers.py b/bundle/python-cpu/Lib/site-packages/flask/helpers.py deleted file mode 100644 index 5d412c90f2bde5ab32adcf6126908330bb2880d6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/helpers.py +++ /dev/null @@ -1,641 +0,0 @@ -from __future__ import annotations - -import importlib.util -import os -import sys -import typing as t -from datetime import datetime -from functools import cache -from functools import update_wrapper - -import werkzeug.utils -from werkzeug.exceptions import abort as _wz_abort -from werkzeug.utils import redirect as _wz_redirect -from werkzeug.wrappers import Response as BaseResponse - -from .globals import _cv_app -from .globals import _cv_request -from .globals import current_app -from .globals import request -from .globals import request_ctx -from .globals import session -from .signals import message_flashed - -if t.TYPE_CHECKING: # pragma: no cover - from .wrappers import Response - - -def get_debug_flag() -> bool: - """Get whether debug mode should be enabled for the app, indicated by the - :envvar:`FLASK_DEBUG` environment variable. The default is ``False``. - """ - val = os.environ.get("FLASK_DEBUG") - return bool(val and val.lower() not in {"0", "false", "no"}) - - -def get_load_dotenv(default: bool = True) -> bool: - """Get whether the user has disabled loading default dotenv files by - setting :envvar:`FLASK_SKIP_DOTENV`. The default is ``True``, load - the files. - - :param default: What to return if the env var isn't set. - """ - val = os.environ.get("FLASK_SKIP_DOTENV") - - if not val: - return default - - return val.lower() in ("0", "false", "no") - - -@t.overload -def stream_with_context( - generator_or_function: t.Iterator[t.AnyStr], -) -> t.Iterator[t.AnyStr]: ... - - -@t.overload -def stream_with_context( - generator_or_function: t.Callable[..., t.Iterator[t.AnyStr]], -) -> t.Callable[[t.Iterator[t.AnyStr]], t.Iterator[t.AnyStr]]: ... - - -def stream_with_context( - generator_or_function: t.Iterator[t.AnyStr] | t.Callable[..., t.Iterator[t.AnyStr]], -) -> t.Iterator[t.AnyStr] | t.Callable[[t.Iterator[t.AnyStr]], t.Iterator[t.AnyStr]]: - """Wrap a response generator function so that it runs inside the current - request context. This keeps :data:`request`, :data:`session`, and :data:`g` - available, even though at the point the generator runs the request context - will typically have ended. - - Use it as a decorator on a generator function: - - .. code-block:: python - - from flask import stream_with_context, request, Response - - @app.get("/stream") - def streamed_response(): - @stream_with_context - def generate(): - yield "Hello " - yield request.args["name"] - yield "!" - - return Response(generate()) - - Or use it as a wrapper around a created generator: - - .. code-block:: python - - from flask import stream_with_context, request, Response - - @app.get("/stream") - def streamed_response(): - def generate(): - yield "Hello " - yield request.args["name"] - yield "!" - - return Response(stream_with_context(generate())) - - .. versionadded:: 0.9 - """ - try: - gen = iter(generator_or_function) # type: ignore[arg-type] - except TypeError: - - def decorator(*args: t.Any, **kwargs: t.Any) -> t.Any: - gen = generator_or_function(*args, **kwargs) # type: ignore[operator] - return stream_with_context(gen) - - return update_wrapper(decorator, generator_or_function) # type: ignore[arg-type] - - def generator() -> t.Iterator[t.AnyStr]: - if (req_ctx := _cv_request.get(None)) is None: - raise RuntimeError( - "'stream_with_context' can only be used when a request" - " context is active, such as in a view function." - ) - - app_ctx = _cv_app.get() - # Setup code below will run the generator to this point, so that the - # current contexts are recorded. The contexts must be pushed after, - # otherwise their ContextVar will record the wrong event loop during - # async view functions. - yield None # type: ignore[misc] - - # Push the app context first, so that the request context does not - # automatically create and push a different app context. - with app_ctx, req_ctx: - try: - yield from gen - finally: - # Clean up in case the user wrapped a WSGI iterator. - if hasattr(gen, "close"): - gen.close() - - # Execute the generator to the sentinel value. This ensures the context is - # preserved in the generator's state. Further iteration will push the - # context and yield from the original iterator. - wrapped_g = generator() - next(wrapped_g) - return wrapped_g - - -def make_response(*args: t.Any) -> Response: - """Sometimes it is necessary to set additional headers in a view. Because - views do not have to return response objects but can return a value that - is converted into a response object by Flask itself, it becomes tricky to - add headers to it. This function can be called instead of using a return - and you will get a response object which you can use to attach headers. - - If view looked like this and you want to add a new header:: - - def index(): - return render_template('index.html', foo=42) - - You can now do something like this:: - - def index(): - response = make_response(render_template('index.html', foo=42)) - response.headers['X-Parachutes'] = 'parachutes are cool' - return response - - This function accepts the very same arguments you can return from a - view function. This for example creates a response with a 404 error - code:: - - response = make_response(render_template('not_found.html'), 404) - - The other use case of this function is to force the return value of a - view function into a response which is helpful with view - decorators:: - - response = make_response(view_function()) - response.headers['X-Parachutes'] = 'parachutes are cool' - - Internally this function does the following things: - - - if no arguments are passed, it creates a new response argument - - if one argument is passed, :meth:`flask.Flask.make_response` - is invoked with it. - - if more than one argument is passed, the arguments are passed - to the :meth:`flask.Flask.make_response` function as tuple. - - .. versionadded:: 0.6 - """ - if not args: - return current_app.response_class() - if len(args) == 1: - args = args[0] - return current_app.make_response(args) - - -def url_for( - endpoint: str, - *, - _anchor: str | None = None, - _method: str | None = None, - _scheme: str | None = None, - _external: bool | None = None, - **values: t.Any, -) -> str: - """Generate a URL to the given endpoint with the given values. - - This requires an active request or application context, and calls - :meth:`current_app.url_for() `. See that method - for full documentation. - - :param endpoint: The endpoint name associated with the URL to - generate. If this starts with a ``.``, the current blueprint - name (if any) will be used. - :param _anchor: If given, append this as ``#anchor`` to the URL. - :param _method: If given, generate the URL associated with this - method for the endpoint. - :param _scheme: If given, the URL will have this scheme if it is - external. - :param _external: If given, prefer the URL to be internal (False) or - require it to be external (True). External URLs include the - scheme and domain. When not in an active request, URLs are - external by default. - :param values: Values to use for the variable parts of the URL rule. - Unknown keys are appended as query string arguments, like - ``?a=b&c=d``. - - .. versionchanged:: 2.2 - Calls ``current_app.url_for``, allowing an app to override the - behavior. - - .. versionchanged:: 0.10 - The ``_scheme`` parameter was added. - - .. versionchanged:: 0.9 - The ``_anchor`` and ``_method`` parameters were added. - - .. versionchanged:: 0.9 - Calls ``app.handle_url_build_error`` on build errors. - """ - return current_app.url_for( - endpoint, - _anchor=_anchor, - _method=_method, - _scheme=_scheme, - _external=_external, - **values, - ) - - -def redirect( - location: str, code: int = 302, Response: type[BaseResponse] | None = None -) -> BaseResponse: - """Create a redirect response object. - - If :data:`~flask.current_app` is available, it will use its - :meth:`~flask.Flask.redirect` method, otherwise it will use - :func:`werkzeug.utils.redirect`. - - :param location: The URL to redirect to. - :param code: The status code for the redirect. - :param Response: The response class to use. Not used when - ``current_app`` is active, which uses ``app.response_class``. - - .. versionadded:: 2.2 - Calls ``current_app.redirect`` if available instead of always - using Werkzeug's default ``redirect``. - """ - if current_app: - return current_app.redirect(location, code=code) - - return _wz_redirect(location, code=code, Response=Response) - - -def abort(code: int | BaseResponse, *args: t.Any, **kwargs: t.Any) -> t.NoReturn: - """Raise an :exc:`~werkzeug.exceptions.HTTPException` for the given - status code. - - If :data:`~flask.current_app` is available, it will call its - :attr:`~flask.Flask.aborter` object, otherwise it will use - :func:`werkzeug.exceptions.abort`. - - :param code: The status code for the exception, which must be - registered in ``app.aborter``. - :param args: Passed to the exception. - :param kwargs: Passed to the exception. - - .. versionadded:: 2.2 - Calls ``current_app.aborter`` if available instead of always - using Werkzeug's default ``abort``. - """ - if current_app: - current_app.aborter(code, *args, **kwargs) - - _wz_abort(code, *args, **kwargs) - - -def get_template_attribute(template_name: str, attribute: str) -> t.Any: - """Loads a macro (or variable) a template exports. This can be used to - invoke a macro from within Python code. If you for example have a - template named :file:`_cider.html` with the following contents: - - .. sourcecode:: html+jinja - - {% macro hello(name) %}Hello {{ name }}!{% endmacro %} - - You can access this from Python code like this:: - - hello = get_template_attribute('_cider.html', 'hello') - return hello('World') - - .. versionadded:: 0.2 - - :param template_name: the name of the template - :param attribute: the name of the variable of macro to access - """ - return getattr(current_app.jinja_env.get_template(template_name).module, attribute) - - -def flash(message: str, category: str = "message") -> None: - """Flashes a message to the next request. In order to remove the - flashed message from the session and to display it to the user, - the template has to call :func:`get_flashed_messages`. - - .. versionchanged:: 0.3 - `category` parameter added. - - :param message: the message to be flashed. - :param category: the category for the message. The following values - are recommended: ``'message'`` for any kind of message, - ``'error'`` for errors, ``'info'`` for information - messages and ``'warning'`` for warnings. However any - kind of string can be used as category. - """ - # Original implementation: - # - # session.setdefault('_flashes', []).append((category, message)) - # - # This assumed that changes made to mutable structures in the session are - # always in sync with the session object, which is not true for session - # implementations that use external storage for keeping their keys/values. - flashes = session.get("_flashes", []) - flashes.append((category, message)) - session["_flashes"] = flashes - app = current_app._get_current_object() # type: ignore - message_flashed.send( - app, - _async_wrapper=app.ensure_sync, - message=message, - category=category, - ) - - -def get_flashed_messages( - with_categories: bool = False, category_filter: t.Iterable[str] = () -) -> list[str] | list[tuple[str, str]]: - """Pulls all flashed messages from the session and returns them. - Further calls in the same request to the function will return - the same messages. By default just the messages are returned, - but when `with_categories` is set to ``True``, the return value will - be a list of tuples in the form ``(category, message)`` instead. - - Filter the flashed messages to one or more categories by providing those - categories in `category_filter`. This allows rendering categories in - separate html blocks. The `with_categories` and `category_filter` - arguments are distinct: - - * `with_categories` controls whether categories are returned with message - text (``True`` gives a tuple, where ``False`` gives just the message text). - * `category_filter` filters the messages down to only those matching the - provided categories. - - See :doc:`/patterns/flashing` for examples. - - .. versionchanged:: 0.3 - `with_categories` parameter added. - - .. versionchanged:: 0.9 - `category_filter` parameter added. - - :param with_categories: set to ``True`` to also receive categories. - :param category_filter: filter of categories to limit return values. Only - categories in the list will be returned. - """ - flashes = request_ctx.flashes - if flashes is None: - flashes = session.pop("_flashes") if "_flashes" in session else [] - request_ctx.flashes = flashes - if category_filter: - flashes = list(filter(lambda f: f[0] in category_filter, flashes)) - if not with_categories: - return [x[1] for x in flashes] - return flashes - - -def _prepare_send_file_kwargs(**kwargs: t.Any) -> dict[str, t.Any]: - if kwargs.get("max_age") is None: - kwargs["max_age"] = current_app.get_send_file_max_age - - kwargs.update( - environ=request.environ, - use_x_sendfile=current_app.config["USE_X_SENDFILE"], - response_class=current_app.response_class, - _root_path=current_app.root_path, - ) - return kwargs - - -def send_file( - path_or_file: os.PathLike[t.AnyStr] | str | t.IO[bytes], - mimetype: str | None = None, - as_attachment: bool = False, - download_name: str | None = None, - conditional: bool = True, - etag: bool | str = True, - last_modified: datetime | int | float | None = None, - max_age: None | (int | t.Callable[[str | None], int | None]) = None, -) -> Response: - """Send the contents of a file to the client. - - The first argument can be a file path or a file-like object. Paths - are preferred in most cases because Werkzeug can manage the file and - get extra information from the path. Passing a file-like object - requires that the file is opened in binary mode, and is mostly - useful when building a file in memory with :class:`io.BytesIO`. - - Never pass file paths provided by a user. The path is assumed to be - trusted, so a user could craft a path to access a file you didn't - intend. Use :func:`send_from_directory` to safely serve - user-requested paths from within a directory. - - If the WSGI server sets a ``file_wrapper`` in ``environ``, it is - used, otherwise Werkzeug's built-in wrapper is used. Alternatively, - if the HTTP server supports ``X-Sendfile``, configuring Flask with - ``USE_X_SENDFILE = True`` will tell the server to send the given - path, which is much more efficient than reading it in Python. - - :param path_or_file: The path to the file to send, relative to the - current working directory if a relative path is given. - Alternatively, a file-like object opened in binary mode. Make - sure the file pointer is seeked to the start of the data. - :param mimetype: The MIME type to send for the file. If not - provided, it will try to detect it from the file name. - :param as_attachment: Indicate to a browser that it should offer to - save the file instead of displaying it. - :param download_name: The default name browsers will use when saving - the file. Defaults to the passed file name. - :param conditional: Enable conditional and range responses based on - request headers. Requires passing a file path and ``environ``. - :param etag: Calculate an ETag for the file, which requires passing - a file path. Can also be a string to use instead. - :param last_modified: The last modified time to send for the file, - in seconds. If not provided, it will try to detect it from the - file path. - :param max_age: How long the client should cache the file, in - seconds. If set, ``Cache-Control`` will be ``public``, otherwise - it will be ``no-cache`` to prefer conditional caching. - - .. versionchanged:: 2.0 - ``download_name`` replaces the ``attachment_filename`` - parameter. If ``as_attachment=False``, it is passed with - ``Content-Disposition: inline`` instead. - - .. versionchanged:: 2.0 - ``max_age`` replaces the ``cache_timeout`` parameter. - ``conditional`` is enabled and ``max_age`` is not set by - default. - - .. versionchanged:: 2.0 - ``etag`` replaces the ``add_etags`` parameter. It can be a - string to use instead of generating one. - - .. versionchanged:: 2.0 - Passing a file-like object that inherits from - :class:`~io.TextIOBase` will raise a :exc:`ValueError` rather - than sending an empty file. - - .. versionadded:: 2.0 - Moved the implementation to Werkzeug. This is now a wrapper to - pass some Flask-specific arguments. - - .. versionchanged:: 1.1 - ``filename`` may be a :class:`~os.PathLike` object. - - .. versionchanged:: 1.1 - Passing a :class:`~io.BytesIO` object supports range requests. - - .. versionchanged:: 1.0.3 - Filenames are encoded with ASCII instead of Latin-1 for broader - compatibility with WSGI servers. - - .. versionchanged:: 1.0 - UTF-8 filenames as specified in :rfc:`2231` are supported. - - .. versionchanged:: 0.12 - The filename is no longer automatically inferred from file - objects. If you want to use automatic MIME and etag support, - pass a filename via ``filename_or_fp`` or - ``attachment_filename``. - - .. versionchanged:: 0.12 - ``attachment_filename`` is preferred over ``filename`` for MIME - detection. - - .. versionchanged:: 0.9 - ``cache_timeout`` defaults to - :meth:`Flask.get_send_file_max_age`. - - .. versionchanged:: 0.7 - MIME guessing and etag support for file-like objects was - removed because it was unreliable. Pass a filename if you are - able to, otherwise attach an etag yourself. - - .. versionchanged:: 0.5 - The ``add_etags``, ``cache_timeout`` and ``conditional`` - parameters were added. The default behavior is to add etags. - - .. versionadded:: 0.2 - """ - return werkzeug.utils.send_file( # type: ignore[return-value] - **_prepare_send_file_kwargs( - path_or_file=path_or_file, - environ=request.environ, - mimetype=mimetype, - as_attachment=as_attachment, - download_name=download_name, - conditional=conditional, - etag=etag, - last_modified=last_modified, - max_age=max_age, - ) - ) - - -def send_from_directory( - directory: os.PathLike[str] | str, - path: os.PathLike[str] | str, - **kwargs: t.Any, -) -> Response: - """Send a file from within a directory using :func:`send_file`. - - .. code-block:: python - - @app.route("/uploads/") - def download_file(name): - return send_from_directory( - app.config['UPLOAD_FOLDER'], name, as_attachment=True - ) - - This is a secure way to serve files from a folder, such as static - files or uploads. Uses :func:`~werkzeug.security.safe_join` to - ensure the path coming from the client is not maliciously crafted to - point outside the specified directory. - - If the final path does not point to an existing regular file, - raises a 404 :exc:`~werkzeug.exceptions.NotFound` error. - - :param directory: The directory that ``path`` must be located under, - relative to the current application's root path. This *must not* - be a value provided by the client, otherwise it becomes insecure. - :param path: The path to the file to send, relative to - ``directory``. - :param kwargs: Arguments to pass to :func:`send_file`. - - .. versionchanged:: 2.0 - ``path`` replaces the ``filename`` parameter. - - .. versionadded:: 2.0 - Moved the implementation to Werkzeug. This is now a wrapper to - pass some Flask-specific arguments. - - .. versionadded:: 0.5 - """ - return werkzeug.utils.send_from_directory( # type: ignore[return-value] - directory, path, **_prepare_send_file_kwargs(**kwargs) - ) - - -def get_root_path(import_name: str) -> str: - """Find the root path of a package, or the path that contains a - module. If it cannot be found, returns the current working - directory. - - Not to be confused with the value returned by :func:`find_package`. - - :meta private: - """ - # Module already imported and has a file attribute. Use that first. - mod = sys.modules.get(import_name) - - if mod is not None and hasattr(mod, "__file__") and mod.__file__ is not None: - return os.path.dirname(os.path.abspath(mod.__file__)) - - # Next attempt: check the loader. - try: - spec = importlib.util.find_spec(import_name) - - if spec is None: - raise ValueError - except (ImportError, ValueError): - loader = None - else: - loader = spec.loader - - # Loader does not exist or we're referring to an unloaded main - # module or a main module without path (interactive sessions), go - # with the current working directory. - if loader is None: - return os.getcwd() - - if hasattr(loader, "get_filename"): - filepath = loader.get_filename(import_name) # pyright: ignore - else: - # Fall back to imports. - __import__(import_name) - mod = sys.modules[import_name] - filepath = getattr(mod, "__file__", None) - - # If we don't have a file path it might be because it is a - # namespace package. In this case pick the root path from the - # first module that is contained in the package. - if filepath is None: - raise RuntimeError( - "No root path can be found for the provided module" - f" {import_name!r}. This can happen because the module" - " came from an import hook that does not provide file" - " name information or because it's a namespace package." - " In this case the root path needs to be explicitly" - " provided." - ) - - # filepath is import_name.py for a module, or __init__.py for a package. - return os.path.dirname(os.path.abspath(filepath)) # type: ignore[no-any-return] - - -@cache -def _split_blueprint_path(name: str) -> list[str]: - out: list[str] = [name] - - if "." in name: - out.extend(_split_blueprint_path(name.rpartition(".")[0])) - - return out diff --git a/bundle/python-cpu/Lib/site-packages/flask/json/__init__.py b/bundle/python-cpu/Lib/site-packages/flask/json/__init__.py deleted file mode 100644 index c0941d049e7268345acde34667019550dadba0b8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/json/__init__.py +++ /dev/null @@ -1,170 +0,0 @@ -from __future__ import annotations - -import json as _json -import typing as t - -from ..globals import current_app -from .provider import _default - -if t.TYPE_CHECKING: # pragma: no cover - from ..wrappers import Response - - -def dumps(obj: t.Any, **kwargs: t.Any) -> str: - """Serialize data as JSON. - - If :data:`~flask.current_app` is available, it will use its - :meth:`app.json.dumps() ` - method, otherwise it will use :func:`json.dumps`. - - :param obj: The data to serialize. - :param kwargs: Arguments passed to the ``dumps`` implementation. - - .. versionchanged:: 2.3 - The ``app`` parameter was removed. - - .. versionchanged:: 2.2 - Calls ``current_app.json.dumps``, allowing an app to override - the behavior. - - .. versionchanged:: 2.0.2 - :class:`decimal.Decimal` is supported by converting to a string. - - .. versionchanged:: 2.0 - ``encoding`` will be removed in Flask 2.1. - - .. versionchanged:: 1.0.3 - ``app`` can be passed directly, rather than requiring an app - context for configuration. - """ - if current_app: - return current_app.json.dumps(obj, **kwargs) - - kwargs.setdefault("default", _default) - return _json.dumps(obj, **kwargs) - - -def dump(obj: t.Any, fp: t.IO[str], **kwargs: t.Any) -> None: - """Serialize data as JSON and write to a file. - - If :data:`~flask.current_app` is available, it will use its - :meth:`app.json.dump() ` - method, otherwise it will use :func:`json.dump`. - - :param obj: The data to serialize. - :param fp: A file opened for writing text. Should use the UTF-8 - encoding to be valid JSON. - :param kwargs: Arguments passed to the ``dump`` implementation. - - .. versionchanged:: 2.3 - The ``app`` parameter was removed. - - .. versionchanged:: 2.2 - Calls ``current_app.json.dump``, allowing an app to override - the behavior. - - .. versionchanged:: 2.0 - Writing to a binary file, and the ``encoding`` argument, will be - removed in Flask 2.1. - """ - if current_app: - current_app.json.dump(obj, fp, **kwargs) - else: - kwargs.setdefault("default", _default) - _json.dump(obj, fp, **kwargs) - - -def loads(s: str | bytes, **kwargs: t.Any) -> t.Any: - """Deserialize data as JSON. - - If :data:`~flask.current_app` is available, it will use its - :meth:`app.json.loads() ` - method, otherwise it will use :func:`json.loads`. - - :param s: Text or UTF-8 bytes. - :param kwargs: Arguments passed to the ``loads`` implementation. - - .. versionchanged:: 2.3 - The ``app`` parameter was removed. - - .. versionchanged:: 2.2 - Calls ``current_app.json.loads``, allowing an app to override - the behavior. - - .. versionchanged:: 2.0 - ``encoding`` will be removed in Flask 2.1. The data must be a - string or UTF-8 bytes. - - .. versionchanged:: 1.0.3 - ``app`` can be passed directly, rather than requiring an app - context for configuration. - """ - if current_app: - return current_app.json.loads(s, **kwargs) - - return _json.loads(s, **kwargs) - - -def load(fp: t.IO[t.AnyStr], **kwargs: t.Any) -> t.Any: - """Deserialize data as JSON read from a file. - - If :data:`~flask.current_app` is available, it will use its - :meth:`app.json.load() ` - method, otherwise it will use :func:`json.load`. - - :param fp: A file opened for reading text or UTF-8 bytes. - :param kwargs: Arguments passed to the ``load`` implementation. - - .. versionchanged:: 2.3 - The ``app`` parameter was removed. - - .. versionchanged:: 2.2 - Calls ``current_app.json.load``, allowing an app to override - the behavior. - - .. versionchanged:: 2.2 - The ``app`` parameter will be removed in Flask 2.3. - - .. versionchanged:: 2.0 - ``encoding`` will be removed in Flask 2.1. The file must be text - mode, or binary mode with UTF-8 bytes. - """ - if current_app: - return current_app.json.load(fp, **kwargs) - - return _json.load(fp, **kwargs) - - -def jsonify(*args: t.Any, **kwargs: t.Any) -> Response: - """Serialize the given arguments as JSON, and return a - :class:`~flask.Response` object with the ``application/json`` - mimetype. A dict or list returned from a view will be converted to a - JSON response automatically without needing to call this. - - This requires an active request or application context, and calls - :meth:`app.json.response() `. - - In debug mode, the output is formatted with indentation to make it - easier to read. This may also be controlled by the provider. - - Either positional or keyword arguments can be given, not both. - If no arguments are given, ``None`` is serialized. - - :param args: A single value to serialize, or multiple values to - treat as a list to serialize. - :param kwargs: Treat as a dict to serialize. - - .. versionchanged:: 2.2 - Calls ``current_app.json.response``, allowing an app to override - the behavior. - - .. versionchanged:: 2.0.2 - :class:`decimal.Decimal` is supported by converting to a string. - - .. versionchanged:: 0.11 - Added support for serializing top-level arrays. This was a - security risk in ancient browsers. See :ref:`security-json`. - - .. versionadded:: 0.2 - """ - return current_app.json.response(*args, **kwargs) # type: ignore[return-value] diff --git a/bundle/python-cpu/Lib/site-packages/flask/json/provider.py b/bundle/python-cpu/Lib/site-packages/flask/json/provider.py deleted file mode 100644 index ea7e4753ffb65255ae48efa481b4d5362e476a01..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/json/provider.py +++ /dev/null @@ -1,215 +0,0 @@ -from __future__ import annotations - -import dataclasses -import decimal -import json -import typing as t -import uuid -import weakref -from datetime import date - -from werkzeug.http import http_date - -if t.TYPE_CHECKING: # pragma: no cover - from werkzeug.sansio.response import Response - - from ..sansio.app import App - - -class JSONProvider: - """A standard set of JSON operations for an application. Subclasses - of this can be used to customize JSON behavior or use different - JSON libraries. - - To implement a provider for a specific library, subclass this base - class and implement at least :meth:`dumps` and :meth:`loads`. All - other methods have default implementations. - - To use a different provider, either subclass ``Flask`` and set - :attr:`~flask.Flask.json_provider_class` to a provider class, or set - :attr:`app.json ` to an instance of the class. - - :param app: An application instance. This will be stored as a - :class:`weakref.proxy` on the :attr:`_app` attribute. - - .. versionadded:: 2.2 - """ - - def __init__(self, app: App) -> None: - self._app: App = weakref.proxy(app) - - def dumps(self, obj: t.Any, **kwargs: t.Any) -> str: - """Serialize data as JSON. - - :param obj: The data to serialize. - :param kwargs: May be passed to the underlying JSON library. - """ - raise NotImplementedError - - def dump(self, obj: t.Any, fp: t.IO[str], **kwargs: t.Any) -> None: - """Serialize data as JSON and write to a file. - - :param obj: The data to serialize. - :param fp: A file opened for writing text. Should use the UTF-8 - encoding to be valid JSON. - :param kwargs: May be passed to the underlying JSON library. - """ - fp.write(self.dumps(obj, **kwargs)) - - def loads(self, s: str | bytes, **kwargs: t.Any) -> t.Any: - """Deserialize data as JSON. - - :param s: Text or UTF-8 bytes. - :param kwargs: May be passed to the underlying JSON library. - """ - raise NotImplementedError - - def load(self, fp: t.IO[t.AnyStr], **kwargs: t.Any) -> t.Any: - """Deserialize data as JSON read from a file. - - :param fp: A file opened for reading text or UTF-8 bytes. - :param kwargs: May be passed to the underlying JSON library. - """ - return self.loads(fp.read(), **kwargs) - - def _prepare_response_obj( - self, args: tuple[t.Any, ...], kwargs: dict[str, t.Any] - ) -> t.Any: - if args and kwargs: - raise TypeError("app.json.response() takes either args or kwargs, not both") - - if not args and not kwargs: - return None - - if len(args) == 1: - return args[0] - - return args or kwargs - - def response(self, *args: t.Any, **kwargs: t.Any) -> Response: - """Serialize the given arguments as JSON, and return a - :class:`~flask.Response` object with the ``application/json`` - mimetype. - - The :func:`~flask.json.jsonify` function calls this method for - the current application. - - Either positional or keyword arguments can be given, not both. - If no arguments are given, ``None`` is serialized. - - :param args: A single value to serialize, or multiple values to - treat as a list to serialize. - :param kwargs: Treat as a dict to serialize. - """ - obj = self._prepare_response_obj(args, kwargs) - return self._app.response_class(self.dumps(obj), mimetype="application/json") - - -def _default(o: t.Any) -> t.Any: - if isinstance(o, date): - return http_date(o) - - if isinstance(o, (decimal.Decimal, uuid.UUID)): - return str(o) - - if dataclasses and dataclasses.is_dataclass(o): - return dataclasses.asdict(o) # type: ignore[arg-type] - - if hasattr(o, "__html__"): - return str(o.__html__()) - - raise TypeError(f"Object of type {type(o).__name__} is not JSON serializable") - - -class DefaultJSONProvider(JSONProvider): - """Provide JSON operations using Python's built-in :mod:`json` - library. Serializes the following additional data types: - - - :class:`datetime.datetime` and :class:`datetime.date` are - serialized to :rfc:`822` strings. This is the same as the HTTP - date format. - - :class:`uuid.UUID` is serialized to a string. - - :class:`dataclasses.dataclass` is passed to - :func:`dataclasses.asdict`. - - :class:`~markupsafe.Markup` (or any object with a ``__html__`` - method) will call the ``__html__`` method to get a string. - """ - - default: t.Callable[[t.Any], t.Any] = staticmethod(_default) # type: ignore[assignment] - """Apply this function to any object that :meth:`json.dumps` does - not know how to serialize. It should return a valid JSON type or - raise a ``TypeError``. - """ - - ensure_ascii = True - """Replace non-ASCII characters with escape sequences. This may be - more compatible with some clients, but can be disabled for better - performance and size. - """ - - sort_keys = True - """Sort the keys in any serialized dicts. This may be useful for - some caching situations, but can be disabled for better performance. - When enabled, keys must all be strings, they are not converted - before sorting. - """ - - compact: bool | None = None - """If ``True``, or ``None`` out of debug mode, the :meth:`response` - output will not add indentation, newlines, or spaces. If ``False``, - or ``None`` in debug mode, it will use a non-compact representation. - """ - - mimetype = "application/json" - """The mimetype set in :meth:`response`.""" - - def dumps(self, obj: t.Any, **kwargs: t.Any) -> str: - """Serialize data as JSON to a string. - - Keyword arguments are passed to :func:`json.dumps`. Sets some - parameter defaults from the :attr:`default`, - :attr:`ensure_ascii`, and :attr:`sort_keys` attributes. - - :param obj: The data to serialize. - :param kwargs: Passed to :func:`json.dumps`. - """ - kwargs.setdefault("default", self.default) - kwargs.setdefault("ensure_ascii", self.ensure_ascii) - kwargs.setdefault("sort_keys", self.sort_keys) - return json.dumps(obj, **kwargs) - - def loads(self, s: str | bytes, **kwargs: t.Any) -> t.Any: - """Deserialize data as JSON from a string or bytes. - - :param s: Text or UTF-8 bytes. - :param kwargs: Passed to :func:`json.loads`. - """ - return json.loads(s, **kwargs) - - def response(self, *args: t.Any, **kwargs: t.Any) -> Response: - """Serialize the given arguments as JSON, and return a - :class:`~flask.Response` object with it. The response mimetype - will be "application/json" and can be changed with - :attr:`mimetype`. - - If :attr:`compact` is ``False`` or debug mode is enabled, the - output will be formatted to be easier to read. - - Either positional or keyword arguments can be given, not both. - If no arguments are given, ``None`` is serialized. - - :param args: A single value to serialize, or multiple values to - treat as a list to serialize. - :param kwargs: Treat as a dict to serialize. - """ - obj = self._prepare_response_obj(args, kwargs) - dump_args: dict[str, t.Any] = {} - - if (self.compact is None and self._app.debug) or self.compact is False: - dump_args.setdefault("indent", 2) - else: - dump_args.setdefault("separators", (",", ":")) - - return self._app.response_class( - f"{self.dumps(obj, **dump_args)}\n", mimetype=self.mimetype - ) diff --git a/bundle/python-cpu/Lib/site-packages/flask/json/tag.py b/bundle/python-cpu/Lib/site-packages/flask/json/tag.py deleted file mode 100644 index 8dc3629bf9e5e4788e3ef69e528dfd72675cc0c6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/json/tag.py +++ /dev/null @@ -1,327 +0,0 @@ -""" -Tagged JSON -~~~~~~~~~~~ - -A compact representation for lossless serialization of non-standard JSON -types. :class:`~flask.sessions.SecureCookieSessionInterface` uses this -to serialize the session data, but it may be useful in other places. It -can be extended to support other types. - -.. autoclass:: TaggedJSONSerializer - :members: - -.. autoclass:: JSONTag - :members: - -Let's see an example that adds support for -:class:`~collections.OrderedDict`. Dicts don't have an order in JSON, so -to handle this we will dump the items as a list of ``[key, value]`` -pairs. Subclass :class:`JSONTag` and give it the new key ``' od'`` to -identify the type. The session serializer processes dicts first, so -insert the new tag at the front of the order since ``OrderedDict`` must -be processed before ``dict``. - -.. code-block:: python - - from flask.json.tag import JSONTag - - class TagOrderedDict(JSONTag): - __slots__ = ('serializer',) - key = ' od' - - def check(self, value): - return isinstance(value, OrderedDict) - - def to_json(self, value): - return [[k, self.serializer.tag(v)] for k, v in iteritems(value)] - - def to_python(self, value): - return OrderedDict(value) - - app.session_interface.serializer.register(TagOrderedDict, index=0) -""" - -from __future__ import annotations - -import typing as t -from base64 import b64decode -from base64 import b64encode -from datetime import datetime -from uuid import UUID - -from markupsafe import Markup -from werkzeug.http import http_date -from werkzeug.http import parse_date - -from ..json import dumps -from ..json import loads - - -class JSONTag: - """Base class for defining type tags for :class:`TaggedJSONSerializer`.""" - - __slots__ = ("serializer",) - - #: The tag to mark the serialized object with. If empty, this tag is - #: only used as an intermediate step during tagging. - key: str = "" - - def __init__(self, serializer: TaggedJSONSerializer) -> None: - """Create a tagger for the given serializer.""" - self.serializer = serializer - - def check(self, value: t.Any) -> bool: - """Check if the given value should be tagged by this tag.""" - raise NotImplementedError - - def to_json(self, value: t.Any) -> t.Any: - """Convert the Python object to an object that is a valid JSON type. - The tag will be added later.""" - raise NotImplementedError - - def to_python(self, value: t.Any) -> t.Any: - """Convert the JSON representation back to the correct type. The tag - will already be removed.""" - raise NotImplementedError - - def tag(self, value: t.Any) -> dict[str, t.Any]: - """Convert the value to a valid JSON type and add the tag structure - around it.""" - return {self.key: self.to_json(value)} - - -class TagDict(JSONTag): - """Tag for 1-item dicts whose only key matches a registered tag. - - Internally, the dict key is suffixed with `__`, and the suffix is removed - when deserializing. - """ - - __slots__ = () - key = " di" - - def check(self, value: t.Any) -> bool: - return ( - isinstance(value, dict) - and len(value) == 1 - and next(iter(value)) in self.serializer.tags - ) - - def to_json(self, value: t.Any) -> t.Any: - key = next(iter(value)) - return {f"{key}__": self.serializer.tag(value[key])} - - def to_python(self, value: t.Any) -> t.Any: - key = next(iter(value)) - return {key[:-2]: value[key]} - - -class PassDict(JSONTag): - __slots__ = () - - def check(self, value: t.Any) -> bool: - return isinstance(value, dict) - - def to_json(self, value: t.Any) -> t.Any: - # JSON objects may only have string keys, so don't bother tagging the - # key here. - return {k: self.serializer.tag(v) for k, v in value.items()} - - tag = to_json - - -class TagTuple(JSONTag): - __slots__ = () - key = " t" - - def check(self, value: t.Any) -> bool: - return isinstance(value, tuple) - - def to_json(self, value: t.Any) -> t.Any: - return [self.serializer.tag(item) for item in value] - - def to_python(self, value: t.Any) -> t.Any: - return tuple(value) - - -class PassList(JSONTag): - __slots__ = () - - def check(self, value: t.Any) -> bool: - return isinstance(value, list) - - def to_json(self, value: t.Any) -> t.Any: - return [self.serializer.tag(item) for item in value] - - tag = to_json - - -class TagBytes(JSONTag): - __slots__ = () - key = " b" - - def check(self, value: t.Any) -> bool: - return isinstance(value, bytes) - - def to_json(self, value: t.Any) -> t.Any: - return b64encode(value).decode("ascii") - - def to_python(self, value: t.Any) -> t.Any: - return b64decode(value) - - -class TagMarkup(JSONTag): - """Serialize anything matching the :class:`~markupsafe.Markup` API by - having a ``__html__`` method to the result of that method. Always - deserializes to an instance of :class:`~markupsafe.Markup`.""" - - __slots__ = () - key = " m" - - def check(self, value: t.Any) -> bool: - return callable(getattr(value, "__html__", None)) - - def to_json(self, value: t.Any) -> t.Any: - return str(value.__html__()) - - def to_python(self, value: t.Any) -> t.Any: - return Markup(value) - - -class TagUUID(JSONTag): - __slots__ = () - key = " u" - - def check(self, value: t.Any) -> bool: - return isinstance(value, UUID) - - def to_json(self, value: t.Any) -> t.Any: - return value.hex - - def to_python(self, value: t.Any) -> t.Any: - return UUID(value) - - -class TagDateTime(JSONTag): - __slots__ = () - key = " d" - - def check(self, value: t.Any) -> bool: - return isinstance(value, datetime) - - def to_json(self, value: t.Any) -> t.Any: - return http_date(value) - - def to_python(self, value: t.Any) -> t.Any: - return parse_date(value) - - -class TaggedJSONSerializer: - """Serializer that uses a tag system to compactly represent objects that - are not JSON types. Passed as the intermediate serializer to - :class:`itsdangerous.Serializer`. - - The following extra types are supported: - - * :class:`dict` - * :class:`tuple` - * :class:`bytes` - * :class:`~markupsafe.Markup` - * :class:`~uuid.UUID` - * :class:`~datetime.datetime` - """ - - __slots__ = ("tags", "order") - - #: Tag classes to bind when creating the serializer. Other tags can be - #: added later using :meth:`~register`. - default_tags = [ - TagDict, - PassDict, - TagTuple, - PassList, - TagBytes, - TagMarkup, - TagUUID, - TagDateTime, - ] - - def __init__(self) -> None: - self.tags: dict[str, JSONTag] = {} - self.order: list[JSONTag] = [] - - for cls in self.default_tags: - self.register(cls) - - def register( - self, - tag_class: type[JSONTag], - force: bool = False, - index: int | None = None, - ) -> None: - """Register a new tag with this serializer. - - :param tag_class: tag class to register. Will be instantiated with this - serializer instance. - :param force: overwrite an existing tag. If false (default), a - :exc:`KeyError` is raised. - :param index: index to insert the new tag in the tag order. Useful when - the new tag is a special case of an existing tag. If ``None`` - (default), the tag is appended to the end of the order. - - :raise KeyError: if the tag key is already registered and ``force`` is - not true. - """ - tag = tag_class(self) - key = tag.key - - if key: - if not force and key in self.tags: - raise KeyError(f"Tag '{key}' is already registered.") - - self.tags[key] = tag - - if index is None: - self.order.append(tag) - else: - self.order.insert(index, tag) - - def tag(self, value: t.Any) -> t.Any: - """Convert a value to a tagged representation if necessary.""" - for tag in self.order: - if tag.check(value): - return tag.tag(value) - - return value - - def untag(self, value: dict[str, t.Any]) -> t.Any: - """Convert a tagged representation back to the original type.""" - if len(value) != 1: - return value - - key = next(iter(value)) - - if key not in self.tags: - return value - - return self.tags[key].to_python(value[key]) - - def _untag_scan(self, value: t.Any) -> t.Any: - if isinstance(value, dict): - # untag each item recursively - value = {k: self._untag_scan(v) for k, v in value.items()} - # untag the dict itself - value = self.untag(value) - elif isinstance(value, list): - # untag each item recursively - value = [self._untag_scan(item) for item in value] - - return value - - def dumps(self, value: t.Any) -> str: - """Tag the value and dump it to a compact JSON string.""" - return dumps(self.tag(value), separators=(",", ":")) - - def loads(self, value: str) -> t.Any: - """Load data from a JSON string and deserialized any tagged objects.""" - return self._untag_scan(loads(value)) diff --git a/bundle/python-cpu/Lib/site-packages/flask/logging.py b/bundle/python-cpu/Lib/site-packages/flask/logging.py deleted file mode 100644 index 0cb8f43746c04b95b277d6dcb7af6d1930f09b62..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/logging.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -import logging -import sys -import typing as t - -from werkzeug.local import LocalProxy - -from .globals import request - -if t.TYPE_CHECKING: # pragma: no cover - from .sansio.app import App - - -@LocalProxy -def wsgi_errors_stream() -> t.TextIO: - """Find the most appropriate error stream for the application. If a request - is active, log to ``wsgi.errors``, otherwise use ``sys.stderr``. - - If you configure your own :class:`logging.StreamHandler`, you may want to - use this for the stream. If you are using file or dict configuration and - can't import this directly, you can refer to it as - ``ext://flask.logging.wsgi_errors_stream``. - """ - if request: - return request.environ["wsgi.errors"] # type: ignore[no-any-return] - - return sys.stderr - - -def has_level_handler(logger: logging.Logger) -> bool: - """Check if there is a handler in the logging chain that will handle the - given logger's :meth:`effective level <~logging.Logger.getEffectiveLevel>`. - """ - level = logger.getEffectiveLevel() - current = logger - - while current: - if any(handler.level <= level for handler in current.handlers): - return True - - if not current.propagate: - break - - current = current.parent # type: ignore - - return False - - -#: Log messages to :func:`~flask.logging.wsgi_errors_stream` with the format -#: ``[%(asctime)s] %(levelname)s in %(module)s: %(message)s``. -default_handler = logging.StreamHandler(wsgi_errors_stream) # type: ignore -default_handler.setFormatter( - logging.Formatter("[%(asctime)s] %(levelname)s in %(module)s: %(message)s") -) - - -def create_logger(app: App) -> logging.Logger: - """Get the Flask app's logger and configure it if needed. - - The logger name will be the same as - :attr:`app.import_name `. - - When :attr:`~flask.Flask.debug` is enabled, set the logger level to - :data:`logging.DEBUG` if it is not set. - - If there is no handler for the logger's effective level, add a - :class:`~logging.StreamHandler` for - :func:`~flask.logging.wsgi_errors_stream` with a basic format. - """ - logger = logging.getLogger(app.name) - - if app.debug and not logger.level: - logger.setLevel(logging.DEBUG) - - if not has_level_handler(logger): - logger.addHandler(default_handler) - - return logger diff --git a/bundle/python-cpu/Lib/site-packages/flask/py.typed b/bundle/python-cpu/Lib/site-packages/flask/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/flask/sansio/README.md b/bundle/python-cpu/Lib/site-packages/flask/sansio/README.md deleted file mode 100644 index 623ac1982366d862db0a9d950fa2f50dd334d32f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/sansio/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Sansio - -This folder contains code that can be used by alternative Flask -implementations, for example Quart. The code therefore cannot do any -IO, nor be part of a likely IO path. Finally this code cannot use the -Flask globals. diff --git a/bundle/python-cpu/Lib/site-packages/flask/sansio/app.py b/bundle/python-cpu/Lib/site-packages/flask/sansio/app.py deleted file mode 100644 index 58cb87306293c40ad3f6f4082ec7beb74e3885d1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/sansio/app.py +++ /dev/null @@ -1,964 +0,0 @@ -from __future__ import annotations - -import logging -import os -import sys -import typing as t -from datetime import timedelta -from itertools import chain - -from werkzeug.exceptions import Aborter -from werkzeug.exceptions import BadRequest -from werkzeug.exceptions import BadRequestKeyError -from werkzeug.routing import BuildError -from werkzeug.routing import Map -from werkzeug.routing import Rule -from werkzeug.sansio.response import Response -from werkzeug.utils import cached_property -from werkzeug.utils import redirect as _wz_redirect - -from .. import typing as ft -from ..config import Config -from ..config import ConfigAttribute -from ..ctx import _AppCtxGlobals -from ..helpers import _split_blueprint_path -from ..helpers import get_debug_flag -from ..json.provider import DefaultJSONProvider -from ..json.provider import JSONProvider -from ..logging import create_logger -from ..templating import DispatchingJinjaLoader -from ..templating import Environment -from .scaffold import _endpoint_from_view_func -from .scaffold import find_package -from .scaffold import Scaffold -from .scaffold import setupmethod - -if t.TYPE_CHECKING: # pragma: no cover - from werkzeug.wrappers import Response as BaseResponse - - from ..testing import FlaskClient - from ..testing import FlaskCliRunner - from .blueprints import Blueprint - -T_shell_context_processor = t.TypeVar( - "T_shell_context_processor", bound=ft.ShellContextProcessorCallable -) -T_teardown = t.TypeVar("T_teardown", bound=ft.TeardownCallable) -T_template_filter = t.TypeVar("T_template_filter", bound=ft.TemplateFilterCallable) -T_template_global = t.TypeVar("T_template_global", bound=ft.TemplateGlobalCallable) -T_template_test = t.TypeVar("T_template_test", bound=ft.TemplateTestCallable) - - -def _make_timedelta(value: timedelta | int | None) -> timedelta | None: - if value is None or isinstance(value, timedelta): - return value - - return timedelta(seconds=value) - - -class App(Scaffold): - """The flask object implements a WSGI application and acts as the central - object. It is passed the name of the module or package of the - application. Once it is created it will act as a central registry for - the view functions, the URL rules, template configuration and much more. - - The name of the package is used to resolve resources from inside the - package or the folder the module is contained in depending on if the - package parameter resolves to an actual python package (a folder with - an :file:`__init__.py` file inside) or a standard module (just a ``.py`` file). - - For more information about resource loading, see :func:`open_resource`. - - Usually you create a :class:`Flask` instance in your main module or - in the :file:`__init__.py` file of your package like this:: - - from flask import Flask - app = Flask(__name__) - - .. admonition:: About the First Parameter - - The idea of the first parameter is to give Flask an idea of what - belongs to your application. This name is used to find resources - on the filesystem, can be used by extensions to improve debugging - information and a lot more. - - So it's important what you provide there. If you are using a single - module, `__name__` is always the correct value. If you however are - using a package, it's usually recommended to hardcode the name of - your package there. - - For example if your application is defined in :file:`yourapplication/app.py` - you should create it with one of the two versions below:: - - app = Flask('yourapplication') - app = Flask(__name__.split('.')[0]) - - Why is that? The application will work even with `__name__`, thanks - to how resources are looked up. However it will make debugging more - painful. Certain extensions can make assumptions based on the - import name of your application. For example the Flask-SQLAlchemy - extension will look for the code in your application that triggered - an SQL query in debug mode. If the import name is not properly set - up, that debugging information is lost. (For example it would only - pick up SQL queries in `yourapplication.app` and not - `yourapplication.views.frontend`) - - .. versionadded:: 0.7 - The `static_url_path`, `static_folder`, and `template_folder` - parameters were added. - - .. versionadded:: 0.8 - The `instance_path` and `instance_relative_config` parameters were - added. - - .. versionadded:: 0.11 - The `root_path` parameter was added. - - .. versionadded:: 1.0 - The ``host_matching`` and ``static_host`` parameters were added. - - .. versionadded:: 1.0 - The ``subdomain_matching`` parameter was added. Subdomain - matching needs to be enabled manually now. Setting - :data:`SERVER_NAME` does not implicitly enable it. - - :param import_name: the name of the application package - :param static_url_path: can be used to specify a different path for the - static files on the web. Defaults to the name - of the `static_folder` folder. - :param static_folder: The folder with static files that is served at - ``static_url_path``. Relative to the application ``root_path`` - or an absolute path. Defaults to ``'static'``. - :param static_host: the host to use when adding the static route. - Defaults to None. Required when using ``host_matching=True`` - with a ``static_folder`` configured. - :param host_matching: set ``url_map.host_matching`` attribute. - Defaults to False. - :param subdomain_matching: consider the subdomain relative to - :data:`SERVER_NAME` when matching routes. Defaults to False. - :param template_folder: the folder that contains the templates that should - be used by the application. Defaults to - ``'templates'`` folder in the root path of the - application. - :param instance_path: An alternative instance path for the application. - By default the folder ``'instance'`` next to the - package or module is assumed to be the instance - path. - :param instance_relative_config: if set to ``True`` relative filenames - for loading the config are assumed to - be relative to the instance path instead - of the application root. - :param root_path: The path to the root of the application files. - This should only be set manually when it can't be detected - automatically, such as for namespace packages. - """ - - #: The class of the object assigned to :attr:`aborter`, created by - #: :meth:`create_aborter`. That object is called by - #: :func:`flask.abort` to raise HTTP errors, and can be - #: called directly as well. - #: - #: Defaults to :class:`werkzeug.exceptions.Aborter`. - #: - #: .. versionadded:: 2.2 - aborter_class = Aborter - - #: The class that is used for the Jinja environment. - #: - #: .. versionadded:: 0.11 - jinja_environment = Environment - - #: The class that is used for the :data:`~flask.g` instance. - #: - #: Example use cases for a custom class: - #: - #: 1. Store arbitrary attributes on flask.g. - #: 2. Add a property for lazy per-request database connectors. - #: 3. Return None instead of AttributeError on unexpected attributes. - #: 4. Raise exception if an unexpected attr is set, a "controlled" flask.g. - #: - #: In Flask 0.9 this property was called `request_globals_class` but it - #: was changed in 0.10 to :attr:`app_ctx_globals_class` because the - #: flask.g object is now application context scoped. - #: - #: .. versionadded:: 0.10 - app_ctx_globals_class = _AppCtxGlobals - - #: The class that is used for the ``config`` attribute of this app. - #: Defaults to :class:`~flask.Config`. - #: - #: Example use cases for a custom class: - #: - #: 1. Default values for certain config options. - #: 2. Access to config values through attributes in addition to keys. - #: - #: .. versionadded:: 0.11 - config_class = Config - - #: The testing flag. Set this to ``True`` to enable the test mode of - #: Flask extensions (and in the future probably also Flask itself). - #: For example this might activate test helpers that have an - #: additional runtime cost which should not be enabled by default. - #: - #: If this is enabled and PROPAGATE_EXCEPTIONS is not changed from the - #: default it's implicitly enabled. - #: - #: This attribute can also be configured from the config with the - #: ``TESTING`` configuration key. Defaults to ``False``. - testing = ConfigAttribute[bool]("TESTING") - - #: If a secret key is set, cryptographic components can use this to - #: sign cookies and other things. Set this to a complex random value - #: when you want to use the secure cookie for instance. - #: - #: This attribute can also be configured from the config with the - #: :data:`SECRET_KEY` configuration key. Defaults to ``None``. - secret_key = ConfigAttribute[t.Union[str, bytes, None]]("SECRET_KEY") - - #: A :class:`~datetime.timedelta` which is used to set the expiration - #: date of a permanent session. The default is 31 days which makes a - #: permanent session survive for roughly one month. - #: - #: This attribute can also be configured from the config with the - #: ``PERMANENT_SESSION_LIFETIME`` configuration key. Defaults to - #: ``timedelta(days=31)`` - permanent_session_lifetime = ConfigAttribute[timedelta]( - "PERMANENT_SESSION_LIFETIME", - get_converter=_make_timedelta, # type: ignore[arg-type] - ) - - json_provider_class: type[JSONProvider] = DefaultJSONProvider - """A subclass of :class:`~flask.json.provider.JSONProvider`. An - instance is created and assigned to :attr:`app.json` when creating - the app. - - The default, :class:`~flask.json.provider.DefaultJSONProvider`, uses - Python's built-in :mod:`json` library. A different provider can use - a different JSON library. - - .. versionadded:: 2.2 - """ - - #: Options that are passed to the Jinja environment in - #: :meth:`create_jinja_environment`. Changing these options after - #: the environment is created (accessing :attr:`jinja_env`) will - #: have no effect. - #: - #: .. versionchanged:: 1.1.0 - #: This is a ``dict`` instead of an ``ImmutableDict`` to allow - #: easier configuration. - #: - jinja_options: dict[str, t.Any] = {} - - #: The rule object to use for URL rules created. This is used by - #: :meth:`add_url_rule`. Defaults to :class:`werkzeug.routing.Rule`. - #: - #: .. versionadded:: 0.7 - url_rule_class = Rule - - #: The map object to use for storing the URL rules and routing - #: configuration parameters. Defaults to :class:`werkzeug.routing.Map`. - #: - #: .. versionadded:: 1.1.0 - url_map_class = Map - - #: The :meth:`test_client` method creates an instance of this test - #: client class. Defaults to :class:`~flask.testing.FlaskClient`. - #: - #: .. versionadded:: 0.7 - test_client_class: type[FlaskClient] | None = None - - #: The :class:`~click.testing.CliRunner` subclass, by default - #: :class:`~flask.testing.FlaskCliRunner` that is used by - #: :meth:`test_cli_runner`. Its ``__init__`` method should take a - #: Flask app object as the first argument. - #: - #: .. versionadded:: 1.0 - test_cli_runner_class: type[FlaskCliRunner] | None = None - - default_config: dict[str, t.Any] - response_class: type[Response] - - def __init__( - self, - import_name: str, - static_url_path: str | None = None, - static_folder: str | os.PathLike[str] | None = "static", - static_host: str | None = None, - host_matching: bool = False, - subdomain_matching: bool = False, - template_folder: str | os.PathLike[str] | None = "templates", - instance_path: str | None = None, - instance_relative_config: bool = False, - root_path: str | None = None, - ) -> None: - super().__init__( - import_name=import_name, - static_folder=static_folder, - static_url_path=static_url_path, - template_folder=template_folder, - root_path=root_path, - ) - - if instance_path is None: - instance_path = self.auto_find_instance_path() - elif not os.path.isabs(instance_path): - raise ValueError( - "If an instance path is provided it must be absolute." - " A relative path was given instead." - ) - - #: Holds the path to the instance folder. - #: - #: .. versionadded:: 0.8 - self.instance_path = instance_path - - #: The configuration dictionary as :class:`Config`. This behaves - #: exactly like a regular dictionary but supports additional methods - #: to load a config from files. - self.config = self.make_config(instance_relative_config) - - #: An instance of :attr:`aborter_class` created by - #: :meth:`make_aborter`. This is called by :func:`flask.abort` - #: to raise HTTP errors, and can be called directly as well. - #: - #: .. versionadded:: 2.2 - #: Moved from ``flask.abort``, which calls this object. - self.aborter = self.make_aborter() - - self.json: JSONProvider = self.json_provider_class(self) - """Provides access to JSON methods. Functions in ``flask.json`` - will call methods on this provider when the application context - is active. Used for handling JSON requests and responses. - - An instance of :attr:`json_provider_class`. Can be customized by - changing that attribute on a subclass, or by assigning to this - attribute afterwards. - - The default, :class:`~flask.json.provider.DefaultJSONProvider`, - uses Python's built-in :mod:`json` library. A different provider - can use a different JSON library. - - .. versionadded:: 2.2 - """ - - #: A list of functions that are called by - #: :meth:`handle_url_build_error` when :meth:`.url_for` raises a - #: :exc:`~werkzeug.routing.BuildError`. Each function is called - #: with ``error``, ``endpoint`` and ``values``. If a function - #: returns ``None`` or raises a ``BuildError``, it is skipped. - #: Otherwise, its return value is returned by ``url_for``. - #: - #: .. versionadded:: 0.9 - self.url_build_error_handlers: list[ - t.Callable[[Exception, str, dict[str, t.Any]], str] - ] = [] - - #: A list of functions that are called when the application context - #: is destroyed. Since the application context is also torn down - #: if the request ends this is the place to store code that disconnects - #: from databases. - #: - #: .. versionadded:: 0.9 - self.teardown_appcontext_funcs: list[ft.TeardownCallable] = [] - - #: A list of shell context processor functions that should be run - #: when a shell context is created. - #: - #: .. versionadded:: 0.11 - self.shell_context_processors: list[ft.ShellContextProcessorCallable] = [] - - #: Maps registered blueprint names to blueprint objects. The - #: dict retains the order the blueprints were registered in. - #: Blueprints can be registered multiple times, this dict does - #: not track how often they were attached. - #: - #: .. versionadded:: 0.7 - self.blueprints: dict[str, Blueprint] = {} - - #: a place where extensions can store application specific state. For - #: example this is where an extension could store database engines and - #: similar things. - #: - #: The key must match the name of the extension module. For example in - #: case of a "Flask-Foo" extension in `flask_foo`, the key would be - #: ``'foo'``. - #: - #: .. versionadded:: 0.7 - self.extensions: dict[str, t.Any] = {} - - #: The :class:`~werkzeug.routing.Map` for this instance. You can use - #: this to change the routing converters after the class was created - #: but before any routes are connected. Example:: - #: - #: from werkzeug.routing import BaseConverter - #: - #: class ListConverter(BaseConverter): - #: def to_python(self, value): - #: return value.split(',') - #: def to_url(self, values): - #: return ','.join(super(ListConverter, self).to_url(value) - #: for value in values) - #: - #: app = Flask(__name__) - #: app.url_map.converters['list'] = ListConverter - self.url_map = self.url_map_class(host_matching=host_matching) - - self.subdomain_matching = subdomain_matching - - # tracks internally if the application already handled at least one - # request. - self._got_first_request = False - - def _check_setup_finished(self, f_name: str) -> None: - if self._got_first_request: - raise AssertionError( - f"The setup method '{f_name}' can no longer be called" - " on the application. It has already handled its first" - " request, any changes will not be applied" - " consistently.\n" - "Make sure all imports, decorators, functions, etc." - " needed to set up the application are done before" - " running it." - ) - - @cached_property - def name(self) -> str: - """The name of the application. This is usually the import name - with the difference that it's guessed from the run file if the - import name is main. This name is used as a display name when - Flask needs the name of the application. It can be set and overridden - to change the value. - - .. versionadded:: 0.8 - """ - if self.import_name == "__main__": - fn: str | None = getattr(sys.modules["__main__"], "__file__", None) - if fn is None: - return "__main__" - return os.path.splitext(os.path.basename(fn))[0] - return self.import_name - - @cached_property - def logger(self) -> logging.Logger: - """A standard Python :class:`~logging.Logger` for the app, with - the same name as :attr:`name`. - - In debug mode, the logger's :attr:`~logging.Logger.level` will - be set to :data:`~logging.DEBUG`. - - If there are no handlers configured, a default handler will be - added. See :doc:`/logging` for more information. - - .. versionchanged:: 1.1.0 - The logger takes the same name as :attr:`name` rather than - hard-coding ``"flask.app"``. - - .. versionchanged:: 1.0.0 - Behavior was simplified. The logger is always named - ``"flask.app"``. The level is only set during configuration, - it doesn't check ``app.debug`` each time. Only one format is - used, not different ones depending on ``app.debug``. No - handlers are removed, and a handler is only added if no - handlers are already configured. - - .. versionadded:: 0.3 - """ - return create_logger(self) - - @cached_property - def jinja_env(self) -> Environment: - """The Jinja environment used to load templates. - - The environment is created the first time this property is - accessed. Changing :attr:`jinja_options` after that will have no - effect. - """ - return self.create_jinja_environment() - - def create_jinja_environment(self) -> Environment: - raise NotImplementedError() - - def make_config(self, instance_relative: bool = False) -> Config: - """Used to create the config attribute by the Flask constructor. - The `instance_relative` parameter is passed in from the constructor - of Flask (there named `instance_relative_config`) and indicates if - the config should be relative to the instance path or the root path - of the application. - - .. versionadded:: 0.8 - """ - root_path = self.root_path - if instance_relative: - root_path = self.instance_path - defaults = dict(self.default_config) - defaults["DEBUG"] = get_debug_flag() - return self.config_class(root_path, defaults) - - def make_aborter(self) -> Aborter: - """Create the object to assign to :attr:`aborter`. That object - is called by :func:`flask.abort` to raise HTTP errors, and can - be called directly as well. - - By default, this creates an instance of :attr:`aborter_class`, - which defaults to :class:`werkzeug.exceptions.Aborter`. - - .. versionadded:: 2.2 - """ - return self.aborter_class() - - def auto_find_instance_path(self) -> str: - """Tries to locate the instance path if it was not provided to the - constructor of the application class. It will basically calculate - the path to a folder named ``instance`` next to your main file or - the package. - - .. versionadded:: 0.8 - """ - prefix, package_path = find_package(self.import_name) - if prefix is None: - return os.path.join(package_path, "instance") - return os.path.join(prefix, "var", f"{self.name}-instance") - - def create_global_jinja_loader(self) -> DispatchingJinjaLoader: - """Creates the loader for the Jinja environment. Can be used to - override just the loader and keeping the rest unchanged. It's - discouraged to override this function. Instead one should override - the :meth:`jinja_loader` function instead. - - The global loader dispatches between the loaders of the application - and the individual blueprints. - - .. versionadded:: 0.7 - """ - return DispatchingJinjaLoader(self) - - def select_jinja_autoescape(self, filename: str | None) -> bool: - """Returns ``True`` if autoescaping should be active for the given - template name. If no template name is given, returns `True`. - - .. versionchanged:: 2.2 - Autoescaping is now enabled by default for ``.svg`` files. - - .. versionadded:: 0.5 - """ - if filename is None: - return True - return filename.endswith((".html", ".htm", ".xml", ".xhtml", ".svg")) - - @property - def debug(self) -> bool: - """Whether debug mode is enabled. When using ``flask run`` to start the - development server, an interactive debugger will be shown for unhandled - exceptions, and the server will be reloaded when code changes. This maps to the - :data:`DEBUG` config key. It may not behave as expected if set late. - - **Do not enable debug mode when deploying in production.** - - Default: ``False`` - """ - return self.config["DEBUG"] # type: ignore[no-any-return] - - @debug.setter - def debug(self, value: bool) -> None: - self.config["DEBUG"] = value - - if self.config["TEMPLATES_AUTO_RELOAD"] is None: - self.jinja_env.auto_reload = value - - @setupmethod - def register_blueprint(self, blueprint: Blueprint, **options: t.Any) -> None: - """Register a :class:`~flask.Blueprint` on the application. Keyword - arguments passed to this method will override the defaults set on the - blueprint. - - Calls the blueprint's :meth:`~flask.Blueprint.register` method after - recording the blueprint in the application's :attr:`blueprints`. - - :param blueprint: The blueprint to register. - :param url_prefix: Blueprint routes will be prefixed with this. - :param subdomain: Blueprint routes will match on this subdomain. - :param url_defaults: Blueprint routes will use these default values for - view arguments. - :param options: Additional keyword arguments are passed to - :class:`~flask.blueprints.BlueprintSetupState`. They can be - accessed in :meth:`~flask.Blueprint.record` callbacks. - - .. versionchanged:: 2.0.1 - The ``name`` option can be used to change the (pre-dotted) - name the blueprint is registered with. This allows the same - blueprint to be registered multiple times with unique names - for ``url_for``. - - .. versionadded:: 0.7 - """ - blueprint.register(self, options) - - def iter_blueprints(self) -> t.ValuesView[Blueprint]: - """Iterates over all blueprints by the order they were registered. - - .. versionadded:: 0.11 - """ - return self.blueprints.values() - - @setupmethod - def add_url_rule( - self, - rule: str, - endpoint: str | None = None, - view_func: ft.RouteCallable | None = None, - provide_automatic_options: bool | None = None, - **options: t.Any, - ) -> None: - if endpoint is None: - endpoint = _endpoint_from_view_func(view_func) # type: ignore - options["endpoint"] = endpoint - methods = options.pop("methods", None) - - # if the methods are not given and the view_func object knows its - # methods we can use that instead. If neither exists, we go with - # a tuple of only ``GET`` as default. - if methods is None: - methods = getattr(view_func, "methods", None) or ("GET",) - if isinstance(methods, str): - raise TypeError( - "Allowed methods must be a list of strings, for" - ' example: @app.route(..., methods=["POST"])' - ) - methods = {item.upper() for item in methods} - - # Methods that should always be added - required_methods: set[str] = set(getattr(view_func, "required_methods", ())) - - # starting with Flask 0.8 the view_func object can disable and - # force-enable the automatic options handling. - if provide_automatic_options is None: - provide_automatic_options = getattr( - view_func, "provide_automatic_options", None - ) - - if provide_automatic_options is None: - if "OPTIONS" not in methods and self.config["PROVIDE_AUTOMATIC_OPTIONS"]: - provide_automatic_options = True - required_methods.add("OPTIONS") - else: - provide_automatic_options = False - - # Add the required methods now. - methods |= required_methods - - rule_obj = self.url_rule_class(rule, methods=methods, **options) - rule_obj.provide_automatic_options = provide_automatic_options # type: ignore[attr-defined] - - self.url_map.add(rule_obj) - if view_func is not None: - old_func = self.view_functions.get(endpoint) - if old_func is not None and old_func != view_func: - raise AssertionError( - "View function mapping is overwriting an existing" - f" endpoint function: {endpoint}" - ) - self.view_functions[endpoint] = view_func - - @setupmethod - def template_filter( - self, name: str | None = None - ) -> t.Callable[[T_template_filter], T_template_filter]: - """A decorator that is used to register custom template filter. - You can specify a name for the filter, otherwise the function - name will be used. Example:: - - @app.template_filter() - def reverse(s): - return s[::-1] - - :param name: the optional name of the filter, otherwise the - function name will be used. - """ - - def decorator(f: T_template_filter) -> T_template_filter: - self.add_template_filter(f, name=name) - return f - - return decorator - - @setupmethod - def add_template_filter( - self, f: ft.TemplateFilterCallable, name: str | None = None - ) -> None: - """Register a custom template filter. Works exactly like the - :meth:`template_filter` decorator. - - :param name: the optional name of the filter, otherwise the - function name will be used. - """ - self.jinja_env.filters[name or f.__name__] = f - - @setupmethod - def template_test( - self, name: str | None = None - ) -> t.Callable[[T_template_test], T_template_test]: - """A decorator that is used to register custom template test. - You can specify a name for the test, otherwise the function - name will be used. Example:: - - @app.template_test() - def is_prime(n): - if n == 2: - return True - for i in range(2, int(math.ceil(math.sqrt(n))) + 1): - if n % i == 0: - return False - return True - - .. versionadded:: 0.10 - - :param name: the optional name of the test, otherwise the - function name will be used. - """ - - def decorator(f: T_template_test) -> T_template_test: - self.add_template_test(f, name=name) - return f - - return decorator - - @setupmethod - def add_template_test( - self, f: ft.TemplateTestCallable, name: str | None = None - ) -> None: - """Register a custom template test. Works exactly like the - :meth:`template_test` decorator. - - .. versionadded:: 0.10 - - :param name: the optional name of the test, otherwise the - function name will be used. - """ - self.jinja_env.tests[name or f.__name__] = f - - @setupmethod - def template_global( - self, name: str | None = None - ) -> t.Callable[[T_template_global], T_template_global]: - """A decorator that is used to register a custom template global function. - You can specify a name for the global function, otherwise the function - name will be used. Example:: - - @app.template_global() - def double(n): - return 2 * n - - .. versionadded:: 0.10 - - :param name: the optional name of the global function, otherwise the - function name will be used. - """ - - def decorator(f: T_template_global) -> T_template_global: - self.add_template_global(f, name=name) - return f - - return decorator - - @setupmethod - def add_template_global( - self, f: ft.TemplateGlobalCallable, name: str | None = None - ) -> None: - """Register a custom template global function. Works exactly like the - :meth:`template_global` decorator. - - .. versionadded:: 0.10 - - :param name: the optional name of the global function, otherwise the - function name will be used. - """ - self.jinja_env.globals[name or f.__name__] = f - - @setupmethod - def teardown_appcontext(self, f: T_teardown) -> T_teardown: - """Registers a function to be called when the application - context is popped. The application context is typically popped - after the request context for each request, at the end of CLI - commands, or after a manually pushed context ends. - - .. code-block:: python - - with app.app_context(): - ... - - When the ``with`` block exits (or ``ctx.pop()`` is called), the - teardown functions are called just before the app context is - made inactive. Since a request context typically also manages an - application context it would also be called when you pop a - request context. - - When a teardown function was called because of an unhandled - exception it will be passed an error object. If an - :meth:`errorhandler` is registered, it will handle the exception - and the teardown will not receive it. - - Teardown functions must avoid raising exceptions. If they - execute code that might fail they must surround that code with a - ``try``/``except`` block and log any errors. - - The return values of teardown functions are ignored. - - .. versionadded:: 0.9 - """ - self.teardown_appcontext_funcs.append(f) - return f - - @setupmethod - def shell_context_processor( - self, f: T_shell_context_processor - ) -> T_shell_context_processor: - """Registers a shell context processor function. - - .. versionadded:: 0.11 - """ - self.shell_context_processors.append(f) - return f - - def _find_error_handler( - self, e: Exception, blueprints: list[str] - ) -> ft.ErrorHandlerCallable | None: - """Return a registered error handler for an exception in this order: - blueprint handler for a specific code, app handler for a specific code, - blueprint handler for an exception class, app handler for an exception - class, or ``None`` if a suitable handler is not found. - """ - exc_class, code = self._get_exc_class_and_code(type(e)) - names = (*blueprints, None) - - for c in (code, None) if code is not None else (None,): - for name in names: - handler_map = self.error_handler_spec[name][c] - - if not handler_map: - continue - - for cls in exc_class.__mro__: - handler = handler_map.get(cls) - - if handler is not None: - return handler - return None - - def trap_http_exception(self, e: Exception) -> bool: - """Checks if an HTTP exception should be trapped or not. By default - this will return ``False`` for all exceptions except for a bad request - key error if ``TRAP_BAD_REQUEST_ERRORS`` is set to ``True``. It - also returns ``True`` if ``TRAP_HTTP_EXCEPTIONS`` is set to ``True``. - - This is called for all HTTP exceptions raised by a view function. - If it returns ``True`` for any exception the error handler for this - exception is not called and it shows up as regular exception in the - traceback. This is helpful for debugging implicitly raised HTTP - exceptions. - - .. versionchanged:: 1.0 - Bad request errors are not trapped by default in debug mode. - - .. versionadded:: 0.8 - """ - if self.config["TRAP_HTTP_EXCEPTIONS"]: - return True - - trap_bad_request = self.config["TRAP_BAD_REQUEST_ERRORS"] - - # if unset, trap key errors in debug mode - if ( - trap_bad_request is None - and self.debug - and isinstance(e, BadRequestKeyError) - ): - return True - - if trap_bad_request: - return isinstance(e, BadRequest) - - return False - - def should_ignore_error(self, error: BaseException | None) -> bool: - """This is called to figure out if an error should be ignored - or not as far as the teardown system is concerned. If this - function returns ``True`` then the teardown handlers will not be - passed the error. - - .. versionadded:: 0.10 - """ - return False - - def redirect(self, location: str, code: int = 302) -> BaseResponse: - """Create a redirect response object. - - This is called by :func:`flask.redirect`, and can be called - directly as well. - - :param location: The URL to redirect to. - :param code: The status code for the redirect. - - .. versionadded:: 2.2 - Moved from ``flask.redirect``, which calls this method. - """ - return _wz_redirect( - location, - code=code, - Response=self.response_class, # type: ignore[arg-type] - ) - - def inject_url_defaults(self, endpoint: str, values: dict[str, t.Any]) -> None: - """Injects the URL defaults for the given endpoint directly into - the values dictionary passed. This is used internally and - automatically called on URL building. - - .. versionadded:: 0.7 - """ - names: t.Iterable[str | None] = (None,) - - # url_for may be called outside a request context, parse the - # passed endpoint instead of using request.blueprints. - if "." in endpoint: - names = chain( - names, reversed(_split_blueprint_path(endpoint.rpartition(".")[0])) - ) - - for name in names: - if name in self.url_default_functions: - for func in self.url_default_functions[name]: - func(endpoint, values) - - def handle_url_build_error( - self, error: BuildError, endpoint: str, values: dict[str, t.Any] - ) -> str: - """Called by :meth:`.url_for` if a - :exc:`~werkzeug.routing.BuildError` was raised. If this returns - a value, it will be returned by ``url_for``, otherwise the error - will be re-raised. - - Each function in :attr:`url_build_error_handlers` is called with - ``error``, ``endpoint`` and ``values``. If a function returns - ``None`` or raises a ``BuildError``, it is skipped. Otherwise, - its return value is returned by ``url_for``. - - :param error: The active ``BuildError`` being handled. - :param endpoint: The endpoint being built. - :param values: The keyword arguments passed to ``url_for``. - """ - for handler in self.url_build_error_handlers: - try: - rv = handler(error, endpoint, values) - except BuildError as e: - # make error available outside except block - error = e - else: - if rv is not None: - return rv - - # Re-raise if called with an active exception, otherwise raise - # the passed in exception. - if error is sys.exc_info()[1]: - raise - - raise error diff --git a/bundle/python-cpu/Lib/site-packages/flask/sansio/blueprints.py b/bundle/python-cpu/Lib/site-packages/flask/sansio/blueprints.py deleted file mode 100644 index 4f912cca0565669ac6360982716b3bc19c72a5ca..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/sansio/blueprints.py +++ /dev/null @@ -1,632 +0,0 @@ -from __future__ import annotations - -import os -import typing as t -from collections import defaultdict -from functools import update_wrapper - -from .. import typing as ft -from .scaffold import _endpoint_from_view_func -from .scaffold import _sentinel -from .scaffold import Scaffold -from .scaffold import setupmethod - -if t.TYPE_CHECKING: # pragma: no cover - from .app import App - -DeferredSetupFunction = t.Callable[["BlueprintSetupState"], None] -T_after_request = t.TypeVar("T_after_request", bound=ft.AfterRequestCallable[t.Any]) -T_before_request = t.TypeVar("T_before_request", bound=ft.BeforeRequestCallable) -T_error_handler = t.TypeVar("T_error_handler", bound=ft.ErrorHandlerCallable) -T_teardown = t.TypeVar("T_teardown", bound=ft.TeardownCallable) -T_template_context_processor = t.TypeVar( - "T_template_context_processor", bound=ft.TemplateContextProcessorCallable -) -T_template_filter = t.TypeVar("T_template_filter", bound=ft.TemplateFilterCallable) -T_template_global = t.TypeVar("T_template_global", bound=ft.TemplateGlobalCallable) -T_template_test = t.TypeVar("T_template_test", bound=ft.TemplateTestCallable) -T_url_defaults = t.TypeVar("T_url_defaults", bound=ft.URLDefaultCallable) -T_url_value_preprocessor = t.TypeVar( - "T_url_value_preprocessor", bound=ft.URLValuePreprocessorCallable -) - - -class BlueprintSetupState: - """Temporary holder object for registering a blueprint with the - application. An instance of this class is created by the - :meth:`~flask.Blueprint.make_setup_state` method and later passed - to all register callback functions. - """ - - def __init__( - self, - blueprint: Blueprint, - app: App, - options: t.Any, - first_registration: bool, - ) -> None: - #: a reference to the current application - self.app = app - - #: a reference to the blueprint that created this setup state. - self.blueprint = blueprint - - #: a dictionary with all options that were passed to the - #: :meth:`~flask.Flask.register_blueprint` method. - self.options = options - - #: as blueprints can be registered multiple times with the - #: application and not everything wants to be registered - #: multiple times on it, this attribute can be used to figure - #: out if the blueprint was registered in the past already. - self.first_registration = first_registration - - subdomain = self.options.get("subdomain") - if subdomain is None: - subdomain = self.blueprint.subdomain - - #: The subdomain that the blueprint should be active for, ``None`` - #: otherwise. - self.subdomain = subdomain - - url_prefix = self.options.get("url_prefix") - if url_prefix is None: - url_prefix = self.blueprint.url_prefix - #: The prefix that should be used for all URLs defined on the - #: blueprint. - self.url_prefix = url_prefix - - self.name = self.options.get("name", blueprint.name) - self.name_prefix = self.options.get("name_prefix", "") - - #: A dictionary with URL defaults that is added to each and every - #: URL that was defined with the blueprint. - self.url_defaults = dict(self.blueprint.url_values_defaults) - self.url_defaults.update(self.options.get("url_defaults", ())) - - def add_url_rule( - self, - rule: str, - endpoint: str | None = None, - view_func: ft.RouteCallable | None = None, - **options: t.Any, - ) -> None: - """A helper method to register a rule (and optionally a view function) - to the application. The endpoint is automatically prefixed with the - blueprint's name. - """ - if self.url_prefix is not None: - if rule: - rule = "/".join((self.url_prefix.rstrip("/"), rule.lstrip("/"))) - else: - rule = self.url_prefix - options.setdefault("subdomain", self.subdomain) - if endpoint is None: - endpoint = _endpoint_from_view_func(view_func) # type: ignore - defaults = self.url_defaults - if "defaults" in options: - defaults = dict(defaults, **options.pop("defaults")) - - self.app.add_url_rule( - rule, - f"{self.name_prefix}.{self.name}.{endpoint}".lstrip("."), - view_func, - defaults=defaults, - **options, - ) - - -class Blueprint(Scaffold): - """Represents a blueprint, a collection of routes and other - app-related functions that can be registered on a real application - later. - - A blueprint is an object that allows defining application functions - without requiring an application object ahead of time. It uses the - same decorators as :class:`~flask.Flask`, but defers the need for an - application by recording them for later registration. - - Decorating a function with a blueprint creates a deferred function - that is called with :class:`~flask.blueprints.BlueprintSetupState` - when the blueprint is registered on an application. - - See :doc:`/blueprints` for more information. - - :param name: The name of the blueprint. Will be prepended to each - endpoint name. - :param import_name: The name of the blueprint package, usually - ``__name__``. This helps locate the ``root_path`` for the - blueprint. - :param static_folder: A folder with static files that should be - served by the blueprint's static route. The path is relative to - the blueprint's root path. Blueprint static files are disabled - by default. - :param static_url_path: The url to serve static files from. - Defaults to ``static_folder``. If the blueprint does not have - a ``url_prefix``, the app's static route will take precedence, - and the blueprint's static files won't be accessible. - :param template_folder: A folder with templates that should be added - to the app's template search path. The path is relative to the - blueprint's root path. Blueprint templates are disabled by - default. Blueprint templates have a lower precedence than those - in the app's templates folder. - :param url_prefix: A path to prepend to all of the blueprint's URLs, - to make them distinct from the rest of the app's routes. - :param subdomain: A subdomain that blueprint routes will match on by - default. - :param url_defaults: A dict of default values that blueprint routes - will receive by default. - :param root_path: By default, the blueprint will automatically set - this based on ``import_name``. In certain situations this - automatic detection can fail, so the path can be specified - manually instead. - - .. versionchanged:: 1.1.0 - Blueprints have a ``cli`` group to register nested CLI commands. - The ``cli_group`` parameter controls the name of the group under - the ``flask`` command. - - .. versionadded:: 0.7 - """ - - _got_registered_once = False - - def __init__( - self, - name: str, - import_name: str, - static_folder: str | os.PathLike[str] | None = None, - static_url_path: str | None = None, - template_folder: str | os.PathLike[str] | None = None, - url_prefix: str | None = None, - subdomain: str | None = None, - url_defaults: dict[str, t.Any] | None = None, - root_path: str | None = None, - cli_group: str | None = _sentinel, # type: ignore[assignment] - ): - super().__init__( - import_name=import_name, - static_folder=static_folder, - static_url_path=static_url_path, - template_folder=template_folder, - root_path=root_path, - ) - - if not name: - raise ValueError("'name' may not be empty.") - - if "." in name: - raise ValueError("'name' may not contain a dot '.' character.") - - self.name = name - self.url_prefix = url_prefix - self.subdomain = subdomain - self.deferred_functions: list[DeferredSetupFunction] = [] - - if url_defaults is None: - url_defaults = {} - - self.url_values_defaults = url_defaults - self.cli_group = cli_group - self._blueprints: list[tuple[Blueprint, dict[str, t.Any]]] = [] - - def _check_setup_finished(self, f_name: str) -> None: - if self._got_registered_once: - raise AssertionError( - f"The setup method '{f_name}' can no longer be called on the blueprint" - f" '{self.name}'. It has already been registered at least once, any" - " changes will not be applied consistently.\n" - "Make sure all imports, decorators, functions, etc. needed to set up" - " the blueprint are done before registering it." - ) - - @setupmethod - def record(self, func: DeferredSetupFunction) -> None: - """Registers a function that is called when the blueprint is - registered on the application. This function is called with the - state as argument as returned by the :meth:`make_setup_state` - method. - """ - self.deferred_functions.append(func) - - @setupmethod - def record_once(self, func: DeferredSetupFunction) -> None: - """Works like :meth:`record` but wraps the function in another - function that will ensure the function is only called once. If the - blueprint is registered a second time on the application, the - function passed is not called. - """ - - def wrapper(state: BlueprintSetupState) -> None: - if state.first_registration: - func(state) - - self.record(update_wrapper(wrapper, func)) - - def make_setup_state( - self, app: App, options: dict[str, t.Any], first_registration: bool = False - ) -> BlueprintSetupState: - """Creates an instance of :meth:`~flask.blueprints.BlueprintSetupState` - object that is later passed to the register callback functions. - Subclasses can override this to return a subclass of the setup state. - """ - return BlueprintSetupState(self, app, options, first_registration) - - @setupmethod - def register_blueprint(self, blueprint: Blueprint, **options: t.Any) -> None: - """Register a :class:`~flask.Blueprint` on this blueprint. Keyword - arguments passed to this method will override the defaults set - on the blueprint. - - .. versionchanged:: 2.0.1 - The ``name`` option can be used to change the (pre-dotted) - name the blueprint is registered with. This allows the same - blueprint to be registered multiple times with unique names - for ``url_for``. - - .. versionadded:: 2.0 - """ - if blueprint is self: - raise ValueError("Cannot register a blueprint on itself") - self._blueprints.append((blueprint, options)) - - def register(self, app: App, options: dict[str, t.Any]) -> None: - """Called by :meth:`Flask.register_blueprint` to register all - views and callbacks registered on the blueprint with the - application. Creates a :class:`.BlueprintSetupState` and calls - each :meth:`record` callback with it. - - :param app: The application this blueprint is being registered - with. - :param options: Keyword arguments forwarded from - :meth:`~Flask.register_blueprint`. - - .. versionchanged:: 2.3 - Nested blueprints now correctly apply subdomains. - - .. versionchanged:: 2.1 - Registering the same blueprint with the same name multiple - times is an error. - - .. versionchanged:: 2.0.1 - Nested blueprints are registered with their dotted name. - This allows different blueprints with the same name to be - nested at different locations. - - .. versionchanged:: 2.0.1 - The ``name`` option can be used to change the (pre-dotted) - name the blueprint is registered with. This allows the same - blueprint to be registered multiple times with unique names - for ``url_for``. - """ - name_prefix = options.get("name_prefix", "") - self_name = options.get("name", self.name) - name = f"{name_prefix}.{self_name}".lstrip(".") - - if name in app.blueprints: - bp_desc = "this" if app.blueprints[name] is self else "a different" - existing_at = f" '{name}'" if self_name != name else "" - - raise ValueError( - f"The name '{self_name}' is already registered for" - f" {bp_desc} blueprint{existing_at}. Use 'name=' to" - f" provide a unique name." - ) - - first_bp_registration = not any(bp is self for bp in app.blueprints.values()) - first_name_registration = name not in app.blueprints - - app.blueprints[name] = self - self._got_registered_once = True - state = self.make_setup_state(app, options, first_bp_registration) - - if self.has_static_folder: - state.add_url_rule( - f"{self.static_url_path}/", - view_func=self.send_static_file, # type: ignore[attr-defined] - endpoint="static", - ) - - # Merge blueprint data into parent. - if first_bp_registration or first_name_registration: - self._merge_blueprint_funcs(app, name) - - for deferred in self.deferred_functions: - deferred(state) - - cli_resolved_group = options.get("cli_group", self.cli_group) - - if self.cli.commands: - if cli_resolved_group is None: - app.cli.commands.update(self.cli.commands) - elif cli_resolved_group is _sentinel: - self.cli.name = name - app.cli.add_command(self.cli) - else: - self.cli.name = cli_resolved_group - app.cli.add_command(self.cli) - - for blueprint, bp_options in self._blueprints: - bp_options = bp_options.copy() - bp_url_prefix = bp_options.get("url_prefix") - bp_subdomain = bp_options.get("subdomain") - - if bp_subdomain is None: - bp_subdomain = blueprint.subdomain - - if state.subdomain is not None and bp_subdomain is not None: - bp_options["subdomain"] = bp_subdomain + "." + state.subdomain - elif bp_subdomain is not None: - bp_options["subdomain"] = bp_subdomain - elif state.subdomain is not None: - bp_options["subdomain"] = state.subdomain - - if bp_url_prefix is None: - bp_url_prefix = blueprint.url_prefix - - if state.url_prefix is not None and bp_url_prefix is not None: - bp_options["url_prefix"] = ( - state.url_prefix.rstrip("/") + "/" + bp_url_prefix.lstrip("/") - ) - elif bp_url_prefix is not None: - bp_options["url_prefix"] = bp_url_prefix - elif state.url_prefix is not None: - bp_options["url_prefix"] = state.url_prefix - - bp_options["name_prefix"] = name - blueprint.register(app, bp_options) - - def _merge_blueprint_funcs(self, app: App, name: str) -> None: - def extend( - bp_dict: dict[ft.AppOrBlueprintKey, list[t.Any]], - parent_dict: dict[ft.AppOrBlueprintKey, list[t.Any]], - ) -> None: - for key, values in bp_dict.items(): - key = name if key is None else f"{name}.{key}" - parent_dict[key].extend(values) - - for key, value in self.error_handler_spec.items(): - key = name if key is None else f"{name}.{key}" - value = defaultdict( - dict, - { - code: {exc_class: func for exc_class, func in code_values.items()} - for code, code_values in value.items() - }, - ) - app.error_handler_spec[key] = value - - for endpoint, func in self.view_functions.items(): - app.view_functions[endpoint] = func - - extend(self.before_request_funcs, app.before_request_funcs) - extend(self.after_request_funcs, app.after_request_funcs) - extend( - self.teardown_request_funcs, - app.teardown_request_funcs, - ) - extend(self.url_default_functions, app.url_default_functions) - extend(self.url_value_preprocessors, app.url_value_preprocessors) - extend(self.template_context_processors, app.template_context_processors) - - @setupmethod - def add_url_rule( - self, - rule: str, - endpoint: str | None = None, - view_func: ft.RouteCallable | None = None, - provide_automatic_options: bool | None = None, - **options: t.Any, - ) -> None: - """Register a URL rule with the blueprint. See :meth:`.Flask.add_url_rule` for - full documentation. - - The URL rule is prefixed with the blueprint's URL prefix. The endpoint name, - used with :func:`url_for`, is prefixed with the blueprint's name. - """ - if endpoint and "." in endpoint: - raise ValueError("'endpoint' may not contain a dot '.' character.") - - if view_func and hasattr(view_func, "__name__") and "." in view_func.__name__: - raise ValueError("'view_func' name may not contain a dot '.' character.") - - self.record( - lambda s: s.add_url_rule( - rule, - endpoint, - view_func, - provide_automatic_options=provide_automatic_options, - **options, - ) - ) - - @setupmethod - def app_template_filter( - self, name: str | None = None - ) -> t.Callable[[T_template_filter], T_template_filter]: - """Register a template filter, available in any template rendered by the - application. Equivalent to :meth:`.Flask.template_filter`. - - :param name: the optional name of the filter, otherwise the - function name will be used. - """ - - def decorator(f: T_template_filter) -> T_template_filter: - self.add_app_template_filter(f, name=name) - return f - - return decorator - - @setupmethod - def add_app_template_filter( - self, f: ft.TemplateFilterCallable, name: str | None = None - ) -> None: - """Register a template filter, available in any template rendered by the - application. Works like the :meth:`app_template_filter` decorator. Equivalent to - :meth:`.Flask.add_template_filter`. - - :param name: the optional name of the filter, otherwise the - function name will be used. - """ - - def register_template(state: BlueprintSetupState) -> None: - state.app.jinja_env.filters[name or f.__name__] = f - - self.record_once(register_template) - - @setupmethod - def app_template_test( - self, name: str | None = None - ) -> t.Callable[[T_template_test], T_template_test]: - """Register a template test, available in any template rendered by the - application. Equivalent to :meth:`.Flask.template_test`. - - .. versionadded:: 0.10 - - :param name: the optional name of the test, otherwise the - function name will be used. - """ - - def decorator(f: T_template_test) -> T_template_test: - self.add_app_template_test(f, name=name) - return f - - return decorator - - @setupmethod - def add_app_template_test( - self, f: ft.TemplateTestCallable, name: str | None = None - ) -> None: - """Register a template test, available in any template rendered by the - application. Works like the :meth:`app_template_test` decorator. Equivalent to - :meth:`.Flask.add_template_test`. - - .. versionadded:: 0.10 - - :param name: the optional name of the test, otherwise the - function name will be used. - """ - - def register_template(state: BlueprintSetupState) -> None: - state.app.jinja_env.tests[name or f.__name__] = f - - self.record_once(register_template) - - @setupmethod - def app_template_global( - self, name: str | None = None - ) -> t.Callable[[T_template_global], T_template_global]: - """Register a template global, available in any template rendered by the - application. Equivalent to :meth:`.Flask.template_global`. - - .. versionadded:: 0.10 - - :param name: the optional name of the global, otherwise the - function name will be used. - """ - - def decorator(f: T_template_global) -> T_template_global: - self.add_app_template_global(f, name=name) - return f - - return decorator - - @setupmethod - def add_app_template_global( - self, f: ft.TemplateGlobalCallable, name: str | None = None - ) -> None: - """Register a template global, available in any template rendered by the - application. Works like the :meth:`app_template_global` decorator. Equivalent to - :meth:`.Flask.add_template_global`. - - .. versionadded:: 0.10 - - :param name: the optional name of the global, otherwise the - function name will be used. - """ - - def register_template(state: BlueprintSetupState) -> None: - state.app.jinja_env.globals[name or f.__name__] = f - - self.record_once(register_template) - - @setupmethod - def before_app_request(self, f: T_before_request) -> T_before_request: - """Like :meth:`before_request`, but before every request, not only those handled - by the blueprint. Equivalent to :meth:`.Flask.before_request`. - """ - self.record_once( - lambda s: s.app.before_request_funcs.setdefault(None, []).append(f) - ) - return f - - @setupmethod - def after_app_request(self, f: T_after_request) -> T_after_request: - """Like :meth:`after_request`, but after every request, not only those handled - by the blueprint. Equivalent to :meth:`.Flask.after_request`. - """ - self.record_once( - lambda s: s.app.after_request_funcs.setdefault(None, []).append(f) - ) - return f - - @setupmethod - def teardown_app_request(self, f: T_teardown) -> T_teardown: - """Like :meth:`teardown_request`, but after every request, not only those - handled by the blueprint. Equivalent to :meth:`.Flask.teardown_request`. - """ - self.record_once( - lambda s: s.app.teardown_request_funcs.setdefault(None, []).append(f) - ) - return f - - @setupmethod - def app_context_processor( - self, f: T_template_context_processor - ) -> T_template_context_processor: - """Like :meth:`context_processor`, but for templates rendered by every view, not - only by the blueprint. Equivalent to :meth:`.Flask.context_processor`. - """ - self.record_once( - lambda s: s.app.template_context_processors.setdefault(None, []).append(f) - ) - return f - - @setupmethod - def app_errorhandler( - self, code: type[Exception] | int - ) -> t.Callable[[T_error_handler], T_error_handler]: - """Like :meth:`errorhandler`, but for every request, not only those handled by - the blueprint. Equivalent to :meth:`.Flask.errorhandler`. - """ - - def decorator(f: T_error_handler) -> T_error_handler: - def from_blueprint(state: BlueprintSetupState) -> None: - state.app.errorhandler(code)(f) - - self.record_once(from_blueprint) - return f - - return decorator - - @setupmethod - def app_url_value_preprocessor( - self, f: T_url_value_preprocessor - ) -> T_url_value_preprocessor: - """Like :meth:`url_value_preprocessor`, but for every request, not only those - handled by the blueprint. Equivalent to :meth:`.Flask.url_value_preprocessor`. - """ - self.record_once( - lambda s: s.app.url_value_preprocessors.setdefault(None, []).append(f) - ) - return f - - @setupmethod - def app_url_defaults(self, f: T_url_defaults) -> T_url_defaults: - """Like :meth:`url_defaults`, but for every request, not only those handled by - the blueprint. Equivalent to :meth:`.Flask.url_defaults`. - """ - self.record_once( - lambda s: s.app.url_default_functions.setdefault(None, []).append(f) - ) - return f diff --git a/bundle/python-cpu/Lib/site-packages/flask/sansio/scaffold.py b/bundle/python-cpu/Lib/site-packages/flask/sansio/scaffold.py deleted file mode 100644 index 0e96f15b74a4ca6096c5eaa9f041be909b5e01f1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/sansio/scaffold.py +++ /dev/null @@ -1,792 +0,0 @@ -from __future__ import annotations - -import importlib.util -import os -import pathlib -import sys -import typing as t -from collections import defaultdict -from functools import update_wrapper - -from jinja2 import BaseLoader -from jinja2 import FileSystemLoader -from werkzeug.exceptions import default_exceptions -from werkzeug.exceptions import HTTPException -from werkzeug.utils import cached_property - -from .. import typing as ft -from ..helpers import get_root_path -from ..templating import _default_template_ctx_processor - -if t.TYPE_CHECKING: # pragma: no cover - from click import Group - -# a singleton sentinel value for parameter defaults -_sentinel = object() - -F = t.TypeVar("F", bound=t.Callable[..., t.Any]) -T_after_request = t.TypeVar("T_after_request", bound=ft.AfterRequestCallable[t.Any]) -T_before_request = t.TypeVar("T_before_request", bound=ft.BeforeRequestCallable) -T_error_handler = t.TypeVar("T_error_handler", bound=ft.ErrorHandlerCallable) -T_teardown = t.TypeVar("T_teardown", bound=ft.TeardownCallable) -T_template_context_processor = t.TypeVar( - "T_template_context_processor", bound=ft.TemplateContextProcessorCallable -) -T_url_defaults = t.TypeVar("T_url_defaults", bound=ft.URLDefaultCallable) -T_url_value_preprocessor = t.TypeVar( - "T_url_value_preprocessor", bound=ft.URLValuePreprocessorCallable -) -T_route = t.TypeVar("T_route", bound=ft.RouteCallable) - - -def setupmethod(f: F) -> F: - f_name = f.__name__ - - def wrapper_func(self: Scaffold, *args: t.Any, **kwargs: t.Any) -> t.Any: - self._check_setup_finished(f_name) - return f(self, *args, **kwargs) - - return t.cast(F, update_wrapper(wrapper_func, f)) - - -class Scaffold: - """Common behavior shared between :class:`~flask.Flask` and - :class:`~flask.blueprints.Blueprint`. - - :param import_name: The import name of the module where this object - is defined. Usually :attr:`__name__` should be used. - :param static_folder: Path to a folder of static files to serve. - If this is set, a static route will be added. - :param static_url_path: URL prefix for the static route. - :param template_folder: Path to a folder containing template files. - for rendering. If this is set, a Jinja loader will be added. - :param root_path: The path that static, template, and resource files - are relative to. Typically not set, it is discovered based on - the ``import_name``. - - .. versionadded:: 2.0 - """ - - cli: Group - name: str - _static_folder: str | None = None - _static_url_path: str | None = None - - def __init__( - self, - import_name: str, - static_folder: str | os.PathLike[str] | None = None, - static_url_path: str | None = None, - template_folder: str | os.PathLike[str] | None = None, - root_path: str | None = None, - ): - #: The name of the package or module that this object belongs - #: to. Do not change this once it is set by the constructor. - self.import_name = import_name - - self.static_folder = static_folder - self.static_url_path = static_url_path - - #: The path to the templates folder, relative to - #: :attr:`root_path`, to add to the template loader. ``None`` if - #: templates should not be added. - self.template_folder = template_folder - - if root_path is None: - root_path = get_root_path(self.import_name) - - #: Absolute path to the package on the filesystem. Used to look - #: up resources contained in the package. - self.root_path = root_path - - #: A dictionary mapping endpoint names to view functions. - #: - #: To register a view function, use the :meth:`route` decorator. - #: - #: This data structure is internal. It should not be modified - #: directly and its format may change at any time. - self.view_functions: dict[str, ft.RouteCallable] = {} - - #: A data structure of registered error handlers, in the format - #: ``{scope: {code: {class: handler}}}``. The ``scope`` key is - #: the name of a blueprint the handlers are active for, or - #: ``None`` for all requests. The ``code`` key is the HTTP - #: status code for ``HTTPException``, or ``None`` for - #: other exceptions. The innermost dictionary maps exception - #: classes to handler functions. - #: - #: To register an error handler, use the :meth:`errorhandler` - #: decorator. - #: - #: This data structure is internal. It should not be modified - #: directly and its format may change at any time. - self.error_handler_spec: dict[ - ft.AppOrBlueprintKey, - dict[int | None, dict[type[Exception], ft.ErrorHandlerCallable]], - ] = defaultdict(lambda: defaultdict(dict)) - - #: A data structure of functions to call at the beginning of - #: each request, in the format ``{scope: [functions]}``. The - #: ``scope`` key is the name of a blueprint the functions are - #: active for, or ``None`` for all requests. - #: - #: To register a function, use the :meth:`before_request` - #: decorator. - #: - #: This data structure is internal. It should not be modified - #: directly and its format may change at any time. - self.before_request_funcs: dict[ - ft.AppOrBlueprintKey, list[ft.BeforeRequestCallable] - ] = defaultdict(list) - - #: A data structure of functions to call at the end of each - #: request, in the format ``{scope: [functions]}``. The - #: ``scope`` key is the name of a blueprint the functions are - #: active for, or ``None`` for all requests. - #: - #: To register a function, use the :meth:`after_request` - #: decorator. - #: - #: This data structure is internal. It should not be modified - #: directly and its format may change at any time. - self.after_request_funcs: dict[ - ft.AppOrBlueprintKey, list[ft.AfterRequestCallable[t.Any]] - ] = defaultdict(list) - - #: A data structure of functions to call at the end of each - #: request even if an exception is raised, in the format - #: ``{scope: [functions]}``. The ``scope`` key is the name of a - #: blueprint the functions are active for, or ``None`` for all - #: requests. - #: - #: To register a function, use the :meth:`teardown_request` - #: decorator. - #: - #: This data structure is internal. It should not be modified - #: directly and its format may change at any time. - self.teardown_request_funcs: dict[ - ft.AppOrBlueprintKey, list[ft.TeardownCallable] - ] = defaultdict(list) - - #: A data structure of functions to call to pass extra context - #: values when rendering templates, in the format - #: ``{scope: [functions]}``. The ``scope`` key is the name of a - #: blueprint the functions are active for, or ``None`` for all - #: requests. - #: - #: To register a function, use the :meth:`context_processor` - #: decorator. - #: - #: This data structure is internal. It should not be modified - #: directly and its format may change at any time. - self.template_context_processors: dict[ - ft.AppOrBlueprintKey, list[ft.TemplateContextProcessorCallable] - ] = defaultdict(list, {None: [_default_template_ctx_processor]}) - - #: A data structure of functions to call to modify the keyword - #: arguments passed to the view function, in the format - #: ``{scope: [functions]}``. The ``scope`` key is the name of a - #: blueprint the functions are active for, or ``None`` for all - #: requests. - #: - #: To register a function, use the - #: :meth:`url_value_preprocessor` decorator. - #: - #: This data structure is internal. It should not be modified - #: directly and its format may change at any time. - self.url_value_preprocessors: dict[ - ft.AppOrBlueprintKey, - list[ft.URLValuePreprocessorCallable], - ] = defaultdict(list) - - #: A data structure of functions to call to modify the keyword - #: arguments when generating URLs, in the format - #: ``{scope: [functions]}``. The ``scope`` key is the name of a - #: blueprint the functions are active for, or ``None`` for all - #: requests. - #: - #: To register a function, use the :meth:`url_defaults` - #: decorator. - #: - #: This data structure is internal. It should not be modified - #: directly and its format may change at any time. - self.url_default_functions: dict[ - ft.AppOrBlueprintKey, list[ft.URLDefaultCallable] - ] = defaultdict(list) - - def __repr__(self) -> str: - return f"<{type(self).__name__} {self.name!r}>" - - def _check_setup_finished(self, f_name: str) -> None: - raise NotImplementedError - - @property - def static_folder(self) -> str | None: - """The absolute path to the configured static folder. ``None`` - if no static folder is set. - """ - if self._static_folder is not None: - return os.path.join(self.root_path, self._static_folder) - else: - return None - - @static_folder.setter - def static_folder(self, value: str | os.PathLike[str] | None) -> None: - if value is not None: - value = os.fspath(value).rstrip(r"\/") - - self._static_folder = value - - @property - def has_static_folder(self) -> bool: - """``True`` if :attr:`static_folder` is set. - - .. versionadded:: 0.5 - """ - return self.static_folder is not None - - @property - def static_url_path(self) -> str | None: - """The URL prefix that the static route will be accessible from. - - If it was not configured during init, it is derived from - :attr:`static_folder`. - """ - if self._static_url_path is not None: - return self._static_url_path - - if self.static_folder is not None: - basename = os.path.basename(self.static_folder) - return f"/{basename}".rstrip("/") - - return None - - @static_url_path.setter - def static_url_path(self, value: str | None) -> None: - if value is not None: - value = value.rstrip("/") - - self._static_url_path = value - - @cached_property - def jinja_loader(self) -> BaseLoader | None: - """The Jinja loader for this object's templates. By default this - is a class :class:`jinja2.loaders.FileSystemLoader` to - :attr:`template_folder` if it is set. - - .. versionadded:: 0.5 - """ - if self.template_folder is not None: - return FileSystemLoader(os.path.join(self.root_path, self.template_folder)) - else: - return None - - def _method_route( - self, - method: str, - rule: str, - options: dict[str, t.Any], - ) -> t.Callable[[T_route], T_route]: - if "methods" in options: - raise TypeError("Use the 'route' decorator to use the 'methods' argument.") - - return self.route(rule, methods=[method], **options) - - @setupmethod - def get(self, rule: str, **options: t.Any) -> t.Callable[[T_route], T_route]: - """Shortcut for :meth:`route` with ``methods=["GET"]``. - - .. versionadded:: 2.0 - """ - return self._method_route("GET", rule, options) - - @setupmethod - def post(self, rule: str, **options: t.Any) -> t.Callable[[T_route], T_route]: - """Shortcut for :meth:`route` with ``methods=["POST"]``. - - .. versionadded:: 2.0 - """ - return self._method_route("POST", rule, options) - - @setupmethod - def put(self, rule: str, **options: t.Any) -> t.Callable[[T_route], T_route]: - """Shortcut for :meth:`route` with ``methods=["PUT"]``. - - .. versionadded:: 2.0 - """ - return self._method_route("PUT", rule, options) - - @setupmethod - def delete(self, rule: str, **options: t.Any) -> t.Callable[[T_route], T_route]: - """Shortcut for :meth:`route` with ``methods=["DELETE"]``. - - .. versionadded:: 2.0 - """ - return self._method_route("DELETE", rule, options) - - @setupmethod - def patch(self, rule: str, **options: t.Any) -> t.Callable[[T_route], T_route]: - """Shortcut for :meth:`route` with ``methods=["PATCH"]``. - - .. versionadded:: 2.0 - """ - return self._method_route("PATCH", rule, options) - - @setupmethod - def route(self, rule: str, **options: t.Any) -> t.Callable[[T_route], T_route]: - """Decorate a view function to register it with the given URL - rule and options. Calls :meth:`add_url_rule`, which has more - details about the implementation. - - .. code-block:: python - - @app.route("/") - def index(): - return "Hello, World!" - - See :ref:`url-route-registrations`. - - The endpoint name for the route defaults to the name of the view - function if the ``endpoint`` parameter isn't passed. - - The ``methods`` parameter defaults to ``["GET"]``. ``HEAD`` and - ``OPTIONS`` are added automatically. - - :param rule: The URL rule string. - :param options: Extra options passed to the - :class:`~werkzeug.routing.Rule` object. - """ - - def decorator(f: T_route) -> T_route: - endpoint = options.pop("endpoint", None) - self.add_url_rule(rule, endpoint, f, **options) - return f - - return decorator - - @setupmethod - def add_url_rule( - self, - rule: str, - endpoint: str | None = None, - view_func: ft.RouteCallable | None = None, - provide_automatic_options: bool | None = None, - **options: t.Any, - ) -> None: - """Register a rule for routing incoming requests and building - URLs. The :meth:`route` decorator is a shortcut to call this - with the ``view_func`` argument. These are equivalent: - - .. code-block:: python - - @app.route("/") - def index(): - ... - - .. code-block:: python - - def index(): - ... - - app.add_url_rule("/", view_func=index) - - See :ref:`url-route-registrations`. - - The endpoint name for the route defaults to the name of the view - function if the ``endpoint`` parameter isn't passed. An error - will be raised if a function has already been registered for the - endpoint. - - The ``methods`` parameter defaults to ``["GET"]``. ``HEAD`` is - always added automatically, and ``OPTIONS`` is added - automatically by default. - - ``view_func`` does not necessarily need to be passed, but if the - rule should participate in routing an endpoint name must be - associated with a view function at some point with the - :meth:`endpoint` decorator. - - .. code-block:: python - - app.add_url_rule("/", endpoint="index") - - @app.endpoint("index") - def index(): - ... - - If ``view_func`` has a ``required_methods`` attribute, those - methods are added to the passed and automatic methods. If it - has a ``provide_automatic_methods`` attribute, it is used as the - default if the parameter is not passed. - - :param rule: The URL rule string. - :param endpoint: The endpoint name to associate with the rule - and view function. Used when routing and building URLs. - Defaults to ``view_func.__name__``. - :param view_func: The view function to associate with the - endpoint name. - :param provide_automatic_options: Add the ``OPTIONS`` method and - respond to ``OPTIONS`` requests automatically. - :param options: Extra options passed to the - :class:`~werkzeug.routing.Rule` object. - """ - raise NotImplementedError - - @setupmethod - def endpoint(self, endpoint: str) -> t.Callable[[F], F]: - """Decorate a view function to register it for the given - endpoint. Used if a rule is added without a ``view_func`` with - :meth:`add_url_rule`. - - .. code-block:: python - - app.add_url_rule("/ex", endpoint="example") - - @app.endpoint("example") - def example(): - ... - - :param endpoint: The endpoint name to associate with the view - function. - """ - - def decorator(f: F) -> F: - self.view_functions[endpoint] = f - return f - - return decorator - - @setupmethod - def before_request(self, f: T_before_request) -> T_before_request: - """Register a function to run before each request. - - For example, this can be used to open a database connection, or - to load the logged in user from the session. - - .. code-block:: python - - @app.before_request - def load_user(): - if "user_id" in session: - g.user = db.session.get(session["user_id"]) - - The function will be called without any arguments. If it returns - a non-``None`` value, the value is handled as if it was the - return value from the view, and further request handling is - stopped. - - This is available on both app and blueprint objects. When used on an app, this - executes before every request. When used on a blueprint, this executes before - every request that the blueprint handles. To register with a blueprint and - execute before every request, use :meth:`.Blueprint.before_app_request`. - """ - self.before_request_funcs.setdefault(None, []).append(f) - return f - - @setupmethod - def after_request(self, f: T_after_request) -> T_after_request: - """Register a function to run after each request to this object. - - The function is called with the response object, and must return - a response object. This allows the functions to modify or - replace the response before it is sent. - - If a function raises an exception, any remaining - ``after_request`` functions will not be called. Therefore, this - should not be used for actions that must execute, such as to - close resources. Use :meth:`teardown_request` for that. - - This is available on both app and blueprint objects. When used on an app, this - executes after every request. When used on a blueprint, this executes after - every request that the blueprint handles. To register with a blueprint and - execute after every request, use :meth:`.Blueprint.after_app_request`. - """ - self.after_request_funcs.setdefault(None, []).append(f) - return f - - @setupmethod - def teardown_request(self, f: T_teardown) -> T_teardown: - """Register a function to be called when the request context is - popped. Typically this happens at the end of each request, but - contexts may be pushed manually as well during testing. - - .. code-block:: python - - with app.test_request_context(): - ... - - When the ``with`` block exits (or ``ctx.pop()`` is called), the - teardown functions are called just before the request context is - made inactive. - - When a teardown function was called because of an unhandled - exception it will be passed an error object. If an - :meth:`errorhandler` is registered, it will handle the exception - and the teardown will not receive it. - - Teardown functions must avoid raising exceptions. If they - execute code that might fail they must surround that code with a - ``try``/``except`` block and log any errors. - - The return values of teardown functions are ignored. - - This is available on both app and blueprint objects. When used on an app, this - executes after every request. When used on a blueprint, this executes after - every request that the blueprint handles. To register with a blueprint and - execute after every request, use :meth:`.Blueprint.teardown_app_request`. - """ - self.teardown_request_funcs.setdefault(None, []).append(f) - return f - - @setupmethod - def context_processor( - self, - f: T_template_context_processor, - ) -> T_template_context_processor: - """Registers a template context processor function. These functions run before - rendering a template. The keys of the returned dict are added as variables - available in the template. - - This is available on both app and blueprint objects. When used on an app, this - is called for every rendered template. When used on a blueprint, this is called - for templates rendered from the blueprint's views. To register with a blueprint - and affect every template, use :meth:`.Blueprint.app_context_processor`. - """ - self.template_context_processors[None].append(f) - return f - - @setupmethod - def url_value_preprocessor( - self, - f: T_url_value_preprocessor, - ) -> T_url_value_preprocessor: - """Register a URL value preprocessor function for all view - functions in the application. These functions will be called before the - :meth:`before_request` functions. - - The function can modify the values captured from the matched url before - they are passed to the view. For example, this can be used to pop a - common language code value and place it in ``g`` rather than pass it to - every view. - - The function is passed the endpoint name and values dict. The return - value is ignored. - - This is available on both app and blueprint objects. When used on an app, this - is called for every request. When used on a blueprint, this is called for - requests that the blueprint handles. To register with a blueprint and affect - every request, use :meth:`.Blueprint.app_url_value_preprocessor`. - """ - self.url_value_preprocessors[None].append(f) - return f - - @setupmethod - def url_defaults(self, f: T_url_defaults) -> T_url_defaults: - """Callback function for URL defaults for all view functions of the - application. It's called with the endpoint and values and should - update the values passed in place. - - This is available on both app and blueprint objects. When used on an app, this - is called for every request. When used on a blueprint, this is called for - requests that the blueprint handles. To register with a blueprint and affect - every request, use :meth:`.Blueprint.app_url_defaults`. - """ - self.url_default_functions[None].append(f) - return f - - @setupmethod - def errorhandler( - self, code_or_exception: type[Exception] | int - ) -> t.Callable[[T_error_handler], T_error_handler]: - """Register a function to handle errors by code or exception class. - - A decorator that is used to register a function given an - error code. Example:: - - @app.errorhandler(404) - def page_not_found(error): - return 'This page does not exist', 404 - - You can also register handlers for arbitrary exceptions:: - - @app.errorhandler(DatabaseError) - def special_exception_handler(error): - return 'Database connection failed', 500 - - This is available on both app and blueprint objects. When used on an app, this - can handle errors from every request. When used on a blueprint, this can handle - errors from requests that the blueprint handles. To register with a blueprint - and affect every request, use :meth:`.Blueprint.app_errorhandler`. - - .. versionadded:: 0.7 - Use :meth:`register_error_handler` instead of modifying - :attr:`error_handler_spec` directly, for application wide error - handlers. - - .. versionadded:: 0.7 - One can now additionally also register custom exception types - that do not necessarily have to be a subclass of the - :class:`~werkzeug.exceptions.HTTPException` class. - - :param code_or_exception: the code as integer for the handler, or - an arbitrary exception - """ - - def decorator(f: T_error_handler) -> T_error_handler: - self.register_error_handler(code_or_exception, f) - return f - - return decorator - - @setupmethod - def register_error_handler( - self, - code_or_exception: type[Exception] | int, - f: ft.ErrorHandlerCallable, - ) -> None: - """Alternative error attach function to the :meth:`errorhandler` - decorator that is more straightforward to use for non decorator - usage. - - .. versionadded:: 0.7 - """ - exc_class, code = self._get_exc_class_and_code(code_or_exception) - self.error_handler_spec[None][code][exc_class] = f - - @staticmethod - def _get_exc_class_and_code( - exc_class_or_code: type[Exception] | int, - ) -> tuple[type[Exception], int | None]: - """Get the exception class being handled. For HTTP status codes - or ``HTTPException`` subclasses, return both the exception and - status code. - - :param exc_class_or_code: Any exception class, or an HTTP status - code as an integer. - """ - exc_class: type[Exception] - - if isinstance(exc_class_or_code, int): - try: - exc_class = default_exceptions[exc_class_or_code] - except KeyError: - raise ValueError( - f"'{exc_class_or_code}' is not a recognized HTTP" - " error code. Use a subclass of HTTPException with" - " that code instead." - ) from None - else: - exc_class = exc_class_or_code - - if isinstance(exc_class, Exception): - raise TypeError( - f"{exc_class!r} is an instance, not a class. Handlers" - " can only be registered for Exception classes or HTTP" - " error codes." - ) - - if not issubclass(exc_class, Exception): - raise ValueError( - f"'{exc_class.__name__}' is not a subclass of Exception." - " Handlers can only be registered for Exception classes" - " or HTTP error codes." - ) - - if issubclass(exc_class, HTTPException): - return exc_class, exc_class.code - else: - return exc_class, None - - -def _endpoint_from_view_func(view_func: ft.RouteCallable) -> str: - """Internal helper that returns the default endpoint for a given - function. This always is the function name. - """ - assert view_func is not None, "expected view func if endpoint is not provided." - return view_func.__name__ - - -def _find_package_path(import_name: str) -> str: - """Find the path that contains the package or module.""" - root_mod_name, _, _ = import_name.partition(".") - - try: - root_spec = importlib.util.find_spec(root_mod_name) - - if root_spec is None: - raise ValueError("not found") - except (ImportError, ValueError): - # ImportError: the machinery told us it does not exist - # ValueError: - # - the module name was invalid - # - the module name is __main__ - # - we raised `ValueError` due to `root_spec` being `None` - return os.getcwd() - - if root_spec.submodule_search_locations: - if root_spec.origin is None or root_spec.origin == "namespace": - # namespace package - package_spec = importlib.util.find_spec(import_name) - - if package_spec is not None and package_spec.submodule_search_locations: - # Pick the path in the namespace that contains the submodule. - package_path = pathlib.Path( - os.path.commonpath(package_spec.submodule_search_locations) - ) - search_location = next( - location - for location in root_spec.submodule_search_locations - if package_path.is_relative_to(location) - ) - else: - # Pick the first path. - search_location = root_spec.submodule_search_locations[0] - - return os.path.dirname(search_location) - else: - # package with __init__.py - return os.path.dirname(os.path.dirname(root_spec.origin)) - else: - # module - return os.path.dirname(root_spec.origin) # type: ignore[type-var, return-value] - - -def find_package(import_name: str) -> tuple[str | None, str]: - """Find the prefix that a package is installed under, and the path - that it would be imported from. - - The prefix is the directory containing the standard directory - hierarchy (lib, bin, etc.). If the package is not installed to the - system (:attr:`sys.prefix`) or a virtualenv (``site-packages``), - ``None`` is returned. - - The path is the entry in :attr:`sys.path` that contains the package - for import. If the package is not installed, it's assumed that the - package was imported from the current working directory. - """ - package_path = _find_package_path(import_name) - py_prefix = os.path.abspath(sys.prefix) - - # installed to the system - if pathlib.PurePath(package_path).is_relative_to(py_prefix): - return py_prefix, package_path - - site_parent, site_folder = os.path.split(package_path) - - # installed to a virtualenv - if site_folder.lower() == "site-packages": - parent, folder = os.path.split(site_parent) - - # Windows (prefix/lib/site-packages) - if folder.lower() == "lib": - return parent, package_path - - # Unix (prefix/lib/pythonX.Y/site-packages) - if os.path.basename(parent).lower() == "lib": - return os.path.dirname(parent), package_path - - # something else (prefix/site-packages) - return site_parent, package_path - - # not installed - return None, package_path diff --git a/bundle/python-cpu/Lib/site-packages/flask/sessions.py b/bundle/python-cpu/Lib/site-packages/flask/sessions.py deleted file mode 100644 index ad357706ff59ae07ee2c8b2a850624a33e19e220..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/sessions.py +++ /dev/null @@ -1,385 +0,0 @@ -from __future__ import annotations - -import collections.abc as c -import hashlib -import typing as t -from collections.abc import MutableMapping -from datetime import datetime -from datetime import timezone - -from itsdangerous import BadSignature -from itsdangerous import URLSafeTimedSerializer -from werkzeug.datastructures import CallbackDict - -from .json.tag import TaggedJSONSerializer - -if t.TYPE_CHECKING: # pragma: no cover - import typing_extensions as te - - from .app import Flask - from .wrappers import Request - from .wrappers import Response - - -class SessionMixin(MutableMapping[str, t.Any]): - """Expands a basic dictionary with session attributes.""" - - @property - def permanent(self) -> bool: - """This reflects the ``'_permanent'`` key in the dict.""" - return self.get("_permanent", False) # type: ignore[no-any-return] - - @permanent.setter - def permanent(self, value: bool) -> None: - self["_permanent"] = bool(value) - - #: Some implementations can detect whether a session is newly - #: created, but that is not guaranteed. Use with caution. The mixin - # default is hard-coded ``False``. - new = False - - #: Some implementations can detect changes to the session and set - #: this when that happens. The mixin default is hard coded to - #: ``True``. - modified = True - - accessed = False - """Indicates if the session was accessed, even if it was not modified. This - is set when the session object is accessed through the request context, - including the global :data:`.session` proxy. A ``Vary: cookie`` header will - be added if this is ``True``. - - .. versionchanged:: 3.1.3 - This is tracked by the request context. - """ - - -class SecureCookieSession(CallbackDict[str, t.Any], SessionMixin): - """Base class for sessions based on signed cookies. - - This session backend will set the :attr:`modified` and - :attr:`accessed` attributes. It cannot reliably track whether a - session is new (vs. empty), so :attr:`new` remains hard coded to - ``False``. - """ - - #: When data is changed, this is set to ``True``. Only the session - #: dictionary itself is tracked; if the session contains mutable - #: data (for example a nested dict) then this must be set to - #: ``True`` manually when modifying that data. The session cookie - #: will only be written to the response if this is ``True``. - modified = False - - def __init__( - self, - initial: c.Mapping[str, t.Any] | None = None, - ) -> None: - def on_update(self: te.Self) -> None: - self.modified = True - - super().__init__(initial, on_update) - - -class NullSession(SecureCookieSession): - """Class used to generate nicer error messages if sessions are not - available. Will still allow read-only access to the empty session - but fail on setting. - """ - - def _fail(self, *args: t.Any, **kwargs: t.Any) -> t.NoReturn: - raise RuntimeError( - "The session is unavailable because no secret " - "key was set. Set the secret_key on the " - "application to something unique and secret." - ) - - __setitem__ = __delitem__ = clear = pop = popitem = update = setdefault = _fail - del _fail - - -class SessionInterface: - """The basic interface you have to implement in order to replace the - default session interface which uses werkzeug's securecookie - implementation. The only methods you have to implement are - :meth:`open_session` and :meth:`save_session`, the others have - useful defaults which you don't need to change. - - The session object returned by the :meth:`open_session` method has to - provide a dictionary like interface plus the properties and methods - from the :class:`SessionMixin`. We recommend just subclassing a dict - and adding that mixin:: - - class Session(dict, SessionMixin): - pass - - If :meth:`open_session` returns ``None`` Flask will call into - :meth:`make_null_session` to create a session that acts as replacement - if the session support cannot work because some requirement is not - fulfilled. The default :class:`NullSession` class that is created - will complain that the secret key was not set. - - To replace the session interface on an application all you have to do - is to assign :attr:`flask.Flask.session_interface`:: - - app = Flask(__name__) - app.session_interface = MySessionInterface() - - Multiple requests with the same session may be sent and handled - concurrently. When implementing a new session interface, consider - whether reads or writes to the backing store must be synchronized. - There is no guarantee on the order in which the session for each - request is opened or saved, it will occur in the order that requests - begin and end processing. - - .. versionadded:: 0.8 - """ - - #: :meth:`make_null_session` will look here for the class that should - #: be created when a null session is requested. Likewise the - #: :meth:`is_null_session` method will perform a typecheck against - #: this type. - null_session_class = NullSession - - #: A flag that indicates if the session interface is pickle based. - #: This can be used by Flask extensions to make a decision in regards - #: to how to deal with the session object. - #: - #: .. versionadded:: 0.10 - pickle_based = False - - def make_null_session(self, app: Flask) -> NullSession: - """Creates a null session which acts as a replacement object if the - real session support could not be loaded due to a configuration - error. This mainly aids the user experience because the job of the - null session is to still support lookup without complaining but - modifications are answered with a helpful error message of what - failed. - - This creates an instance of :attr:`null_session_class` by default. - """ - return self.null_session_class() - - def is_null_session(self, obj: object) -> bool: - """Checks if a given object is a null session. Null sessions are - not asked to be saved. - - This checks if the object is an instance of :attr:`null_session_class` - by default. - """ - return isinstance(obj, self.null_session_class) - - def get_cookie_name(self, app: Flask) -> str: - """The name of the session cookie. Uses``app.config["SESSION_COOKIE_NAME"]``.""" - return app.config["SESSION_COOKIE_NAME"] # type: ignore[no-any-return] - - def get_cookie_domain(self, app: Flask) -> str | None: - """The value of the ``Domain`` parameter on the session cookie. If not set, - browsers will only send the cookie to the exact domain it was set from. - Otherwise, they will send it to any subdomain of the given value as well. - - Uses the :data:`SESSION_COOKIE_DOMAIN` config. - - .. versionchanged:: 2.3 - Not set by default, does not fall back to ``SERVER_NAME``. - """ - return app.config["SESSION_COOKIE_DOMAIN"] # type: ignore[no-any-return] - - def get_cookie_path(self, app: Flask) -> str: - """Returns the path for which the cookie should be valid. The - default implementation uses the value from the ``SESSION_COOKIE_PATH`` - config var if it's set, and falls back to ``APPLICATION_ROOT`` or - uses ``/`` if it's ``None``. - """ - return app.config["SESSION_COOKIE_PATH"] or app.config["APPLICATION_ROOT"] # type: ignore[no-any-return] - - def get_cookie_httponly(self, app: Flask) -> bool: - """Returns True if the session cookie should be httponly. This - currently just returns the value of the ``SESSION_COOKIE_HTTPONLY`` - config var. - """ - return app.config["SESSION_COOKIE_HTTPONLY"] # type: ignore[no-any-return] - - def get_cookie_secure(self, app: Flask) -> bool: - """Returns True if the cookie should be secure. This currently - just returns the value of the ``SESSION_COOKIE_SECURE`` setting. - """ - return app.config["SESSION_COOKIE_SECURE"] # type: ignore[no-any-return] - - def get_cookie_samesite(self, app: Flask) -> str | None: - """Return ``'Strict'`` or ``'Lax'`` if the cookie should use the - ``SameSite`` attribute. This currently just returns the value of - the :data:`SESSION_COOKIE_SAMESITE` setting. - """ - return app.config["SESSION_COOKIE_SAMESITE"] # type: ignore[no-any-return] - - def get_cookie_partitioned(self, app: Flask) -> bool: - """Returns True if the cookie should be partitioned. By default, uses - the value of :data:`SESSION_COOKIE_PARTITIONED`. - - .. versionadded:: 3.1 - """ - return app.config["SESSION_COOKIE_PARTITIONED"] # type: ignore[no-any-return] - - def get_expiration_time(self, app: Flask, session: SessionMixin) -> datetime | None: - """A helper method that returns an expiration date for the session - or ``None`` if the session is linked to the browser session. The - default implementation returns now + the permanent session - lifetime configured on the application. - """ - if session.permanent: - return datetime.now(timezone.utc) + app.permanent_session_lifetime - return None - - def should_set_cookie(self, app: Flask, session: SessionMixin) -> bool: - """Used by session backends to determine if a ``Set-Cookie`` header - should be set for this session cookie for this response. If the session - has been modified, the cookie is set. If the session is permanent and - the ``SESSION_REFRESH_EACH_REQUEST`` config is true, the cookie is - always set. - - This check is usually skipped if the session was deleted. - - .. versionadded:: 0.11 - """ - - return session.modified or ( - session.permanent and app.config["SESSION_REFRESH_EACH_REQUEST"] - ) - - def open_session(self, app: Flask, request: Request) -> SessionMixin | None: - """This is called at the beginning of each request, after - pushing the request context, before matching the URL. - - This must return an object which implements a dictionary-like - interface as well as the :class:`SessionMixin` interface. - - This will return ``None`` to indicate that loading failed in - some way that is not immediately an error. The request - context will fall back to using :meth:`make_null_session` - in this case. - """ - raise NotImplementedError() - - def save_session( - self, app: Flask, session: SessionMixin, response: Response - ) -> None: - """This is called at the end of each request, after generating - a response, before removing the request context. It is skipped - if :meth:`is_null_session` returns ``True``. - """ - raise NotImplementedError() - - -session_json_serializer = TaggedJSONSerializer() - - -def _lazy_sha1(string: bytes = b"") -> t.Any: - """Don't access ``hashlib.sha1`` until runtime. FIPS builds may not include - SHA-1, in which case the import and use as a default would fail before the - developer can configure something else. - """ - return hashlib.sha1(string) - - -class SecureCookieSessionInterface(SessionInterface): - """The default session interface that stores sessions in signed cookies - through the :mod:`itsdangerous` module. - """ - - #: the salt that should be applied on top of the secret key for the - #: signing of cookie based sessions. - salt = "cookie-session" - #: the hash function to use for the signature. The default is sha1 - digest_method = staticmethod(_lazy_sha1) - #: the name of the itsdangerous supported key derivation. The default - #: is hmac. - key_derivation = "hmac" - #: A python serializer for the payload. The default is a compact - #: JSON derived serializer with support for some extra Python types - #: such as datetime objects or tuples. - serializer = session_json_serializer - session_class = SecureCookieSession - - def get_signing_serializer(self, app: Flask) -> URLSafeTimedSerializer | None: - if not app.secret_key: - return None - - keys: list[str | bytes] = [] - - if fallbacks := app.config["SECRET_KEY_FALLBACKS"]: - keys.extend(fallbacks) - - keys.append(app.secret_key) # itsdangerous expects current key at top - return URLSafeTimedSerializer( - keys, # type: ignore[arg-type] - salt=self.salt, - serializer=self.serializer, - signer_kwargs={ - "key_derivation": self.key_derivation, - "digest_method": self.digest_method, - }, - ) - - def open_session(self, app: Flask, request: Request) -> SecureCookieSession | None: - s = self.get_signing_serializer(app) - if s is None: - return None - val = request.cookies.get(self.get_cookie_name(app)) - if not val: - return self.session_class() - max_age = int(app.permanent_session_lifetime.total_seconds()) - try: - data = s.loads(val, max_age=max_age) - return self.session_class(data) - except BadSignature: - return self.session_class() - - def save_session( - self, app: Flask, session: SessionMixin, response: Response - ) -> None: - name = self.get_cookie_name(app) - domain = self.get_cookie_domain(app) - path = self.get_cookie_path(app) - secure = self.get_cookie_secure(app) - partitioned = self.get_cookie_partitioned(app) - samesite = self.get_cookie_samesite(app) - httponly = self.get_cookie_httponly(app) - - # Add a "Vary: Cookie" header if the session was accessed at all. - if session.accessed: - response.vary.add("Cookie") - - # If the session is modified to be empty, remove the cookie. - # If the session is empty, return without setting the cookie. - if not session: - if session.modified: - response.delete_cookie( - name, - domain=domain, - path=path, - secure=secure, - partitioned=partitioned, - samesite=samesite, - httponly=httponly, - ) - response.vary.add("Cookie") - - return - - if not self.should_set_cookie(app, session): - return - - expires = self.get_expiration_time(app, session) - val = self.get_signing_serializer(app).dumps(dict(session)) # type: ignore[union-attr] - response.set_cookie( - name, - val, - expires=expires, - httponly=httponly, - domain=domain, - path=path, - secure=secure, - partitioned=partitioned, - samesite=samesite, - ) - response.vary.add("Cookie") diff --git a/bundle/python-cpu/Lib/site-packages/flask/signals.py b/bundle/python-cpu/Lib/site-packages/flask/signals.py deleted file mode 100644 index 444fda9987b0e77d78afdd08d74d31b5516c8642..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/signals.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - -from blinker import Namespace - -# This namespace is only for signals provided by Flask itself. -_signals = Namespace() - -template_rendered = _signals.signal("template-rendered") -before_render_template = _signals.signal("before-render-template") -request_started = _signals.signal("request-started") -request_finished = _signals.signal("request-finished") -request_tearing_down = _signals.signal("request-tearing-down") -got_request_exception = _signals.signal("got-request-exception") -appcontext_tearing_down = _signals.signal("appcontext-tearing-down") -appcontext_pushed = _signals.signal("appcontext-pushed") -appcontext_popped = _signals.signal("appcontext-popped") -message_flashed = _signals.signal("message-flashed") diff --git a/bundle/python-cpu/Lib/site-packages/flask/templating.py b/bundle/python-cpu/Lib/site-packages/flask/templating.py deleted file mode 100644 index c5fb5b99050824efd4d09453e22b2877c599f6cf..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/templating.py +++ /dev/null @@ -1,220 +0,0 @@ -from __future__ import annotations - -import typing as t - -from jinja2 import BaseLoader -from jinja2 import Environment as BaseEnvironment -from jinja2 import Template -from jinja2 import TemplateNotFound - -from .globals import _cv_app -from .globals import _cv_request -from .globals import current_app -from .globals import request -from .helpers import stream_with_context -from .signals import before_render_template -from .signals import template_rendered - -if t.TYPE_CHECKING: # pragma: no cover - from .app import Flask - from .sansio.app import App - from .sansio.scaffold import Scaffold - - -def _default_template_ctx_processor() -> dict[str, t.Any]: - """Default template context processor. Replaces the ``request`` and ``g`` - proxies with their concrete objects for faster access. - """ - appctx = _cv_app.get(None) - reqctx = _cv_request.get(None) - rv: dict[str, t.Any] = {} - if appctx is not None: - rv["g"] = appctx.g - if reqctx is not None: - rv["request"] = reqctx.request - # The session proxy cannot be replaced, accessing it gets - # RequestContext.session, which sets session.accessed. - return rv - - -class Environment(BaseEnvironment): - """Works like a regular Jinja environment but has some additional - knowledge of how Flask's blueprint works so that it can prepend the - name of the blueprint to referenced templates if necessary. - """ - - def __init__(self, app: App, **options: t.Any) -> None: - if "loader" not in options: - options["loader"] = app.create_global_jinja_loader() - BaseEnvironment.__init__(self, **options) - self.app = app - - -class DispatchingJinjaLoader(BaseLoader): - """A loader that looks for templates in the application and all - the blueprint folders. - """ - - def __init__(self, app: App) -> None: - self.app = app - - def get_source( - self, environment: BaseEnvironment, template: str - ) -> tuple[str, str | None, t.Callable[[], bool] | None]: - if self.app.config["EXPLAIN_TEMPLATE_LOADING"]: - return self._get_source_explained(environment, template) - return self._get_source_fast(environment, template) - - def _get_source_explained( - self, environment: BaseEnvironment, template: str - ) -> tuple[str, str | None, t.Callable[[], bool] | None]: - attempts = [] - rv: tuple[str, str | None, t.Callable[[], bool] | None] | None - trv: None | (tuple[str, str | None, t.Callable[[], bool] | None]) = None - - for srcobj, loader in self._iter_loaders(template): - try: - rv = loader.get_source(environment, template) - if trv is None: - trv = rv - except TemplateNotFound: - rv = None - attempts.append((loader, srcobj, rv)) - - from .debughelpers import explain_template_loading_attempts - - explain_template_loading_attempts(self.app, template, attempts) - - if trv is not None: - return trv - raise TemplateNotFound(template) - - def _get_source_fast( - self, environment: BaseEnvironment, template: str - ) -> tuple[str, str | None, t.Callable[[], bool] | None]: - for _srcobj, loader in self._iter_loaders(template): - try: - return loader.get_source(environment, template) - except TemplateNotFound: - continue - raise TemplateNotFound(template) - - def _iter_loaders(self, template: str) -> t.Iterator[tuple[Scaffold, BaseLoader]]: - loader = self.app.jinja_loader - if loader is not None: - yield self.app, loader - - for blueprint in self.app.iter_blueprints(): - loader = blueprint.jinja_loader - if loader is not None: - yield blueprint, loader - - def list_templates(self) -> list[str]: - result = set() - loader = self.app.jinja_loader - if loader is not None: - result.update(loader.list_templates()) - - for blueprint in self.app.iter_blueprints(): - loader = blueprint.jinja_loader - if loader is not None: - for template in loader.list_templates(): - result.add(template) - - return list(result) - - -def _render(app: Flask, template: Template, context: dict[str, t.Any]) -> str: - app.update_template_context(context) - before_render_template.send( - app, _async_wrapper=app.ensure_sync, template=template, context=context - ) - rv = template.render(context) - template_rendered.send( - app, _async_wrapper=app.ensure_sync, template=template, context=context - ) - return rv - - -def render_template( - template_name_or_list: str | Template | list[str | Template], - **context: t.Any, -) -> str: - """Render a template by name with the given context. - - :param template_name_or_list: The name of the template to render. If - a list is given, the first name to exist will be rendered. - :param context: The variables to make available in the template. - """ - app = current_app._get_current_object() # type: ignore[attr-defined] - template = app.jinja_env.get_or_select_template(template_name_or_list) - return _render(app, template, context) - - -def render_template_string(source: str, **context: t.Any) -> str: - """Render a template from the given source string with the given - context. - - :param source: The source code of the template to render. - :param context: The variables to make available in the template. - """ - app = current_app._get_current_object() # type: ignore[attr-defined] - template = app.jinja_env.from_string(source) - return _render(app, template, context) - - -def _stream( - app: Flask, template: Template, context: dict[str, t.Any] -) -> t.Iterator[str]: - app.update_template_context(context) - before_render_template.send( - app, _async_wrapper=app.ensure_sync, template=template, context=context - ) - - def generate() -> t.Iterator[str]: - yield from template.generate(context) - template_rendered.send( - app, _async_wrapper=app.ensure_sync, template=template, context=context - ) - - rv = generate() - - # If a request context is active, keep it while generating. - if request: - rv = stream_with_context(rv) - - return rv - - -def stream_template( - template_name_or_list: str | Template | list[str | Template], - **context: t.Any, -) -> t.Iterator[str]: - """Render a template by name with the given context as a stream. - This returns an iterator of strings, which can be used as a - streaming response from a view. - - :param template_name_or_list: The name of the template to render. If - a list is given, the first name to exist will be rendered. - :param context: The variables to make available in the template. - - .. versionadded:: 2.2 - """ - app = current_app._get_current_object() # type: ignore[attr-defined] - template = app.jinja_env.get_or_select_template(template_name_or_list) - return _stream(app, template, context) - - -def stream_template_string(source: str, **context: t.Any) -> t.Iterator[str]: - """Render a template from the given source string with the given - context as a stream. This returns an iterator of strings, which can - be used as a streaming response from a view. - - :param source: The source code of the template to render. - :param context: The variables to make available in the template. - - .. versionadded:: 2.2 - """ - app = current_app._get_current_object() # type: ignore[attr-defined] - template = app.jinja_env.from_string(source) - return _stream(app, template, context) diff --git a/bundle/python-cpu/Lib/site-packages/flask/testing.py b/bundle/python-cpu/Lib/site-packages/flask/testing.py deleted file mode 100644 index 55eb12fe75457dd03345e37dd18b0075e6f578c1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/testing.py +++ /dev/null @@ -1,298 +0,0 @@ -from __future__ import annotations - -import importlib.metadata -import typing as t -from contextlib import contextmanager -from contextlib import ExitStack -from copy import copy -from types import TracebackType -from urllib.parse import urlsplit - -import werkzeug.test -from click.testing import CliRunner -from click.testing import Result -from werkzeug.test import Client -from werkzeug.wrappers import Request as BaseRequest - -from .cli import ScriptInfo -from .sessions import SessionMixin - -if t.TYPE_CHECKING: # pragma: no cover - from _typeshed.wsgi import WSGIEnvironment - from werkzeug.test import TestResponse - - from .app import Flask - - -class EnvironBuilder(werkzeug.test.EnvironBuilder): - """An :class:`~werkzeug.test.EnvironBuilder`, that takes defaults from the - application. - - :param app: The Flask application to configure the environment from. - :param path: URL path being requested. - :param base_url: Base URL where the app is being served, which - ``path`` is relative to. If not given, built from - :data:`PREFERRED_URL_SCHEME`, ``subdomain``, - :data:`SERVER_NAME`, and :data:`APPLICATION_ROOT`. - :param subdomain: Subdomain name to append to :data:`SERVER_NAME`. - :param url_scheme: Scheme to use instead of - :data:`PREFERRED_URL_SCHEME`. - :param json: If given, this is serialized as JSON and passed as - ``data``. Also defaults ``content_type`` to - ``application/json``. - :param args: other positional arguments passed to - :class:`~werkzeug.test.EnvironBuilder`. - :param kwargs: other keyword arguments passed to - :class:`~werkzeug.test.EnvironBuilder`. - """ - - def __init__( - self, - app: Flask, - path: str = "/", - base_url: str | None = None, - subdomain: str | None = None, - url_scheme: str | None = None, - *args: t.Any, - **kwargs: t.Any, - ) -> None: - assert not (base_url or subdomain or url_scheme) or ( - base_url is not None - ) != bool(subdomain or url_scheme), ( - 'Cannot pass "subdomain" or "url_scheme" with "base_url".' - ) - - if base_url is None: - http_host = app.config.get("SERVER_NAME") or "localhost" - app_root = app.config["APPLICATION_ROOT"] - - if subdomain: - http_host = f"{subdomain}.{http_host}" - - if url_scheme is None: - url_scheme = app.config["PREFERRED_URL_SCHEME"] - - url = urlsplit(path) - base_url = ( - f"{url.scheme or url_scheme}://{url.netloc or http_host}" - f"/{app_root.lstrip('/')}" - ) - path = url.path - - if url.query: - path = f"{path}?{url.query}" - - self.app = app - super().__init__(path, base_url, *args, **kwargs) - - def json_dumps(self, obj: t.Any, **kwargs: t.Any) -> str: - """Serialize ``obj`` to a JSON-formatted string. - - The serialization will be configured according to the config associated - with this EnvironBuilder's ``app``. - """ - return self.app.json.dumps(obj, **kwargs) - - -_werkzeug_version = "" - - -def _get_werkzeug_version() -> str: - global _werkzeug_version - - if not _werkzeug_version: - _werkzeug_version = importlib.metadata.version("werkzeug") - - return _werkzeug_version - - -class FlaskClient(Client): - """Works like a regular Werkzeug test client but has knowledge about - Flask's contexts to defer the cleanup of the request context until - the end of a ``with`` block. For general information about how to - use this class refer to :class:`werkzeug.test.Client`. - - .. versionchanged:: 0.12 - `app.test_client()` includes preset default environment, which can be - set after instantiation of the `app.test_client()` object in - `client.environ_base`. - - Basic usage is outlined in the :doc:`/testing` chapter. - """ - - application: Flask - - def __init__(self, *args: t.Any, **kwargs: t.Any) -> None: - super().__init__(*args, **kwargs) - self.preserve_context = False - self._new_contexts: list[t.ContextManager[t.Any]] = [] - self._context_stack = ExitStack() - self.environ_base = { - "REMOTE_ADDR": "127.0.0.1", - "HTTP_USER_AGENT": f"Werkzeug/{_get_werkzeug_version()}", - } - - @contextmanager - def session_transaction( - self, *args: t.Any, **kwargs: t.Any - ) -> t.Iterator[SessionMixin]: - """When used in combination with a ``with`` statement this opens a - session transaction. This can be used to modify the session that - the test client uses. Once the ``with`` block is left the session is - stored back. - - :: - - with client.session_transaction() as session: - session['value'] = 42 - - Internally this is implemented by going through a temporary test - request context and since session handling could depend on - request variables this function accepts the same arguments as - :meth:`~flask.Flask.test_request_context` which are directly - passed through. - """ - if self._cookies is None: - raise TypeError( - "Cookies are disabled. Create a client with 'use_cookies=True'." - ) - - app = self.application - ctx = app.test_request_context(*args, **kwargs) - self._add_cookies_to_wsgi(ctx.request.environ) - - with ctx: - sess = app.session_interface.open_session(app, ctx.request) - - if sess is None: - raise RuntimeError("Session backend did not open a session.") - - yield sess - resp = app.response_class() - - if app.session_interface.is_null_session(sess): - return - - with ctx: - app.session_interface.save_session(app, sess, resp) - - self._update_cookies_from_response( - ctx.request.host.partition(":")[0], - ctx.request.path, - resp.headers.getlist("Set-Cookie"), - ) - - def _copy_environ(self, other: WSGIEnvironment) -> WSGIEnvironment: - out = {**self.environ_base, **other} - - if self.preserve_context: - out["werkzeug.debug.preserve_context"] = self._new_contexts.append - - return out - - def _request_from_builder_args( - self, args: tuple[t.Any, ...], kwargs: dict[str, t.Any] - ) -> BaseRequest: - kwargs["environ_base"] = self._copy_environ(kwargs.get("environ_base", {})) - builder = EnvironBuilder(self.application, *args, **kwargs) - - try: - return builder.get_request() - finally: - builder.close() - - def open( - self, - *args: t.Any, - buffered: bool = False, - follow_redirects: bool = False, - **kwargs: t.Any, - ) -> TestResponse: - if args and isinstance( - args[0], (werkzeug.test.EnvironBuilder, dict, BaseRequest) - ): - if isinstance(args[0], werkzeug.test.EnvironBuilder): - builder = copy(args[0]) - builder.environ_base = self._copy_environ(builder.environ_base or {}) # type: ignore[arg-type] - request = builder.get_request() - elif isinstance(args[0], dict): - request = EnvironBuilder.from_environ( - args[0], app=self.application, environ_base=self._copy_environ({}) - ).get_request() - else: - # isinstance(args[0], BaseRequest) - request = copy(args[0]) - request.environ = self._copy_environ(request.environ) - else: - # request is None - request = self._request_from_builder_args(args, kwargs) - - # Pop any previously preserved contexts. This prevents contexts - # from being preserved across redirects or multiple requests - # within a single block. - self._context_stack.close() - - response = super().open( - request, - buffered=buffered, - follow_redirects=follow_redirects, - ) - response.json_module = self.application.json # type: ignore[assignment] - - # Re-push contexts that were preserved during the request. - for cm in self._new_contexts: - self._context_stack.enter_context(cm) - - self._new_contexts.clear() - return response - - def __enter__(self) -> FlaskClient: - if self.preserve_context: - raise RuntimeError("Cannot nest client invocations") - self.preserve_context = True - return self - - def __exit__( - self, - exc_type: type | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> None: - self.preserve_context = False - self._context_stack.close() - - -class FlaskCliRunner(CliRunner): - """A :class:`~click.testing.CliRunner` for testing a Flask app's - CLI commands. Typically created using - :meth:`~flask.Flask.test_cli_runner`. See :ref:`testing-cli`. - """ - - def __init__(self, app: Flask, **kwargs: t.Any) -> None: - self.app = app - super().__init__(**kwargs) - - def invoke( # type: ignore - self, cli: t.Any = None, args: t.Any = None, **kwargs: t.Any - ) -> Result: - """Invokes a CLI command in an isolated environment. See - :meth:`CliRunner.invoke ` for - full method documentation. See :ref:`testing-cli` for examples. - - If the ``obj`` argument is not given, passes an instance of - :class:`~flask.cli.ScriptInfo` that knows how to load the Flask - app being tested. - - :param cli: Command object to invoke. Default is the app's - :attr:`~flask.app.Flask.cli` group. - :param args: List of strings to invoke the command with. - - :return: a :class:`~click.testing.Result` object. - """ - if cli is None: - cli = self.app.cli - - if "obj" not in kwargs: - kwargs["obj"] = ScriptInfo(create_app=lambda: self.app) - - return super().invoke(cli, args, **kwargs) diff --git a/bundle/python-cpu/Lib/site-packages/flask/typing.py b/bundle/python-cpu/Lib/site-packages/flask/typing.py deleted file mode 100644 index 6b70c409735e07f05b31352437d715aa042fc14f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/typing.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import typing as t - -if t.TYPE_CHECKING: # pragma: no cover - from _typeshed.wsgi import WSGIApplication # noqa: F401 - from werkzeug.datastructures import Headers # noqa: F401 - from werkzeug.sansio.response import Response # noqa: F401 - -# The possible types that are directly convertible or are a Response object. -ResponseValue = t.Union[ - "Response", - str, - bytes, - list[t.Any], - # Only dict is actually accepted, but Mapping allows for TypedDict. - t.Mapping[str, t.Any], - t.Iterator[str], - t.Iterator[bytes], - cabc.AsyncIterable[str], # for Quart, until App is generic. - cabc.AsyncIterable[bytes], -] - -# the possible types for an individual HTTP header -# This should be a Union, but mypy doesn't pass unless it's a TypeVar. -HeaderValue = t.Union[str, list[str], tuple[str, ...]] - -# the possible types for HTTP headers -HeadersValue = t.Union[ - "Headers", - t.Mapping[str, HeaderValue], - t.Sequence[tuple[str, HeaderValue]], -] - -# The possible types returned by a route function. -ResponseReturnValue = t.Union[ - ResponseValue, - tuple[ResponseValue, HeadersValue], - tuple[ResponseValue, int], - tuple[ResponseValue, int, HeadersValue], - "WSGIApplication", -] - -# Allow any subclass of werkzeug.Response, such as the one from Flask, -# as a callback argument. Using werkzeug.Response directly makes a -# callback annotated with flask.Response fail type checking. -ResponseClass = t.TypeVar("ResponseClass", bound="Response") - -AppOrBlueprintKey = t.Optional[str] # The App key is None, whereas blueprints are named -AfterRequestCallable = t.Union[ - t.Callable[[ResponseClass], ResponseClass], - t.Callable[[ResponseClass], t.Awaitable[ResponseClass]], -] -BeforeFirstRequestCallable = t.Union[ - t.Callable[[], None], t.Callable[[], t.Awaitable[None]] -] -BeforeRequestCallable = t.Union[ - t.Callable[[], t.Optional[ResponseReturnValue]], - t.Callable[[], t.Awaitable[t.Optional[ResponseReturnValue]]], -] -ShellContextProcessorCallable = t.Callable[[], dict[str, t.Any]] -TeardownCallable = t.Union[ - t.Callable[[t.Optional[BaseException]], None], - t.Callable[[t.Optional[BaseException]], t.Awaitable[None]], -] -TemplateContextProcessorCallable = t.Union[ - t.Callable[[], dict[str, t.Any]], - t.Callable[[], t.Awaitable[dict[str, t.Any]]], -] -TemplateFilterCallable = t.Callable[..., t.Any] -TemplateGlobalCallable = t.Callable[..., t.Any] -TemplateTestCallable = t.Callable[..., bool] -URLDefaultCallable = t.Callable[[str, dict[str, t.Any]], None] -URLValuePreprocessorCallable = t.Callable[ - [t.Optional[str], t.Optional[dict[str, t.Any]]], None -] - -# This should take Exception, but that either breaks typing the argument -# with a specific exception, or decorating multiple times with different -# exceptions (and using a union type on the argument). -# https://github.com/pallets/flask/issues/4095 -# https://github.com/pallets/flask/issues/4295 -# https://github.com/pallets/flask/issues/4297 -ErrorHandlerCallable = t.Union[ - t.Callable[[t.Any], ResponseReturnValue], - t.Callable[[t.Any], t.Awaitable[ResponseReturnValue]], -] - -RouteCallable = t.Union[ - t.Callable[..., ResponseReturnValue], - t.Callable[..., t.Awaitable[ResponseReturnValue]], -] diff --git a/bundle/python-cpu/Lib/site-packages/flask/views.py b/bundle/python-cpu/Lib/site-packages/flask/views.py deleted file mode 100644 index 53fe976dc2ad658e761690eff40ae3e9e014e1e5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/views.py +++ /dev/null @@ -1,191 +0,0 @@ -from __future__ import annotations - -import typing as t - -from . import typing as ft -from .globals import current_app -from .globals import request - -F = t.TypeVar("F", bound=t.Callable[..., t.Any]) - -http_method_funcs = frozenset( - ["get", "post", "head", "options", "delete", "put", "trace", "patch"] -) - - -class View: - """Subclass this class and override :meth:`dispatch_request` to - create a generic class-based view. Call :meth:`as_view` to create a - view function that creates an instance of the class with the given - arguments and calls its ``dispatch_request`` method with any URL - variables. - - See :doc:`views` for a detailed guide. - - .. code-block:: python - - class Hello(View): - init_every_request = False - - def dispatch_request(self, name): - return f"Hello, {name}!" - - app.add_url_rule( - "/hello/", view_func=Hello.as_view("hello") - ) - - Set :attr:`methods` on the class to change what methods the view - accepts. - - Set :attr:`decorators` on the class to apply a list of decorators to - the generated view function. Decorators applied to the class itself - will not be applied to the generated view function! - - Set :attr:`init_every_request` to ``False`` for efficiency, unless - you need to store request-global data on ``self``. - """ - - #: The methods this view is registered for. Uses the same default - #: (``["GET", "HEAD", "OPTIONS"]``) as ``route`` and - #: ``add_url_rule`` by default. - methods: t.ClassVar[t.Collection[str] | None] = None - - #: Control whether the ``OPTIONS`` method is handled automatically. - #: Uses the same default (``True``) as ``route`` and - #: ``add_url_rule`` by default. - provide_automatic_options: t.ClassVar[bool | None] = None - - #: A list of decorators to apply, in order, to the generated view - #: function. Remember that ``@decorator`` syntax is applied bottom - #: to top, so the first decorator in the list would be the bottom - #: decorator. - #: - #: .. versionadded:: 0.8 - decorators: t.ClassVar[list[t.Callable[..., t.Any]]] = [] - - #: Create a new instance of this view class for every request by - #: default. If a view subclass sets this to ``False``, the same - #: instance is used for every request. - #: - #: A single instance is more efficient, especially if complex setup - #: is done during init. However, storing data on ``self`` is no - #: longer safe across requests, and :data:`~flask.g` should be used - #: instead. - #: - #: .. versionadded:: 2.2 - init_every_request: t.ClassVar[bool] = True - - def dispatch_request(self) -> ft.ResponseReturnValue: - """The actual view function behavior. Subclasses must override - this and return a valid response. Any variables from the URL - rule are passed as keyword arguments. - """ - raise NotImplementedError() - - @classmethod - def as_view( - cls, name: str, *class_args: t.Any, **class_kwargs: t.Any - ) -> ft.RouteCallable: - """Convert the class into a view function that can be registered - for a route. - - By default, the generated view will create a new instance of the - view class for every request and call its - :meth:`dispatch_request` method. If the view class sets - :attr:`init_every_request` to ``False``, the same instance will - be used for every request. - - Except for ``name``, all other arguments passed to this method - are forwarded to the view class ``__init__`` method. - - .. versionchanged:: 2.2 - Added the ``init_every_request`` class attribute. - """ - if cls.init_every_request: - - def view(**kwargs: t.Any) -> ft.ResponseReturnValue: - self = view.view_class( # type: ignore[attr-defined] - *class_args, **class_kwargs - ) - return current_app.ensure_sync(self.dispatch_request)(**kwargs) # type: ignore[no-any-return] - - else: - self = cls(*class_args, **class_kwargs) # pyright: ignore - - def view(**kwargs: t.Any) -> ft.ResponseReturnValue: - return current_app.ensure_sync(self.dispatch_request)(**kwargs) # type: ignore[no-any-return] - - if cls.decorators: - view.__name__ = name - view.__module__ = cls.__module__ - for decorator in cls.decorators: - view = decorator(view) - - # We attach the view class to the view function for two reasons: - # first of all it allows us to easily figure out what class-based - # view this thing came from, secondly it's also used for instantiating - # the view class so you can actually replace it with something else - # for testing purposes and debugging. - view.view_class = cls # type: ignore - view.__name__ = name - view.__doc__ = cls.__doc__ - view.__module__ = cls.__module__ - view.methods = cls.methods # type: ignore - view.provide_automatic_options = cls.provide_automatic_options # type: ignore - return view - - -class MethodView(View): - """Dispatches request methods to the corresponding instance methods. - For example, if you implement a ``get`` method, it will be used to - handle ``GET`` requests. - - This can be useful for defining a REST API. - - :attr:`methods` is automatically set based on the methods defined on - the class. - - See :doc:`views` for a detailed guide. - - .. code-block:: python - - class CounterAPI(MethodView): - def get(self): - return str(session.get("counter", 0)) - - def post(self): - session["counter"] = session.get("counter", 0) + 1 - return redirect(url_for("counter")) - - app.add_url_rule( - "/counter", view_func=CounterAPI.as_view("counter") - ) - """ - - def __init_subclass__(cls, **kwargs: t.Any) -> None: - super().__init_subclass__(**kwargs) - - if "methods" not in cls.__dict__: - methods = set() - - for base in cls.__bases__: - if getattr(base, "methods", None): - methods.update(base.methods) # type: ignore[attr-defined] - - for key in http_method_funcs: - if hasattr(cls, key): - methods.add(key.upper()) - - if methods: - cls.methods = methods - - def dispatch_request(self, **kwargs: t.Any) -> ft.ResponseReturnValue: - meth = getattr(self, request.method.lower(), None) - - # If the request method is HEAD and we don't have a handler for it - # retry with GET. - if meth is None and request.method == "HEAD": - meth = getattr(self, "get", None) - - assert meth is not None, f"Unimplemented method {request.method!r}" - return current_app.ensure_sync(meth)(**kwargs) # type: ignore[no-any-return] diff --git a/bundle/python-cpu/Lib/site-packages/flask/wrappers.py b/bundle/python-cpu/Lib/site-packages/flask/wrappers.py deleted file mode 100644 index bab610291ce3b6134e313de923420549a1f8ffbe..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/flask/wrappers.py +++ /dev/null @@ -1,257 +0,0 @@ -from __future__ import annotations - -import typing as t - -from werkzeug.exceptions import BadRequest -from werkzeug.exceptions import HTTPException -from werkzeug.wrappers import Request as RequestBase -from werkzeug.wrappers import Response as ResponseBase - -from . import json -from .globals import current_app -from .helpers import _split_blueprint_path - -if t.TYPE_CHECKING: # pragma: no cover - from werkzeug.routing import Rule - - -class Request(RequestBase): - """The request object used by default in Flask. Remembers the - matched endpoint and view arguments. - - It is what ends up as :class:`~flask.request`. If you want to replace - the request object used you can subclass this and set - :attr:`~flask.Flask.request_class` to your subclass. - - The request object is a :class:`~werkzeug.wrappers.Request` subclass and - provides all of the attributes Werkzeug defines plus a few Flask - specific ones. - """ - - json_module: t.Any = json - - #: The internal URL rule that matched the request. This can be - #: useful to inspect which methods are allowed for the URL from - #: a before/after handler (``request.url_rule.methods``) etc. - #: Though if the request's method was invalid for the URL rule, - #: the valid list is available in ``routing_exception.valid_methods`` - #: instead (an attribute of the Werkzeug exception - #: :exc:`~werkzeug.exceptions.MethodNotAllowed`) - #: because the request was never internally bound. - #: - #: .. versionadded:: 0.6 - url_rule: Rule | None = None - - #: A dict of view arguments that matched the request. If an exception - #: happened when matching, this will be ``None``. - view_args: dict[str, t.Any] | None = None - - #: If matching the URL failed, this is the exception that will be - #: raised / was raised as part of the request handling. This is - #: usually a :exc:`~werkzeug.exceptions.NotFound` exception or - #: something similar. - routing_exception: HTTPException | None = None - - _max_content_length: int | None = None - _max_form_memory_size: int | None = None - _max_form_parts: int | None = None - - @property - def max_content_length(self) -> int | None: - """The maximum number of bytes that will be read during this request. If - this limit is exceeded, a 413 :exc:`~werkzeug.exceptions.RequestEntityTooLarge` - error is raised. If it is set to ``None``, no limit is enforced at the - Flask application level. However, if it is ``None`` and the request has - no ``Content-Length`` header and the WSGI server does not indicate that - it terminates the stream, then no data is read to avoid an infinite - stream. - - Each request defaults to the :data:`MAX_CONTENT_LENGTH` config, which - defaults to ``None``. It can be set on a specific ``request`` to apply - the limit to that specific view. This should be set appropriately based - on an application's or view's specific needs. - - .. versionchanged:: 3.1 - This can be set per-request. - - .. versionchanged:: 0.6 - This is configurable through Flask config. - """ - if self._max_content_length is not None: - return self._max_content_length - - if not current_app: - return super().max_content_length - - return current_app.config["MAX_CONTENT_LENGTH"] # type: ignore[no-any-return] - - @max_content_length.setter - def max_content_length(self, value: int | None) -> None: - self._max_content_length = value - - @property - def max_form_memory_size(self) -> int | None: - """The maximum size in bytes any non-file form field may be in a - ``multipart/form-data`` body. If this limit is exceeded, a 413 - :exc:`~werkzeug.exceptions.RequestEntityTooLarge` error is raised. If it - is set to ``None``, no limit is enforced at the Flask application level. - - Each request defaults to the :data:`MAX_FORM_MEMORY_SIZE` config, which - defaults to ``500_000``. It can be set on a specific ``request`` to - apply the limit to that specific view. This should be set appropriately - based on an application's or view's specific needs. - - .. versionchanged:: 3.1 - This is configurable through Flask config. - """ - if self._max_form_memory_size is not None: - return self._max_form_memory_size - - if not current_app: - return super().max_form_memory_size - - return current_app.config["MAX_FORM_MEMORY_SIZE"] # type: ignore[no-any-return] - - @max_form_memory_size.setter - def max_form_memory_size(self, value: int | None) -> None: - self._max_form_memory_size = value - - @property # type: ignore[override] - def max_form_parts(self) -> int | None: - """The maximum number of fields that may be present in a - ``multipart/form-data`` body. If this limit is exceeded, a 413 - :exc:`~werkzeug.exceptions.RequestEntityTooLarge` error is raised. If it - is set to ``None``, no limit is enforced at the Flask application level. - - Each request defaults to the :data:`MAX_FORM_PARTS` config, which - defaults to ``1_000``. It can be set on a specific ``request`` to apply - the limit to that specific view. This should be set appropriately based - on an application's or view's specific needs. - - .. versionchanged:: 3.1 - This is configurable through Flask config. - """ - if self._max_form_parts is not None: - return self._max_form_parts - - if not current_app: - return super().max_form_parts - - return current_app.config["MAX_FORM_PARTS"] # type: ignore[no-any-return] - - @max_form_parts.setter - def max_form_parts(self, value: int | None) -> None: - self._max_form_parts = value - - @property - def endpoint(self) -> str | None: - """The endpoint that matched the request URL. - - This will be ``None`` if matching failed or has not been - performed yet. - - This in combination with :attr:`view_args` can be used to - reconstruct the same URL or a modified URL. - """ - if self.url_rule is not None: - return self.url_rule.endpoint # type: ignore[no-any-return] - - return None - - @property - def blueprint(self) -> str | None: - """The registered name of the current blueprint. - - This will be ``None`` if the endpoint is not part of a - blueprint, or if URL matching failed or has not been performed - yet. - - This does not necessarily match the name the blueprint was - created with. It may have been nested, or registered with a - different name. - """ - endpoint = self.endpoint - - if endpoint is not None and "." in endpoint: - return endpoint.rpartition(".")[0] - - return None - - @property - def blueprints(self) -> list[str]: - """The registered names of the current blueprint upwards through - parent blueprints. - - This will be an empty list if there is no current blueprint, or - if URL matching failed. - - .. versionadded:: 2.0.1 - """ - name = self.blueprint - - if name is None: - return [] - - return _split_blueprint_path(name) - - def _load_form_data(self) -> None: - super()._load_form_data() - - # In debug mode we're replacing the files multidict with an ad-hoc - # subclass that raises a different error for key errors. - if ( - current_app - and current_app.debug - and self.mimetype != "multipart/form-data" - and not self.files - ): - from .debughelpers import attach_enctype_error_multidict - - attach_enctype_error_multidict(self) - - def on_json_loading_failed(self, e: ValueError | None) -> t.Any: - try: - return super().on_json_loading_failed(e) - except BadRequest as ebr: - if current_app and current_app.debug: - raise - - raise BadRequest() from ebr - - -class Response(ResponseBase): - """The response object that is used by default in Flask. Works like the - response object from Werkzeug but is set to have an HTML mimetype by - default. Quite often you don't have to create this object yourself because - :meth:`~flask.Flask.make_response` will take care of that for you. - - If you want to replace the response object used you can subclass this and - set :attr:`~flask.Flask.response_class` to your subclass. - - .. versionchanged:: 1.0 - JSON support is added to the response, like the request. This is useful - when testing to get the test client response data as JSON. - - .. versionchanged:: 1.0 - - Added :attr:`max_cookie_size`. - """ - - default_mimetype: str | None = "text/html" - - json_module = json - - autocorrect_location_header = False - - @property - def max_cookie_size(self) -> int: # type: ignore - """Read-only view of the :data:`MAX_COOKIE_SIZE` config key. - - See :attr:`~werkzeug.wrappers.Response.max_cookie_size` in - Werkzeug's docs. - """ - if current_app: - return current_app.config["MAX_COOKIE_SIZE"] # type: ignore[no-any-return] - - # return Werkzeug's default when not in an app context - return super().max_cookie_size diff --git a/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/INSTALLER deleted file mode 100644 index a1b589e38a32041e49332e5e81c2d363dc418d68..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/METADATA deleted file mode 100644 index 369e66b1d0f65050a4b3d7541ed4fec1ad3cd069..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/METADATA +++ /dev/null @@ -1,257 +0,0 @@ -Metadata-Version: 2.4 -Name: fsspec -Version: 2026.7.0 -Summary: File-system specification -Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html -Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/ -Project-URL: Homepage, https://github.com/fsspec/filesystem_spec -Maintainer-email: Martin Durant -License-Expression: BSD-3-Clause -License-File: LICENSE -Keywords: file -Classifier: Development Status :: 4 - Beta -Classifier: Intended Audience :: Developers -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Requires-Python: >=3.10 -Provides-Extra: abfs -Requires-Dist: adlfs; extra == 'abfs' -Provides-Extra: adl -Requires-Dist: adlfs; extra == 'adl' -Provides-Extra: arrow -Requires-Dist: pyarrow>=1; extra == 'arrow' -Provides-Extra: dask -Requires-Dist: dask; extra == 'dask' -Requires-Dist: distributed; extra == 'dask' -Provides-Extra: dev -Requires-Dist: pre-commit; extra == 'dev' -Requires-Dist: ruff>=0.5; extra == 'dev' -Provides-Extra: doc -Requires-Dist: numpydoc; extra == 'doc' -Requires-Dist: sphinx; extra == 'doc' -Requires-Dist: sphinx-design; extra == 'doc' -Requires-Dist: sphinx-rtd-theme; extra == 'doc' -Requires-Dist: yarl; extra == 'doc' -Provides-Extra: dropbox -Requires-Dist: dropbox; extra == 'dropbox' -Requires-Dist: dropboxdrivefs; extra == 'dropbox' -Requires-Dist: requests; extra == 'dropbox' -Provides-Extra: entrypoints -Provides-Extra: full -Requires-Dist: adlfs; extra == 'full' -Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'full' -Requires-Dist: dask; extra == 'full' -Requires-Dist: distributed; extra == 'full' -Requires-Dist: dropbox; extra == 'full' -Requires-Dist: dropboxdrivefs; extra == 'full' -Requires-Dist: fusepy; extra == 'full' -Requires-Dist: gcsfs>=2026.4.0; extra == 'full' -Requires-Dist: libarchive-c; extra == 'full' -Requires-Dist: ocifs; extra == 'full' -Requires-Dist: panel; extra == 'full' -Requires-Dist: paramiko; extra == 'full' -Requires-Dist: pyarrow>=1; extra == 'full' -Requires-Dist: pygit2; extra == 'full' -Requires-Dist: requests; extra == 'full' -Requires-Dist: s3fs>=2026.6.0; extra == 'full' -Requires-Dist: smbprotocol; extra == 'full' -Requires-Dist: tqdm; extra == 'full' -Provides-Extra: fuse -Requires-Dist: fusepy; extra == 'fuse' -Provides-Extra: gcs -Requires-Dist: gcsfs>=2026.4.0; extra == 'gcs' -Provides-Extra: git -Requires-Dist: pygit2; extra == 'git' -Provides-Extra: github -Requires-Dist: requests; extra == 'github' -Provides-Extra: gs -Requires-Dist: gcsfs>=2026.4.0; extra == 'gs' -Provides-Extra: gui -Requires-Dist: panel; extra == 'gui' -Provides-Extra: hdfs -Requires-Dist: pyarrow>=1; extra == 'hdfs' -Provides-Extra: http -Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'http' -Provides-Extra: libarchive -Requires-Dist: libarchive-c; extra == 'libarchive' -Provides-Extra: oci -Requires-Dist: ocifs; extra == 'oci' -Provides-Extra: s3 -Requires-Dist: s3fs>=2026.6.0; extra == 's3' -Provides-Extra: sftp -Requires-Dist: paramiko; extra == 'sftp' -Provides-Extra: smb -Requires-Dist: smbprotocol; extra == 'smb' -Provides-Extra: ssh -Requires-Dist: paramiko; extra == 'ssh' -Provides-Extra: test -Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'test' -Requires-Dist: numpy; extra == 'test' -Requires-Dist: pytest; extra == 'test' -Requires-Dist: pytest-asyncio!=0.22.0; extra == 'test' -Requires-Dist: pytest-benchmark; extra == 'test' -Requires-Dist: pytest-cov; extra == 'test' -Requires-Dist: pytest-mock; extra == 'test' -Requires-Dist: pytest-recording; extra == 'test' -Requires-Dist: pytest-rerunfailures; extra == 'test' -Requires-Dist: requests; extra == 'test' -Provides-Extra: test-downstream -Requires-Dist: aiobotocore<3.0.0,>=2.5.4; extra == 'test-downstream' -Requires-Dist: dask[dataframe,test]; extra == 'test-downstream' -Requires-Dist: moto[server]<5,>4; extra == 'test-downstream' -Requires-Dist: pytest-timeout; extra == 'test-downstream' -Requires-Dist: xarray; extra == 'test-downstream' -Provides-Extra: test-full -Requires-Dist: adlfs; extra == 'test-full' -Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'test-full' -Requires-Dist: backports-zstd; (python_version < '3.14') and extra == 'test-full' -Requires-Dist: cloudpickle; extra == 'test-full' -Requires-Dist: dask; extra == 'test-full' -Requires-Dist: distributed; extra == 'test-full' -Requires-Dist: dropbox; extra == 'test-full' -Requires-Dist: dropboxdrivefs; extra == 'test-full' -Requires-Dist: fastparquet; extra == 'test-full' -Requires-Dist: fusepy; extra == 'test-full' -Requires-Dist: gcsfs>=2026.4.0; extra == 'test-full' -Requires-Dist: jinja2; extra == 'test-full' -Requires-Dist: kerchunk; extra == 'test-full' -Requires-Dist: libarchive-c; extra == 'test-full' -Requires-Dist: lz4; extra == 'test-full' -Requires-Dist: notebook; extra == 'test-full' -Requires-Dist: numpy; extra == 'test-full' -Requires-Dist: ocifs; extra == 'test-full' -Requires-Dist: pandas<3.0.0; extra == 'test-full' -Requires-Dist: panel; extra == 'test-full' -Requires-Dist: paramiko; extra == 'test-full' -Requires-Dist: pyarrow>=1; extra == 'test-full' -Requires-Dist: pyftpdlib; extra == 'test-full' -Requires-Dist: pygit2; extra == 'test-full' -Requires-Dist: pytest; extra == 'test-full' -Requires-Dist: pytest-asyncio!=0.22.0; extra == 'test-full' -Requires-Dist: pytest-benchmark; extra == 'test-full' -Requires-Dist: pytest-cov; extra == 'test-full' -Requires-Dist: pytest-mock; extra == 'test-full' -Requires-Dist: pytest-recording; extra == 'test-full' -Requires-Dist: pytest-rerunfailures; extra == 'test-full' -Requires-Dist: python-snappy; extra == 'test-full' -Requires-Dist: requests; extra == 'test-full' -Requires-Dist: s3fs>=2026.6.0; extra == 'test-full' -Requires-Dist: smbprotocol; extra == 'test-full' -Requires-Dist: tqdm; extra == 'test-full' -Requires-Dist: urllib3; extra == 'test-full' -Requires-Dist: zarr<3.2.0; extra == 'test-full' -Requires-Dist: zstandard; (python_version < '3.14') and extra == 'test-full' -Provides-Extra: tqdm -Requires-Dist: tqdm; extra == 'tqdm' -Description-Content-Type: text/markdown - -# filesystem_spec - -[![PyPI version](https://badge.fury.io/py/fsspec.svg)](https://pypi.python.org/pypi/fsspec/) -[![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec) -![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg) -[![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest) - -A specification for pythonic filesystems. - -## Install - -```bash -pip install fsspec -``` - -would install the base fsspec. Various optionally supported features might require specification of custom -extra require, e.g. `pip install fsspec[ssh]` will install dependencies for `ssh` backends support. -Use `pip install fsspec[full]` for installation of all known extra dependencies. - -Up-to-date package also provided through conda-forge distribution: - -```bash -conda install -c conda-forge fsspec -``` - - -## Purpose - -To produce a template or specification for a file-system interface, that specific implementations should follow, -so that applications making use of them can rely on a common behaviour and not have to worry about the specific -internal implementation decisions with any given backend. Many such implementations are included in this package, -or in sister projects such as `s3fs` and `gcsfs`. - -In addition, if this is well-designed, then additional functionality, such as a key-value store or FUSE -mounting of the file-system implementation may be available for all implementations "for free". - -## Documentation - -Please refer to [RTD](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest) - -## Develop - -fsspec uses GitHub Actions for CI. Environment files can be found -in the "ci/" directory. Note that the main environment is called "py38", -but it is expected that the version of python installed be adjustable at -CI runtime. For local use, pick a version suitable for you. - -```bash -# For a new environment (mamba / conda). -mamba create -n fsspec -c conda-forge python=3.10 -y -conda activate fsspec - -# Standard dev install with docs and tests. -pip install -e ".[dev,doc,test]" - -# Full tests except for downstream -pip install s3fs -pip uninstall s3fs -pip install -e .[dev,doc,test_full] -pip install s3fs --no-deps -pytest -v - -# Downstream tests. -sh install_s3fs.sh -# Windows powershell. -install_s3fs.sh -``` - -### Testing - -Tests can be run in the dev environment, if activated, via ``pytest fsspec``. - -The full fsspec suite requires a system-level docker, docker-compose, and fuse -installation. If only making changes to one backend implementation, it is -not generally necessary to run all tests locally. - -It is expected that contributors ensure that any change to fsspec does not -cause issues or regressions for either other fsspec-related packages such -as gcsfs and s3fs, nor for downstream users of fsspec. The "downstream" CI -run and corresponding environment file run a set of tests from the dask -test suite, and very minimal tests against pandas and zarr from the -test_downstream.py module in this repo. - -### Code Formatting - -fsspec uses [Black](https://black.readthedocs.io/en/stable) to ensure -a consistent code format throughout the project. -Run ``black fsspec`` from the root of the filesystem_spec repository to -auto-format your code. Additionally, many editors have plugins that will apply -``black`` as you edit files. ``black`` is included in the ``tox`` environments. - -Optionally, you may wish to setup [pre-commit hooks](https://pre-commit.com) to -automatically run ``black`` when you make a git commit. -Run ``pre-commit install --install-hooks`` from the root of the -filesystem_spec repository to setup pre-commit hooks. ``black`` will now be run -before you commit, reformatting any changed files. You can format without -committing via ``pre-commit run`` or skip these checks with ``git commit ---no-verify``. - -## Support - -Work on this repository is supported in part by: - -"Anaconda, Inc. - Advancing AI through open source." - -anaconda logo diff --git a/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/RECORD deleted file mode 100644 index bafcf89bed3b634b73c4ff8b00ed07f988f88d5b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/RECORD +++ /dev/null @@ -1,119 +0,0 @@ -fsspec-2026.7.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -fsspec-2026.7.0.dist-info/METADATA,sha256=iLFldYXeFOO7gqpSeKqVlbMKpMp06tBz2JUGFcgICZ0,10561 -fsspec-2026.7.0.dist-info/RECORD,, -fsspec-2026.7.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87 -fsspec-2026.7.0.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513 -fsspec/__init__.py,sha256=xFcoHBD17fc5KamKUQAIA0rePGjLqIN1WBAd9NAoAi8,2111 -fsspec/__pycache__/__init__.cpython-310.pyc,, -fsspec/__pycache__/_version.cpython-310.pyc,, -fsspec/__pycache__/archive.cpython-310.pyc,, -fsspec/__pycache__/asyn.cpython-310.pyc,, -fsspec/__pycache__/caching.cpython-310.pyc,, -fsspec/__pycache__/callbacks.cpython-310.pyc,, -fsspec/__pycache__/compression.cpython-310.pyc,, -fsspec/__pycache__/config.cpython-310.pyc,, -fsspec/__pycache__/conftest.cpython-310.pyc,, -fsspec/__pycache__/core.cpython-310.pyc,, -fsspec/__pycache__/dircache.cpython-310.pyc,, -fsspec/__pycache__/exceptions.cpython-310.pyc,, -fsspec/__pycache__/fuse.cpython-310.pyc,, -fsspec/__pycache__/generic.cpython-310.pyc,, -fsspec/__pycache__/gui.cpython-310.pyc,, -fsspec/__pycache__/json.cpython-310.pyc,, -fsspec/__pycache__/mapping.cpython-310.pyc,, -fsspec/__pycache__/parquet.cpython-310.pyc,, -fsspec/__pycache__/registry.cpython-310.pyc,, -fsspec/__pycache__/spec.cpython-310.pyc,, -fsspec/__pycache__/transaction.cpython-310.pyc,, -fsspec/__pycache__/utils.cpython-310.pyc,, -fsspec/_version.py,sha256=XYrsnKX7E2XNoSiwqs7ALMtioidXl8y8KSliz3pL7S0,526 -fsspec/archive.py,sha256=vM6t_lgV6lBWbBYwpm3S4ofBQFQxUPr5KkDQrrQcQro,2411 -fsspec/asyn.py,sha256=V1Kig3HVzgWwZ-VtUYhys3QfdmiWRLG7EcP1E1PwM6M,39105 -fsspec/caching.py,sha256=Glmc9tDtS8Ctp44sd7mJURJYq7sHkH_yD480q6IgbxE,34777 -fsspec/callbacks.py,sha256=BDIwLzK6rr_0V5ch557fSzsivCElpdqhXr5dZ9Te-EE,9210 -fsspec/compression.py,sha256=3v_Fe39gzRRWfaeXpzNjAGPqgTzmETYRCo3qHVqD3po,5132 -fsspec/config.py,sha256=mHKzAgjXa_LwqtO_VoTJJ568VqwYkbK3gUp4J6JZJp4,4238 -fsspec/conftest.py,sha256=uWfm_Qs5alPRxOhRpDfQ0-1jqSJ54pni4y96IxOREXM,3446 -fsspec/core.py,sha256=EapTTO4bGR9G12rC90FiaBRemMhqo0ypx3ZqXj1zVuk,24189 -fsspec/dircache.py,sha256=YzogWJrhEastHU7vWz-cJiJ7sdtLXFXhEpInGKd4EcM,2717 -fsspec/exceptions.py,sha256=pauSLDMxzTJMOjvX1WEUK0cMyFkrFxpWJsyFywav7A8,331 -fsspec/fuse.py,sha256=Q-3NOOyLqBfYa4Db5E19z_ZY36zzYHtIs1mOUasItBQ,10177 -fsspec/generic.py,sha256=-LIADa9qRN79bfa3Hl9m3HaN6rlU2GNivawLNmybqxE,13752 -fsspec/gui.py,sha256=CQ7QsrTpaDlWSLNOpwNoJc7khOcYXIZxmrAJN9bHWQU,14002 -fsspec/implementations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fsspec/implementations/__pycache__/__init__.cpython-310.pyc,, -fsspec/implementations/__pycache__/arrow.cpython-310.pyc,, -fsspec/implementations/__pycache__/asyn_wrapper.cpython-310.pyc,, -fsspec/implementations/__pycache__/cache_mapper.cpython-310.pyc,, -fsspec/implementations/__pycache__/cache_metadata.cpython-310.pyc,, -fsspec/implementations/__pycache__/cached.cpython-310.pyc,, -fsspec/implementations/__pycache__/chained.cpython-310.pyc,, -fsspec/implementations/__pycache__/dask.cpython-310.pyc,, -fsspec/implementations/__pycache__/data.cpython-310.pyc,, -fsspec/implementations/__pycache__/dbfs.cpython-310.pyc,, -fsspec/implementations/__pycache__/dirfs.cpython-310.pyc,, -fsspec/implementations/__pycache__/ftp.cpython-310.pyc,, -fsspec/implementations/__pycache__/gist.cpython-310.pyc,, -fsspec/implementations/__pycache__/git.cpython-310.pyc,, -fsspec/implementations/__pycache__/github.cpython-310.pyc,, -fsspec/implementations/__pycache__/http.cpython-310.pyc,, -fsspec/implementations/__pycache__/http_sync.cpython-310.pyc,, -fsspec/implementations/__pycache__/jupyter.cpython-310.pyc,, -fsspec/implementations/__pycache__/libarchive.cpython-310.pyc,, -fsspec/implementations/__pycache__/local.cpython-310.pyc,, -fsspec/implementations/__pycache__/memory.cpython-310.pyc,, -fsspec/implementations/__pycache__/reference.cpython-310.pyc,, -fsspec/implementations/__pycache__/sftp.cpython-310.pyc,, -fsspec/implementations/__pycache__/smb.cpython-310.pyc,, -fsspec/implementations/__pycache__/tar.cpython-310.pyc,, -fsspec/implementations/__pycache__/webhdfs.cpython-310.pyc,, -fsspec/implementations/__pycache__/zip.cpython-310.pyc,, -fsspec/implementations/arrow.py,sha256=HSXhExZ3HKNkWEm6m36h2jrbVilt3u8MvJI60u1_6fU,9236 -fsspec/implementations/asyn_wrapper.py,sha256=3lfJkGs6D_AwRBdxTSYlL-RCVdaXBZ9Itys2P5o5Si0,3738 -fsspec/implementations/cache_mapper.py,sha256=W4wlxyPxZbSp9ItJ0pYRVBMh6bw9eFypgP6kUYuuiI4,2421 -fsspec/implementations/cache_metadata.py,sha256=EPhQV27lgPlYREYprDET1vcMed15pVNMit-Ta1ljoyE,7942 -fsspec/implementations/cached.py,sha256=A_QyiukZzVznBWSZgMXSLd6h6K7k-ueroqnsoVY4bZg,37134 -fsspec/implementations/chained.py,sha256=iGivpNaHUFjB_ea0-HAPhcmm6CL8qnDf270PSj7JwuE,680 -fsspec/implementations/dask.py,sha256=CXZbJzIVOhKV8ILcxuy3bTvcacCueAbyQxmvAkbPkrk,4466 -fsspec/implementations/data.py,sha256=9tMcy1fyX1auvzFX2aANPK9CfPoarnxTCDLhX12usvs,2131 -fsspec/implementations/dbfs.py,sha256=yHuLUGNmQd5pUCt9vpXleQNpjOpIZZwVZA08YauQU4U,16249 -fsspec/implementations/dirfs.py,sha256=KaIHW18i4-eNqdv7NR9qXEvoobQ88trGCNrqb4XW9ew,13449 -fsspec/implementations/ftp.py,sha256=S_y5XOv_QhBiYBw430qzFB5MfS9AzxNBOs3f87XehC0,13539 -fsspec/implementations/gist.py,sha256=Y6jTDrE-wuTwvpPyAQDuuOMBGxlajafKWoB1_yX6jdY,8528 -fsspec/implementations/git.py,sha256=qBDWMz5LNllPqVjr5jf_1FuNha4P5lyQI3IlhYg-wUE,3731 -fsspec/implementations/github.py,sha256=aCsZL8UvXZgdkcB1RUs3DdLeNrjLKcFsFYeQFDWbBFo,11653 -fsspec/implementations/http.py,sha256=1uWwxllOjPe8mQD54IrCYQJHU1YaAXh_cDmXBk-AoQ8,30894 -fsspec/implementations/http_sync.py,sha256=UmBqd938ebwVjYgVtzg-ysG3ZoGhIJw0wFtQAfxV3Aw,30332 -fsspec/implementations/jupyter.py,sha256=q1PlQ66AAswGFyr8MFKWyobaV2YekMWRtqENBDQtD28,4002 -fsspec/implementations/libarchive.py,sha256=SpIA1F-zf7kb2-VYUVuhMrXTBOhBxUXKgEW1RaAdDoA,7098 -fsspec/implementations/local.py,sha256=jpvvSYj8RaocLB8YuHjGlWOTMJAJNTnUBj9D_Zn32BA,17216 -fsspec/implementations/memory.py,sha256=fXUYjKF1xr4_9FX26vDCazEhmFbE7Om4xUufIOQe2vA,14088 -fsspec/implementations/reference.py,sha256=YKLzPBF6bLpOnHY9iM0MR5JLGRi5CB6x1cWIgMb6Ahg,49974 -fsspec/implementations/sftp.py,sha256=v7t-LpD4hEycHnw1kbepyoNgrhClshXQXitFuY4mBWE,5951 -fsspec/implementations/smb.py,sha256=5fhu8h06nOLBPh2c48aT7WBRqh9cEcbIwtyu06wTjec,15236 -fsspec/implementations/tar.py,sha256=adO9r_Badc4ucfApmK-mrvbv1re0_qoxwBjgnSMIrh0,4601 -fsspec/implementations/webhdfs.py,sha256=osF2m0nhDil6sbMzYW_4DZzhxF4ygtb59XDiybd9Fyg,17589 -fsspec/implementations/zip.py,sha256=GPDJh4UtUNJmaqijUQ0eiEZIGpwusQjkIcbhsrmLSi0,6263 -fsspec/json.py,sha256=4EBZ-xOmRiyxmIqPIwxmDImosRQ7io7qBM2xjJPsEE4,3768 -fsspec/mapping.py,sha256=m2ndB_gtRBXYmNJg0Ie1-BVR75TFleHmIQBzC-yWhjU,8343 -fsspec/parquet.py,sha256=0SzfswUTc2k5nAj7xZNYPPyvUkFNXznFRXXtBRhr-iE,20506 -fsspec/registry.py,sha256=KeDNL9HOyHX_D4D4qQGAI-Y3DQhiLeUC1VK2I81NZaE,12344 -fsspec/spec.py,sha256=3LTaRdAhOTIPCetwPCS9HC6MR1VlcdfLVBh4mWFrXns,79674 -fsspec/tests/abstract/__init__.py,sha256=V0-8FqQPfHlHZ3xomJ0hyWl6t1QvqsDnQFJYHeSTlEw,10197 -fsspec/tests/abstract/__pycache__/__init__.cpython-310.pyc,, -fsspec/tests/abstract/__pycache__/common.cpython-310.pyc,, -fsspec/tests/abstract/__pycache__/copy.cpython-310.pyc,, -fsspec/tests/abstract/__pycache__/get.cpython-310.pyc,, -fsspec/tests/abstract/__pycache__/mv.cpython-310.pyc,, -fsspec/tests/abstract/__pycache__/open.cpython-310.pyc,, -fsspec/tests/abstract/__pycache__/pipe.cpython-310.pyc,, -fsspec/tests/abstract/__pycache__/put.cpython-310.pyc,, -fsspec/tests/abstract/common.py,sha256=1GQwNo5AONzAnzZj0fWgn8NJPLXALehbsuGxS3FzWVU,4973 -fsspec/tests/abstract/copy.py,sha256=gU5-d97U3RSde35Vp4RxPY4rWwL744HiSrJ8IBOp9-8,19967 -fsspec/tests/abstract/get.py,sha256=vNR4HztvTR7Cj56AMo7_tx7TeYz1Jgr_2Wb8Lv-UiBY,20755 -fsspec/tests/abstract/mv.py,sha256=k8eUEBIrRrGMsBY5OOaDXdGnQUKGwDIfQyduB6YD3Ns,1982 -fsspec/tests/abstract/open.py,sha256=Fi2PBPYLbRqysF8cFm0rwnB41kMdQVYjq8cGyDXp3BU,329 -fsspec/tests/abstract/pipe.py,sha256=LFzIrLCB5GLXf9rzFKJmE8AdG7LQ_h4bJo70r8FLPqM,402 -fsspec/tests/abstract/put.py,sha256=7aih17OKB_IZZh1Mkq1eBDIjobhtMQmI8x-Pw-S_aZk,21201 -fsspec/transaction.py,sha256=xliRG6U2Zf3khG4xcw9WiB-yAoqJSHEGK_VjHOdtgo0,2398 -fsspec/utils.py,sha256=T6x7JS-qHGdZ6hMPrT1tH6_41lHFw88g3WxgNIItYPc,24114 diff --git a/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/WHEEL deleted file mode 100644 index 7401812e19af977cc5088f3b8fb1ef6bc0441c0a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: hatchling 1.31.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/licenses/LICENSE b/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/licenses/LICENSE deleted file mode 100644 index 67590a5e5be5a5a2dde3fe53a7512e404a896c22..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec-2026.7.0.dist-info/licenses/LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2018, Martin Durant -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/__init__.py b/bundle/python-cpu/Lib/site-packages/fsspec/__init__.py deleted file mode 100644 index 5de87eda16630e55f6a48eae3e145515c167d9cb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -from . import caching -from .callbacks import Callback -from .compression import available_compressions -from .core import get_fs_token_paths, open, open_files, open_local, url_to_fs -from .exceptions import FSTimeoutError -from .mapping import FSMap, get_mapper -from .registry import ( - available_protocols, - filesystem, - get_filesystem_class, - register_implementation, - registry, -) -from .spec import AbstractFileSystem - -try: - from ._version import __version__ # noqa: F401 -except ImportError: - __version__ = "unknown" - -__all__ = [ - "AbstractFileSystem", - "FSTimeoutError", - "FSMap", - "filesystem", - "register_implementation", - "get_filesystem_class", - "get_fs_token_paths", - "get_mapper", - "open", - "open_files", - "open_local", - "registry", - "caching", - "Callback", - "available_protocols", - "available_compressions", - "url_to_fs", -] - - -def process_entries(): - try: - from importlib.metadata import entry_points - except ImportError: - return - if entry_points is not None: - try: - eps = entry_points() - except TypeError: - pass # importlib-metadata < 0.8 - else: - if hasattr(eps, "select"): # Python 3.10+ / importlib_metadata >= 3.9.0 - specs = eps.select(group="fsspec.specs") - else: - specs = eps.get("fsspec.specs", []) - registered_names = {} - for spec in specs: - err_msg = f"Unable to load filesystem from {spec}" - name = spec.name - if name in registered_names: - continue - registered_names[name] = True - register_implementation( - name, - spec.value.replace(":", "."), - errtxt=err_msg, - # We take our implementations as the ones to overload with if - # for some reason we encounter some, may be the same, already - # registered - clobber=True, - ) - - -process_entries() diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/_version.py b/bundle/python-cpu/Lib/site-packages/fsspec/_version.py deleted file mode 100644 index 64642af9bccd08c5c0c2193d1702d36418b25349..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/_version.py +++ /dev/null @@ -1,24 +0,0 @@ -# file generated by vcs-versioning -# don't change, don't track in version control -from __future__ import annotations - -__all__ = [ - "__version__", - "__version_tuple__", - "version", - "version_tuple", - "__commit_id__", - "commit_id", -] - -version: str -__version__: str -__version_tuple__: tuple[int | str, ...] -version_tuple: tuple[int | str, ...] -commit_id: str | None -__commit_id__: str | None - -__version__ = version = '2026.7.0' -__version_tuple__ = version_tuple = (2026, 7, 0) - -__commit_id__ = commit_id = None diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/archive.py b/bundle/python-cpu/Lib/site-packages/fsspec/archive.py deleted file mode 100644 index 13a4da8df7c9405297cdd7d37476be2f725b2f57..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/archive.py +++ /dev/null @@ -1,75 +0,0 @@ -import operator - -from fsspec import AbstractFileSystem -from fsspec.utils import tokenize - - -class AbstractArchiveFileSystem(AbstractFileSystem): - """ - A generic superclass for implementing Archive-based filesystems. - - Currently, it is shared amongst - :class:`~fsspec.implementations.zip.ZipFileSystem`, - :class:`~fsspec.implementations.libarchive.LibArchiveFileSystem` and - :class:`~fsspec.implementations.tar.TarFileSystem`. - """ - - def __str__(self): - return f"" - - __repr__ = __str__ - - def ukey(self, path): - return tokenize(path, self.fo, self.protocol) - - def _all_dirnames(self, paths): - """Returns *all* directory names for each path in paths, including intermediate - ones. - - Parameters - ---------- - paths: Iterable of path strings - """ - if len(paths) == 0: - return set() - - dirnames = {self._parent(path) for path in paths} - {self.root_marker} - return dirnames | self._all_dirnames(dirnames) - - def info(self, path, **kwargs): - self._get_dirs() - path = self._strip_protocol(path) - if path in {"", "/"} and self.dir_cache: - return {"name": "", "type": "directory", "size": 0} - if path in self.dir_cache: - return self.dir_cache[path] - elif path + "/" in self.dir_cache: - return self.dir_cache[path + "/"] - else: - raise FileNotFoundError(path) - - def ls(self, path, detail=True, **kwargs): - self._get_dirs() - paths = {} - for p, f in self.dir_cache.items(): - p = p.rstrip("/") - if "/" in p: - root = p.rsplit("/", 1)[0] - else: - root = "" - if root == path.rstrip("/"): - paths[p] = f - elif all( - (a == b) - for a, b in zip(path.split("/"), [""] + p.strip("/").split("/")) - ): - # root directory entry - ppath = p.rstrip("/").split("/", 1)[0] - if ppath not in paths: - out = {"name": ppath, "size": 0, "type": "directory"} - paths[ppath] = out - if detail: - out = sorted(paths.values(), key=operator.itemgetter("name")) - return out - else: - return sorted(paths) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/asyn.py b/bundle/python-cpu/Lib/site-packages/fsspec/asyn.py deleted file mode 100644 index 32ad3d35d132355feb5989c3c999599392a56ea5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/asyn.py +++ /dev/null @@ -1,1171 +0,0 @@ -import asyncio -import asyncio.events -import functools -import inspect -import io -import numbers -import os -import re -import threading -from collections.abc import Iterable -from glob import has_magic -from typing import TYPE_CHECKING - -from .callbacks import DEFAULT_CALLBACK -from .exceptions import FSTimeoutError -from .implementations.local import LocalFileSystem, make_path_posix, trailing_sep -from .spec import AbstractBufferedFile, AbstractFileSystem -from .utils import glob_translate, is_exception, other_paths - -private = re.compile("_[^_]") -iothread = [None] # dedicated fsspec IO thread -loop = [None] # global event loop for any non-async instance -_lock = None # global lock placeholder -get_running_loop = asyncio.get_running_loop - - -def get_lock(): - """Allocate or return a threading lock. - - The lock is allocated on first use to allow setting one lock per forked process. - """ - global _lock - if not _lock: - _lock = threading.Lock() - return _lock - - -def reset_lock(): - """Reset the global lock. - - This should be called only on the init of a forked process to reset the lock to - None, enabling the new forked process to get a new lock. - """ - global _lock - - iothread[0] = None - loop[0] = None - _lock = None - - -async def _runner(event, coro, result, timeout=None): - timeout = timeout if timeout else None # convert 0 or 0.0 to None - if timeout is not None: - coro = asyncio.wait_for(coro, timeout=timeout) - try: - result[0] = await coro - except Exception as ex: - result[0] = ex - finally: - event.set() - - -def sync(loop, func, *args, timeout=None, **kwargs): - """ - Make loop run coroutine until it returns. Runs in other thread - - Examples - -------- - >>> fsspec.asyn.sync(fsspec.asyn.get_loop(), func, *args, - timeout=timeout, **kwargs) - """ - timeout = timeout if timeout else None # convert 0 or 0.0 to None - # NB: if the loop is not running *yet*, it is OK to submit work - # and we will wait for it - if loop is None or loop.is_closed(): - raise RuntimeError("Loop is not running") - try: - loop0 = asyncio.events.get_running_loop() - if loop0 is loop: - raise NotImplementedError("Calling sync() from within a running loop") - except NotImplementedError: - raise - except RuntimeError: - pass - coro = func(*args, **kwargs) - result = [None] - event = threading.Event() - asyncio.run_coroutine_threadsafe(_runner(event, coro, result, timeout), loop) - while True: - # this loops allows thread to get interrupted - if event.wait(1): - break - if timeout is not None: - timeout -= 1 - if timeout < 0: - raise FSTimeoutError - - return_result = result[0] - if isinstance(return_result, asyncio.TimeoutError): - # suppress asyncio.TimeoutError, raise FSTimeoutError - raise FSTimeoutError from return_result - elif isinstance(return_result, BaseException): - raise return_result - else: - return return_result - - -def sync_wrapper(func, obj=None): - """Given a function, make so can be called in blocking contexts - - Leave obj=None if defining within a class. Pass the instance if attaching - as an attribute of the instance. - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - self = obj or args[0] - return sync(self.loop, func, *args, **kwargs) - - return wrapper - - -def async_gen_wrapper(func, obj=None): - """Given a async generator, make so can be called in blocking contexts""" - - @functools.wraps(func) - def wrapper(*args, **kwargs): - self = obj or args[0] - gen = func(*args, **kwargs) - while True: - try: - yield sync(self.loop, gen.__anext__) - except StopAsyncIteration: - break - - return wrapper - - -def get_loop(): - """Create or return the default fsspec IO loop - - The loop will be running on a separate thread. - """ - if loop[0] is None: - with get_lock(): - # repeat the check just in case the loop got filled between the - # previous two calls from another thread - if loop[0] is None: - loop[0] = asyncio.new_event_loop() - th = threading.Thread(target=loop[0].run_forever, name="fsspecIO") - th.daemon = True - th.start() - iothread[0] = th - return loop[0] - - -def reset_after_fork(): - global lock - loop[0] = None - iothread[0] = None - lock = None - - -if hasattr(os, "register_at_fork"): - # should be posix; this will do nothing for spawn or forkserver subprocesses - os.register_at_fork(after_in_child=reset_after_fork) - - -if TYPE_CHECKING: - import resource - - ResourceError = resource.error -else: - try: - import resource - except ImportError: - resource = None - ResourceError = OSError - else: - ResourceError = getattr(resource, "error", OSError) - -_DEFAULT_BATCH_SIZE = 128 -_NOFILES_DEFAULT_BATCH_SIZE = 1280 - - -def _get_batch_size(nofiles=False): - from fsspec.config import conf - - if nofiles: - if "nofiles_gather_batch_size" in conf: - return conf["nofiles_gather_batch_size"] - else: - if "gather_batch_size" in conf: - return conf["gather_batch_size"] - if nofiles: - return _NOFILES_DEFAULT_BATCH_SIZE - if resource is None: - return _DEFAULT_BATCH_SIZE - - try: - soft_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE) - except (ImportError, ValueError, ResourceError): - return _DEFAULT_BATCH_SIZE - - if soft_limit == resource.RLIM_INFINITY: - return -1 - else: - return soft_limit // 8 - - -def running_async() -> bool: - """Being executed by an event loop?""" - try: - asyncio.get_running_loop() - return True - except RuntimeError: - return False - - -async def _run_coros_in_chunks( - coros, - batch_size=None, - callback=DEFAULT_CALLBACK, - timeout=None, - return_exceptions=False, - nofiles=False, -): - """Run the given coroutines in chunks. - - Parameters - ---------- - coros: list of coroutines to run - batch_size: int or None - Number of coroutines to submit/wait on simultaneously. - If -1, then it will not be any throttling. If - None, it will be inferred from _get_batch_size() - callback: fsspec.callbacks.Callback instance - Gets a relative_update when each coroutine completes - timeout: number or None - If given, each coroutine times out after this time. Note that, since - there are multiple batches, the total run time of this function will in - general be longer - return_exceptions: bool - Same meaning as in asyncio.gather - nofiles: bool - If inferring the batch_size, does this operation involve local files? - If yes, you normally expect smaller batches. - """ - - if batch_size is None: - batch_size = _get_batch_size(nofiles=nofiles) - - if batch_size == -1: - batch_size = len(coros) - elif batch_size <= 0: - raise ValueError - - async def _run_coro(coro, i): - try: - return await asyncio.wait_for(coro, timeout=timeout), i - except Exception as e: - if not return_exceptions: - raise - return e, i - finally: - callback.relative_update(1) - - i = 0 - n = len(coros) - results = [None] * n - pending = set() - - while pending or i < n: - while len(pending) < batch_size and i < n: - pending.add(asyncio.ensure_future(_run_coro(coros[i], i))) - i += 1 - - if not pending: - break - - done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) - first_exc = None - while done: - task = done.pop() - try: - result, k = await task - results[k] = result - except Exception as exc: - if first_exc is None: - first_exc = exc - - if first_exc is not None: - for task in pending: - task.cancel() - if pending: - await asyncio.gather(*pending, return_exceptions=True) - raise first_exc - - return results - - -# these methods should be implemented as async by any async-able backend -async_methods = [ - "_ls", - "_cat_file", - "_get_file", - "_put_file", - "_rm_file", - "_cp_file", - "_pipe_file", - "_expand_path", - "_info", - "_isfile", - "_isdir", - "_exists", - "_walk", - "_glob", - "_find", - "_du", - "_size", - "_mkdir", - "_makedirs", -] - - -class AsyncFileSystem(AbstractFileSystem): - """Async file operations, default implementations - - Passes bulk operations to asyncio.gather for concurrent operation. - - Implementations that have concurrent batch operations and/or async methods - should inherit from this class instead of AbstractFileSystem. Docstrings are - copied from the un-underscored method in AbstractFileSystem, if not given. - """ - - # note that methods do not have docstring here; they will be copied - # for _* methods and inferred for overridden methods. - - async_impl = True - mirror_sync_methods = True - disable_throttling = False - - def __init__(self, *args, asynchronous=False, loop=None, batch_size=None, **kwargs): - self.asynchronous = asynchronous - self._pid = os.getpid() - if not asynchronous: - self._loop = loop or get_loop() - else: - self._loop = None - self.batch_size = batch_size - super().__init__(*args, **kwargs) - - @property - def loop(self): - if self._pid != os.getpid(): - raise RuntimeError("This class is not fork-safe") - return self._loop - - async def _rm_file(self, path, **kwargs): - if ( - inspect.iscoroutinefunction(self._rm) - and type(self)._rm is not AsyncFileSystem._rm - ): - return await self._rm(path, recursive=False, batch_size=1, **kwargs) - raise NotImplementedError - - async def _rm(self, path, recursive=False, batch_size=None, **kwargs): - # TODO: implement on_error - batch_size = batch_size or self.batch_size - path = await self._expand_path(path, recursive=recursive) - return await _run_coros_in_chunks( - [self._rm_file(p, **kwargs) for p in reversed(path)], - batch_size=batch_size, - nofiles=True, - ) - - async def _cp_file(self, path1, path2, **kwargs): - raise NotImplementedError - - async def _mv_file(self, path1, path2): - await self._cp_file(path1, path2) - await self._rm_file(path1) - - async def _copy( - self, - path1, - path2, - recursive=False, - on_error=None, - maxdepth=None, - batch_size=None, - **kwargs, - ): - if on_error is None and recursive: - on_error = "ignore" - elif on_error is None: - on_error = "raise" - - if isinstance(path1, list) and isinstance(path2, list): - # No need to expand paths when both source and destination - # are provided as lists - paths1 = path1 - paths2 = path2 - else: - source_is_str = isinstance(path1, str) - paths1 = await self._expand_path( - path1, maxdepth=maxdepth, recursive=recursive - ) - if source_is_str and (not recursive or maxdepth is not None): - # Non-recursive glob does not copy directories - paths1 = [ - p for p in paths1 if not (trailing_sep(p) or await self._isdir(p)) - ] - if not paths1: - return - - source_is_file = len(paths1) == 1 - dest_is_dir = isinstance(path2, str) and ( - trailing_sep(path2) or await self._isdir(path2) - ) - - exists = source_is_str and ( - (has_magic(path1) and source_is_file) - or (not has_magic(path1) and dest_is_dir and not trailing_sep(path1)) - ) - paths2 = other_paths( - paths1, - path2, - exists=exists, - flatten=not source_is_str, - ) - - batch_size = batch_size or self.batch_size - coros = [self._cp_file(p1, p2, **kwargs) for p1, p2 in zip(paths1, paths2)] - result = await _run_coros_in_chunks( - coros, batch_size=batch_size, return_exceptions=True, nofiles=True - ) - - for ex in filter(is_exception, result): - if on_error == "ignore" and isinstance(ex, FileNotFoundError): - continue - raise ex - - async def _pipe_file(self, path, value, mode="overwrite", **kwargs): - raise NotImplementedError - - async def _pipe(self, path, value=None, batch_size=None, **kwargs): - if isinstance(path, str): - path = {path: value} - batch_size = batch_size or self.batch_size - return await _run_coros_in_chunks( - [self._pipe_file(k, v, **kwargs) for k, v in path.items()], - batch_size=batch_size, - nofiles=True, - ) - - async def _process_limits(self, url, start, end): - """Helper for "Range"-based _cat_file""" - size = None - suff = False - if start is not None and start < 0: - # if start is negative and end None, end is the "suffix length" - if end is None: - end = -start - start = "" - suff = True - else: - size = size or (await self._info(url))["size"] - start = size + start - elif start is None: - start = 0 - if not suff: - if end is not None and end < 0: - if start is not None: - size = size or (await self._info(url))["size"] - end = size + end - elif end is None: - end = "" - if isinstance(end, numbers.Integral): - end -= 1 # bytes range is inclusive - return f"bytes={start}-{end}" - - async def _cat_file(self, path, start=None, end=None, **kwargs): - raise NotImplementedError - - async def _cat( - self, path, recursive=False, on_error="raise", batch_size=None, **kwargs - ): - paths = await self._expand_path(path, recursive=recursive) - coros = [self._cat_file(path, **kwargs) for path in paths] - batch_size = batch_size or self.batch_size - out = await _run_coros_in_chunks( - coros, batch_size=batch_size, nofiles=True, return_exceptions=True - ) - if on_error == "raise": - ex = next(filter(is_exception, out), False) - if ex: - raise ex - if ( - len(paths) > 1 - or isinstance(path, list) - or paths[0] != self._strip_protocol(path) - ): - return { - k: v - for k, v in zip(paths, out) - if on_error != "omit" or not is_exception(v) - } - else: - return out[0] - - async def _cat_ranges( - self, - paths, - starts, - ends, - max_gap=None, - batch_size=None, - on_error="return", - **kwargs, - ): - """Get the contents of byte ranges from one or more files - - Parameters - ---------- - paths: list - A list of of filepaths on this filesystems - starts, ends: int or list - Bytes limits of the read. If using a single int, the same value will be - used to read all the specified files. - on_error: "return" or "raise" - If "return" (default), any per-range exception is placed in the output - list at the corresponding position. Otherwise the first such exception - is raised. Matches ``AbstractFileSystem.cat_ranges``. - """ - if max_gap is not None: - # use utils.merge_offset_ranges - raise NotImplementedError - if not isinstance(paths, list): - raise TypeError - if not isinstance(starts, Iterable): - starts = [starts] * len(paths) - if not isinstance(ends, Iterable): - ends = [ends] * len(paths) - if len(starts) != len(paths) or len(ends) != len(paths): - raise ValueError - coros = [ - self._cat_file(p, start=s, end=e, **kwargs) - for p, s, e in zip(paths, starts, ends) - ] - batch_size = batch_size or self.batch_size - out = await _run_coros_in_chunks( - coros, batch_size=batch_size, nofiles=True, return_exceptions=True - ) - if on_error != "return": - ex = next(filter(is_exception, out), None) - if ex is not None: - raise ex - return out - - async def _put_file(self, lpath, rpath, mode="overwrite", **kwargs): - raise NotImplementedError - - async def _put( - self, - lpath, - rpath, - recursive=False, - callback=DEFAULT_CALLBACK, - batch_size=None, - maxdepth=None, - **kwargs, - ): - """Copy file(s) from local. - - Copies a specific file or tree of files (if recursive=True). If rpath - ends with a "/", it will be assumed to be a directory, and target files - will go within. - - The put_file method will be called concurrently on a batch of files. The - batch_size option can configure the amount of futures that can be executed - at the same time. If it is -1, then all the files will be uploaded concurrently. - The default can be set for this instance by passing "batch_size" in the - constructor, or for all instances by setting the "gather_batch_size" key - in ``fsspec.config.conf``, falling back to 1/8th of the system limit . - """ - if isinstance(lpath, list) and isinstance(rpath, list): - # No need to expand paths when both source and destination - # are provided as lists - rpaths = rpath - lpaths = lpath - else: - source_is_str = isinstance(lpath, str) - if source_is_str: - lpath = make_path_posix(lpath) - fs = LocalFileSystem() - lpaths = fs.expand_path(lpath, recursive=recursive, maxdepth=maxdepth) - if source_is_str and (not recursive or maxdepth is not None): - # Non-recursive glob does not copy directories - lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))] - if not lpaths: - return - - source_is_file = len(lpaths) == 1 - dest_is_dir = isinstance(rpath, str) and ( - trailing_sep(rpath) or await self._isdir(rpath) - ) - - rpath = self._strip_protocol(rpath) - exists = source_is_str and ( - (has_magic(lpath) and source_is_file) - or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath)) - ) - rpaths = other_paths( - lpaths, - rpath, - exists=exists, - flatten=not source_is_str, - ) - - is_dir = {l: os.path.isdir(l) for l in lpaths} - rdirs = [r for l, r in zip(lpaths, rpaths) if is_dir[l]] - file_pairs = [(l, r) for l, r in zip(lpaths, rpaths) if not is_dir[l]] - - await asyncio.gather(*[self._makedirs(d, exist_ok=True) for d in rdirs]) - batch_size = batch_size or self.batch_size - - coros = [] - callback.set_size(len(file_pairs)) - for lfile, rfile in file_pairs: - put_file = callback.branch_coro(self._put_file) - coros.append(put_file(lfile, rfile, **kwargs)) - - return await _run_coros_in_chunks( - coros, batch_size=batch_size, callback=callback - ) - - async def _get_file(self, rpath, lpath, **kwargs): - raise NotImplementedError - - async def _get( - self, - rpath, - lpath, - recursive=False, - callback=DEFAULT_CALLBACK, - maxdepth=None, - **kwargs, - ): - """Copy file(s) to local. - - Copies a specific file or tree of files (if recursive=True). If lpath - ends with a "/", it will be assumed to be a directory, and target files - will go within. Can submit a list of paths, which may be glob-patterns - and will be expanded. - - The get_file method will be called concurrently on a batch of files. The - batch_size option can configure the amount of futures that can be executed - at the same time. If it is -1, then all the files will be uploaded concurrently. - The default can be set for this instance by passing "batch_size" in the - constructor, or for all instances by setting the "gather_batch_size" key - in ``fsspec.config.conf``, falling back to 1/8th of the system limit . - """ - if isinstance(lpath, list) and isinstance(rpath, list): - # No need to expand paths when both source and destination - # are provided as lists - rpaths = rpath - lpaths = lpath - else: - source_is_str = isinstance(rpath, str) - # First check for rpath trailing slash as _strip_protocol removes it. - source_not_trailing_sep = source_is_str and not trailing_sep(rpath) - rpath = self._strip_protocol(rpath) - rpaths = await self._expand_path( - rpath, recursive=recursive, maxdepth=maxdepth - ) - if source_is_str and (not recursive or maxdepth is not None): - # Non-recursive glob does not copy directories - rpaths = [ - p for p in rpaths if not (trailing_sep(p) or await self._isdir(p)) - ] - if not rpaths: - return - - lpath = make_path_posix(lpath) - source_is_file = len(rpaths) == 1 - dest_is_dir = isinstance(lpath, str) and ( - trailing_sep(lpath) or LocalFileSystem().isdir(lpath) - ) - - exists = source_is_str and ( - (has_magic(rpath) and source_is_file) - or (not has_magic(rpath) and dest_is_dir and source_not_trailing_sep) - ) - lpaths = other_paths( - rpaths, - lpath, - exists=exists, - flatten=not source_is_str, - ) - - [os.makedirs(os.path.dirname(lp), exist_ok=True) for lp in lpaths] - batch_size = kwargs.pop("batch_size", self.batch_size) - - coros = [] - callback.set_size(len(lpaths)) - for lpath, rpath in zip(lpaths, rpaths): - get_file = callback.branch_coro(self._get_file) - coros.append(get_file(rpath, lpath, **kwargs)) - return await _run_coros_in_chunks( - coros, batch_size=batch_size, callback=callback - ) - - async def _isfile(self, path): - try: - return (await self._info(path))["type"] == "file" - except: # noqa: E722 - return False - - async def _isdir(self, path): - try: - return (await self._info(path))["type"] == "directory" - except OSError: - return False - - async def _size(self, path): - return (await self._info(path)).get("size", None) - - async def _sizes(self, paths, batch_size=None): - batch_size = batch_size or self.batch_size - return await _run_coros_in_chunks( - [self._size(p) for p in paths], batch_size=batch_size - ) - - async def _exists(self, path, **kwargs): - try: - await self._info(path, **kwargs) - return True - except FileNotFoundError: - return False - - async def _info(self, path, **kwargs): - raise NotImplementedError - - async def _ls(self, path, detail=True, **kwargs): - raise NotImplementedError - - async def _walk(self, path, maxdepth=None, topdown=True, on_error="omit", **kwargs): - if maxdepth is not None and maxdepth < 1: - raise ValueError("maxdepth must be at least 1") - - path = self._strip_protocol(path) - full_dirs = {} - dirs = {} - files = {} - - detail = kwargs.pop("detail", False) - try: - listing = await self._ls(path, detail=True, **kwargs) - except (FileNotFoundError, OSError) as e: - if on_error == "raise": - raise - elif callable(on_error): - on_error(e) - if detail: - yield path, {}, {} - else: - yield path, [], [] - return - - for info in listing: - # each info name must be at least [path]/part , but here - # we check also for names like [path]/part/ - pathname = info["name"].rstrip("/") - name = pathname.rsplit("/", 1)[-1] - if info["type"] == "directory" and pathname != path: - # do not include "self" path - full_dirs[name] = pathname - dirs[name] = info - elif pathname == path: - # file-like with same name as give path - files[""] = info - else: - files[name] = info - - if not detail: - dirs = list(dirs) - files = list(files) - - if topdown: - # Yield before recursion if walking top down - yield path, dirs, files - - if maxdepth is not None: - maxdepth -= 1 - if maxdepth < 1: - if not topdown: - yield path, dirs, files - return - - for d in dirs: - async for _ in self._walk( - full_dirs[d], - maxdepth=maxdepth, - detail=detail, - topdown=topdown, - **kwargs, - ): - yield _ - - if not topdown: - # Yield after recursion if walking bottom up - yield path, dirs, files - - async def _glob(self, path, maxdepth=None, **kwargs): - if maxdepth is not None and maxdepth < 1: - raise ValueError("maxdepth must be at least 1") - - import re - - seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,) - ends_with_sep = path.endswith(seps) # _strip_protocol strips trailing slash - path = self._strip_protocol(path) - append_slash_to_dirname = ends_with_sep or path.endswith( - tuple(sep + "**" for sep in seps) - ) - idx_star = path.find("*") if path.find("*") >= 0 else len(path) - idx_qmark = path.find("?") if path.find("?") >= 0 else len(path) - idx_brace = path.find("[") if path.find("[") >= 0 else len(path) - - min_idx = min(idx_star, idx_qmark, idx_brace) - - detail = kwargs.pop("detail", False) - withdirs = kwargs.pop("withdirs", True) - - if not has_magic(path): - if await self._exists(path, **kwargs): - if not detail: - return [path] - else: - return {path: await self._info(path, **kwargs)} - else: - if not detail: - return [] # glob of non-existent returns empty - else: - return {} - elif "/" in path[:min_idx]: - first_wildcard_idx = min_idx - min_idx = path[:min_idx].rindex("/") - root = path[ - : min_idx + 1 - ] # everything up to the last / before the first wildcard - prefix = path[ - min_idx + 1 : first_wildcard_idx - ] # stem between last "/" and first wildcard - depth = path[min_idx + 1 :].count("/") + 1 - else: - root = "" - prefix = path[:min_idx] # stem up to the first wildcard - depth = path[min_idx + 1 :].count("/") + 1 - - if "**" in path: - if maxdepth is not None: - idx_double_stars = path.find("**") - depth_double_stars = path[idx_double_stars:].count("/") + 1 - depth = depth - depth_double_stars + maxdepth - else: - depth = None - - # Pass the filename stem as prefix= so backends that support it such as - # gcsfs, s3fs and adlfs can filter server-side up to the first wildcard. - if prefix: - kwargs["prefix"] = prefix - allpaths = await self._find( - root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs - ) - - pattern = glob_translate(path + ("/" if ends_with_sep else "")) - pattern = re.compile(pattern) - - out = { - p: info - for p, info in sorted(allpaths.items()) - if pattern.match( - p + "/" - if append_slash_to_dirname and info["type"] == "directory" - else p - ) - } - - if detail: - return out - else: - return list(out) - - async def _du(self, path, total=True, maxdepth=None, **kwargs): - sizes = {} - # async for? - for f in await self._find(path, maxdepth=maxdepth, **kwargs): - info = await self._info(f) - sizes[info["name"]] = info["size"] - if total: - return sum(sizes.values()) - else: - return sizes - - async def _find(self, path, maxdepth=None, withdirs=False, **kwargs): - path = self._strip_protocol(path) - out = {} - detail = kwargs.pop("detail", False) - - # Add the root directory if withdirs is requested - # This is needed for posix glob compliance - if withdirs and path != "" and await self._isdir(path): - out[path] = await self._info(path) - - # async for? - async for _, dirs, files in self._walk(path, maxdepth, detail=True, **kwargs): - if withdirs: - files.update(dirs) - out.update({info["name"]: info for name, info in files.items()}) - if not out and (await self._isfile(path)): - # walk works on directories, but find should also return [path] - # when path happens to be a file - out[path] = {} - names = sorted(out) - if not detail: - return names - else: - return {name: out[name] for name in names} - - async def _expand_path( - self, path, recursive=False, maxdepth=None, assume_literal=False - ): - if maxdepth is not None and maxdepth < 1: - raise ValueError("maxdepth must be at least 1") - - if isinstance(path, str): - out = await self._expand_path([path], recursive, maxdepth) - else: - out = set() - path = [self._strip_protocol(p) for p in path] - for p in path: # can gather here - if not assume_literal and has_magic(p): - bit = set(await self._glob(p, maxdepth=maxdepth)) - out |= bit - if recursive: - # glob call above expanded one depth so if maxdepth is defined - # then decrement it in expand_path call below. If it is zero - # after decrementing then avoid expand_path call. - if maxdepth is not None and maxdepth <= 1: - continue - out |= set( - await self._expand_path( - list(bit), - recursive=recursive, - maxdepth=maxdepth - 1 if maxdepth is not None else None, - assume_literal=True, - ) - ) - continue - elif recursive: - rec = set(await self._find(p, maxdepth=maxdepth, withdirs=True)) - out |= rec - if p not in out and (recursive is False or (await self._exists(p))): - # should only check once, for the root - out.add(p) - if not out: - raise FileNotFoundError(path) - return sorted(out) - - async def _mkdir(self, path, create_parents=True, **kwargs): - pass # not necessary to implement, may not have directories - - async def _makedirs(self, path, exist_ok=False): - pass # not necessary to implement, may not have directories - - async def open_async(self, path, mode="rb", **kwargs): - if "b" not in mode or kwargs.get("compression"): - raise ValueError - raise NotImplementedError - - -def mirror_sync_methods(obj): - """Populate sync and async methods for obj - - For each method will create a sync version if the name refers to an async method - (coroutine) and there is no override in the child class; will create an async - method for the corresponding sync method if there is no implementation. - - Uses the methods specified in - - async_methods: the set that an implementation is expected to provide - - default_async_methods: that can be derived from their sync version in - AbstractFileSystem - - AsyncFileSystem: async-specific default coroutines - """ - from fsspec import AbstractFileSystem - - for method in set(async_methods + dir(AsyncFileSystem)): - if not method.startswith("_"): - continue - smethod = method[1:] - if private.match(method): - isco = inspect.iscoroutinefunction(getattr(obj, method, None)) - unsync = getattr(getattr(obj, smethod, False), "__func__", None) - is_default = unsync is getattr(AbstractFileSystem, smethod, "") - if isco and is_default: - mth = sync_wrapper(getattr(obj, method), obj=obj) - elif inspect.isasyncgenfunction(getattr(obj, method, None)) and is_default: - mth = async_gen_wrapper(getattr(obj, method), obj=obj) - else: - continue - setattr(obj, smethod, mth) - if not mth.__doc__: - mth.__doc__ = getattr( - getattr(AbstractFileSystem, smethod, None), "__doc__", "" - ) - - -class FSSpecCoroutineCancel(Exception): - pass - - -def _dump_running_tasks( - printout=True, cancel=True, exc=FSSpecCoroutineCancel, with_task=False -): - import traceback - - tasks = [t for t in asyncio.tasks.all_tasks(loop[0]) if not t.done()] - if printout: - [task.print_stack() for task in tasks] - out = [ - { - "locals": task._coro.cr_frame.f_locals, - "file": task._coro.cr_frame.f_code.co_filename, - "firstline": task._coro.cr_frame.f_code.co_firstlineno, - "linelo": task._coro.cr_frame.f_lineno, - "stack": traceback.format_stack(task._coro.cr_frame), - "task": task if with_task else None, - } - for task in tasks - ] - if cancel: - for t in tasks: - cbs = t._callbacks - t.cancel() - asyncio.futures.Future.set_exception(t, exc) - asyncio.futures.Future.cancel(t) - [cb[0](t) for cb in cbs] # cancels any dependent concurrent.futures - try: - t._coro.throw(exc) # exits coro, unless explicitly handled - except exc: - pass - return out - - -class AbstractAsyncStreamedFile(AbstractBufferedFile): - # no read buffering, and always auto-commit - # TODO: readahead might still be useful here, but needs async version - - async def read(self, length=-1): - """ - Return data from cache, or fetch pieces as necessary - - Parameters - ---------- - length: int (-1) - Number of bytes to read; if <0, all remaining bytes. - """ - length = -1 if length is None else int(length) - if self.mode != "rb": - raise ValueError("File not in read mode") - if length < 0: - length = self.size - self.loc - if self.closed: - raise ValueError("I/O operation on closed file.") - if length == 0: - # don't even bother calling fetch - return b"" - out = await self._fetch_range(self.loc, self.loc + length) - self.loc += len(out) - return out - - async def write(self, data): - """ - Write data to buffer. - - Buffer only sent on flush() or if buffer is greater than - or equal to blocksize. - - Parameters - ---------- - data: bytes - Set of bytes to be written. - """ - if self.mode not in {"wb", "ab"}: - raise ValueError("File not in write mode") - if self.closed: - raise ValueError("I/O operation on closed file.") - if self.forced: - raise ValueError("This file has been force-flushed, can only close") - out = self.buffer.write(data) - self.loc += out - if self.buffer.tell() >= self.blocksize: - await self.flush() - return out - - async def close(self): - """Close file - - Finalizes writes, discards cache - """ - if getattr(self, "_unclosable", False): - return - if self.closed: - return - if self.mode == "rb": - self.cache = None - else: - if not self.forced: - await self.flush(force=True) - - if self.fs is not None: - self.fs.invalidate_cache(self.path) - self.fs.invalidate_cache(self.fs._parent(self.path)) - - self.closed = True - - async def flush(self, force=False): - if self.closed: - raise ValueError("Flush on closed file") - if force and self.forced: - raise ValueError("Force flush cannot be called more than once") - if force: - self.forced = True - - if self.mode not in {"wb", "ab"}: - # no-op to flush on read-mode - return - - if not force and self.buffer.tell() < self.blocksize: - # Defer write on small block - return - - if self.offset is None: - # Initialize a multipart upload - self.offset = 0 - try: - await self._initiate_upload() - except: - self.closed = True - raise - - if await self._upload_chunk(final=force) is not False: - self.offset += self.buffer.seek(0, 2) - self.buffer = io.BytesIO() - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.close() - - async def _fetch_range(self, start, end): - raise NotImplementedError - - async def _initiate_upload(self): - pass - - async def _upload_chunk(self, final=False): - raise NotImplementedError diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/caching.py b/bundle/python-cpu/Lib/site-packages/fsspec/caching.py deleted file mode 100644 index 3cfb343a6dd092587ba4d0c10b41a6b30d84a381..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/caching.py +++ /dev/null @@ -1,1025 +0,0 @@ -from __future__ import annotations - -import collections -import functools -import logging -import math -import os -import threading -from collections import OrderedDict -from collections.abc import Callable -from concurrent.futures import Future, ThreadPoolExecutor -from itertools import groupby -from operator import itemgetter -from typing import TYPE_CHECKING, Any, ClassVar, Generic, NamedTuple, TypeVar - -if TYPE_CHECKING: - import mmap - - from typing_extensions import ParamSpec - - P = ParamSpec("P") -else: - P = TypeVar("P") - -T = TypeVar("T") - - -logger = logging.getLogger("fsspec.caching") - -Fetcher = Callable[[int, int], bytes] # Maps (start, end) to bytes -MultiFetcher = Callable[[list[int, int]], bytes] # Maps [(start, end)] to bytes - - -class BaseCache: - """Pass-though cache: doesn't keep anything, calls every time - - Acts as base class for other cachers - - Parameters - ---------- - blocksize: int - How far to read ahead in numbers of bytes - fetcher: func - Function of the form f(start, end) which gets bytes from remote as - specified - size: int - How big this file is - """ - - name: ClassVar[str] = "none" - - def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: - self.blocksize = blocksize - self.nblocks = 0 - self.fetcher = fetcher - self.size = size - self.hit_count = 0 - self.miss_count = 0 - # the bytes that we actually requested - self.total_requested_bytes = 0 - - def _fetch(self, start: int | None, stop: int | None) -> bytes: - if start is None: - start = 0 - if stop is None: - stop = self.size - if start >= self.size or start >= stop: - return b"" - return self.fetcher(start, stop) - - def _reset_stats(self) -> None: - """Reset hit and miss counts for a more ganular report e.g. by file.""" - self.hit_count = 0 - self.miss_count = 0 - self.total_requested_bytes = 0 - - def _log_stats(self) -> str: - """Return a formatted string of the cache statistics.""" - if self.hit_count == 0 and self.miss_count == 0: - # a cache that does nothing, this is for logs only - return "" - return f" , {self.name}: {self.hit_count} hits, {self.miss_count} misses, {self.total_requested_bytes} total requested bytes" - - def __repr__(self) -> str: - # TODO: use rich for better formatting - return f""" - <{self.__class__.__name__}: - block size : {self.blocksize} - block count : {self.nblocks} - file size : {self.size} - cache hits : {self.hit_count} - cache misses: {self.miss_count} - total requested bytes: {self.total_requested_bytes}> - """ - - -class MMapCache(BaseCache): - """memory-mapped sparse file cache - - Opens temporary file, which is filled blocks-wise when data is requested. - Ensure there is enough disc space in the temporary location. - - This cache method might only work on posix - - Parameters - ---------- - blocksize: int - How far to read ahead in numbers of bytes - fetcher: Fetcher - Function of the form f(start, end) which gets bytes from remote as - specified - size: int - How big this file is - location: str - Where to create the temporary file. If None, a temporary file is - created using tempfile.TemporaryFile(). - blocks: set[int] - Set of block numbers that have already been fetched. If None, an empty - set is created. - multi_fetcher: MultiFetcher - Function of the form f([(start, end)]) which gets bytes from remote - as specified. This function is used to fetch multiple blocks at once. - If not specified, the fetcher function is used instead. - """ - - name = "mmap" - - def __init__( - self, - blocksize: int, - fetcher: Fetcher, - size: int, - location: str | None = None, - blocks: set[int] | None = None, - multi_fetcher: MultiFetcher | None = None, - ) -> None: - super().__init__(blocksize, fetcher, size) - self.blocks = set() if blocks is None else blocks - self.location = location - self.multi_fetcher = multi_fetcher - self.cache = self._makefile() - - def _makefile(self) -> mmap.mmap | bytearray: - import mmap - import tempfile - - if self.size == 0: - return bytearray() - - # posix version - if self.location is None or not os.path.exists(self.location): - if self.location is None: - fd = tempfile.TemporaryFile() - self.blocks = set() - else: - fd = open(self.location, "wb+") - fd.seek(self.size - 1) - fd.write(b"1") - fd.flush() - else: - fd = open(self.location, "r+b") - - return mmap.mmap(fd.fileno(), self.size) - - def _fetch(self, start: int | None, end: int | None) -> bytes: - logger.debug(f"MMap cache fetching {start}-{end}") - if start is None: - start = 0 - if end is None: - end = self.size - if start >= self.size or start >= end: - return b"" - start_block = start // self.blocksize - end_block = end // self.blocksize - block_range = range(start_block, end_block + 1) - # Determine which blocks need to be fetched. This sequence is sorted by construction. - need = (i for i in block_range if i not in self.blocks) - # Count the number of blocks already cached - self.hit_count += sum(1 for i in block_range if i in self.blocks) - - ranges = [] - - # Consolidate needed blocks. - # Algorithm adapted from Python 2.x itertools documentation. - # We are grouping an enumerated sequence of blocks. By comparing when the difference - # between an ascending range (provided by enumerate) and the needed block numbers - # we can detect when the block number skips values. The key computes this difference. - # Whenever the difference changes, we know that we have previously cached block(s), - # and a new group is started. In other words, this algorithm neatly groups - # runs of consecutive block numbers so they can be fetched together. - for _, _blocks in groupby(enumerate(need), key=lambda x: x[0] - x[1]): - # Extract the blocks from the enumerated sequence - _blocks = tuple(map(itemgetter(1), _blocks)) - # Compute start of first block - sstart = _blocks[0] * self.blocksize - # Compute the end of the last block. Last block may not be full size. - send = min(_blocks[-1] * self.blocksize + self.blocksize, self.size) - - # Fetch bytes (could be multiple consecutive blocks) - self.total_requested_bytes += send - sstart - logger.debug( - f"MMap get blocks {_blocks[0]}-{_blocks[-1]} ({sstart}-{send})" - ) - ranges.append((sstart, send)) - - # Update set of cached blocks - self.blocks.update(_blocks) - # Update cache statistics with number of blocks we had to cache - self.miss_count += len(_blocks) - - if not ranges: - return self.cache[start:end] - - if self.multi_fetcher: - logger.debug(f"MMap get blocks {ranges}") - for idx, r in enumerate(self.multi_fetcher(ranges)): - sstart, send = ranges[idx] - logger.debug(f"MMap copy block ({sstart}-{send}") - self.cache[sstart:send] = r - else: - for sstart, send in ranges: - logger.debug(f"MMap get block ({sstart}-{send}") - self.cache[sstart:send] = self.fetcher(sstart, send) - - return self.cache[start:end] - - def __getstate__(self) -> dict[str, Any]: - state = self.__dict__.copy() - # Remove the unpicklable entries. - del state["cache"] - return state - - def __setstate__(self, state: dict[str, Any]) -> None: - # Restore instance attributes - self.__dict__.update(state) - self.cache = self._makefile() - - -class ReadAheadCache(BaseCache): - """Cache which reads only when we get beyond a block of data - - This is a much simpler version of BytesCache, and does not attempt to - fill holes in the cache or keep fragments alive. It is best suited to - many small reads in a sequential order (e.g., reading lines from a file). - """ - - name = "readahead" - - def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: - super().__init__(blocksize, fetcher, size) - self.cache = b"" - self.start = 0 - self.end = 0 - - def _fetch(self, start: int | None, end: int | None) -> bytes: - if start is None: - start = 0 - if end is None or end > self.size: - end = self.size - if start >= self.size or start >= end: - return b"" - l = end - start - if start >= self.start and end <= self.end: - # cache hit - self.hit_count += 1 - return self.cache[start - self.start : end - self.start] - elif self.start <= start < self.end: - # partial hit - self.miss_count += 1 - part = self.cache[start - self.start :] - l -= len(part) - start = self.end - else: - # miss - self.miss_count += 1 - part = b"" - end = min(self.size, end + self.blocksize) - self.total_requested_bytes += end - start - self.cache = self.fetcher(start, end) # new block replaces old - self.start = start - self.end = self.start + len(self.cache) - return part + self.cache[:l] - - -class FirstChunkCache(BaseCache): - """Caches the first block of a file only - - This may be useful for file types where the metadata is stored in the header, - but is randomly accessed. - """ - - name = "first" - - def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: - if blocksize > size: - # this will buffer the whole thing - blocksize = size - super().__init__(blocksize, fetcher, size) - self.cache: bytes | None = None - - def _fetch(self, start: int | None, end: int | None) -> bytes: - start = start or 0 - if start > self.size: - logger.debug("FirstChunkCache: requested start > file size") - return b"" - - if end is None: - end = self.size - end = min(end, self.size) - - if start < self.blocksize: - if self.cache is None: - self.miss_count += 1 - if end > self.blocksize: - self.total_requested_bytes += end - data = self.fetcher(0, end) - self.cache = data[: self.blocksize] - return data[start:] - self.cache = self.fetcher(0, self.blocksize) - self.total_requested_bytes += self.blocksize - part = self.cache[start:end] - if end > self.blocksize: - self.total_requested_bytes += end - self.blocksize - part += self.fetcher(self.blocksize, end) - self.hit_count += 1 - return part - else: - self.miss_count += 1 - self.total_requested_bytes += end - start - return self.fetcher(start, end) - - -class BlockCache(BaseCache): - """ - Cache holding memory as a set of blocks. - - Requests are only ever made ``blocksize`` at a time, and are - stored in an LRU cache. The least recently accessed block is - discarded when more than ``maxblocks`` are stored. - - Parameters - ---------- - blocksize : int - The number of bytes to store in each block. - Requests are only ever made for ``blocksize``, so this - should balance the overhead of making a request against - the granularity of the blocks. - fetcher : Callable - size : int - The total size of the file being cached. - maxblocks : int - The maximum number of blocks to cache for. The maximum memory - use for this cache is then ``blocksize * maxblocks``. - """ - - name = "blockcache" - - def __init__( - self, blocksize: int, fetcher: Fetcher, size: int, maxblocks: int = 32 - ) -> None: - super().__init__(blocksize, fetcher, size) - self.nblocks = math.ceil(size / blocksize) - self.maxblocks = maxblocks - self._fetch_block_cached = functools.lru_cache(maxblocks)(self._fetch_block) - - def cache_info(self): - """ - The statistics on the block cache. - - Returns - ------- - NamedTuple - Returned directly from the LRU Cache used internally. - """ - return self._fetch_block_cached.cache_info() - - def __getstate__(self) -> dict[str, Any]: - state = self.__dict__ - del state["_fetch_block_cached"] - return state - - def __setstate__(self, state: dict[str, Any]) -> None: - self.__dict__.update(state) - self._fetch_block_cached = functools.lru_cache(state["maxblocks"])( - self._fetch_block - ) - - def _fetch(self, start: int | None, end: int | None) -> bytes: - if start is None: - start = 0 - if end is None or end > self.size: - end = self.size - if start >= self.size or start >= end: - return b"" - - return self._read_cache( - start, end, start // self.blocksize, (end - 1) // self.blocksize - ) - - def _fetch_block(self, block_number: int) -> bytes: - """ - Fetch the block of data for `block_number`. - """ - if block_number > self.nblocks: - raise ValueError( - f"'block_number={block_number}' is greater than " - f"the number of blocks ({self.nblocks})" - ) - - start = block_number * self.blocksize - end = start + self.blocksize - self.total_requested_bytes += end - start - self.miss_count += 1 - logger.info("BlockCache fetching block %d", block_number) - block_contents = super()._fetch(start, end) - return block_contents - - def _read_cache( - self, start: int, end: int, start_block_number: int, end_block_number: int - ) -> bytes: - """ - Read from our block cache. - - Parameters - ---------- - start, end : int - The start and end byte positions. - start_block_number, end_block_number : int - The start and end block numbers. - """ - start_pos = start % self.blocksize - end_pos = end % self.blocksize - if end_pos == 0: - end_pos = self.blocksize - - self.hit_count += 1 - if start_block_number == end_block_number: - block: bytes = self._fetch_block_cached(start_block_number) - return block[start_pos:end_pos] - - else: - # read from the initial - out = [self._fetch_block_cached(start_block_number)[start_pos:]] - - # intermediate blocks - # Note: it'd be nice to combine these into one big request. However - # that doesn't play nicely with our LRU cache. - out.extend( - map( - self._fetch_block_cached, - range(start_block_number + 1, end_block_number), - ) - ) - - # final block - out.append(self._fetch_block_cached(end_block_number)[:end_pos]) - - return b"".join(out) - - -class BytesCache(BaseCache): - """Cache which holds data in a in-memory bytes object - - Implements read-ahead by the block size, for semi-random reads progressing - through the file. - - Parameters - ---------- - trim: bool - As we read more data, whether to discard the start of the buffer when - we are more than a blocksize ahead of it. - """ - - name: ClassVar[str] = "bytes" - - def __init__( - self, blocksize: int, fetcher: Fetcher, size: int, trim: bool = True - ) -> None: - super().__init__(blocksize, fetcher, size) - self.cache = b"" - self.start: int | None = None - self.end: int | None = None - self.trim = trim - - def _fetch(self, start: int | None, end: int | None) -> bytes: - # TODO: only set start/end after fetch, in case it fails? - # is this where retry logic might go? - if start is None: - start = 0 - if end is None: - end = self.size - if start >= self.size or start >= end: - return b"" - if ( - self.start is not None - and start >= self.start - and self.end is not None - and end < self.end - ): - # cache hit: we have all the required data - offset = start - self.start - self.hit_count += 1 - return self.cache[offset : offset + end - start] - - if self.blocksize: - bend = min(self.size, end + self.blocksize) - else: - bend = end - - if bend == start or start > self.size: - return b"" - - if (self.start is None or start < self.start) and ( - self.end is None or end > self.end - ): - # First read, or extending both before and after - self.total_requested_bytes += bend - start - self.miss_count += 1 - self.cache = self.fetcher(start, bend) - self.start = start - else: - assert self.start is not None - assert self.end is not None - self.miss_count += 1 - - if start < self.start: - if self.end is None or self.end - end > self.blocksize: - self.total_requested_bytes += bend - start - self.cache = self.fetcher(start, bend) - self.start = start - else: - self.total_requested_bytes += self.start - start - new = self.fetcher(start, self.start) - self.start = start - self.cache = new + self.cache - elif self.end is not None and bend > self.end: - if self.end > self.size: - pass - elif end - self.end > self.blocksize: - self.total_requested_bytes += bend - start - self.cache = self.fetcher(start, bend) - self.start = start - else: - self.total_requested_bytes += bend - self.end - new = self.fetcher(self.end, bend) - self.cache = self.cache + new - - self.end = self.start + len(self.cache) - offset = start - self.start - out = self.cache[offset : offset + end - start] - if self.trim: - num = (self.end - self.start) // (self.blocksize + 1) - if num > 1: - self.start += self.blocksize * num - self.cache = self.cache[self.blocksize * num :] - return out - - def __len__(self) -> int: - return len(self.cache) - - -class AllBytes(BaseCache): - """Cache entire contents of the file""" - - name: ClassVar[str] = "all" - - def __init__( - self, - blocksize: int | None = None, - fetcher: Fetcher | None = None, - size: int | None = None, - data: bytes | None = None, - ) -> None: - super().__init__(blocksize, fetcher, size) # type: ignore[arg-type] - if data is None: - self.miss_count += 1 - self.total_requested_bytes += self.size - data = self.fetcher(0, self.size) - self.data = data - - def _fetch(self, start: int | None, stop: int | None) -> bytes: - self.hit_count += 1 - return self.data[start:stop] - - -class KnownPartsOfAFile(BaseCache): - """ - Cache holding known file parts. - - Parameters - ---------- - blocksize: int - How far to read ahead in numbers of bytes - fetcher: func - Function of the form f(start, end) which gets bytes from remote as - specified - size: int - How big this file is - data: dict - A dictionary mapping explicit `(start, stop)` file-offset tuples - with known bytes. - strict: bool, default True - Whether to fetch reads that go beyond a known byte-range boundary. - If `False`, any read that ends outside a known part will be zero - padded. Note that zero padding will not be used for reads that - begin outside a known byte-range. - """ - - name: ClassVar[str] = "parts" - - def __init__( - self, - blocksize: int, - fetcher: Fetcher, - size: int, - data: dict[tuple[int, int], bytes] | None = None, - strict: bool = False, - **_: Any, - ): - super().__init__(blocksize, fetcher, size) - self.strict = strict - - # simple consolidation of contiguous blocks - if data: - old_offsets = sorted(data.keys()) - offsets = [old_offsets[0]] - blocks = [data.pop(old_offsets[0])] - for start, stop in old_offsets[1:]: - start0, stop0 = offsets[-1] - if start == stop0: - offsets[-1] = (start0, stop) - blocks[-1] += data.pop((start, stop)) - else: - offsets.append((start, stop)) - blocks.append(data.pop((start, stop))) - - self.data = dict(zip(offsets, blocks)) - else: - self.data = {} - - @property - def size(self): - return sum(_[1] - _[0] for _ in self.data) - - @size.setter - def size(self, value): - pass - - @property - def nblocks(self): - return len(self.data) - - @nblocks.setter - def nblocks(self, value): - pass - - def _fetch(self, start: int | None, stop: int | None) -> bytes: - logger.debug("Known parts request %s %s", start, stop) - if start is None: - start = 0 - if stop is None: - stop = self.size - self.total_requested_bytes += stop - start - out = b"" - started = False - loc_old = 0 - for loc0, loc1 in sorted(self.data): - if (loc0 <= start < loc1) and (loc0 <= stop <= loc1): - # entirely within the block - off = start - loc0 - self.hit_count += 1 - return self.data[(loc0, loc1)][off : off + stop - start] - if stop <= loc0: - break - if started and loc0 > loc_old: - # a gap where we need data - self.miss_count += 1 - if self.strict: - raise ValueError - out += b"\x00" * (loc0 - loc_old) - if loc0 <= start < loc1: - # found the start - self.hit_count += 1 - off = start - loc0 - out = self.data[(loc0, loc1)][off : off + stop - start] - started = True - elif start < loc0 and stop > loc1: - # the whole block - self.hit_count += 1 - out += self.data[(loc0, loc1)] - elif loc0 <= stop <= loc1: - # end block - self.hit_count += 1 - out = out + self.data[(loc0, loc1)][: stop - loc0] - return out - loc_old = loc1 - self.miss_count += 1 - if started and not self.strict: - out = out + b"\x00" * (stop - loc_old) - return out - raise ValueError - - -class UpdatableLRU(Generic[P, T]): - """ - Custom implementation of LRU cache that allows updating keys - - Used by BackgroundBlockCache - """ - - class CacheInfo(NamedTuple): - hits: int - misses: int - maxsize: int - currsize: int - - def __init__(self, func: Callable[P, T], max_size: int = 128) -> None: - self._cache: OrderedDict[Any, T] = collections.OrderedDict() - self._func = func - self._max_size = max_size - self._hits = 0 - self._misses = 0 - self._lock = threading.Lock() - - def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: - if kwargs: - raise TypeError(f"Got unexpected keyword argument {kwargs.keys()}") - with self._lock: - if args in self._cache: - self._cache.move_to_end(args) - self._hits += 1 - return self._cache[args] - - result = self._func(*args, **kwargs) - - with self._lock: - self._cache[args] = result - self._misses += 1 - if len(self._cache) > self._max_size: - self._cache.popitem(last=False) - - return result - - def is_key_cached(self, *args: Any) -> bool: - with self._lock: - return args in self._cache - - def add_key(self, result: T, *args: Any) -> None: - with self._lock: - self._cache[args] = result - if len(self._cache) > self._max_size: - self._cache.popitem(last=False) - - def cache_info(self) -> UpdatableLRU.CacheInfo: - with self._lock: - return self.CacheInfo( - maxsize=self._max_size, - currsize=len(self._cache), - hits=self._hits, - misses=self._misses, - ) - - -class BackgroundBlockCache(BaseCache): - """ - Cache holding memory as a set of blocks with pre-loading of - the next block in the background. - - Requests are only ever made ``blocksize`` at a time, and are - stored in an LRU cache. The least recently accessed block is - discarded when more than ``maxblocks`` are stored. If the - next block is not in cache, it is loaded in a separate thread - in non-blocking way. - - Parameters - ---------- - blocksize : int - The number of bytes to store in each block. - Requests are only ever made for ``blocksize``, so this - should balance the overhead of making a request against - the granularity of the blocks. - fetcher : Callable - size : int - The total size of the file being cached. - maxblocks : int - The maximum number of blocks to cache for. The maximum memory - use for this cache is then ``blocksize * maxblocks``. - """ - - name: ClassVar[str] = "background" - - def __init__( - self, blocksize: int, fetcher: Fetcher, size: int, maxblocks: int = 32 - ) -> None: - super().__init__(blocksize, fetcher, size) - self.nblocks = math.ceil(size / blocksize) - self.maxblocks = maxblocks - self._fetch_block_cached = UpdatableLRU(self._fetch_block, maxblocks) - - self._thread_executor = ThreadPoolExecutor(max_workers=1) - self._fetch_future_block_number: int | None = None - self._fetch_future: Future[bytes] | None = None - self._fetch_future_lock = threading.Lock() - self._closed = False - - def cache_info(self) -> UpdatableLRU.CacheInfo: - """ - The statistics on the block cache. - - Returns - ------- - NamedTuple - Returned directly from the LRU Cache used internally. - """ - return self._fetch_block_cached.cache_info() - - def close(self) -> None: - """Cancel pending work and shut down the background worker.""" - with self._fetch_future_lock: - if self._closed: - return - self._closed = True - future = self._fetch_future - self._fetch_future = None - self._fetch_future_block_number = None - - if future is not None: - future.cancel() - self._thread_executor.shutdown(wait=True, cancel_futures=True) - - # UpdatableLRU stores a bound method and otherwise forms a reference cycle. - del self._fetch_block_cached - - def __getstate__(self) -> dict[str, Any]: - state = self.__dict__ - del state["_fetch_block_cached"] - del state["_thread_executor"] - del state["_fetch_future_block_number"] - del state["_fetch_future"] - del state["_fetch_future_lock"] - return state - - def __setstate__(self, state) -> None: - self.__dict__.update(state) - self._fetch_block_cached = UpdatableLRU(self._fetch_block, state["maxblocks"]) - self._thread_executor = ThreadPoolExecutor(max_workers=1) - self._fetch_future_block_number = None - self._fetch_future = None - self._fetch_future_lock = threading.Lock() - self._closed = False - - def _fetch(self, start: int | None, end: int | None) -> bytes: - if start is None: - start = 0 - if end is None or end > self.size: - end = self.size - if start >= self.size or start >= end: - return b"" - - # byte position -> block numbers - start_block_number = start // self.blocksize - end_block_number = end // self.blocksize - - fetch_future_block_number = None - fetch_future = None - with self._fetch_future_lock: - # Background thread is running. Check we we can or must join it. - if self._fetch_future is not None: - assert self._fetch_future_block_number is not None - if self._fetch_future.done(): - logger.info("BlockCache joined background fetch without waiting.") - self._fetch_block_cached.add_key( - self._fetch_future.result(), self._fetch_future_block_number - ) - # Cleanup the fetch variables. Done with fetching the block. - self._fetch_future_block_number = None - self._fetch_future = None - else: - # Must join if we need the block for the current fetch - must_join = bool( - start_block_number - <= self._fetch_future_block_number - <= end_block_number - ) - if must_join: - # Copy to the local variables to release lock - # before waiting for result - fetch_future_block_number = self._fetch_future_block_number - fetch_future = self._fetch_future - - # Cleanup the fetch variables. Have a local copy. - self._fetch_future_block_number = None - self._fetch_future = None - - # Need to wait for the future for the current read - if fetch_future is not None: - logger.info("BlockCache waiting for background fetch.") - # Wait until result and put it in cache - self._fetch_block_cached.add_key( - fetch_future.result(), fetch_future_block_number - ) - - # these are cached, so safe to do multiple calls for the same start and end. - for block_number in range(start_block_number, end_block_number + 1): - self._fetch_block_cached(block_number) - - # fetch next block in the background if nothing is running in the background, - # the block is within file and it is not already cached - end_block_plus_1 = end_block_number + 1 - with self._fetch_future_lock: - if ( - self._fetch_future is None - and end_block_plus_1 <= self.nblocks - and not self._fetch_block_cached.is_key_cached(end_block_plus_1) - ): - self._fetch_future_block_number = end_block_plus_1 - self._fetch_future = self._thread_executor.submit( - self._fetch_block, end_block_plus_1, "async" - ) - - return self._read_cache( - start, - end, - start_block_number=start_block_number, - end_block_number=end_block_number, - ) - - def _fetch_block(self, block_number: int, log_info: str = "sync") -> bytes: - """ - Fetch the block of data for `block_number`. - """ - if block_number > self.nblocks: - raise ValueError( - f"'block_number={block_number}' is greater than " - f"the number of blocks ({self.nblocks})" - ) - - start = block_number * self.blocksize - end = start + self.blocksize - logger.info("BlockCache fetching block (%s) %d", log_info, block_number) - self.total_requested_bytes += end - start - self.miss_count += 1 - block_contents = super()._fetch(start, end) - return block_contents - - def _read_cache( - self, start: int, end: int, start_block_number: int, end_block_number: int - ) -> bytes: - """ - Read from our block cache. - - Parameters - ---------- - start, end : int - The start and end byte positions. - start_block_number, end_block_number : int - The start and end block numbers. - """ - start_pos = start % self.blocksize - end_pos = end % self.blocksize - - # kind of pointless to count this as a hit, but it is - self.hit_count += 1 - - if start_block_number == end_block_number: - block = self._fetch_block_cached(start_block_number) - return block[start_pos:end_pos] - - else: - # read from the initial - out = [self._fetch_block_cached(start_block_number)[start_pos:]] - - # intermediate blocks - # Note: it'd be nice to combine these into one big request. However - # that doesn't play nicely with our LRU cache. - out.extend( - map( - self._fetch_block_cached, - range(start_block_number + 1, end_block_number), - ) - ) - - # final block - out.append(self._fetch_block_cached(end_block_number)[:end_pos]) - - return b"".join(out) - - -caches: dict[str | None, type[BaseCache]] = { - # one custom case - None: BaseCache, -} - - -def register_cache(cls: type[BaseCache], clobber: bool = False) -> None: - """'Register' cache implementation. - - Parameters - ---------- - clobber: bool, optional - If set to True (default is False) - allow to overwrite existing - entry. - - Raises - ------ - ValueError - """ - name = cls.name - if not clobber and name in caches: - raise ValueError(f"Cache with name {name!r} is already known: {caches[name]}") - caches[name] = cls - - -for c in ( - BaseCache, - MMapCache, - BytesCache, - ReadAheadCache, - BlockCache, - FirstChunkCache, - AllBytes, - KnownPartsOfAFile, - BackgroundBlockCache, -): - register_cache(c) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/callbacks.py b/bundle/python-cpu/Lib/site-packages/fsspec/callbacks.py deleted file mode 100644 index 7ca99ca6ac3cd69b28bcd1550f6550e8e648c5fe..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/callbacks.py +++ /dev/null @@ -1,324 +0,0 @@ -from functools import wraps - - -class Callback: - """ - Base class and interface for callback mechanism - - This class can be used directly for monitoring file transfers by - providing ``callback=Callback(hooks=...)`` (see the ``hooks`` argument, - below), or subclassed for more specialised behaviour. - - Parameters - ---------- - size: int (optional) - Nominal quantity for the value that corresponds to a complete - transfer, e.g., total number of tiles or total number of - bytes - value: int (0) - Starting internal counter value - hooks: dict or None - A dict of named functions to be called on each update. The signature - of these must be ``f(size, value, **kwargs)`` - """ - - def __init__(self, size=None, value=0, hooks=None, **kwargs): - self.size = size - self.value = value - self.hooks = hooks or {} - self.kw = kwargs - - def __enter__(self): - return self - - def __exit__(self, *exc_args): - self.close() - - def close(self): - """Close callback.""" - - def branched(self, path_1, path_2, **kwargs): - """ - Return callback for child transfers - - If this callback is operating at a higher level, e.g., put, which may - trigger transfers that can also be monitored. The function returns a callback - that has to be passed to the child method, e.g., put_file, - as `callback=` argument. - - The implementation uses `callback.branch` for compatibility. - When implementing callbacks, it is recommended to override this function instead - of `branch` and avoid calling `super().branched(...)`. - - Prefer using this function over `branch`. - - Parameters - ---------- - path_1: str - Child's source path - path_2: str - Child's destination path - **kwargs: - Arbitrary keyword arguments - - Returns - ------- - callback: Callback - A callback instance to be passed to the child method - """ - self.branch(path_1, path_2, kwargs) - # mutate kwargs so that we can force the caller to pass "callback=" explicitly - return kwargs.pop("callback", DEFAULT_CALLBACK) - - def branch_coro(self, fn): - """ - Wraps a coroutine, and pass a new child callback to it. - """ - - @wraps(fn) - async def func(path1, path2: str, **kwargs): - with self.branched(path1, path2, **kwargs) as child: - return await fn(path1, path2, callback=child, **kwargs) - - return func - - def set_size(self, size): - """ - Set the internal maximum size attribute - - Usually called if not initially set at instantiation. Note that this - triggers a ``call()``. - - Parameters - ---------- - size: int - """ - self.size = size - self.call() - - def absolute_update(self, value): - """ - Set the internal value state - - Triggers ``call()`` - - Parameters - ---------- - value: int - """ - self.value = value - self.call() - - def relative_update(self, inc=1): - """ - Delta increment the internal counter - - Triggers ``call()`` - - Parameters - ---------- - inc: int - """ - self.value += inc - self.call() - - def call(self, hook_name=None, **kwargs): - """ - Execute hook(s) with current state - - Each function is passed the internal size and current value - - Parameters - ---------- - hook_name: str or None - If given, execute on this hook - kwargs: passed on to (all) hook(s) - """ - if not self.hooks: - return - kw = self.kw.copy() - kw.update(kwargs) - if hook_name: - if hook_name not in self.hooks: - return - return self.hooks[hook_name](self.size, self.value, **kw) - for hook in self.hooks.values() or []: - hook(self.size, self.value, **kw) - - def wrap(self, iterable): - """ - Wrap an iterable to call ``relative_update`` on each iterations - - Parameters - ---------- - iterable: Iterable - The iterable that is being wrapped - """ - for item in iterable: - self.relative_update() - yield item - - def branch(self, path_1, path_2, kwargs): - """ - Set callbacks for child transfers - - If this callback is operating at a higher level, e.g., put, which may - trigger transfers that can also be monitored. The passed kwargs are - to be *mutated* to add ``callback=``, if this class supports branching - to children. - - Parameters - ---------- - path_1: str - Child's source path - path_2: str - Child's destination path - kwargs: dict - arguments passed to child method, e.g., put_file. - - Returns - ------- - - """ - return None - - def no_op(self, *_, **__): - pass - - def __getattr__(self, item): - """ - If undefined methods are called on this class, nothing happens - """ - return self.no_op - - @classmethod - def as_callback(cls, maybe_callback=None): - """Transform callback=... into Callback instance - - For the special value of ``None``, return the global instance of - ``NoOpCallback``. This is an alternative to including - ``callback=DEFAULT_CALLBACK`` directly in a method signature. - """ - if maybe_callback is None: - return DEFAULT_CALLBACK - return maybe_callback - - -class NoOpCallback(Callback): - """ - This implementation of Callback does exactly nothing - """ - - def call(self, *args, **kwargs): - return None - - -class DotPrinterCallback(Callback): - """ - Simple example Callback implementation - - Almost identical to Callback with a hook that prints a char; here we - demonstrate how the outer layer may print "#" and the inner layer "." - """ - - def __init__(self, chr_to_print="#", **kwargs): - self.chr = chr_to_print - super().__init__(**kwargs) - - def branch(self, path_1, path_2, kwargs): - """Mutate kwargs to add new instance with different print char""" - kwargs["callback"] = DotPrinterCallback(".") - - def call(self, **kwargs): - """Just outputs a character""" - print(self.chr, end="") - - -class TqdmCallback(Callback): - """ - A callback to display a progress bar using tqdm - - Parameters - ---------- - tqdm_kwargs : dict, (optional) - Any argument accepted by the tqdm constructor. - See the `tqdm doc `_. - Will be forwarded to `tqdm_cls`. - tqdm_cls: (optional) - subclass of `tqdm.tqdm`. If not passed, it will default to `tqdm.tqdm`. - - Examples - -------- - >>> import fsspec - >>> from fsspec.callbacks import TqdmCallback - >>> fs = fsspec.filesystem("memory") - >>> path2distant_data = "/your-path" - >>> fs.upload( - ".", - path2distant_data, - recursive=True, - callback=TqdmCallback(), - ) - - You can forward args to tqdm using the ``tqdm_kwargs`` parameter. - - >>> fs.upload( - ".", - path2distant_data, - recursive=True, - callback=TqdmCallback(tqdm_kwargs={"desc": "Your tqdm description"}), - ) - - You can also customize the progress bar by passing a subclass of `tqdm`. - - .. code-block:: python - - class TqdmFormat(tqdm): - '''Provides a `total_time` format parameter''' - @property - def format_dict(self): - d = super().format_dict - total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1) - d.update(total_time=self.format_interval(total_time) + " in total") - return d - - >>> with TqdmCallback( - tqdm_kwargs={ - "desc": "desc", - "bar_format": "{total_time}: {percentage:.0f}%|{bar}{r_bar}", - }, - tqdm_cls=TqdmFormat, - ) as callback: - fs.upload(".", path2distant_data, recursive=True, callback=callback) - """ - - def __init__(self, tqdm_kwargs=None, *args, **kwargs): - try: - from tqdm import tqdm - - except ImportError as exce: - raise ImportError( - "Using TqdmCallback requires tqdm to be installed" - ) from exce - - self._tqdm_cls = kwargs.pop("tqdm_cls", tqdm) - self._tqdm_kwargs = tqdm_kwargs or {} - self.tqdm = None - super().__init__(*args, **kwargs) - - def call(self, *args, **kwargs): - if self.tqdm is None: - self.tqdm = self._tqdm_cls(total=self.size, **self._tqdm_kwargs) - self.tqdm.total = self.size - self.tqdm.update(self.value - self.tqdm.n) - - def close(self): - if self.tqdm is not None: - self.tqdm.close() - self.tqdm = None - - def __del__(self): - return self.close() - - -DEFAULT_CALLBACK = _DEFAULT_CALLBACK = NoOpCallback() diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/compression.py b/bundle/python-cpu/Lib/site-packages/fsspec/compression.py deleted file mode 100644 index 11c2e3d3f142d95186663fa5a747911e66832266..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/compression.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Helper functions for a standard streaming compression API""" - -import sys -from zipfile import ZipFile - -import fsspec.utils -from fsspec.spec import AbstractBufferedFile - - -def noop_file(file, mode, **kwargs): - return file - - -# TODO: files should also be available as contexts -# should be functions of the form func(infile, mode=, **kwargs) -> file-like -compr = {None: noop_file} - - -def register_compression(name, callback, extensions, force=False): - """Register an "inferable" file compression type. - - Registers transparent file compression type for use with fsspec.open. - Compression can be specified by name in open, or "infer"-ed for any files - ending with the given extensions. - - Args: - name: (str) The compression type name. Eg. "gzip". - callback: A callable of form (infile, mode, **kwargs) -> file-like. - Accepts an input file-like object, the target mode and kwargs. - Returns a wrapped file-like object. - extensions: (str, Iterable[str]) A file extension, or list of file - extensions for which to infer this compression scheme. Eg. "gz". - force: (bool) Force re-registration of compression type or extensions. - - Raises: - ValueError: If name or extensions already registered, and not force. - - """ - if isinstance(extensions, str): - extensions = [extensions] - - # Validate registration - if name in compr and not force: - raise ValueError(f"Duplicate compression registration: {name}") - - for ext in extensions: - if ext in fsspec.utils.compressions and not force: - raise ValueError(f"Duplicate compression file extension: {ext} ({name})") - - compr[name] = callback - - for ext in extensions: - fsspec.utils.compressions[ext] = name - - -def unzip(infile, mode="rb", filename=None, **kwargs): - if "r" not in mode: - filename = filename or "file" - z = ZipFile(infile, mode="w", **kwargs) - fo = z.open(filename, mode="w") - fo.close = lambda closer=fo.close: closer() or z.close() - return fo - z = ZipFile(infile) - if filename is None: - filename = z.namelist()[0] - return z.open(filename, mode="r", **kwargs) - - -register_compression("zip", unzip, "zip") - -try: - from bz2 import BZ2File -except ImportError: - pass -else: - register_compression("bz2", BZ2File, "bz2") - -try: # pragma: no cover - from isal import igzip - - def isal(infile, mode="rb", **kwargs): - return igzip.IGzipFile(fileobj=infile, mode=mode, **kwargs) - - register_compression("gzip", isal, "gz") -except ImportError: - from gzip import GzipFile - - register_compression( - "gzip", lambda f, **kwargs: GzipFile(fileobj=f, **kwargs), "gz" - ) - -try: - from lzma import LZMAFile - - register_compression("lzma", LZMAFile, "lzma") - register_compression("xz", LZMAFile, "xz") -except ImportError: - pass - -try: - import lzmaffi - - register_compression("lzma", lzmaffi.LZMAFile, "lzma", force=True) - register_compression("xz", lzmaffi.LZMAFile, "xz", force=True) -except ImportError: - pass - - -class SnappyFile(AbstractBufferedFile): - def __init__(self, infile, mode, **kwargs): - import snappy - - super().__init__( - fs=None, path="snappy", mode=mode.strip("b") + "b", size=999999999, **kwargs - ) - self.infile = infile - if "r" in mode: - self.codec = snappy.StreamDecompressor() - else: - self.codec = snappy.StreamCompressor() - - def _upload_chunk(self, final=False): - self.buffer.seek(0) - out = self.codec.add_chunk(self.buffer.read()) - self.infile.write(out) - return True - - def seek(self, loc, whence=0): - raise NotImplementedError("SnappyFile is not seekable") - - def seekable(self): - return False - - def _fetch_range(self, start, end): - """Get the specified set of bytes from remote""" - data = self.infile.read(end - start) - return self.codec.decompress(data) - - -try: - import snappy - - snappy.compress(b"") - # Snappy may use the .sz file extension, but this is not part of the - # standard implementation. - register_compression("snappy", SnappyFile, []) - -except (ImportError, NameError, AttributeError): - pass - -try: - import lz4.frame - - register_compression("lz4", lz4.frame.open, "lz4") -except ImportError: - pass - -try: - if sys.version_info >= (3, 14): - from compression import zstd - else: - from backports import zstd - - register_compression("zstd", zstd.ZstdFile, "zst") -except ImportError: - try: - import zstandard as zstd - - def zstandard_file(infile, mode="rb"): - if "r" in mode: - cctx = zstd.ZstdDecompressor() - return cctx.stream_reader(infile) - else: - cctx = zstd.ZstdCompressor(level=10) - return cctx.stream_writer(infile) - - register_compression("zstd", zstandard_file, "zst") - except ImportError: - pass - pass - - -def available_compressions(): - """Return a list of the implemented compressions.""" - return list(compr) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/config.py b/bundle/python-cpu/Lib/site-packages/fsspec/config.py deleted file mode 100644 index 19f68071ca026dc2bb11da70effb0c517de5380b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/config.py +++ /dev/null @@ -1,131 +0,0 @@ -from __future__ import annotations - -import configparser -import json -import os -import warnings -from typing import Any - -conf: dict[str, dict[str, Any]] = {} -default_conf_dir = os.path.join(os.path.expanduser("~"), ".config/fsspec") -conf_dir = os.environ.get("FSSPEC_CONFIG_DIR", default_conf_dir) - - -def set_conf_env(conf_dict, envdict=os.environ): - """Set config values from environment variables - - Looks for variables of the form ``FSSPEC_`` and - ``FSSPEC__``. For ``FSSPEC_`` the value is parsed - as a json dictionary and used to ``update`` the config of the - corresponding protocol. For ``FSSPEC__`` there is no - attempt to convert the string value, but the kwarg keys will be lower-cased. - - The ``FSSPEC__`` variables are applied after the - ``FSSPEC_`` ones. - - Parameters - ---------- - conf_dict : dict(str, dict) - This dict will be mutated - envdict : dict-like(str, str) - Source for the values - usually the real environment - """ - envdict = dict(envdict) - kwarg_keys = [] - for key in envdict: - if key.startswith("FSSPEC_") and len(key) > 7 and key[7] != "_": - try: - value = json.loads(envdict[key]) - envdict[key] = value - except json.decoder.JSONDecodeError: - value = envdict[key] - if key.count("_") > 1: - kwarg_keys.append(key) - continue - else: - if isinstance(value, dict): - _, proto = key.split("_", 1) - conf_dict.setdefault(proto.lower(), {}).update(value) - else: - warnings.warn( - f"Ignoring environment variable {key} due to not being a dict:" - f" {type(value)}" - ) - elif key.startswith("FSSPEC"): - warnings.warn( - f"Ignoring environment variable {key} due to having an unexpected name" - ) - - for key in kwarg_keys: - _, proto, kwarg = key.split("_", 2) - conf_dict.setdefault(proto.lower(), {})[kwarg.lower()] = envdict[key] - - -def set_conf_files(cdir, conf_dict): - """Set config values from files - - Scans for INI and JSON files in the given dictionary, and uses their - contents to set the config. In case of repeated values, later values - win. - - In the case of INI files, all values are strings, and these will not - be converted. - - Parameters - ---------- - cdir : str - Directory to search - conf_dict : dict(str, dict) - This dict will be mutated - """ - if not os.path.isdir(cdir): - return - allfiles = sorted(os.listdir(cdir)) - for fn in allfiles: - if fn.endswith(".ini"): - ini = configparser.ConfigParser() - ini.read(os.path.join(cdir, fn)) - for key in ini: - if key == "DEFAULT": - continue - conf_dict.setdefault(key, {}).update(dict(ini[key])) - if fn.endswith(".json"): - with open(os.path.join(cdir, fn)) as f: - js = json.load(f) - for key in js: - conf_dict.setdefault(key, {}).update(dict(js[key])) - - -def apply_config(cls, kwargs, conf_dict=None): - """Supply default values for kwargs when instantiating class - - Augments the passed kwargs, by finding entries in the config dict - which match the classes ``.protocol`` attribute (one or more str) - - Parameters - ---------- - cls : file system implementation - kwargs : dict - conf_dict : dict of dict - Typically this is the global configuration - - Returns - ------- - dict : the modified set of kwargs - """ - if conf_dict is None: - conf_dict = conf - protos = cls.protocol if isinstance(cls.protocol, (tuple, list)) else [cls.protocol] - kw = {} - for proto in protos: - # default kwargs from the current state of the config - if proto in conf_dict: - kw.update(conf_dict[proto]) - # explicit kwargs always win - kw.update(**kwargs) - kwargs = kw - return kwargs - - -set_conf_files(conf_dir, conf) -set_conf_env(conf) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/conftest.py b/bundle/python-cpu/Lib/site-packages/fsspec/conftest.py deleted file mode 100644 index f05eb5c30d42b0c1c5cc432f9c217d8f0e01f412..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/conftest.py +++ /dev/null @@ -1,125 +0,0 @@ -import os -import shutil -import subprocess -import sys -import time -from collections import deque -from collections.abc import Generator, Sequence - -import pytest - -import fsspec - - -@pytest.fixture() -def m(): - """ - Fixture providing a memory filesystem. - """ - m = fsspec.filesystem("memory") - m.store.clear() - m.pseudo_dirs.clear() - m.pseudo_dirs.append("") - try: - yield m - finally: - m.store.clear() - m.pseudo_dirs.clear() - m.pseudo_dirs.append("") - - -class InstanceCacheInspector: - """ - Helper class to inspect instance caches of filesystem classes in tests. - """ - - def clear(self) -> None: - """ - Clear instance caches of all currently imported filesystem classes. - """ - classes = deque([fsspec.spec.AbstractFileSystem]) - while classes: - cls = classes.popleft() - cls.clear_instance_cache() - classes.extend(cls.__subclasses__()) - - def gather_counts(self, *, omit_zero: bool = True) -> dict[str, int]: - """ - Gather counts of filesystem instances in the instance caches - of all currently imported filesystem classes. - - Parameters - ---------- - omit_zero: - Whether to omit instance types with no cached instances. - """ - out: dict[str, int] = {} - classes = deque([fsspec.spec.AbstractFileSystem]) - while classes: - cls = classes.popleft() - count = len(cls._cache) # there is no public interface for the cache - # note: skip intermediate AbstractFileSystem subclasses - # if they proxy the protocol attribute via a property. - if isinstance(cls.protocol, (Sequence, str)): - key = cls.protocol if isinstance(cls.protocol, str) else cls.protocol[0] - if count or not omit_zero: - out[key] = count - classes.extend(cls.__subclasses__()) - return out - - -@pytest.fixture(scope="function", autouse=True) -def instance_caches() -> Generator[InstanceCacheInspector, None, None]: - """ - Fixture to ensure empty filesystem instance caches before and after a test. - - Used by default for all tests. - Clears caches of all imported filesystem classes. - Can be used to write test assertions about instance caches. - - Usage: - - def test_something(instance_caches): - # Test code here - fsspec.open("file://abc") - fsspec.open("memory://foo/bar") - - # Test assertion - assert instance_caches.gather_counts() == {"file": 1, "memory": 1} - - Returns - ------- - instance_caches: An instance cache inspector for clearing and inspecting caches. - """ - ic = InstanceCacheInspector() - - ic.clear() - try: - yield ic - finally: - ic.clear() - - -@pytest.fixture(scope="function") -def ftp_writable(tmpdir): - """ - Fixture providing a writable FTP filesystem. - """ - pytest.importorskip("pyftpdlib") - - d = str(tmpdir) - with open(os.path.join(d, "out"), "wb") as f: - f.write(b"hello" * 10000) - P = subprocess.Popen( - [sys.executable, "-m", "pyftpdlib", "-d", d, "-u", "user", "-P", "pass", "-w"] - ) - try: - time.sleep(1) - yield "localhost", 2121, "user", "pass" - finally: - P.terminate() - P.wait() - try: - shutil.rmtree(tmpdir) - except Exception: - pass diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/core.py b/bundle/python-cpu/Lib/site-packages/fsspec/core.py deleted file mode 100644 index 881b07db38b6f99bd92ed1747086a366df798ea8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/core.py +++ /dev/null @@ -1,760 +0,0 @@ -from __future__ import annotations - -import io -import logging -import os -import re -from glob import has_magic -from pathlib import Path - -# for backwards compat, we export cache things from here too -from fsspec.caching import ( # noqa: F401 - BaseCache, - BlockCache, - BytesCache, - MMapCache, - ReadAheadCache, - caches, -) -from fsspec.compression import compr -from fsspec.config import conf -from fsspec.registry import available_protocols, filesystem, get_filesystem_class -from fsspec.utils import ( - _unstrip_protocol, - build_name_function, - infer_compression, - stringify_path, -) - -logger = logging.getLogger("fsspec") - - -class OpenFile: - """ - File-like object to be used in a context - - Can layer (buffered) text-mode and compression over any file-system, which - are typically binary-only. - - These instances are safe to serialize, as the low-level file object - is not created until invoked using ``with``. - - Parameters - ---------- - fs: FileSystem - The file system to use for opening the file. Should be a subclass or duck-type - with ``fsspec.spec.AbstractFileSystem`` - path: str - Location to open - mode: str like 'rb', optional - Mode of the opened file - compression: str or None, optional - Compression to apply - encoding: str or None, optional - The encoding to use if opened in text mode. - errors: str or None, optional - How to handle encoding errors if opened in text mode. - newline: None or str - Passed to TextIOWrapper in text mode, how to handle line endings. - autoopen: bool - If True, calls open() immediately. Mostly used by pickle - pos: int - If given and autoopen is True, seek to this location immediately - """ - - def __init__( - self, - fs, - path, - mode="rb", - compression=None, - encoding=None, - errors=None, - newline=None, - ): - self.fs = fs - self.path = path - self.mode = mode - self.compression = get_compression(path, compression) - self.encoding = encoding - self.errors = errors - self.newline = newline - self.fobjects = [] - - def __reduce__(self): - return ( - OpenFile, - ( - self.fs, - self.path, - self.mode, - self.compression, - self.encoding, - self.errors, - self.newline, - ), - ) - - def __repr__(self): - return f"" - - def __enter__(self): - mode = self.mode.replace("t", "").replace("b", "") + "b" - - try: - f = self.fs.open(self.path, mode=mode) - except FileNotFoundError as e: - if has_magic(self.path): - raise FileNotFoundError( - "%s not found. The URL contains glob characters: you maybe needed\n" - "to pass expand=True in fsspec.open() or the storage_options of \n" - "your library. You can also set the config value 'open_expand'\n" - "before import, or fsspec.core.DEFAULT_EXPAND at runtime, to True.", - self.path, - ) from e - raise - - self.fobjects = [f] - - if self.compression is not None: - compress = compr[self.compression] - f = compress(f, mode=mode[0]) - self.fobjects.append(f) - - if "b" not in self.mode: - # assume, for example, that 'r' is equivalent to 'rt' as in builtin - f = PickleableTextIOWrapper( - f, encoding=self.encoding, errors=self.errors, newline=self.newline - ) - self.fobjects.append(f) - - return self.fobjects[-1] - - def __exit__(self, *args): - self.close() - - @property - def full_name(self): - return _unstrip_protocol(self.path, self.fs) - - def open(self): - """Materialise this as a real open file without context - - The OpenFile object should be explicitly closed to avoid enclosed file - instances persisting. You must, therefore, keep a reference to the OpenFile - during the life of the file-like it generates. - """ - return self.__enter__() - - def close(self): - """Close all encapsulated file objects""" - for f in reversed(self.fobjects): - if "r" not in self.mode and not f.closed: - f.flush() - f.close() - self.fobjects.clear() - - -class OpenFiles(list): - """List of OpenFile instances - - Can be used in a single context, which opens and closes all of the - contained files. Normal list access to get the elements works as - normal. - - A special case is made for caching filesystems - the files will - be down/uploaded together at the start or end of the context, and - this may happen concurrently, if the target filesystem supports it. - """ - - def __init__(self, *args, mode="rb", fs=None): - self.mode = mode - self.fs = fs - self.files = [] - super().__init__(*args) - - def __enter__(self): - if self.fs is None: - raise ValueError("Context has already been used") - - fs = self.fs - while True: - if hasattr(fs, "open_many"): - # check for concurrent cache download; or set up for upload - self.files = fs.open_many(self) - return self.files - if hasattr(fs, "fs") and fs.fs is not None: - fs = fs.fs - else: - break - return [s.__enter__() for s in self] - - def __exit__(self, *args): - fs = self.fs - [s.__exit__(*args) for s in self] - if "r" not in self.mode: - while True: - if hasattr(fs, "open_many"): - # check for concurrent cache upload - fs.commit_many(self.files) - return - if hasattr(fs, "fs") and fs.fs is not None: - fs = fs.fs - else: - break - - def __getitem__(self, item): - out = super().__getitem__(item) - if isinstance(item, slice): - return OpenFiles(out, mode=self.mode, fs=self.fs) - return out - - def __repr__(self): - return f"" - - -def open_files( - urlpath, - mode="rb", - compression=None, - encoding="utf8", - errors=None, - name_function=None, - num=1, - protocol=None, - newline=None, - auto_mkdir=True, - expand=True, - **kwargs, -): - """Given a path or paths, return a list of ``OpenFile`` objects. - - For writing, a str path must contain the "*" character, which will be filled - in by increasing numbers, e.g., "part*" -> "part1", "part2" if num=2. - - For either reading or writing, can instead provide explicit list of paths. - - Parameters - ---------- - urlpath: string or list - Absolute or relative filepath(s). Prefix with a protocol like ``s3://`` - to read from alternative filesystems. To read from multiple files you - can pass a globstring or a list of paths, with the caveat that they - must all have the same protocol. - mode: 'rb', 'wt', etc. - compression: string or None - If given, open file using compression codec. Can either be a compression - name (a key in ``fsspec.compression.compr``) or "infer" to guess the - compression from the filename suffix. - encoding: str - For text mode only - errors: None or str - Passed to TextIOWrapper in text mode - name_function: function or None - if opening a set of files for writing, those files do not yet exist, - so we need to generate their names by formatting the urlpath for - each sequence number - num: int [1] - if writing mode, number of files we expect to create (passed to - name+function) - protocol: str or None - If given, overrides the protocol found in the URL. - newline: bytes or None - Used for line terminator in text mode. If None, uses system default; - if blank, uses no translation. - auto_mkdir: bool (True) - If in write mode, this will ensure the target directory exists before - writing, by calling ``fs.mkdirs(exist_ok=True)``. - expand: bool - **kwargs: dict - Extra options that make sense to a particular storage connection, e.g. - host, port, username, password, etc. - - Examples - -------- - >>> files = open_files('2015-*-*.csv') # doctest: +SKIP - >>> files = open_files( - ... 's3://bucket/2015-*-*.csv.gz', compression='gzip' - ... ) # doctest: +SKIP - - Returns - ------- - An ``OpenFiles`` instance, which is a list of ``OpenFile`` objects that can - be used as a single context - - Notes - ----- - For a full list of the available protocols and the implementations that - they map across to see the latest online documentation: - - - For implementations built into ``fsspec`` see - https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - - For implementations in separate packages see - https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations - """ - fs, fs_token, paths = get_fs_token_paths( - urlpath, - mode, - num=num, - name_function=name_function, - storage_options=kwargs, - protocol=protocol, - expand=expand, - ) - if fs.protocol == "file": - fs.auto_mkdir = auto_mkdir - elif "r" not in mode and auto_mkdir: - parents = {fs._parent(path) for path in paths} - for parent in parents: - try: - fs.makedirs(parent, exist_ok=True) - except PermissionError: - pass - return OpenFiles( - [ - OpenFile( - fs, - path, - mode=mode, - compression=compression, - encoding=encoding, - errors=errors, - newline=newline, - ) - for path in paths - ], - mode=mode, - fs=fs, - ) - - -def _un_chain(path, kwargs): - # Avoid a circular import - from fsspec.implementations.chained import ChainedFileSystem - - if "::" in path: - x = re.compile(".*[^a-z]+.*") # test for non protocol-like single word - known_protocols = set(available_protocols()) - bits = [] - - # split on '::', then ensure each bit has a protocol - for p in path.split("::"): - if p in known_protocols: - bits.append(p + "://") - elif "://" in p or x.match(p): - bits.append(p) - else: - bits.append(p + "://") - else: - bits = [path] - - # [[url, protocol, kwargs], ...] - out = [] - previous_bit = None - kwargs = kwargs.copy() - - for bit in reversed(bits): - protocol = kwargs.pop("protocol", None) or split_protocol(bit)[0] or "file" - cls = get_filesystem_class(protocol) - extra_kwargs = cls._get_kwargs_from_urls(bit) - kws = kwargs.pop(protocol, {}) - - if bit is bits[0]: - kws.update(kwargs) - - kw = dict( - **{k: v for k, v in extra_kwargs.items() if k not in kws or v != kws[k]}, - **kws, - ) - bit = cls._strip_protocol(bit) - - if ( - "target_protocol" not in kw - and issubclass(cls, ChainedFileSystem) - and not bit - ): - # replace bit if we are chaining and no path given - bit = previous_bit - - out.append((bit, protocol, kw)) - previous_bit = bit - - out.reverse() - return out - - -def url_to_fs(url, **kwargs): - """ - Turn fully-qualified and potentially chained URL into filesystem instance - - Parameters - ---------- - url : str - The fsspec-compatible URL - **kwargs: dict - Extra options that make sense to a particular storage connection, e.g. - host, port, username, password, etc. - - Returns - ------- - filesystem : FileSystem - The new filesystem discovered from ``url`` and created with - ``**kwargs``. - urlpath : str - The file-systems-specific URL for ``url``. - """ - url = stringify_path(url) - # non-FS arguments that appear in fsspec.open() - # inspect could keep this in sync with open()'s signature - known_kwargs = { - "compression", - "encoding", - "errors", - "expand", - "mode", - "name_function", - "newline", - "num", - } - kwargs = {k: v for k, v in kwargs.items() if k not in known_kwargs} - chain = _un_chain(url, kwargs) - inkwargs = {} - # Reverse iterate the chain, creating a nested target_* structure - for i, ch in enumerate(reversed(chain)): - urls, protocol, kw = ch - if i == len(chain) - 1: - inkwargs = dict(**kw, **inkwargs) - continue - inkwargs["target_options"] = dict(**kw, **inkwargs) - inkwargs["target_protocol"] = protocol - inkwargs["fo"] = urls - urlpath, protocol, _ = chain[0] - fs = filesystem(protocol, **inkwargs) - return fs, urlpath - - -DEFAULT_EXPAND = conf.get("open_expand", False) - - -def open( - urlpath, - mode="rb", - compression=None, - encoding="utf8", - errors=None, - protocol=None, - newline=None, - expand=None, - **kwargs, -): - """Given a path or paths, return one ``OpenFile`` object. - - Parameters - ---------- - urlpath: string or list - Absolute or relative filepath. Prefix with a protocol like ``s3://`` - to read from alternative filesystems. Should not include glob - character(s). - mode: 'rb', 'wt', etc. - compression: string or None - If given, open file using compression codec. Can either be a compression - name (a key in ``fsspec.compression.compr``) or "infer" to guess the - compression from the filename suffix. - encoding: str - For text mode only - errors: None or str - Passed to TextIOWrapper in text mode - protocol: str or None - If given, overrides the protocol found in the URL. - newline: bytes or None - Used for line terminator in text mode. If None, uses system default; - if blank, uses no translation. - expand: bool or None - Whether to regard file paths containing special glob characters as needing - expansion (finding the first match) or absolute. Setting False allows using - paths which do embed such characters. If None (default), this argument - takes its value from the DEFAULT_EXPAND module variable, which takes - its initial value from the "open_expand" config value at startup, which will - be False if not set. - **kwargs: dict - Extra options that make sense to a particular storage connection, e.g. - host, port, username, password, etc. - - Examples - -------- - >>> openfile = open('2015-01-01.csv') # doctest: +SKIP - >>> openfile = open( - ... 's3://bucket/2015-01-01.csv.gz', compression='gzip' - ... ) # doctest: +SKIP - >>> with openfile as f: - ... df = pd.read_csv(f) # doctest: +SKIP - ... - - Returns - ------- - ``OpenFile`` object. - - Notes - ----- - For a full list of the available protocols and the implementations that - they map across to see the latest online documentation: - - - For implementations built into ``fsspec`` see - https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - - For implementations in separate packages see - https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations - """ - expand = DEFAULT_EXPAND if expand is None else expand - out = open_files( - urlpath=[urlpath], - mode=mode, - compression=compression, - encoding=encoding, - errors=errors, - protocol=protocol, - newline=newline, - expand=expand, - **kwargs, - ) - if not out: - raise FileNotFoundError(urlpath) - return out[0] - - -def open_local( - url: str | list[str] | Path | list[Path], - mode: str = "rb", - **storage_options: dict, -) -> str | list[str]: - """Open file(s) which can be resolved to local - - For files which either are local, or get downloaded upon open - (e.g., by file caching) - - Parameters - ---------- - url: str or list(str) - mode: str - Must be read mode - storage_options: - passed on to FS for or used by open_files (e.g., compression) - """ - if "r" not in mode: - raise ValueError("Can only ensure local files when reading") - of = open_files(url, mode=mode, **storage_options) - if not getattr(of[0].fs, "local_file", False): - raise ValueError( - "open_local can only be used on a filesystem which" - " has attribute local_file=True" - ) - with of as files: - paths = [f.name for f in files] - if (isinstance(url, str) and not has_magic(url)) or isinstance(url, Path): - return paths[0] - return paths - - -def get_compression(urlpath, compression): - if compression == "infer": - compression = infer_compression(urlpath) - if compression is not None and compression not in compr: - raise ValueError(f"Compression type {compression} not supported") - return compression - - -def split_protocol(urlpath): - """Return protocol, path pair""" - urlpath = stringify_path(urlpath) - if "://" in urlpath: - protocol, path = urlpath.split("://", 1) - if len(protocol) > 1: - # excludes Windows paths - return protocol, path - if urlpath.startswith("data:"): - return urlpath.split(":", 1) - return None, urlpath - - -def strip_protocol(urlpath): - """Return only path part of full URL, according to appropriate backend""" - protocol, _ = split_protocol(urlpath) - cls = get_filesystem_class(protocol) - return cls._strip_protocol(urlpath) - - -def expand_paths_if_needed(paths, mode, num, fs, name_function): - """Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]`` - in them (read mode). - - :param paths: list of paths - mode: str - Mode in which to open files. - num: int - If opening in writing mode, number of files we expect to create. - fs: filesystem object - name_function: callable - If opening in writing mode, this callable is used to generate path - names. Names are generated for each partition by - ``urlpath.replace('*', name_function(partition_index))``. - :return: list of paths - """ - expanded_paths = [] - paths = list(paths) - - if "w" in mode or "x" in mode: # write mode - if sum(1 for p in paths if "*" in p) > 1: - raise ValueError( - "When writing data, only one filename mask can be specified." - ) - num = max(num, len(paths)) - - for curr_path in paths: - if "*" in curr_path: - # expand using name_function - expanded_paths.extend(_expand_paths(curr_path, name_function, num)) - else: - expanded_paths.append(curr_path) - # if we generated more paths that asked for, trim the list - if len(expanded_paths) > num: - expanded_paths = expanded_paths[:num] - - else: # read mode - for curr_path in paths: - if has_magic(curr_path): - # expand using glob - expanded_paths.extend(fs.glob(curr_path)) - else: - expanded_paths.append(curr_path) - - return expanded_paths - - -def get_fs_token_paths( - urlpath, - mode="rb", - num=1, - name_function=None, - storage_options=None, - protocol=None, - expand=True, -): - """Filesystem, deterministic token, and paths from a urlpath and options. - - Parameters - ---------- - urlpath: string or iterable - Absolute or relative filepath, URL (may include protocols like - ``s3://``), or globstring pointing to data. - mode: str, optional - Mode in which to open files. - num: int, optional - If opening in writing mode, number of files we expect to create. - name_function: callable, optional - If opening in writing mode, this callable is used to generate path - names. Names are generated for each partition by - ``urlpath.replace('*', name_function(partition_index))``. - storage_options: dict, optional - Additional keywords to pass to the filesystem class. - protocol: str or None - To override the protocol specifier in the URL - expand: bool - Expand string paths for writing, assuming the path is a directory - """ - if isinstance(urlpath, (list, tuple, set)): - if not urlpath: - raise ValueError("empty urlpath sequence") - urlpath0 = stringify_path(next(iter(urlpath))) - else: - urlpath0 = stringify_path(urlpath) - storage_options = storage_options or {} - if protocol: - storage_options["protocol"] = protocol - chain = _un_chain(urlpath0, storage_options or {}) - inkwargs = {} - # Reverse iterate the chain, creating a nested target_* structure - for i, ch in enumerate(reversed(chain)): - urls, nested_protocol, kw = ch - if i == len(chain) - 1: - inkwargs = dict(**kw, **inkwargs) - continue - inkwargs["target_options"] = dict(**kw, **inkwargs) - inkwargs["target_protocol"] = nested_protocol - inkwargs["fo"] = urls - paths, protocol, _ = chain[0] - fs = filesystem(protocol, **inkwargs) - if isinstance(urlpath, (list, tuple, set)): - pchains = [ - _un_chain(stringify_path(u), storage_options or {})[0] for u in urlpath - ] - if len({pc[1] for pc in pchains}) > 1: - raise ValueError("Protocol mismatch getting fs from %s", urlpath) - paths = [pc[0] for pc in pchains] - else: - paths = fs._strip_protocol(paths) - if isinstance(paths, (list, tuple, set)): - if expand: - paths = expand_paths_if_needed(paths, mode, num, fs, name_function) - elif not isinstance(paths, list): - paths = list(paths) - else: - if ("w" in mode or "x" in mode) and expand: - paths = _expand_paths(paths, name_function, num) - elif "*" in paths: - paths = [f for f in sorted(fs.glob(paths)) if not fs.isdir(f)] - else: - paths = [paths] - - return fs, fs._fs_token, paths - - -def _expand_paths(path, name_function, num): - if isinstance(path, str): - if path.count("*") > 1: - raise ValueError("Output path spec must contain exactly one '*'.") - elif "*" not in path: - path = os.path.join(path, "*.part") - - if name_function is None: - name_function = build_name_function(num - 1) - - paths = [path.replace("*", name_function(i)) for i in range(num)] - if paths != sorted(paths): - logger.warning( - "In order to preserve order between partitions" - " paths created with ``name_function`` should " - "sort to partition order" - ) - elif isinstance(path, (tuple, list)): - assert len(path) == num - paths = list(path) - else: - raise ValueError( - "Path should be either\n" - "1. A list of paths: ['foo.json', 'bar.json', ...]\n" - "2. A directory: 'foo/\n" - "3. A path with a '*' in it: 'foo.*.json'" - ) - return paths - - -class PickleableTextIOWrapper(io.TextIOWrapper): - """TextIOWrapper cannot be pickled. This solves it. - - Requires that ``buffer`` be pickleable, which all instances of - AbstractBufferedFile are. - """ - - def __init__( - self, - buffer, - encoding=None, - errors=None, - newline=None, - line_buffering=False, - write_through=False, - ): - self.args = buffer, encoding, errors, newline, line_buffering, write_through - super().__init__(*self.args) - - def __reduce__(self): - return PickleableTextIOWrapper, self.args diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/dircache.py b/bundle/python-cpu/Lib/site-packages/fsspec/dircache.py deleted file mode 100644 index eca19566b135e5a7a4f6e7407d56411ec58bfe44..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/dircache.py +++ /dev/null @@ -1,98 +0,0 @@ -import time -from collections.abc import MutableMapping -from functools import lru_cache - - -class DirCache(MutableMapping): - """ - Caching of directory listings, in a structure like:: - - {"path0": [ - {"name": "path0/file0", - "size": 123, - "type": "file", - ... - }, - {"name": "path0/file1", - }, - ... - ], - "path1": [...] - } - - Parameters to this class control listing expiry or indeed turn - caching off - """ - - def __init__( - self, - use_listings_cache=True, - listings_expiry_time=None, - max_paths=None, - **kwargs, - ): - """ - - Parameters - ---------- - use_listings_cache: bool - If False, this cache never returns items, but always reports KeyError, - and setting items has no effect - listings_expiry_time: int or float (optional) - Time in seconds that a listing is considered valid. If None, - listings do not expire. - max_paths: int (optional) - The number of most recent listings that are considered valid; 'recent' - refers to when the entry was set. - """ - self._cache = {} - self._times = {} - if max_paths: - self._q = lru_cache(max_paths + 1)(lambda key: self._cache.pop(key, None)) - self.use_listings_cache = use_listings_cache - self.listings_expiry_time = listings_expiry_time - self.max_paths = max_paths - - def __getitem__(self, item): - if self.listings_expiry_time is not None: - if self._times.get(item, 0) - time.time() < -self.listings_expiry_time: - del self._cache[item] - if self.max_paths: - self._q(item) - return self._cache[item] # maybe raises KeyError - - def clear(self): - self._cache.clear() - - def __len__(self): - return len(self._cache) - - def __contains__(self, item): - try: - self[item] - return True - except KeyError: - return False - - def __setitem__(self, key, value): - if not self.use_listings_cache: - return - if self.max_paths: - self._q(key) - self._cache[key] = value - if self.listings_expiry_time is not None: - self._times[key] = time.time() - - def __delitem__(self, key): - del self._cache[key] - - def __iter__(self): - entries = list(self._cache) - - return (k for k in entries if k in self) - - def __reduce__(self): - return ( - DirCache, - (self.use_listings_cache, self.listings_expiry_time, self.max_paths), - ) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/exceptions.py b/bundle/python-cpu/Lib/site-packages/fsspec/exceptions.py deleted file mode 100644 index ae8905475f02655f4fc5863931d99ca9da55db78..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/exceptions.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -fsspec user-defined exception classes -""" - -import asyncio - - -class BlocksizeMismatchError(ValueError): - """ - Raised when a cached file is opened with a different blocksize than it was - written with - """ - - -class FSTimeoutError(asyncio.TimeoutError): - """ - Raised when a fsspec function timed out occurs - """ diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/fuse.py b/bundle/python-cpu/Lib/site-packages/fsspec/fuse.py deleted file mode 100644 index 566d520fce3e94e3bbaee48c3c6acc9f1db315a8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/fuse.py +++ /dev/null @@ -1,324 +0,0 @@ -import argparse -import logging -import os -import stat -import threading -import time -from errno import EIO, ENOENT - -from fuse import FUSE, FuseOSError, LoggingMixIn, Operations - -from fsspec import __version__ -from fsspec.core import url_to_fs - -logger = logging.getLogger("fsspec.fuse") - - -class FUSEr(Operations): - def __init__(self, fs, path, ready_file=False): - self.fs = fs - self.cache = {} - self.root = path.rstrip("/") + "/" - self.counter = 0 - logger.info("Starting FUSE at %s", path) - self._ready_file = ready_file - - def getattr(self, path, fh=None): - logger.debug("getattr %s", path) - if self._ready_file and path in ["/.fuse_ready", ".fuse_ready"]: - return {"type": "file", "st_size": 5} - - path = "".join([self.root, path.lstrip("/")]).rstrip("/") - try: - info = self.fs.info(path) - except FileNotFoundError as exc: - raise FuseOSError(ENOENT) from exc - - data = {"st_uid": info.get("uid", 1000), "st_gid": info.get("gid", 1000)} - perm = info.get("mode", 0o777) - - if info["type"] != "file": - data["st_mode"] = stat.S_IFDIR | perm - data["st_size"] = 0 - data["st_blksize"] = 0 - else: - data["st_mode"] = stat.S_IFREG | perm - data["st_size"] = info["size"] - data["st_blksize"] = 5 * 2**20 - data["st_nlink"] = 1 - data["st_atime"] = info["atime"] if "atime" in info else time.time() - data["st_ctime"] = info["ctime"] if "ctime" in info else time.time() - data["st_mtime"] = info["mtime"] if "mtime" in info else time.time() - return data - - def readdir(self, path, fh): - logger.debug("readdir %s", path) - path = "".join([self.root, path.lstrip("/")]) - files = self.fs.ls(path, False) - files = [os.path.basename(f.rstrip("/")) for f in files] - return [".", ".."] + files - - def mkdir(self, path, mode): - path = "".join([self.root, path.lstrip("/")]) - self.fs.mkdir(path) - return 0 - - def rmdir(self, path): - path = "".join([self.root, path.lstrip("/")]) - self.fs.rmdir(path) - return 0 - - def read(self, path, size, offset, fh): - logger.debug("read %s", (path, size, offset)) - if self._ready_file and path in ["/.fuse_ready", ".fuse_ready"]: - # status indicator - return b"ready" - - f = self.cache[fh] - f.seek(offset) - out = f.read(size) - return out - - def write(self, path, data, offset, fh): - logger.debug("write %s", (path, offset)) - f = self.cache[fh] - f.seek(offset) - f.write(data) - return len(data) - - def create(self, path, flags, fi=None): - logger.debug("create %s", (path, flags)) - fn = "".join([self.root, path.lstrip("/")]) - self.fs.touch(fn) # OS will want to get attributes immediately - f = self.fs.open(fn, "wb") - self.cache[self.counter] = f - self.counter += 1 - return self.counter - 1 - - def open(self, path, flags): - logger.debug("open %s", (path, flags)) - fn = "".join([self.root, path.lstrip("/")]) - if flags % 2 == 0: - # read - mode = "rb" - else: - # write/create - mode = "wb" - self.cache[self.counter] = self.fs.open(fn, mode) - self.counter += 1 - return self.counter - 1 - - def truncate(self, path, length, fh=None): - fn = "".join([self.root, path.lstrip("/")]) - if length != 0: - raise NotImplementedError - # maybe should be no-op since open with write sets size to zero anyway - self.fs.touch(fn) - - def unlink(self, path): - fn = "".join([self.root, path.lstrip("/")]) - try: - self.fs.rm(fn, False) - except (OSError, FileNotFoundError) as exc: - raise FuseOSError(EIO) from exc - - def release(self, path, fh): - try: - if fh in self.cache: - f = self.cache[fh] - f.close() - self.cache.pop(fh) - except Exception as e: - print(e) - return 0 - - def chmod(self, path, mode): - if hasattr(self.fs, "chmod"): - path = "".join([self.root, path.lstrip("/")]) - return self.fs.chmod(path, mode) - raise NotImplementedError - - -def run( - fs, - path, - mount_point, - foreground=True, - threads=False, - ready_file=False, - ops_class=FUSEr, -): - """Mount stuff in a local directory - - This uses fusepy to make it appear as if a given path on an fsspec - instance is in fact resident within the local file-system. - - This requires that fusepy by installed, and that FUSE be available on - the system (typically requiring a package to be installed with - apt, yum, brew, etc.). - - Parameters - ---------- - fs: file-system instance - From one of the compatible implementations - path: str - Location on that file-system to regard as the root directory to - mount. Note that you typically should include the terminating "/" - character. - mount_point: str - An empty directory on the local file-system where the contents of - the remote path will appear. - foreground: bool - Whether or not calling this function will block. Operation will - typically be more stable if True. - threads: bool - Whether or not to create threads when responding to file operations - within the mounter directory. Operation will typically be more - stable if False. - ready_file: bool - Whether the FUSE process is ready. The ``.fuse_ready`` file will - exist in the ``mount_point`` directory if True. Debugging purpose. - ops_class: FUSEr or Subclass of FUSEr - To override the default behavior of FUSEr. For Example, logging - to file. - - """ - func = lambda: FUSE( - ops_class(fs, path, ready_file=ready_file), - mount_point, - nothreads=not threads, - foreground=foreground, - ) - if not foreground: - th = threading.Thread(target=func) - th.daemon = True - th.start() - return th - else: # pragma: no cover - try: - func() - except KeyboardInterrupt: - pass - - -def main(args): - """Mount filesystem from chained URL to MOUNT_POINT. - - Examples: - - python3 -m fsspec.fuse memory /usr/share /tmp/mem - - python3 -m fsspec.fuse local /tmp/source /tmp/local \\ - -l /tmp/fsspecfuse.log - - You can also mount chained-URLs and use special settings: - - python3 -m fsspec.fuse 'filecache::zip::file://data.zip' \\ - / /tmp/zip \\ - -o 'filecache-cache_storage=/tmp/simplecache' - - You can specify the type of the setting by using `[int]` or `[bool]`, - (`true`, `yes`, `1` represents the Boolean value `True`): - - python3 -m fsspec.fuse 'simplecache::ftp://ftp1.at.proftpd.org' \\ - /historic/packages/RPMS /tmp/ftp \\ - -o 'simplecache-cache_storage=/tmp/simplecache' \\ - -o 'simplecache-check_files=false[bool]' \\ - -o 'ftp-listings_expiry_time=60[int]' \\ - -o 'ftp-username=anonymous' \\ - -o 'ftp-password=xieyanbo' - """ - - class RawDescriptionArgumentParser(argparse.ArgumentParser): - def format_help(self): - usage = super().format_help() - parts = usage.split("\n\n") - parts[1] = self.description.rstrip() - return "\n\n".join(parts) - - parser = RawDescriptionArgumentParser(prog="fsspec.fuse", description=main.__doc__) - parser.add_argument("--version", action="version", version=__version__) - parser.add_argument("url", type=str, help="fs url") - parser.add_argument("source_path", type=str, help="source directory in fs") - parser.add_argument("mount_point", type=str, help="local directory") - parser.add_argument( - "-o", - "--option", - action="append", - help="Any options of protocol included in the chained URL", - ) - parser.add_argument( - "-l", "--log-file", type=str, help="Logging FUSE debug info (Default: '')" - ) - parser.add_argument( - "-f", - "--foreground", - action="store_false", - help="Running in foreground or not (Default: False)", - ) - parser.add_argument( - "-t", - "--threads", - action="store_false", - help="Running with threads support (Default: False)", - ) - parser.add_argument( - "-r", - "--ready-file", - action="store_false", - help="The `.fuse_ready` file will exist after FUSE is ready. " - "(Debugging purpose, Default: False)", - ) - args = parser.parse_args(args) - - kwargs = {} - for item in args.option or []: - key, sep, value = item.partition("=") - if not sep: - parser.error(message=f"Wrong option: {item!r}") - val = value.lower() - if val.endswith("[int]"): - value = int(value[: -len("[int]")]) - elif val.endswith("[bool]"): - value = val[: -len("[bool]")] in ["1", "yes", "true"] - - if "-" in key: - fs_name, setting_name = key.split("-", 1) - if fs_name in kwargs: - kwargs[fs_name][setting_name] = value - else: - kwargs[fs_name] = {setting_name: value} - else: - kwargs[key] = value - - if args.log_file: - logging.basicConfig( - level=logging.DEBUG, - filename=args.log_file, - format="%(asctime)s %(message)s", - ) - - class LoggingFUSEr(FUSEr, LoggingMixIn): - pass - - fuser = LoggingFUSEr - else: - fuser = FUSEr - - fs, url_path = url_to_fs(args.url, **kwargs) - logger.debug("Mounting %s to %s", url_path, str(args.mount_point)) - run( - fs, - args.source_path, - args.mount_point, - foreground=args.foreground, - threads=args.threads, - ready_file=args.ready_file, - ops_class=fuser, - ) - - -if __name__ == "__main__": - import sys - - main(sys.argv[1:]) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/generic.py b/bundle/python-cpu/Lib/site-packages/fsspec/generic.py deleted file mode 100644 index 0600e942d08e8974203795e28d43cfa0928a60ea..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/generic.py +++ /dev/null @@ -1,396 +0,0 @@ -from __future__ import annotations - -import inspect -import logging -import os -import shutil -import uuid - -from .asyn import AsyncFileSystem, _run_coros_in_chunks, sync_wrapper -from .callbacks import DEFAULT_CALLBACK -from .core import filesystem, get_filesystem_class, split_protocol, url_to_fs - -_generic_fs = {} -logger = logging.getLogger("fsspec.generic") - - -def set_generic_fs(protocol, **storage_options): - """Populate the dict used for method=="generic" lookups""" - _generic_fs[protocol] = filesystem(protocol, **storage_options) - - -def _resolve_fs(url, method, protocol=None, storage_options=None): - """Pick instance of backend FS""" - url = url[0] if isinstance(url, (list, tuple)) else url - protocol = protocol or split_protocol(url)[0] - storage_options = storage_options or {} - if method == "default": - return filesystem(protocol) - if method == "generic": - return _generic_fs[protocol] - if method == "current": - cls = get_filesystem_class(protocol) - return cls.current() - if method == "options": - fs, _ = url_to_fs(url, **storage_options.get(protocol, {})) - return fs - raise ValueError(f"Unknown FS resolution method: {method}") - - -def rsync( - source, - destination, - delete_missing=False, - source_field="size", - dest_field="size", - update_cond="different", - inst_kwargs=None, - fs=None, - **kwargs, -): - """Sync files between two directory trees - - (experimental) - - Parameters - ---------- - source: str - Root of the directory tree to take files from. This must be a directory, but - do not include any terminating "/" character - destination: str - Root path to copy into. The contents of this location should be - identical to the contents of ``source`` when done. This will be made a - directory, and the terminal "/" should not be included. - delete_missing: bool - If there are paths in the destination that don't exist in the - source and this is True, delete them. Otherwise, leave them alone. - source_field: str | callable - If ``update_field`` is "different", this is the key in the info - of source files to consider for difference. Maybe a function of the - info dict. - dest_field: str | callable - If ``update_field`` is "different", this is the key in the info - of destination files to consider for difference. May be a function of - the info dict. - update_cond: "different"|"always"|"never" - If "always", every file is copied, regardless of whether it exists in - the destination. If "never", files that exist in the destination are - not copied again. If "different" (default), only copy if the info - fields given by ``source_field`` and ``dest_field`` (usually "size") - are different. Other comparisons may be added in the future. - inst_kwargs: dict|None - If ``fs`` is None, use this set of keyword arguments to make a - GenericFileSystem instance - fs: GenericFileSystem|None - Instance to use if explicitly given. The instance defines how to - to make downstream file system instances from paths. - - Returns - ------- - dict of the copy operations that were performed, {source: destination} - """ - fs = fs or GenericFileSystem(**(inst_kwargs or {})) - source = fs._strip_protocol(source) - destination = fs._strip_protocol(destination) - allfiles = fs.find(source, withdirs=True, detail=True) - if not fs.isdir(source): - raise ValueError("Can only rsync on a directory") - otherfiles = fs.find(destination, withdirs=True, detail=True) - dirs = [ - a - for a, v in allfiles.items() - if v["type"] == "directory" and a.replace(source, destination) not in otherfiles - ] - logger.debug(f"{len(dirs)} directories to create") - if dirs: - fs.make_many_dirs( - [dirn.replace(source, destination) for dirn in dirs], exist_ok=True - ) - allfiles = {a: v for a, v in allfiles.items() if v["type"] == "file"} - logger.debug(f"{len(allfiles)} files to consider for copy") - to_delete = [ - o - for o, v in otherfiles.items() - if o.replace(destination, source) not in allfiles and v["type"] == "file" - ] - for k, v in allfiles.copy().items(): - otherfile = k.replace(source, destination) - if otherfile in otherfiles: - if update_cond == "always": - allfiles[k] = otherfile - elif update_cond == "never": - allfiles.pop(k) - elif update_cond == "different": - inf1 = source_field(v) if callable(source_field) else v[source_field] - v2 = otherfiles[otherfile] - inf2 = dest_field(v2) if callable(dest_field) else v2[dest_field] - if inf1 != inf2: - # details mismatch, make copy - allfiles[k] = otherfile - else: - # details match, don't copy - allfiles.pop(k) - else: - # file not in target yet - allfiles[k] = otherfile - logger.debug(f"{len(allfiles)} files to copy") - if allfiles: - source_files, target_files = zip(*allfiles.items()) - fs.cp(source_files, target_files, **kwargs) - logger.debug(f"{len(to_delete)} files to delete") - if delete_missing and to_delete: - fs.rm(to_delete) - return allfiles - - -class GenericFileSystem(AsyncFileSystem): - """Wrapper over all other FS types - - - - This implementation is a single unified interface to be able to run FS operations - over generic URLs, and dispatch to the specific implementations using the URL - protocol prefix. - - Note: instances of this FS are always async, even if you never use it with any async - backend. - """ - - protocol = "generic" # there is no real reason to ever use a protocol with this FS - - def __init__(self, default_method="default", storage_options=None, **kwargs): - """ - - Parameters - ---------- - default_method: str (optional) - Defines how to configure backend FS instances. Options are: - - "default": instantiate like FSClass(), with no - extra arguments; this is the default instance of that FS, and can be - configured via the config system - - "generic": takes instances from the `_generic_fs` dict in this module, - which you must populate before use. Keys are by protocol - - "options": expects storage_options, a dict mapping protocol to - kwargs to use when constructing the filesystem - - "current": takes the most recently instantiated version of each FS - """ - self.method = default_method - self.st_opts = storage_options - super().__init__(**kwargs) - - def _parent(self, path): - fs = _resolve_fs(path, self.method, storage_options=self.st_opts) - return fs.unstrip_protocol(fs._parent(path)) - - def _strip_protocol(self, path): - # normalization only - fs = _resolve_fs(path, self.method, storage_options=self.st_opts) - return fs.unstrip_protocol(fs._strip_protocol(path)) - - async def _find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): - fs = _resolve_fs(path, self.method, storage_options=self.st_opts) - if fs.async_impl: - out = await fs._find( - path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs - ) - else: - out = fs.find( - path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs - ) - result = {} - for k, v in out.items(): - v = v.copy() # don't corrupt target FS dircache - name = fs.unstrip_protocol(k) - v["name"] = name - result[name] = v - if detail: - return result - return list(result) - - async def _info(self, url, **kwargs): - fs = _resolve_fs(url, self.method, storage_options=self.st_opts) - if fs.async_impl: - out = await fs._info(url, **kwargs) - else: - out = fs.info(url, **kwargs) - out = out.copy() # don't edit originals - out["name"] = fs.unstrip_protocol(out["name"]) - return out - - async def _ls( - self, - url, - detail=True, - **kwargs, - ): - fs = _resolve_fs(url, self.method, storage_options=self.st_opts) - if fs.async_impl: - out = await fs._ls(url, detail=True, **kwargs) - else: - out = fs.ls(url, detail=True, **kwargs) - out = [o.copy() for o in out] # don't edit originals - for o in out: - o["name"] = fs.unstrip_protocol(o["name"]) - if detail: - return out - else: - return [o["name"] for o in out] - - async def _cat_file( - self, - url, - **kwargs, - ): - fs = _resolve_fs(url, self.method, storage_options=self.st_opts) - if fs.async_impl: - return await fs._cat_file(url, **kwargs) - else: - return fs.cat_file(url, **kwargs) - - async def _pipe_file( - self, - path, - value, - **kwargs, - ): - fs = _resolve_fs(path, self.method, storage_options=self.st_opts) - if fs.async_impl: - return await fs._pipe_file(path, value, **kwargs) - else: - return fs.pipe_file(path, value, **kwargs) - - async def _rm(self, url, **kwargs): - urls = url - if isinstance(urls, str): - urls = [urls] - fs = _resolve_fs(urls[0], self.method, storage_options=self.st_opts) - if fs.async_impl: - await fs._rm(urls, **kwargs) - else: - fs.rm(url, **kwargs) - - async def _makedirs(self, path, exist_ok=False): - logger.debug("Make dir %s", path) - fs = _resolve_fs(path, self.method, storage_options=self.st_opts) - if fs.async_impl: - await fs._makedirs(path, exist_ok=exist_ok) - else: - fs.makedirs(path, exist_ok=exist_ok) - - def rsync(self, source, destination, **kwargs): - """Sync files between two directory trees - - See `func:rsync` for more details. - """ - rsync(source, destination, fs=self, **kwargs) - - async def _cp_file( - self, - url, - url2, - blocksize=2**20, - callback=DEFAULT_CALLBACK, - tempdir: str | None = None, - **kwargs, - ): - fs = _resolve_fs(url, self.method, storage_options=self.st_opts) - fs2 = _resolve_fs(url2, self.method, storage_options=self.st_opts) - if fs is fs2: - # pure remote - if fs.async_impl: - return await fs._copy(url, url2, **kwargs) - else: - return fs.copy(url, url2, **kwargs) - await copy_file_op(fs, [url], fs2, [url2], tempdir, 1, on_error="raise") - - async def _make_many_dirs(self, urls, exist_ok=True): - fs = _resolve_fs(urls[0], self.method, storage_options=self.st_opts) - if fs.async_impl: - coros = [fs._makedirs(u, exist_ok=exist_ok) for u in urls] - await _run_coros_in_chunks(coros) - else: - for u in urls: - fs.makedirs(u, exist_ok=exist_ok) - - make_many_dirs = sync_wrapper(_make_many_dirs) - - async def _copy( - self, - path1: list[str], - path2: list[str], - recursive: bool = False, - on_error: str = "ignore", - maxdepth: int | None = None, - batch_size: int | None = None, - tempdir: str | None = None, - **kwargs, - ): - # TODO: special case for one FS being local, which can use get/put - # TODO: special case for one being memFS, which can use cat/pipe - if recursive: - raise NotImplementedError("Please use fsspec.generic.rsync") - path1 = [path1] if isinstance(path1, str) else path1 - path2 = [path2] if isinstance(path2, str) else path2 - - fs = _resolve_fs(path1, self.method, storage_options=self.st_opts) - fs2 = _resolve_fs(path2, self.method, storage_options=self.st_opts) - - if fs is fs2: - if fs.async_impl: - return await fs._copy(path1, path2, **kwargs) - else: - return fs.copy(path1, path2, **kwargs) - - await copy_file_op( - fs, path1, fs2, path2, tempdir, batch_size, on_error=on_error - ) - - -async def copy_file_op( - fs1, url1, fs2, url2, tempdir=None, batch_size=20, on_error="ignore" -): - import tempfile - - tempdir = tempdir or tempfile.mkdtemp() - try: - coros = [ - _copy_file_op( - fs1, - u1, - fs2, - u2, - os.path.join(tempdir, uuid.uuid4().hex), - ) - for u1, u2 in zip(url1, url2) - ] - out = await _run_coros_in_chunks( - coros, batch_size=batch_size, return_exceptions=True - ) - finally: - shutil.rmtree(tempdir) - if on_error == "return": - return out - elif on_error == "raise": - for o in out: - if isinstance(o, Exception): - raise o - - -async def _copy_file_op(fs1, url1, fs2, url2, local, on_error="ignore"): - if fs1.async_impl: - await fs1._get_file(url1, local) - else: - fs1.get_file(url1, local) - if fs2.async_impl: - await fs2._put_file(local, url2) - else: - fs2.put_file(local, url2) - os.unlink(local) - logger.debug("Copy %s -> %s; done", url1, url2) - - -async def maybe_await(cor): - if inspect.iscoroutine(cor): - return await cor - else: - return cor diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/gui.py b/bundle/python-cpu/Lib/site-packages/fsspec/gui.py deleted file mode 100644 index 9d914c8beb6cabb2c2700eb8eee31028559be2bd..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/gui.py +++ /dev/null @@ -1,417 +0,0 @@ -import ast -import contextlib -import logging -import os -import re -from collections.abc import Sequence -from typing import ClassVar - -import panel as pn - -from .core import OpenFile, get_filesystem_class, split_protocol -from .registry import known_implementations - -pn.extension() -logger = logging.getLogger("fsspec.gui") - - -class SigSlot: - """Signal-slot mixin, for Panel event passing - - Include this class in a widget manager's superclasses to be able to - register events and callbacks on Panel widgets managed by that class. - - The method ``_register`` should be called as widgets are added, and external - code should call ``connect`` to associate callbacks. - - By default, all signals emit a DEBUG logging statement. - """ - - # names of signals that this class may emit each of which must be - # set by _register for any new instance - signals: ClassVar[Sequence[str]] = [] - # names of actions that this class may respond to - slots: ClassVar[Sequence[str]] = [] - - # each of which must be a method name - - def __init__(self): - self._ignoring_events = False - self._sigs = {} - self._map = {} - self._setup() - - def _setup(self): - """Create GUI elements and register signals""" - self.panel = pn.pane.PaneBase() - # no signals to set up in the base class - - def _register( - self, widget, name, thing="value", log_level=logging.DEBUG, auto=False - ): - """Watch the given attribute of a widget and assign it a named event - - This is normally called at the time a widget is instantiated, in the - class which owns it. - - Parameters - ---------- - widget : pn.layout.Panel or None - Widget to watch. If None, an anonymous signal not associated with - any widget. - name : str - Name of this event - thing : str - Attribute of the given widget to watch - log_level : int - When the signal is triggered, a logging event of the given level - will be fired in the dfviz logger. - auto : bool - If True, automatically connects with a method in this class of the - same name. - """ - if name not in self.signals: - raise ValueError(f"Attempt to assign an undeclared signal: {name}") - self._sigs[name] = { - "widget": widget, - "callbacks": [], - "thing": thing, - "log": log_level, - } - wn = "-".join( - [ - getattr(widget, "name", str(widget)) if widget is not None else "none", - thing, - ] - ) - self._map[wn] = name - if widget is not None: - widget.param.watch(self._signal, thing, onlychanged=True) - if auto and hasattr(self, name): - self.connect(name, getattr(self, name)) - - def _repr_mimebundle_(self, *args, **kwargs): - """Display in a notebook or a server""" - try: - return self.panel._repr_mimebundle_(*args, **kwargs) - except (ValueError, AttributeError) as exc: - raise NotImplementedError( - "Panel does not seem to be set up properly" - ) from exc - - def connect(self, signal, slot): - """Associate call back with given event - - The callback must be a function which takes the "new" value of the - watched attribute as the only parameter. If the callback return False, - this cancels any further processing of the given event. - - Alternatively, the callback can be a string, in which case it means - emitting the correspondingly-named event (i.e., connect to self) - """ - self._sigs[signal]["callbacks"].append(slot) - - def _signal(self, event): - """This is called by a an action on a widget - - Within an self.ignore_events context, nothing happens. - - Tests can execute this method by directly changing the values of - widget components. - """ - if not self._ignoring_events: - wn = "-".join([event.obj.name, event.name]) - if wn in self._map and self._map[wn] in self._sigs: - self._emit(self._map[wn], event.new) - - @contextlib.contextmanager - def ignore_events(self): - """Temporarily turn off events processing in this instance - - (does not propagate to children) - """ - self._ignoring_events = True - try: - yield - finally: - self._ignoring_events = False - - def _emit(self, sig, value=None): - """An event happened, call its callbacks - - This method can be used in tests to simulate message passing without - directly changing visual elements. - - Calling of callbacks will halt whenever one returns False. - """ - logger.log(self._sigs[sig]["log"], f"{sig}: {value}") - for callback in self._sigs[sig]["callbacks"]: - if isinstance(callback, str): - self._emit(callback) - else: - try: - # running callbacks should not break the interface - ret = callback(value) - if ret is False: - break - except Exception as e: - logger.exception( - "Exception (%s) while executing callback for signal: %s", - e, - sig, - ) - - def show(self, threads=False): - """Open a new browser tab and display this instance's interface""" - self.panel.show(threads=threads, verbose=False) - return self - - -class SingleSelect(SigSlot): - """A multiselect which only allows you to select one item for an event""" - - signals = ["_selected", "selected"] # the first is internal - slots = ["set_options", "set_selection", "add", "clear", "select"] - - def __init__(self, **kwargs): - self.kwargs = kwargs - super().__init__() - - def _setup(self): - self.panel = pn.widgets.MultiSelect(**self.kwargs) - self._register(self.panel, "_selected", "value") - self._register(None, "selected") - self.connect("_selected", self.select_one) - - def _signal(self, *args, **kwargs): - super()._signal(*args, **kwargs) - - def select_one(self, *_): - with self.ignore_events(): - val = [self.panel.value[-1]] if self.panel.value else [] - self.panel.value = val - self._emit("selected", self.panel.value) - - def set_options(self, options): - self.panel.options = options - - def clear(self): - self.panel.options = [] - - @property - def value(self): - return self.panel.value - - def set_selection(self, selection): - self.panel.value = [selection] - - -class FileSelector(SigSlot): - """Panel-based graphical file selector widget - - Instances of this widget are interactive and can be displayed in jupyter by having - them as the output of a cell, or in a separate browser tab using ``.show()``. - """ - - signals = [ - "protocol_changed", - "selection_changed", - "directory_entered", - "home_clicked", - "up_clicked", - "go_clicked", - "filters_changed", - ] - slots = ["set_filters", "go_home"] - - def __init__(self, url=None, filters=None, ignore=None, kwargs=None): - """ - - Parameters - ---------- - url : str (optional) - Initial value of the URL to populate the dialog; should include protocol - filters : list(str) (optional) - File endings to include in the listings. If not included, all files are - allowed. Does not affect directories. - If given, the endings will appear as checkboxes in the interface - ignore : list(str) (optional) - Regex(s) of file basename patterns to ignore, e.g., "\\." for typical - hidden files on posix - kwargs : dict (optional) - To pass to file system instance - """ - if url: - self.init_protocol, url = split_protocol(url) - else: - self.init_protocol, url = "file", os.getcwd() - self.init_url = url - self.init_kwargs = (kwargs if isinstance(kwargs, str) else str(kwargs)) or "{}" - self.filters = filters - self.ignore = [re.compile(i) for i in ignore or []] - self._fs = None - super().__init__() - - def _setup(self): - self.url = pn.widgets.TextInput( - name="url", - value=self.init_url, - align="end", - sizing_mode="stretch_width", - width_policy="max", - ) - self.protocol = pn.widgets.Select( - options=sorted(known_implementations), - value=self.init_protocol, - name="protocol", - align="center", - ) - self.kwargs = pn.widgets.TextInput( - name="kwargs", value=self.init_kwargs, align="center" - ) - self.go = pn.widgets.Button(name="⇨", align="end", width=45) - self.main = SingleSelect(size=10) - self.home = pn.widgets.Button(name="🏠", width=40, height=30, align="end") - self.up = pn.widgets.Button(name="‹", width=30, height=30, align="end") - - self._register(self.protocol, "protocol_changed", auto=True) - self._register(self.go, "go_clicked", "clicks", auto=True) - self._register(self.up, "up_clicked", "clicks", auto=True) - self._register(self.home, "home_clicked", "clicks", auto=True) - self._register(None, "selection_changed") - self.main.connect("selected", self.selection_changed) - self._register(None, "directory_entered") - self.prev_protocol = self.protocol.value - self.prev_kwargs = self.storage_options - - self.filter_sel = pn.widgets.CheckBoxGroup( - value=[], options=[], inline=False, align="end", width_policy="min" - ) - self._register(self.filter_sel, "filters_changed", auto=True) - - self.panel = pn.Column( - pn.Row(self.protocol, self.kwargs), - pn.Row(self.home, self.up, self.url, self.go, self.filter_sel), - self.main.panel, - ) - self.set_filters(self.filters) - self.go_clicked() - - def set_filters(self, filters=None): - self.filters = filters - if filters: - self.filter_sel.options = filters - self.filter_sel.value = filters - else: - self.filter_sel.options = [] - self.filter_sel.value = [] - - @property - def storage_options(self): - """Value of the kwargs box as a dictionary""" - return ast.literal_eval(self.kwargs.value) or {} - - @property - def fs(self): - """Current filesystem instance""" - if self._fs is None: - cls = get_filesystem_class(self.protocol.value) - self._fs = cls(**self.storage_options) - return self._fs - - @property - def urlpath(self): - """URL of currently selected item""" - return ( - (f"{self.protocol.value}://{self.main.value[0]}") - if self.main.value - else None - ) - - def open_file(self, mode="rb", compression=None, encoding=None): - """Create OpenFile instance for the currently selected item - - For example, in a notebook you might do something like - - .. code-block:: - - [ ]: sel = FileSelector(); sel - - # user selects their file - - [ ]: with sel.open_file('rb') as f: - ... out = f.read() - - Parameters - ---------- - mode: str (optional) - Open mode for the file. - compression: str (optional) - The interact with the file as compressed. Set to 'infer' to guess - compression from the file ending - encoding: str (optional) - If using text mode, use this encoding; defaults to UTF8. - """ - if self.urlpath is None: - raise ValueError("No file selected") - return OpenFile(self.fs, self.urlpath, mode, compression, encoding) - - def filters_changed(self, values): - self.filters = values - self.go_clicked() - - def selection_changed(self, *_): - if self.urlpath is None: - return - if self.fs.isdir(self.urlpath): - self.url.value = self.fs._strip_protocol(self.urlpath) - self.go_clicked() - - def go_clicked(self, *_): - if ( - self.prev_protocol != self.protocol.value - or self.prev_kwargs != self.storage_options - ): - self._fs = None # causes fs to be recreated - self.prev_protocol = self.protocol.value - self.prev_kwargs = self.storage_options - listing = sorted( - self.fs.ls(self.url.value, detail=True), key=lambda x: x["name"] - ) - listing = [ - l - for l in listing - if not any(i.match(l["name"].rsplit("/", 1)[-1]) for i in self.ignore) - ] - folders = { - "📁 " + o["name"].rsplit("/", 1)[-1]: o["name"] - for o in listing - if o["type"] == "directory" - } - files = { - "📄 " + o["name"].rsplit("/", 1)[-1]: o["name"] - for o in listing - if o["type"] == "file" - } - if self.filters: - files = { - k: v - for k, v in files.items() - if any(v.endswith(ext) for ext in self.filters) - } - self.main.set_options(dict(**folders, **files)) - - def protocol_changed(self, *_): - self._fs = None - self.main.options = [] - self.url.value = "" - - def home_clicked(self, *_): - self.protocol.value = self.init_protocol - self.kwargs.value = self.init_kwargs - self.url.value = self.init_url - self.go_clicked() - - def up_clicked(self, *_): - self.url.value = self.fs._parent(self.url.value) - self.go_clicked() diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/__init__.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/arrow.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/arrow.py deleted file mode 100644 index c312cd4a57b92d92e936d0cf97129f4fef0624fc..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/arrow.py +++ /dev/null @@ -1,322 +0,0 @@ -import errno -import io -import os -import secrets -import shutil -from contextlib import suppress -from functools import cached_property, wraps -from urllib.parse import parse_qs - -from fsspec.spec import AbstractFileSystem -from fsspec.utils import ( - get_package_version_without_import, - infer_storage_options, - mirror_from, - tokenize, -) - - -def wrap_exceptions(func): - @wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except OSError as exception: - if not exception.args: - raise - - message, *args = exception.args - if isinstance(message, str) and "does not exist" in message: - raise FileNotFoundError(errno.ENOENT, message) from exception - else: - raise - - return wrapper - - -PYARROW_VERSION = None -_EMPTY_ROOT_MARKER_FILESYSTEMS = {"gcs", "s3"} - - -class ArrowFSWrapper(AbstractFileSystem): - """FSSpec-compatible wrapper of pyarrow.fs.FileSystem. - - Parameters - ---------- - fs : pyarrow.fs.FileSystem - - """ - - root_marker = "/" - - def __init__(self, fs, **kwargs): - global PYARROW_VERSION - PYARROW_VERSION = get_package_version_without_import("pyarrow") - self.fs = fs - if fs.type_name in _EMPTY_ROOT_MARKER_FILESYSTEMS: - self.root_marker = "" - super().__init__(**kwargs) - - @property - def protocol(self): - return self.fs.type_name - - @cached_property - def fsid(self): - return "hdfs_" + tokenize(self.fs.host, self.fs.port) - - def _parent(self, path): - if self.root_marker: - return super()._parent(path) - - path = self._strip_protocol(path).lstrip("/") - return path.rsplit("/", 1)[0] if "/" in path else "" - - @classmethod - def _strip_protocol(cls, path): - ops = infer_storage_options(path) - path = ops["path"] - if path.startswith("//"): - # special case for "hdfs://path" (without the triple slash) - path = path[1:] - return path - - def ls(self, path, detail=False, **kwargs): - path = self._strip_protocol(path) - from pyarrow.fs import FileSelector - - try: - entries = [ - self._make_entry(entry) - for entry in self.fs.get_file_info(FileSelector(path)) - ] - except (FileNotFoundError, NotADirectoryError): - entries = [self.info(path, **kwargs)] - if detail: - return entries - else: - return [entry["name"] for entry in entries] - - def info(self, path, **kwargs): - path = self._strip_protocol(path) - [info] = self.fs.get_file_info([path]) - return self._make_entry(info) - - def exists(self, path): - path = self._strip_protocol(path) - try: - self.info(path) - except FileNotFoundError: - return False - else: - return True - - def _make_entry(self, info): - from pyarrow.fs import FileType - - if info.type is FileType.Directory: - kind = "directory" - elif info.type is FileType.File: - kind = "file" - elif info.type is FileType.NotFound: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), info.path) - else: - kind = "other" - - return { - "name": info.path, - "size": info.size, - "type": kind, - "mtime": info.mtime, - } - - @wrap_exceptions - def cp_file(self, path1, path2, **kwargs): - path1 = self._strip_protocol(path1).rstrip("/") - path2 = self._strip_protocol(path2).rstrip("/") - - with self._open(path1, "rb") as lstream: - tmp_fname = f"{path2}.tmp.{secrets.token_hex(6)}" - try: - with self.open(tmp_fname, "wb") as rstream: - shutil.copyfileobj(lstream, rstream) - self.fs.move(tmp_fname, path2) - except BaseException: - with suppress(FileNotFoundError): - self.fs.delete_file(tmp_fname) - raise - - @wrap_exceptions - def mv(self, path1, path2, **kwargs): - path1 = self._strip_protocol(path1).rstrip("/") - path2 = self._strip_protocol(path2).rstrip("/") - self.fs.move(path1, path2) - - @wrap_exceptions - def rm_file(self, path): - path = self._strip_protocol(path) - self.fs.delete_file(path) - - @wrap_exceptions - def rm(self, path, recursive=False, maxdepth=None): - path = self._strip_protocol(path).rstrip("/") - if self.isdir(path): - if recursive: - self.fs.delete_dir(path) - else: - raise ValueError("Can't delete directories without recursive=False") - else: - self.fs.delete_file(path) - - @wrap_exceptions - def _open(self, path, mode="rb", block_size=None, seekable=True, **kwargs): - if mode == "rb": - if seekable: - method = self.fs.open_input_file - else: - method = self.fs.open_input_stream - elif mode == "wb": - method = self.fs.open_output_stream - elif mode == "ab": - method = self.fs.open_append_stream - else: - raise ValueError(f"unsupported mode for Arrow filesystem: {mode!r}") - - _kwargs = {} - if mode != "rb" or not seekable: - if int(PYARROW_VERSION.split(".")[0]) >= 4: - # disable compression auto-detection - _kwargs["compression"] = None - stream = method(path, **_kwargs) - - return ArrowFile(self, stream, path, mode, block_size, **kwargs) - - @wrap_exceptions - def mkdir(self, path, create_parents=True, **kwargs): - path = self._strip_protocol(path) - if create_parents: - self.makedirs(path, exist_ok=True) - else: - self.fs.create_dir(path, recursive=False) - - @wrap_exceptions - def makedirs(self, path, exist_ok=False): - path = self._strip_protocol(path) - self.fs.create_dir(path, recursive=True) - - @wrap_exceptions - def rmdir(self, path): - path = self._strip_protocol(path) - self.fs.delete_dir(path) - - @wrap_exceptions - def modified(self, path): - path = self._strip_protocol(path) - return self.fs.get_file_info(path).mtime - - def cat_file(self, path, start=None, end=None, **kwargs): - kwargs.setdefault("seekable", start not in [None, 0]) - return super().cat_file(path, start, end, **kwargs) - - def get_file(self, rpath, lpath, **kwargs): - kwargs.setdefault("seekable", False) - super().get_file(rpath, lpath, **kwargs) - - -@mirror_from( - "stream", - [ - "read", - "seek", - "tell", - "write", - "readable", - "writable", - "close", - "seekable", - ], -) -class ArrowFile(io.IOBase): - def __init__(self, fs, stream, path, mode, block_size=None, **kwargs): - self.path = path - self.mode = mode - - self.fs = fs - self.stream = stream - - self.blocksize = self.block_size = block_size - self.kwargs = kwargs - - def __enter__(self): - return self - - @property - def size(self): - if self.stream.seekable(): - return self.stream.size() - return None - - def __exit__(self, *args): - return self.close() - - -class HadoopFileSystem(ArrowFSWrapper): - """A wrapper on top of the pyarrow.fs.HadoopFileSystem - to connect it's interface with fsspec""" - - protocol = "hdfs" - - def __init__( - self, - host="default", - port=0, - user=None, - kerb_ticket=None, - replication=3, - extra_conf=None, - **kwargs, - ): - """ - - Parameters - ---------- - host: str - Hostname, IP or "default" to try to read from Hadoop config - port: int - Port to connect on, or default from Hadoop config if 0 - user: str or None - If given, connect as this username - kerb_ticket: str or None - If given, use this ticket for authentication - replication: int - set replication factor of file for write operations. default value is 3. - extra_conf: None or dict - Passed on to HadoopFileSystem - """ - from pyarrow.fs import HadoopFileSystem - - fs = HadoopFileSystem( - host=host, - port=port, - user=user, - kerb_ticket=kerb_ticket, - replication=replication, - extra_conf=extra_conf, - ) - super().__init__(fs=fs, **kwargs) - - @staticmethod - def _get_kwargs_from_urls(path): - ops = infer_storage_options(path) - out = {} - if ops.get("host", None): - out["host"] = ops["host"] - if ops.get("username", None): - out["user"] = ops["username"] - if ops.get("port", None): - out["port"] = ops["port"] - if ops.get("url_query", None): - queries = parse_qs(ops["url_query"]) - if queries.get("replication", None): - out["replication"] = int(queries["replication"][0]) - return out diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/asyn_wrapper.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/asyn_wrapper.py deleted file mode 100644 index 91db5eb48d00e36b46d9deb49504a7d2ad76d690..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/asyn_wrapper.py +++ /dev/null @@ -1,124 +0,0 @@ -import asyncio -import functools -import inspect - -import fsspec -from fsspec.asyn import AsyncFileSystem, running_async - -from .chained import ChainedFileSystem - - -def async_wrapper(func, obj=None, semaphore=None): - """ - Wraps a synchronous function to make it awaitable. - - Parameters - ---------- - func : callable - The synchronous function to wrap. - obj : object, optional - The instance to bind the function to, if applicable. - semaphore : asyncio.Semaphore, optional - A semaphore to limit concurrent calls. - - Returns - ------- - coroutine - An awaitable version of the function. - """ - - @functools.wraps(func) - async def wrapper(*args, **kwargs): - if semaphore: - async with semaphore: - return await asyncio.to_thread(func, *args, **kwargs) - return await asyncio.to_thread(func, *args, **kwargs) - - return wrapper - - -class AsyncFileSystemWrapper(AsyncFileSystem, ChainedFileSystem): - """ - A wrapper class to convert a synchronous filesystem into an asynchronous one. - - This class takes an existing synchronous filesystem implementation and wraps all - its methods to provide an asynchronous interface. - - Parameters - ---------- - sync_fs : AbstractFileSystem - The synchronous filesystem instance to wrap. - """ - - protocol = "asyncwrapper", "async_wrapper" - cachable = False - - def __init__( - self, - fs=None, - asynchronous=None, - target_protocol=None, - target_options=None, - semaphore=None, - max_concurrent_tasks=None, - **kwargs, - ): - if asynchronous is None: - asynchronous = running_async() - super().__init__(asynchronous=asynchronous, **kwargs) - if fs is not None: - self.sync_fs = fs - else: - self.sync_fs = fsspec.filesystem(target_protocol, **target_options) - self.protocol = self.sync_fs.protocol - self.semaphore = semaphore - self._wrap_all_sync_methods() - - @property - def fsid(self): - return f"async_{self.sync_fs.fsid}" - - def _wrap_all_sync_methods(self): - """ - Wrap all synchronous methods of the underlying filesystem with asynchronous versions. - """ - excluded_methods = {"open"} - for method_name in dir(self.sync_fs): - if method_name.startswith("_") or method_name in excluded_methods: - continue - - attr = inspect.getattr_static(self.sync_fs, method_name) - if isinstance(attr, property): - continue - - method = getattr(self.sync_fs, method_name) - if callable(method) and not inspect.iscoroutinefunction(method): - async_method = async_wrapper(method, obj=self, semaphore=self.semaphore) - setattr(self, f"_{method_name}", async_method) - - @classmethod - def wrap_class(cls, sync_fs_class): - """ - Create a new class that can be used to instantiate an AsyncFileSystemWrapper - with lazy instantiation of the underlying synchronous filesystem. - - Parameters - ---------- - sync_fs_class : type - The class of the synchronous filesystem to wrap. - - Returns - ------- - type - A new class that wraps the provided synchronous filesystem class. - """ - - class GeneratedAsyncFileSystemWrapper(cls): - def __init__(self, *args, **kwargs): - sync_fs = sync_fs_class(*args, **kwargs) - super().__init__(sync_fs) - - GeneratedAsyncFileSystemWrapper.__name__ = ( - f"Async{sync_fs_class.__name__}Wrapper" - ) - return GeneratedAsyncFileSystemWrapper diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/cache_mapper.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/cache_mapper.py deleted file mode 100644 index 6e7c7d88afdddf12f77b26bb635bd8bf1e2bd7f1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/cache_mapper.py +++ /dev/null @@ -1,75 +0,0 @@ -from __future__ import annotations - -import abc -import hashlib - -from fsspec.implementations.local import make_path_posix - - -class AbstractCacheMapper(abc.ABC): - """Abstract super-class for mappers from remote URLs to local cached - basenames. - """ - - @abc.abstractmethod - def __call__(self, path: str) -> str: ... - - def __eq__(self, other: object) -> bool: - # Identity only depends on class. When derived classes have attributes - # they will need to be included. - return isinstance(other, type(self)) - - def __hash__(self) -> int: - # Identity only depends on class. When derived classes have attributes - # they will need to be included. - return hash(type(self)) - - -class BasenameCacheMapper(AbstractCacheMapper): - """Cache mapper that uses the basename of the remote URL and a fixed number - of directory levels above this. - - The default is zero directory levels, meaning different paths with the same - basename will have the same cached basename. - """ - - def __init__(self, directory_levels: int = 0): - if directory_levels < 0: - raise ValueError( - "BasenameCacheMapper requires zero or positive directory_levels" - ) - self.directory_levels = directory_levels - - # Separator for directories when encoded as strings. - self._separator = "_@_" - - def __call__(self, path: str) -> str: - path = make_path_posix(path) - prefix, *bits = path.rsplit("/", self.directory_levels + 1) - if bits: - return self._separator.join(bits) - else: - return prefix # No separator found, simple filename - - def __eq__(self, other: object) -> bool: - return super().__eq__(other) and self.directory_levels == other.directory_levels - - def __hash__(self) -> int: - return super().__hash__() ^ hash(self.directory_levels) - - -class HashCacheMapper(AbstractCacheMapper): - """Cache mapper that uses a hash of the remote URL.""" - - def __call__(self, path: str) -> str: - return hashlib.sha256(path.encode()).hexdigest() - - -def create_cache_mapper(same_names: bool) -> AbstractCacheMapper: - """Factory method to create cache mapper for backward compatibility with - ``CachingFileSystem`` constructor using ``same_names`` kwarg. - """ - if same_names: - return BasenameCacheMapper() - else: - return HashCacheMapper() diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/cache_metadata.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/cache_metadata.py deleted file mode 100644 index 2a48231cac9f984f3e41bc9f81a113ae28a1f5dd..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/cache_metadata.py +++ /dev/null @@ -1,217 +0,0 @@ -from __future__ import annotations - -import os -import time -from typing import TYPE_CHECKING - -from fsspec.utils import atomic_write - -try: - import ujson as json -except ImportError: - if not TYPE_CHECKING: - import json - -if TYPE_CHECKING: - from collections.abc import Iterator - from typing import Any, Literal, TypeAlias - - from .cached import CachingFileSystem - - Detail: TypeAlias = dict[str, Any] - - -class CacheMetadata: - """Cache metadata. - - All reading and writing of cache metadata is performed by this class, - accessing the cached files and blocks is not. - - Metadata is stored in a single file per storage directory in JSON format. - For backward compatibility. No longer supports pickle. - """ - - def __init__(self, storage: list[str]): - """ - - Parameters - ---------- - storage: list[str] - Directories containing cached files, must be at least one. Metadata - is stored in the last of these directories by convention. - """ - if not storage: - raise ValueError("CacheMetadata expects at least one storage location") - - self._storage = storage - self.cached_files: list[Detail] = [{}] - - def _load(self, fn: str) -> Detail: - """Low-level function to load metadata from specific file""" - with open(fn, "r") as f: - loaded = json.load(f) - for c in loaded.values(): - if isinstance(c.get("blocks"), list): - c["blocks"] = set(c["blocks"]) - return loaded - - def _save(self, metadata_to_save: Detail, fn: str) -> None: - """Low-level function to save metadata to specific file""" - with atomic_write(fn, mode="w") as f: - json.dump(metadata_to_save, f) - - def _scan_locations( - self, writable_only: bool = False - ) -> Iterator[tuple[str, str, bool]]: - """Yield locations (filenames) where metadata is stored, and whether - writable or not. - - Parameters - ---------- - writable: bool - Set to True to only yield writable locations. - - Returns - ------- - Yields (str, str, bool) - """ - n = len(self._storage) - for i, storage in enumerate(self._storage): - writable = i == n - 1 - if writable_only and not writable: - continue - yield os.path.join(storage, "cache"), storage, writable - - def check_file( - self, path: str, cfs: CachingFileSystem | None - ) -> Literal[False] | tuple[Detail, str]: - """If path is in cache return its details, otherwise return ``False``. - - If the optional CachingFileSystem is specified then it is used to - perform extra checks to reject possible matches, such as if they are - too old. - """ - for (fn, base, _), cache in zip(self._scan_locations(), self.cached_files): - if path not in cache: - continue - detail = cache[path].copy() - - if cfs is not None: - if cfs.check_files and detail["uid"] != cfs.fs.ukey(path): - # Wrong file as determined by hash of file properties - continue - if cfs.expiry and time.time() - detail["time"] > cfs.expiry: - # Cached file has expired - continue - - fn = os.path.join(base, detail["fn"]) - if os.path.exists(fn): - return detail, fn - return False - - def clear_expired(self, expiry_time: int) -> tuple[list[str], bool]: - """Remove expired metadata from the cache. - - Returns names of files corresponding to expired metadata and a boolean - flag indicating whether the writable cache is empty. Caller is - responsible for deleting the expired files. - """ - expired_files = [] - for path, detail in self.cached_files[-1].copy().items(): - if time.time() - detail["time"] > expiry_time: - fn = detail.get("fn", "") - if not fn: - raise RuntimeError( - f"Cache metadata does not contain 'fn' for {path}" - ) - fn = os.path.join(self._storage[-1], fn) - expired_files.append(fn) - self.cached_files[-1].pop(path) - - if self.cached_files[-1]: - cache_path = os.path.join(self._storage[-1], "cache") - self._save(self.cached_files[-1], cache_path) - - writable_cache_empty = not self.cached_files[-1] - return expired_files, writable_cache_empty - - def load(self) -> None: - """Load all metadata from disk and store in ``self.cached_files``""" - cached_files = [] - for fn, _, _ in self._scan_locations(): - if os.path.exists(fn): - # TODO: consolidate blocks here - cached_files.append(self._load(fn)) - else: - cached_files.append({}) - self.cached_files = cached_files or [{}] - - def on_close_cached_file(self, f: Any, path: str) -> None: - """Perform side-effect actions on closing a cached file. - - The actual closing of the file is the responsibility of the caller. - """ - # File must be writable, so in self.cached_files[-1] - c = self.cached_files[-1][path] - if c["blocks"] is not True and len(c["blocks"]) * f.blocksize >= f.size: - c["blocks"] = True - - def pop_file(self, path: str) -> str | None: - """Remove metadata of cached file. - - If path is in the cache, return the filename of the cached file, - otherwise return ``None``. Caller is responsible for deleting the - cached file. - """ - details = self.check_file(path, None) - if not details: - return None - _, fn = details - if fn.startswith(self._storage[-1]): - self.cached_files[-1].pop(path) - self.save() - else: - raise PermissionError( - "Can only delete cached file in last, writable cache location" - ) - return fn - - def save(self) -> None: - """Save metadata to disk""" - for (fn, _, writable), cache in zip(self._scan_locations(), self.cached_files): - if not writable: - continue - - if os.path.exists(fn): - cached_files = self._load(fn) - for k, c in cached_files.items(): - if k in cache: - if c["blocks"] is True or cache[k]["blocks"] is True: - c["blocks"] = True - else: - # self.cached_files[*][*]["blocks"] must continue to - # point to the same set object so that updates - # performed by MMapCache are propagated back to - # self.cached_files. - blocks = cache[k]["blocks"] - blocks.update(c["blocks"]) - c["blocks"] = blocks - c["time"] = max(c["time"], cache[k]["time"]) - c["uid"] = cache[k]["uid"] - - # Files can be added to cache after it was written once - for k, c in cache.items(): - if k not in cached_files: - cached_files[k] = c - else: - cached_files = cache - cache = {k: v.copy() for k, v in cached_files.items()} - for c in cache.values(): - if isinstance(c["blocks"], set): - c["blocks"] = list(c["blocks"]) - self._save(cache, fn) - self.cached_files[-1] = cached_files - - def update_file(self, path: str, detail: Detail) -> None: - """Update metadata for specific file in memory, do not save""" - self.cached_files[-1][path] = detail diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/cached.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/cached.py deleted file mode 100644 index c01ce9aef88ba62b04bbc308fd1a65bab354782b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/cached.py +++ /dev/null @@ -1,1040 +0,0 @@ -from __future__ import annotations - -import inspect -import logging -import os -import tempfile -import time -import weakref -from collections.abc import Callable -from shutil import rmtree -from typing import TYPE_CHECKING, Any, ClassVar - -from fsspec import filesystem -from fsspec.callbacks import DEFAULT_CALLBACK -from fsspec.compression import compr -from fsspec.core import BaseCache, MMapCache -from fsspec.exceptions import BlocksizeMismatchError -from fsspec.implementations.cache_mapper import create_cache_mapper -from fsspec.implementations.cache_metadata import CacheMetadata -from fsspec.implementations.chained import ChainedFileSystem -from fsspec.implementations.local import LocalFileSystem -from fsspec.spec import AbstractBufferedFile -from fsspec.transaction import Transaction -from fsspec.utils import infer_compression - -if TYPE_CHECKING: - from fsspec.implementations.cache_mapper import AbstractCacheMapper - -logger = logging.getLogger("fsspec.cached") - - -class WriteCachedTransaction(Transaction): - def complete(self, commit=True): - rpaths = [f.path for f in self.files] - lpaths = [f.fn for f in self.files] - if commit: - self.fs.put(lpaths, rpaths) - self.files.clear() - self.fs._intrans = False - self.fs._transaction = None - self.fs = None # break cycle - - -class CachingFileSystem(ChainedFileSystem): - """Locally caching filesystem, layer over any other FS - - This class implements chunk-wise local storage of remote files, for quick - access after the initial download. The files are stored in a given - directory with hashes of URLs for the filenames. If no directory is given, - a temporary one is used, which should be cleaned up by the OS after the - process ends. The files themselves are sparse (as implemented in - :class:`~fsspec.caching.MMapCache`), so only the data which is accessed - takes up space. - - Restrictions: - - - the block-size must be the same for each access of a given file, unless - all blocks of the file have already been read - - caching can only be applied to file-systems which produce files - derived from fsspec.spec.AbstractBufferedFile ; LocalFileSystem is also - allowed, for testing - """ - - protocol: ClassVar[str | tuple[str, ...]] = ("blockcache", "cached") - _strip_tokenize_options = ("fo",) - - def __init__( - self, - target_protocol=None, - cache_storage="TMP", - cache_check=10, - check_files=False, - expiry_time=604800, - target_options=None, - fs=None, - same_names: bool | None = None, - compression=None, - cache_mapper: AbstractCacheMapper | None = None, - cache_storage_mode=None, - **kwargs, - ): - """ - - Parameters - ---------- - target_protocol: str (optional) - Target filesystem protocol. Provide either this or ``fs``. - cache_storage: str or list(str) - Location to store files. If "TMP", this is a temporary directory, - and will be cleaned up by the OS when this process ends (or later). - If a list, each location will be tried in the order given, but - only the last will be considered writable. - cache_check: int - Number of seconds between reload of cache metadata - check_files: bool - Whether to explicitly see if the UID of the remote file matches - the stored one before using. Warning: some file systems such as - HTTP cannot reliably give a unique hash of the contents of some - path, so be sure to set this option to False. - expiry_time: int - The time in seconds after which a local copy is considered useless. - Set to falsy to prevent expiry. The default is equivalent to one - week. - target_options: dict or None - Passed to the instantiation of the FS, if fs is None. - fs: filesystem instance - The target filesystem to run against. Provide this or ``protocol``. - same_names: bool (optional) - By default, target URLs are hashed using a ``HashCacheMapper`` so - that files from different backends with the same basename do not - conflict. If this argument is ``true``, a ``BasenameCacheMapper`` - is used instead. Other cache mapper options are available by using - the ``cache_mapper`` keyword argument. Only one of this and - ``cache_mapper`` should be specified. - compression: str (optional) - To decompress on download. Can be 'infer' (guess from the URL name), - one of the entries in ``fsspec.compression.compr``, or None for no - decompression. - cache_mapper: AbstractCacheMapper (optional) - The object use to map from original filenames to cached filenames. - Only one of this and ``same_names`` should be specified. - cache_storage_mode: int (optional) - Permission mode used when creating the cache storage directory, - e.g. ``0o700``. The default of ``None`` leaves the directory at - the system default (governed by the umask), which is the existing - behaviour. Pass an octal mode such as ``0o700`` to keep the cache - directory, and so the cached data files within it, readable by the - owner only. - """ - super().__init__(**kwargs) - if fs is None and target_protocol is None: - raise ValueError( - "Please provide filesystem instance(fs) or target_protocol" - ) - if not (fs is None) ^ (target_protocol is None): - raise ValueError( - "Both filesystems (fs) and target_protocol may not be both given." - ) - if cache_storage == "TMP": - tempdir = tempfile.mkdtemp() - storage = [tempdir] - weakref.finalize(self, self._remove_tempdir, tempdir) - else: - if isinstance(cache_storage, str): - storage = [cache_storage] - else: - storage = cache_storage - self.cache_storage_mode = cache_storage_mode - self._makedirs(storage[-1]) - self.storage = storage - self.kwargs = target_options or {} - self.cache_check = cache_check - self.check_files = check_files - self.expiry = expiry_time - self.compression = compression - - # Size of cache in bytes. If None then the size is unknown and will be - # recalculated the next time cache_size() is called. On writes to the - # cache this is reset to None. - self._cache_size = None - - if same_names is not None and cache_mapper is not None: - raise ValueError( - "Cannot specify both same_names and cache_mapper in " - "CachingFileSystem.__init__" - ) - if cache_mapper is not None: - self._mapper = cache_mapper - else: - self._mapper = create_cache_mapper( - same_names if same_names is not None else False - ) - - self.target_protocol = ( - target_protocol - if isinstance(target_protocol, str) - else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0]) - ) - self._metadata = CacheMetadata(self.storage) - self.load_cache() - self.fs = fs if fs is not None else filesystem(target_protocol, **self.kwargs) - - def _strip_protocol(path): - # acts as a method, since each instance has a difference target - return self.fs._strip_protocol(type(self)._strip_protocol(path)) - - self._strip_protocol: Callable = _strip_protocol - - @staticmethod - def _remove_tempdir(tempdir): - try: - rmtree(tempdir) - except Exception: - pass - - def _makedirs(self, path): - if self.cache_storage_mode is None: - os.makedirs(path, exist_ok=True) - else: - os.makedirs(path, exist_ok=True, mode=self.cache_storage_mode) - - def _mkcache(self): - self._makedirs(self.storage[-1]) - - def cache_size(self): - """Return size of cache in bytes. - - If more than one cache directory is in use, only the size of the last - one (the writable cache directory) is returned. - """ - if self._cache_size is None: - cache_dir = self.storage[-1] - self._cache_size = filesystem("file").du(cache_dir, withdirs=True) - return self._cache_size - - def load_cache(self): - """Read set of stored blocks from file""" - self._metadata.load() - self._mkcache() - self.last_cache = time.time() - - def save_cache(self): - """Save set of stored blocks from file""" - self._mkcache() - self._metadata.save() - self.last_cache = time.time() - self._cache_size = None - - def _check_cache(self): - """Reload caches if time elapsed or any disappeared""" - self._mkcache() - if not self.cache_check: - # explicitly told not to bother checking - return - timecond = time.time() - self.last_cache > self.cache_check - existcond = all(os.path.exists(storage) for storage in self.storage) - if timecond or not existcond: - self.load_cache() - - def _check_file(self, path): - """Is path in cache and still valid""" - path = self._strip_protocol(path) - self._check_cache() - return self._metadata.check_file(path, self) - - def clear_cache(self): - """Remove all files and metadata from the cache - - In the case of multiple cache locations, this clears only the last one, - which is assumed to be the read/write one. - """ - rmtree(self.storage[-1]) - self.load_cache() - self._cache_size = None - - def clear_expired_cache(self, expiry_time=None): - """Remove all expired files and metadata from the cache - - In the case of multiple cache locations, this clears only the last one, - which is assumed to be the read/write one. - - Parameters - ---------- - expiry_time: int - The time in seconds after which a local copy is considered useless. - If not defined the default is equivalent to the attribute from the - file caching instantiation. - """ - - if not expiry_time: - expiry_time = self.expiry - - self._check_cache() - - expired_files, writable_cache_empty = self._metadata.clear_expired(expiry_time) - for fn in expired_files: - if os.path.exists(fn): - os.remove(fn) - - if writable_cache_empty: - rmtree(self.storage[-1]) - self.load_cache() - - self._cache_size = None - - def pop_from_cache(self, path): - """Remove cached version of given file - - Deletes local copy of the given (remote) path. If it is found in a cache - location which is not the last, it is assumed to be read-only, and - raises PermissionError - """ - path = self._strip_protocol(path) - fn = self._metadata.pop_file(path) - if fn is not None: - os.remove(fn) - self._cache_size = None - - def _open( - self, - path, - mode="rb", - block_size=None, - autocommit=True, - cache_options=None, - **kwargs, - ): - """Wrap the target _open - - If the whole file exists in the cache, just open it locally and - return that. - - Otherwise, open the file on the target FS, and make it have a mmap - cache pointing to the location which we determine, in our cache. - The ``blocks`` instance is shared, so as the mmap cache instance - updates, so does the entry in our ``cached_files`` attribute. - We monkey-patch this file, so that when it closes, we call - ``close_and_update`` to save the state of the blocks. - """ - path = self._strip_protocol(path) - - path = self.fs._strip_protocol(path) - if "r" not in mode: - return self.fs._open( - path, - mode=mode, - block_size=block_size, - autocommit=autocommit, - cache_options=cache_options, - **kwargs, - ) - detail = self._check_file(path) - if detail: - # file is in cache - detail, fn = detail - hash, blocks = detail["fn"], detail["blocks"] - if blocks is True: - # stored file is complete - logger.debug("Opening local copy of %s", path) - return open(fn, mode) - # TODO: action where partial file exists in read-only cache - logger.debug("Opening partially cached copy of %s", path) - else: - hash = self._mapper(path) - fn = os.path.join(self.storage[-1], hash) - blocks = set() - detail = { - "original": path, - "fn": hash, - "blocks": blocks, - "time": time.time(), - "uid": self.fs.ukey(path), - } - self._metadata.update_file(path, detail) - logger.debug("Creating local sparse file for %s", path) - - # explicitly submitting the size to the open call will avoid extra - # operations when opening. This is particularly relevant - # for any file that is read over a network, e.g. S3. - size = detail.get("size") - - # call target filesystems open - self._mkcache() - f = self.fs._open( - path, - mode=mode, - block_size=block_size, - autocommit=autocommit, - cache_options=cache_options, - cache_type="none", - size=size, - **kwargs, - ) - - # set size if not already set - if size is None: - detail["size"] = f.size - self._metadata.update_file(path, detail) - - if self.compression: - comp = ( - infer_compression(path) - if self.compression == "infer" - else self.compression - ) - f = compr[comp](f, mode="rb") - if "blocksize" in detail: - if detail["blocksize"] != f.blocksize: - raise BlocksizeMismatchError( - f"Cached file must be reopened with same block" - f" size as original (old: {detail['blocksize']}," - f" new {f.blocksize})" - ) - else: - detail["blocksize"] = f.blocksize - - def _fetch_ranges(ranges): - return self.fs.cat_ranges( - [path] * len(ranges), - [r[0] for r in ranges], - [r[1] for r in ranges], - **kwargs, - ) - - multi_fetcher = None if self.compression else _fetch_ranges - f.cache = MMapCache( - f.blocksize, f._fetch_range, f.size, fn, blocks, multi_fetcher=multi_fetcher - ) - close = f.close - f.close = lambda: self.close_and_update(f, close) - self.save_cache() - return f - - def _parent(self, path): - return self.fs._parent(path) - - def hash_name(self, path: str, *args: Any) -> str: - # Kept for backward compatibility with downstream libraries. - # Ignores extra arguments, previously same_name boolean. - return self._mapper(path) - - def close_and_update(self, f, close): - """Called when a file is closing, so store the set of blocks""" - if f.closed: - return - path = self._strip_protocol(f.path) - self._metadata.on_close_cached_file(f, path) - try: - logger.debug("going to save") - self.save_cache() - logger.debug("saved") - except OSError: - logger.debug("Cache saving failed while closing file") - except NameError: - logger.debug("Cache save failed due to interpreter shutdown") - close() - f.closed = True - - def ls(self, path, detail=True, **kwargs): - return self.fs.ls(path, detail, **kwargs) - - def __getattribute__(self, item): - if item in { - "load_cache", - "_get_cached_file_before_open", - "_open", - "save_cache", - "close_and_update", - "__init__", - "__getattribute__", - "__reduce__", - "_make_local_details", - "open", - "cat", - "cat_file", - "_cat_file", - "cat_ranges", - "_cat_ranges", - "get", - "read_block", - "tail", - "head", - "info", - "ls", - "exists", - "isfile", - "isdir", - "_check_file", - "_check_cache", - "_makedirs", - "_mkcache", - "clear_cache", - "clear_expired_cache", - "pop_from_cache", - "local_file", - "_paths_from_path", - "get_mapper", - "open_many", - "commit_many", - "hash_name", - "__hash__", - "__eq__", - "to_json", - "to_dict", - "cache_size", - "pipe_file", - "pipe", - "start_transaction", - "end_transaction", - }: - # all the methods defined in this class. Note `open` here, since - # it calls `_open`, but is actually in superclass - if hasattr(type(self), item): - return lambda *args, **kw: getattr(type(self), item).__get__(self)( - *args, **kw - ) - # method is in the whitelist but not defined on this subclass; - # fall through to delegate to the wrapped filesystem below - if item in ["__reduce_ex__"]: - raise AttributeError - if item in ["transaction"]: - # property - return type(self).transaction.__get__(self) - if item in {"_cache", "transaction_type", "protocol"}: - # class attributes - return getattr(type(self), item) - if item == "__class__": - return type(self) - d = object.__getattribute__(self, "__dict__") - fs = d.get("fs", None) # fs is not immediately defined - if item in d: - return d[item] - elif fs is not None: - if item in fs.__dict__: - # attribute of instance - return fs.__dict__[item] - # attributed belonging to the target filesystem - cls = type(fs) - m = getattr(cls, item) - if (inspect.isfunction(m) or inspect.isdatadescriptor(m)) and ( - not hasattr(m, "__self__") or m.__self__ is None - ): - # instance method - return m.__get__(fs, cls) - return m # class method or attribute - else: - # attributes of the superclass, while target is being set up - return super().__getattribute__(item) - - def __eq__(self, other): - """Test for equality.""" - if self is other: - return True - if not isinstance(other, type(self)): - return False - return ( - self.storage == other.storage - and self.kwargs == other.kwargs - and self.cache_check == other.cache_check - and self.check_files == other.check_files - and self.expiry == other.expiry - and self.compression == other.compression - and self._mapper == other._mapper - and self.target_protocol == other.target_protocol - ) - - def __hash__(self): - """Calculate hash.""" - return ( - hash(tuple(self.storage)) - ^ hash(str(self.kwargs)) - ^ hash(self.cache_check) - ^ hash(self.check_files) - ^ hash(self.expiry) - ^ hash(self.compression) - ^ hash(self._mapper) - ^ hash(self.target_protocol) - ) - - -class WholeFileCacheFileSystem(CachingFileSystem): - """Caches whole remote files on first access - - This class is intended as a layer over any other file system, and - will make a local copy of each file accessed, so that all subsequent - reads are local. This is similar to ``CachingFileSystem``, but without - the block-wise functionality and so can work even when sparse files - are not allowed. See its docstring for definition of the init - arguments. - - The class still needs access to the remote store for listing files, - and may refresh cached files. - """ - - protocol = "filecache" - local_file = True - - def open_many(self, open_files, **kwargs): - paths = [of.path for of in open_files] - if "r" in open_files.mode: - self._mkcache() - else: - return [ - LocalTempFile( - self.fs, - path, - mode=open_files.mode, - fn=os.path.join(self.storage[-1], self._mapper(path)), - **kwargs, - ) - for path in paths - ] - - if self.compression: - raise NotImplementedError - details = [self._check_file(sp) for sp in paths] - downpath = [p for p, d in zip(paths, details) if not d] - downfn0 = [ - os.path.join(self.storage[-1], self._mapper(p)) - for p, d in zip(paths, details) - ] # keep these path names for opening later - downfn = [fn for fn, d in zip(downfn0, details) if not d] - if downpath: - # skip if all files are already cached and up to date - self.fs.get(downpath, downfn) - - # update metadata - only happens when downloads are successful - newdetail = [ - { - "original": path, - "fn": self._mapper(path), - "blocks": True, - "time": time.time(), - "uid": self.fs.ukey(path), - } - for path in downpath - ] - for path, detail in zip(downpath, newdetail): - self._metadata.update_file(path, detail) - self.save_cache() - - def firstpart(fn): - # helper to adapt both whole-file and simple-cache - return fn[1] if isinstance(fn, tuple) else fn - - return [ - open(firstpart(fn0) if fn0 else fn1, mode=open_files.mode) - for fn0, fn1 in zip(details, downfn0) - ] - - def commit_many(self, open_files): - self.fs.put([f.fn for f in open_files], [f.path for f in open_files]) - [f.close() for f in open_files] - for f in open_files: - # in case autocommit is off, and so close did not already delete - try: - os.remove(f.name) - except FileNotFoundError: - pass - self._cache_size = None - - def _make_local_details(self, path): - hash = self._mapper(path) - fn = os.path.join(self.storage[-1], hash) - detail = { - "original": path, - "fn": hash, - "blocks": True, - "time": time.time(), - "uid": self.fs.ukey(path), - } - self._metadata.update_file(path, detail) - logger.debug("Copying %s to local cache", path) - return fn - - def cat( - self, - path, - recursive=False, - on_error="raise", - callback=DEFAULT_CALLBACK, - **kwargs, - ): - paths = self.expand_path( - path, recursive=recursive, maxdepth=kwargs.get("maxdepth") - ) - getpaths = [] - storepaths = [] - fns = [] - out = {} - for p in paths.copy(): - try: - detail = self._check_file(p) - if not detail: - fn = self._make_local_details(p) - getpaths.append(p) - storepaths.append(fn) - else: - detail, fn = detail if isinstance(detail, tuple) else (None, detail) - fns.append(fn) - except Exception as e: - if on_error == "raise": - raise - if on_error == "return": - out[p] = e - paths.remove(p) - - if getpaths: - self.fs.get(getpaths, storepaths) - self.save_cache() - - callback.set_size(len(paths)) - for p, fn in zip(paths, fns): - with open(fn, "rb") as f: - out[p] = f.read() - callback.relative_update(1) - if isinstance(path, str) and len(paths) == 1 and recursive is False: - out = out[paths[0]] - return out - - def _get_cached_file_before_open(self, path, **kwargs): - fn = self._make_local_details(path) - # call target filesystems open - self._mkcache() - if self.compression: - with self.fs._open(path, mode="rb", **kwargs) as f, open(fn, "wb") as f2: - if isinstance(f, AbstractBufferedFile): - # want no type of caching if just downloading whole thing - f.cache = BaseCache(0, f.cache.fetcher, f.size) - comp = ( - infer_compression(path) - if self.compression == "infer" - else self.compression - ) - f = compr[comp](f, mode="rb") - data = True - while data: - block = getattr(f, "blocksize", 5 * 2**20) - data = f.read(block) - f2.write(data) - else: - self.fs.get_file(path, fn) - self.save_cache() - - def _open(self, path, mode="rb", **kwargs): - path = self._strip_protocol(path) - # For read (or append), (try) download from remote - if "r" in mode or "a" in mode: - if not self._check_file(path): - if self.fs.exists(path): - self._get_cached_file_before_open(path, **kwargs) - elif "r" in mode: - raise FileNotFoundError(path) - - detail, fn = self._check_file(path) - _, blocks = detail["fn"], detail["blocks"] - if blocks is True: - logger.debug("Opening local copy of %s", path) - else: - raise ValueError( - f"Attempt to open partially cached file {path}" - f" as a wholly cached file" - ) - - # Just reading does not need special file handling - if "r" in mode and "+" not in mode: - # In order to support downstream filesystems to be able to - # infer the compression from the original filename, like - # the `TarFileSystem`, let's extend the `io.BufferedReader` - # fileobject protocol by adding a dedicated attribute - # `original`. - f = open(fn, mode) - f.original = detail.get("original") - return f - - hash = self._mapper(path) - fn = os.path.join(self.storage[-1], hash) - user_specified_kwargs = { - k: v - for k, v in kwargs.items() - # those kwargs were added by open(), we don't want them - if k not in ["autocommit", "block_size", "cache_options"] - } - return LocalTempFile(self, path, mode=mode, fn=fn, **user_specified_kwargs) - - async def _cat_file(self, path, start=None, end=None, **kwargs): - logger.debug("async cat_file %s", path) - path = self._strip_protocol(path) - sha = self._mapper(path) - fn = self._check_file(path) - - if not fn: - fn = os.path.join(self.storage[-1], sha) - await self.fs._get_file(path, fn, **kwargs) - - with open(fn, "rb") as f: # noqa ASYNC230 - if start: - f.seek(start) - size = -1 if end is None else end - f.tell() - return f.read(size) - - async def _cat_ranges( - self, paths, starts, ends, max_gap=None, on_error="return", **kwargs - ): - logger.debug("async cat ranges %s", paths) - lpaths = [] - rset = set() - download = [] - rpaths = [] - for p in paths: - fn = self._check_file(p) - if fn is None and p not in rset: - sha = self._mapper(p) - fn = os.path.join(self.storage[-1], sha) - download.append(fn) - rset.add(p) - rpaths.append(p) - lpaths.append(fn) - if download: - await self.fs._get(rpaths, download, on_error=on_error) - - return LocalFileSystem().cat_ranges( - lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs - ) - - -class SimpleCacheFileSystem(WholeFileCacheFileSystem): - """Caches whole remote files on first access - - This class is intended as a layer over any other file system, and - will make a local copy of each file accessed, so that all subsequent - reads are local. This implementation only copies whole files, and - does not keep any metadata about the download time or file details. - It is therefore safer to use in multi-threaded/concurrent situations. - - This is the only of the caching filesystems that supports write: you will - be given a real local open file, and upon close and commit, it will be - uploaded to the target filesystem; the writability or the target URL is - not checked until that time. - - """ - - protocol = "simplecache" - local_file = True - transaction_type = WriteCachedTransaction - - def __init__(self, **kwargs): - kw = kwargs.copy() - for key in ["cache_check", "expiry_time", "check_files"]: - kw[key] = False - super().__init__(**kw) - for storage in self.storage: - if not os.path.exists(storage): - self._makedirs(storage) - - def _check_file(self, path): - self._check_cache() - sha = self._mapper(path) - for storage in self.storage: - fn = os.path.join(storage, sha) - if os.path.exists(fn): - return fn - - def save_cache(self): - pass - - def load_cache(self): - pass - - def pipe_file(self, path, value=None, **kwargs): - if self._intrans: - with self.open(path, "wb") as f: - f.write(value) - else: - super().pipe_file(path, value) - - def ls(self, path, detail=True, **kwargs): - path = self._strip_protocol(path) - details = [] - try: - details = self.fs.ls( - path, detail=True, **kwargs - ).copy() # don't edit original! - except FileNotFoundError as e: - ex = e - else: - ex = None - if self._intrans: - path1 = path.rstrip("/") + "/" - for f in self.transaction.files: - if f.path == path: - details.append( - {"name": path, "size": f.size or f.tell(), "type": "file"} - ) - elif f.path.startswith(path1): - if f.path.count("/") == path1.count("/"): - details.append( - {"name": f.path, "size": f.size or f.tell(), "type": "file"} - ) - else: - dname = "/".join(f.path.split("/")[: path1.count("/") + 1]) - details.append({"name": dname, "size": 0, "type": "directory"}) - if ex is not None and not details: - raise ex - if detail: - return details - return sorted(_["name"] for _ in details) - - def info(self, path, **kwargs): - path = self._strip_protocol(path) - if self._intrans: - f = [_ for _ in self.transaction.files if _.path == path] - if f: - size = os.path.getsize(f[0].fn) if f[0].closed else f[0].tell() - return {"name": path, "size": size, "type": "file"} - f = any(_.path.startswith(path + "/") for _ in self.transaction.files) - if f: - return {"name": path, "size": 0, "type": "directory"} - return self.fs.info(path, **kwargs) - - def pipe(self, path, value=None, **kwargs): - if isinstance(path, str): - self.pipe_file(self._strip_protocol(path), value, **kwargs) - elif isinstance(path, dict): - for k, v in path.items(): - self.pipe_file(self._strip_protocol(k), v, **kwargs) - else: - raise ValueError("path must be str or dict") - - def cat_ranges( - self, paths, starts, ends, max_gap=None, on_error="return", **kwargs - ): - logger.debug("cat ranges %s", paths) - lpaths = [self._check_file(p) for p in paths] - rpaths = [p for l, p in zip(lpaths, paths) if l is False] - lpaths = [l for l, p in zip(lpaths, paths) if l is False] - self.fs.get(rpaths, lpaths) - paths = [self._check_file(p) for p in paths] - return LocalFileSystem().cat_ranges( - paths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs - ) - - def _get_cached_file_before_open(self, path, **kwargs): - sha = self._mapper(path) - fn = os.path.join(self.storage[-1], sha) - logger.debug("Copying %s to local cache", path) - - self._mkcache() - self._cache_size = None - - if self.compression: - with self.fs._open(path, mode="rb", **kwargs) as f, open(fn, "wb") as f2: - if isinstance(f, AbstractBufferedFile): - # want no type of caching if just downloading whole thing - f.cache = BaseCache(0, f.cache.fetcher, f.size) - comp = ( - infer_compression(path) - if self.compression == "infer" - else self.compression - ) - f = compr[comp](f, mode="rb") - data = True - while data: - block = getattr(f, "blocksize", 5 * 2**20) - data = f.read(block) - f2.write(data) - else: - self.fs.get_file(path, fn) - - def _open(self, path, mode="rb", **kwargs): - path = self._strip_protocol(path) - sha = self._mapper(path) - - # For read (or append), (try) download from remote - if "r" in mode or "a" in mode: - if not self._check_file(path): - # append does not require an existing file but read does - if self.fs.exists(path): - self._get_cached_file_before_open(path, **kwargs) - elif "r" in mode: - raise FileNotFoundError(path) - - fn = self._check_file(path) - # Just reading does not need special file handling - if "r" in mode and "+" not in mode: - return open(fn, mode) - - fn = os.path.join(self.storage[-1], sha) - user_specified_kwargs = { - k: v - for k, v in kwargs.items() - if k not in ["autocommit", "block_size", "cache_options"] - } # those were added by open() - return LocalTempFile( - self, - path, - mode=mode, - autocommit=not self._intrans, - fn=fn, - **user_specified_kwargs, - ) - - -class LocalTempFile: - """A temporary local file, which will be uploaded on commit""" - - def __init__(self, fs, path, fn, mode="wb", autocommit=True, seek=0, **kwargs): - self.fn = fn - self.fh = open(fn, mode) - self.mode = mode - if seek: - self.fh.seek(seek) - self.path = path - self.size = None - self.fs = fs - self.closed = False - self.autocommit = autocommit - self.kwargs = kwargs - - def __reduce__(self): - # always open in r+b to allow continuing writing at a location - return ( - LocalTempFile, - (self.fs, self.path, self.fn, "r+b", self.autocommit, self.tell()), - ) - - def __enter__(self): - return self.fh - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def close(self): - # self.size = self.fh.tell() - if self.closed: - return - self.fh.close() - self.closed = True - if self.autocommit: - self.commit() - - def discard(self): - self.fh.close() - os.remove(self.fn) - - def commit(self): - # calling put() with list arguments avoids path expansion and additional operations - # like isdir() - self.fs.put([self.fn], [self.path], **self.kwargs) - # we do not delete the local copy, it's still in the cache. - - @property - def name(self): - return self.fn - - def __repr__(self) -> str: - return f"LocalTempFile: {self.path}" - - def __getattr__(self, item): - return getattr(self.fh, item) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/chained.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/chained.py deleted file mode 100644 index bfce64334e8db0272eefa96b4428b23524b059f0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/chained.py +++ /dev/null @@ -1,23 +0,0 @@ -from typing import ClassVar - -from fsspec import AbstractFileSystem - -__all__ = ("ChainedFileSystem",) - - -class ChainedFileSystem(AbstractFileSystem): - """Chained filesystem base class. - - A chained filesystem is designed to be layered over another FS. - This is useful to implement things like caching. - - This base class does very little on its own, but is used as a marker - that the class is designed for chaining. - - Right now this is only used in `url_to_fs` to provide the path argument - (`fo`) to the chained filesystem from the underlying filesystem. - - Additional functionality may be added in the future. - """ - - protocol: ClassVar[str] = "chained" diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/dask.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/dask.py deleted file mode 100644 index 3e1276463db6866665e6a0fe114efc247971b57e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/dask.py +++ /dev/null @@ -1,152 +0,0 @@ -import dask -from distributed.client import Client, _get_global_client -from distributed.worker import Worker - -from fsspec import filesystem -from fsspec.spec import AbstractBufferedFile, AbstractFileSystem -from fsspec.utils import infer_storage_options - - -def _get_client(client): - if client is None: - return _get_global_client() - elif isinstance(client, Client): - return client - else: - # e.g., connection string - return Client(client) - - -def _in_worker(): - return bool(Worker._instances) - - -class DaskWorkerFileSystem(AbstractFileSystem): - """View files accessible to a worker as any other remote file-system - - When instances are run on the worker, uses the real filesystem. When - run on the client, they call the worker to provide information or data. - - **Warning** this implementation is experimental, and read-only for now. - """ - - def __init__( - self, target_protocol=None, target_options=None, fs=None, client=None, **kwargs - ): - super().__init__(**kwargs) - if not (fs is None) ^ (target_protocol is None): - raise ValueError( - "Please provide one of filesystem instance (fs) or" - " target_protocol, not both" - ) - self.target_protocol = target_protocol - self.target_options = target_options - self.worker = None - self.client = client - self.fs = fs - self._determine_worker() - - @staticmethod - def _get_kwargs_from_urls(path): - so = infer_storage_options(path) - if "host" in so and "port" in so: - return {"client": f"{so['host']}:{so['port']}"} - else: - return {} - - def _determine_worker(self): - if _in_worker(): - self.worker = True - if self.fs is None: - self.fs = filesystem( - self.target_protocol, **(self.target_options or {}) - ) - else: - self.worker = False - self.client = _get_client(self.client) - self.rfs = dask.delayed(self) - - def mkdir(self, *args, **kwargs): - if self.worker: - self.fs.mkdir(*args, **kwargs) - else: - self.rfs.mkdir(*args, **kwargs).compute() - - def rm(self, *args, **kwargs): - if self.worker: - self.fs.rm(*args, **kwargs) - else: - self.rfs.rm(*args, **kwargs).compute() - - def copy(self, *args, **kwargs): - if self.worker: - self.fs.copy(*args, **kwargs) - else: - self.rfs.copy(*args, **kwargs).compute() - - def mv(self, *args, **kwargs): - if self.worker: - self.fs.mv(*args, **kwargs) - else: - self.rfs.mv(*args, **kwargs).compute() - - def ls(self, *args, **kwargs): - if self.worker: - return self.fs.ls(*args, **kwargs) - else: - return self.rfs.ls(*args, **kwargs).compute() - - def _open( - self, - path, - mode="rb", - block_size=None, - autocommit=True, - cache_options=None, - **kwargs, - ): - if self.worker: - return self.fs._open( - path, - mode=mode, - block_size=block_size, - autocommit=autocommit, - cache_options=cache_options, - **kwargs, - ) - else: - return DaskFile( - fs=self, - path=path, - mode=mode, - block_size=block_size, - autocommit=autocommit, - cache_options=cache_options, - **kwargs, - ) - - def fetch_range(self, path, mode, start, end): - if self.worker: - with self._open(path, mode) as f: - f.seek(start) - return f.read(end - start) - else: - return self.rfs.fetch_range(path, mode, start, end).compute() - - -class DaskFile(AbstractBufferedFile): - def __init__(self, mode="rb", **kwargs): - if mode != "rb": - raise ValueError('Remote dask files can only be opened in "rb" mode') - super().__init__(**kwargs) - - def _upload_chunk(self, final=False): - pass - - def _initiate_upload(self): - """Create remote file/upload""" - pass - - def _fetch_range(self, start, end): - """Get the specified set of bytes from remote""" - return self.fs.fetch_range(self.path, self.mode, start, end) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/data.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/data.py deleted file mode 100644 index fad1b56d0940a6e5232497475ab32a504e19ec9a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/data.py +++ /dev/null @@ -1,71 +0,0 @@ -import base64 -import io -from urllib.parse import unquote - -from fsspec import AbstractFileSystem -from fsspec.utils import stringify_path - - -class DataFileSystem(AbstractFileSystem): - """A handy decoder for data-URLs - - Example - ------- - >>> with fsspec.open("data:,Hello%2C%20World%21") as f: - ... print(f.read()) - b"Hello, World!" - - See https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs - """ - - protocol = "data" - - def __init__(self, **kwargs): - """No parameters for this filesystem""" - super().__init__(**kwargs) - - @classmethod - def _strip_protocol(cls, path): - if isinstance(path, list): - return [cls._strip_protocol(p) for p in path] - path = stringify_path(path) - if path.startswith("data://"): - path = path[7:] - elif path.startswith("data:"): - path = path[5:] - # Do NOT strip trailing slashes, as they may be meaningful base64 characters - # or percent-encoded data content - return path - - def cat_file(self, path, start=None, end=None, **kwargs): - pref, data = path.split(",", 1) - if pref.endswith("base64"): - return base64.b64decode(data)[start:end] - return unquote(data).encode()[start:end] - - def info(self, path, **kwargs): - pref, name = path.split(",", 1) - data = self.cat_file(path) - mime = pref.split(":", 1)[1].split(";", 1)[0] - return {"name": name, "size": len(data), "type": "file", "mimetype": mime} - - def _open( - self, - path, - mode="rb", - block_size=None, - autocommit=True, - cache_options=None, - **kwargs, - ): - if "r" not in mode: - raise ValueError("Read only filesystem") - return io.BytesIO(self.cat_file(path)) - - @staticmethod - def encode(data: bytes, mime: str | None = None): - """Format the given data into data-URL syntax - - This version always base64 encodes, even when the data is ascii/url-safe. - """ - return f"data:{mime or ''};base64,{base64.b64encode(data).decode()}" diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/dbfs.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/dbfs.py deleted file mode 100644 index 1b7f03f66c94dccf7880546728ce1c3f110bc10a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/dbfs.py +++ /dev/null @@ -1,497 +0,0 @@ -from __future__ import annotations - -import base64 -import urllib - -import requests -from requests.adapters import HTTPAdapter, Retry -from typing_extensions import override - -from fsspec import AbstractFileSystem -from fsspec.spec import AbstractBufferedFile - - -class DatabricksException(Exception): - """ - Helper class for exceptions raised in this module. - """ - - def __init__(self, error_code, message, details=None): - """Create a new DatabricksException""" - super().__init__(message) - - self.error_code = error_code - self.message = message - self.details = details - - -class DatabricksFileSystem(AbstractFileSystem): - """ - Get access to the Databricks filesystem implementation over HTTP. - Can be used inside and outside of a databricks cluster. - """ - - def __init__(self, instance, token, **kwargs): - """ - Create a new DatabricksFileSystem. - - Parameters - ---------- - instance: str - The instance URL of the databricks cluster. - For example for an Azure databricks cluster, this - has the form adb-..azuredatabricks.net. - token: str - Your personal token. Find out more - here: https://docs.databricks.com/dev-tools/api/latest/authentication.html - """ - self.instance = instance - self.token = token - self.session = requests.Session() - self.retries = Retry( - total=10, - backoff_factor=0.05, - status_forcelist=[408, 429, 500, 502, 503, 504], - ) - - self.session.mount("https://", HTTPAdapter(max_retries=self.retries)) - self.session.headers.update({"Authorization": f"Bearer {self.token}"}) - - super().__init__(**kwargs) - - @override - def _ls_from_cache(self, path) -> list[dict[str, str | int]] | None: - """Check cache for listing - - Returns listing, if found (may be empty list for a directory that - exists but contains nothing), None if not in cache. - """ - self.dircache.pop(path.rstrip("/"), None) - - parent = self._parent(path) - if parent in self.dircache: - for entry in self.dircache[parent]: - if entry["name"] == path.rstrip("/"): - if entry["type"] != "directory": - return [entry] - return [] - raise FileNotFoundError(path) - - def ls(self, path, detail=True, **kwargs): - """ - List the contents of the given path. - - Parameters - ---------- - path: str - Absolute path - detail: bool - Return not only the list of filenames, - but also additional information on file sizes - and types. - """ - try: - out = self._ls_from_cache(path) - except FileNotFoundError: - # This happens if the `path`'s parent was cached, but `path` is not - # there. This suggests that `path` is new since the parent was - # cached. Attempt to invalidate parent's cache before continuing. - self.dircache.pop(self._parent(path), None) - out = None - - if not out: - try: - r = self._send_to_api( - method="get", endpoint="list", json={"path": path} - ) - except DatabricksException as e: - if e.error_code == "RESOURCE_DOES_NOT_EXIST": - raise FileNotFoundError(e.message) from e - - raise - files = r.get("files", []) - out = [ - { - "name": o["path"], - "type": "directory" if o["is_dir"] else "file", - "size": o["file_size"], - } - for o in files - ] - self.dircache[path] = out - - if detail: - return out - return [o["name"] for o in out] - - def makedirs(self, path, exist_ok=True): - """ - Create a given absolute path and all of its parents. - - Parameters - ---------- - path: str - Absolute path to create - exist_ok: bool - If false, checks if the folder - exists before creating it (and raises an - Exception if this is the case) - """ - if not exist_ok: - try: - # If the following succeeds, the path is already present - self._send_to_api( - method="get", endpoint="get-status", json={"path": path} - ) - raise FileExistsError(f"Path {path} already exists") - except DatabricksException as e: - if e.error_code == "RESOURCE_DOES_NOT_EXIST": - pass - - try: - self._send_to_api(method="post", endpoint="mkdirs", json={"path": path}) - except DatabricksException as e: - if e.error_code == "RESOURCE_ALREADY_EXISTS": - raise FileExistsError(e.message) from e - - raise - self.invalidate_cache(self._parent(path)) - - def mkdir(self, path, create_parents=True, **kwargs): - """ - Create a given absolute path and all of its parents. - - Parameters - ---------- - path: str - Absolute path to create - create_parents: bool - Whether to create all parents or not. - "False" is not implemented so far. - """ - if not create_parents: - raise NotImplementedError - - self.mkdirs(path, **kwargs) - - def rm(self, path, recursive=False, **kwargs): - """ - Remove the file or folder at the given absolute path. - - Parameters - ---------- - path: str - Absolute path what to remove - recursive: bool - Recursively delete all files in a folder. - """ - try: - self._send_to_api( - method="post", - endpoint="delete", - json={"path": path, "recursive": recursive}, - ) - except DatabricksException as e: - # This is not really an exception, it just means - # not everything was deleted so far - if e.error_code == "PARTIAL_DELETE": - self.rm(path=path, recursive=recursive) - elif e.error_code == "IO_ERROR": - # Using the same exception as the os module would use here - raise OSError(e.message) from e - - raise - self.invalidate_cache(self._parent(path)) - - def mv( - self, source_path, destination_path, recursive=False, maxdepth=None, **kwargs - ): - """ - Move a source to a destination path. - - A note from the original [databricks API manual] - (https://docs.databricks.com/dev-tools/api/latest/dbfs.html#move). - - When moving a large number of files the API call will time out after - approximately 60s, potentially resulting in partially moved data. - Therefore, for operations that move more than 10k files, we strongly - discourage using the DBFS REST API. - - Parameters - ---------- - source_path: str - From where to move (absolute path) - destination_path: str - To where to move (absolute path) - recursive: bool - Not implemented to far. - maxdepth: - Not implemented to far. - """ - if recursive: - raise NotImplementedError - if maxdepth: - raise NotImplementedError - - try: - self._send_to_api( - method="post", - endpoint="move", - json={"source_path": source_path, "destination_path": destination_path}, - ) - except DatabricksException as e: - if e.error_code == "RESOURCE_DOES_NOT_EXIST": - raise FileNotFoundError(e.message) from e - elif e.error_code == "RESOURCE_ALREADY_EXISTS": - raise FileExistsError(e.message) from e - - raise - self.invalidate_cache(self._parent(source_path)) - self.invalidate_cache(self._parent(destination_path)) - - def _open(self, path, mode="rb", block_size="default", **kwargs): - """ - Overwrite the base class method to make sure to create a DBFile. - All arguments are copied from the base method. - - Only the default blocksize is allowed. - """ - return DatabricksFile(self, path, mode=mode, block_size=block_size, **kwargs) - - def _send_to_api(self, method, endpoint, json): - """ - Send the given json to the DBFS API - using a get or post request (specified by the argument `method`). - - Parameters - ---------- - method: str - Which http method to use for communication; "get" or "post". - endpoint: str - Where to send the request to (last part of the API URL) - json: dict - Dictionary of information to send - """ - if method == "post": - session_call = self.session.post - elif method == "get": - session_call = self.session.get - else: - raise ValueError(f"Do not understand method {method}") - - url = urllib.parse.urljoin(f"https://{self.instance}/api/2.0/dbfs/", endpoint) - - r = session_call(url, json=json) - - # The DBFS API will return a json, also in case of an exception. - # We want to preserve this information as good as possible. - try: - r.raise_for_status() - except requests.HTTPError as e: - # try to extract json error message - # if that fails, fall back to the original exception - try: - exception_json = e.response.json() - except Exception: - raise e from None - - raise DatabricksException(**exception_json) from e - - return r.json() - - def _create_handle(self, path, overwrite=True): - """ - Internal function to create a handle, which can be used to - write blocks of a file to DBFS. - A handle has a unique identifier which needs to be passed - whenever written during this transaction. - The handle is active for 10 minutes - after that a new - write transaction needs to be created. - Make sure to close the handle after you are finished. - - Parameters - ---------- - path: str - Absolute path for this file. - overwrite: bool - If a file already exist at this location, either overwrite - it or raise an exception. - """ - try: - r = self._send_to_api( - method="post", - endpoint="create", - json={"path": path, "overwrite": overwrite}, - ) - return r["handle"] - except DatabricksException as e: - if e.error_code == "RESOURCE_ALREADY_EXISTS": - raise FileExistsError(e.message) from e - - raise - - def _close_handle(self, handle): - """ - Close a handle, which was opened by :func:`_create_handle`. - - Parameters - ---------- - handle: str - Which handle to close. - """ - try: - self._send_to_api(method="post", endpoint="close", json={"handle": handle}) - except DatabricksException as e: - if e.error_code == "RESOURCE_DOES_NOT_EXIST": - raise FileNotFoundError(e.message) from e - - raise - - def _add_data(self, handle, data): - """ - Upload data to an already opened file handle - (opened by :func:`_create_handle`). - The maximal allowed data size is 1MB after - conversion to base64. - Remember to close the handle when you are finished. - - Parameters - ---------- - handle: str - Which handle to upload data to. - data: bytes - Block of data to add to the handle. - """ - data = base64.b64encode(data).decode() - try: - self._send_to_api( - method="post", - endpoint="add-block", - json={"handle": handle, "data": data}, - ) - except DatabricksException as e: - if e.error_code == "RESOURCE_DOES_NOT_EXIST": - raise FileNotFoundError(e.message) from e - elif e.error_code == "MAX_BLOCK_SIZE_EXCEEDED": - raise ValueError(e.message) from e - - raise - - def _get_data(self, path, start, end): - """ - Download data in bytes from a given absolute path in a block - from [start, start+length]. - The maximum number of allowed bytes to read is 1MB. - - Parameters - ---------- - path: str - Absolute path to download data from - start: int - Start position of the block - end: int - End position of the block - """ - try: - r = self._send_to_api( - method="get", - endpoint="read", - json={"path": path, "offset": start, "length": end - start}, - ) - return base64.b64decode(r["data"]) - except DatabricksException as e: - if e.error_code == "RESOURCE_DOES_NOT_EXIST": - raise FileNotFoundError(e.message) from e - elif e.error_code in ["INVALID_PARAMETER_VALUE", "MAX_READ_SIZE_EXCEEDED"]: - raise ValueError(e.message) from e - - raise - - def invalidate_cache(self, path=None): - if path is None: - self.dircache.clear() - else: - self.dircache.pop(path, None) - super().invalidate_cache(path) - - -class DatabricksFile(AbstractBufferedFile): - """ - Helper class for files referenced in the DatabricksFileSystem. - """ - - DEFAULT_BLOCK_SIZE = 1 * 2**20 # only allowed block size - - def __init__( - self, - fs, - path, - mode="rb", - block_size="default", - autocommit=True, - cache_type="readahead", - cache_options=None, - **kwargs, - ): - """ - Create a new instance of the DatabricksFile. - - The blocksize needs to be the default one. - """ - if block_size is None or block_size == "default": - block_size = self.DEFAULT_BLOCK_SIZE - - if block_size != self.DEFAULT_BLOCK_SIZE: - raise ValueError( - f"Only the default block size is allowed, not {block_size}" - ) - - super().__init__( - fs, - path, - mode=mode, - block_size=block_size, - autocommit=autocommit, - cache_type=cache_type, - cache_options=cache_options or {}, - **kwargs, - ) - - def _initiate_upload(self): - """Internal function to start a file upload""" - self.handle = self.fs._create_handle(self.path) - - def _upload_chunk(self, final=False): - """Internal function to add a chunk of data to a started upload""" - self.buffer.seek(0) - data = self.buffer.getvalue() - - data_chunks = [ - data[start:end] for start, end in self._to_sized_blocks(len(data)) - ] - - for data_chunk in data_chunks: - self.fs._add_data(handle=self.handle, data=data_chunk) - - if final: - self.fs._close_handle(handle=self.handle) - return True - - def _fetch_range(self, start, end): - """Internal function to download a block of data""" - return_buffer = b"" - length = end - start - for chunk_start, chunk_end in self._to_sized_blocks(length, start): - return_buffer += self.fs._get_data( - path=self.path, start=chunk_start, end=chunk_end - ) - - return return_buffer - - def _to_sized_blocks(self, length, start=0): - """Helper function to split a range from 0 to total_length into blocksizes""" - end = start + length - for data_chunk in range(start, end, self.blocksize): - data_start = data_chunk - data_end = min(end, data_chunk + self.blocksize) - yield data_start, data_end diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/dirfs.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/dirfs.py deleted file mode 100644 index 0fe1ababfe5a7749ed75cc1c29bedea492c8f1f4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/dirfs.py +++ /dev/null @@ -1,426 +0,0 @@ -from .. import filesystem -from ..asyn import AsyncFileSystem -from .chained import ChainedFileSystem -from .local import LocalFileSystem - - -def _escapes_root(path): - """Whether a relative path would resolve above its root via ".." segments.""" - depth = 0 - for part in path.split("/"): - if part == "..": - depth -= 1 - if depth < 0: - return True - elif part and part != ".": - depth += 1 - return False - - -class DirFileSystem(AsyncFileSystem, ChainedFileSystem): - """Directory prefix filesystem - - The DirFileSystem is a filesystem-wrapper. It assumes every path it is dealing with - is relative to the `path`. After performing the necessary paths operation it - delegates everything to the wrapped filesystem. - """ - - protocol = "dir" - - def __init__( - self, - path=None, - fs=None, - fo=None, - target_protocol=None, - target_options=None, - **storage_options, - ): - """ - Parameters - ---------- - path: str - Path to the directory. - fs: AbstractFileSystem - An instantiated filesystem to wrap. - target_protocol, target_options: - if fs is none, construct it from these - fo: str - Alternate for path; do not provide both - """ - super().__init__(**storage_options) - if fs is None: - fs = filesystem(protocol=target_protocol, **(target_options or {})) - path = path or fo - - if self.asynchronous and not fs.async_impl: - raise ValueError("can't use asynchronous with non-async fs") - - if fs.async_impl and self.asynchronous != fs.asynchronous: - raise ValueError("both dirfs and fs should be in the same sync/async mode") - - self.path = fs._strip_protocol(path) - self.fs = fs - - def _join(self, path): - if isinstance(path, str): - if not self.path: - return path - if not path: - return self.path - path = self._strip_protocol(path) - # ".." only navigates above the root on filesystems that resolve it - # against a real directory tree; on object stores it is a literal - # path part, so only guard the local case here. - if isinstance(self.fs, LocalFileSystem) and _escapes_root(path): - raise ValueError( - f"path {path!r} escapes the {self.path!r} root of the filesystem" - ) - return self.fs.sep.join((self.path, path)) - if isinstance(path, dict): - return {self._join(_path): value for _path, value in path.items()} - return [self._join(_path) for _path in path] - - def _relpath(self, path): - if isinstance(path, str): - if not self.path: - return path - # We need to account for S3FileSystem returning paths that do not - # start with a '/' - if path == self.path or ( - self.path.startswith(self.fs.sep) and path == self.path[1:] - ): - return "" - prefix = self.path + self.fs.sep - if self.path.startswith(self.fs.sep) and not path.startswith(self.fs.sep): - prefix = prefix[1:] - assert path.startswith(prefix) - return path[len(prefix) :] - return [self._relpath(_path) for _path in path] - - # Wrappers below - - @property - def sep(self): - return self.fs.sep - - async def set_session(self, *args, **kwargs): - return await self.fs.set_session(*args, **kwargs) - - async def _rm_file(self, path, **kwargs): - return await self.fs._rm_file(self._join(path), **kwargs) - - def rm_file(self, path, **kwargs): - return self.fs.rm_file(self._join(path), **kwargs) - - async def _rm(self, path, *args, **kwargs): - return await self.fs._rm(self._join(path), *args, **kwargs) - - def rm(self, path, *args, **kwargs): - return self.fs.rm(self._join(path), *args, **kwargs) - - def delete(self, path, recursive=False, maxdepth=None): - return self.fs.delete(self._join(path), recursive=recursive, maxdepth=maxdepth) - - async def _cp_file(self, path1, path2, **kwargs): - return await self.fs._cp_file(self._join(path1), self._join(path2), **kwargs) - - def cp_file(self, path1, path2, **kwargs): - return self.fs.cp_file(self._join(path1), self._join(path2), **kwargs) - - async def _copy( - self, - path1, - path2, - *args, - **kwargs, - ): - return await self.fs._copy( - self._join(path1), - self._join(path2), - *args, - **kwargs, - ) - - def copy(self, path1, path2, *args, **kwargs): - return self.fs.copy( - self._join(path1), - self._join(path2), - *args, - **kwargs, - ) - - async def _pipe(self, path, *args, **kwargs): - return await self.fs._pipe(self._join(path), *args, **kwargs) - - def pipe(self, path, *args, **kwargs): - return self.fs.pipe(self._join(path), *args, **kwargs) - - async def _pipe_file(self, path, *args, **kwargs): - return await self.fs._pipe_file(self._join(path), *args, **kwargs) - - def pipe_file(self, path, *args, **kwargs): - return self.fs.pipe_file(self._join(path), *args, **kwargs) - - def write_text( - self, path, value, encoding=None, errors=None, newline=None, **kwargs - ): - return self.fs.write_text( - self._join(path), - value, - encoding=encoding, - errors=errors, - newline=newline, - **kwargs, - ) - - async def _cat_file(self, path, *args, **kwargs): - return await self.fs._cat_file(self._join(path), *args, **kwargs) - - def cat_file(self, path, *args, **kwargs): - return self.fs.cat_file(self._join(path), *args, **kwargs) - - async def _cat(self, path, *args, **kwargs): - ret = await self.fs._cat( - self._join(path), - *args, - **kwargs, - ) - - if isinstance(ret, dict): - return {self._relpath(key): value for key, value in ret.items()} - - return ret - - def cat(self, path, *args, **kwargs): - ret = self.fs.cat( - self._join(path), - *args, - **kwargs, - ) - - if isinstance(ret, dict): - return {self._relpath(key): value for key, value in ret.items()} - - return ret - - async def _put_file(self, lpath, rpath, **kwargs): - return await self.fs._put_file(lpath, self._join(rpath), **kwargs) - - def put_file(self, lpath, rpath, **kwargs): - return self.fs.put_file(lpath, self._join(rpath), **kwargs) - - async def _put( - self, - lpath, - rpath, - *args, - **kwargs, - ): - return await self.fs._put( - lpath, - self._join(rpath), - *args, - **kwargs, - ) - - def put(self, lpath, rpath, *args, **kwargs): - return self.fs.put( - lpath, - self._join(rpath), - *args, - **kwargs, - ) - - async def _get_file(self, rpath, lpath, **kwargs): - return await self.fs._get_file(self._join(rpath), lpath, **kwargs) - - def get_file(self, rpath, lpath, **kwargs): - return self.fs.get_file(self._join(rpath), lpath, **kwargs) - - async def _get(self, rpath, *args, **kwargs): - return await self.fs._get(self._join(rpath), *args, **kwargs) - - def get(self, rpath, *args, **kwargs): - return self.fs.get(self._join(rpath), *args, **kwargs) - - async def _isfile(self, path): - return await self.fs._isfile(self._join(path)) - - def isfile(self, path): - return self.fs.isfile(self._join(path)) - - async def _isdir(self, path): - return await self.fs._isdir(self._join(path)) - - def isdir(self, path): - return self.fs.isdir(self._join(path)) - - async def _size(self, path): - return await self.fs._size(self._join(path)) - - def size(self, path): - return self.fs.size(self._join(path)) - - async def _exists(self, path): - return await self.fs._exists(self._join(path)) - - def exists(self, path): - return self.fs.exists(self._join(path)) - - async def _info(self, path, **kwargs): - info = await self.fs._info(self._join(path), **kwargs) - info = info.copy() - info["name"] = self._relpath(info["name"]) - return info - - def info(self, path, **kwargs): - info = self.fs.info(self._join(path), **kwargs) - info = info.copy() - info["name"] = self._relpath(info["name"]) - return info - - async def _ls(self, path, detail=True, **kwargs): - ret = (await self.fs._ls(self._join(path), detail=detail, **kwargs)).copy() - if detail: - out = [] - for entry in ret: - entry = entry.copy() - entry["name"] = self._relpath(entry["name"]) - out.append(entry) - return out - - return self._relpath(ret) - - def ls(self, path, detail=True, **kwargs): - ret = self.fs.ls(self._join(path), detail=detail, **kwargs).copy() - if detail: - out = [] - for entry in ret: - entry = entry.copy() - entry["name"] = self._relpath(entry["name"]) - out.append(entry) - return out - - return self._relpath(ret) - - async def _walk(self, path, *args, **kwargs): - async for root, dirs, files in self.fs._walk(self._join(path), *args, **kwargs): - yield self._relpath(root), dirs, files - - def walk(self, path, *args, **kwargs): - for root, dirs, files in self.fs.walk(self._join(path), *args, **kwargs): - yield self._relpath(root), dirs, files - - async def _glob(self, path, **kwargs): - detail = kwargs.get("detail", False) - ret = await self.fs._glob(self._join(path), **kwargs) - if detail: - return {self._relpath(path): info for path, info in ret.items()} - return self._relpath(ret) - - def glob(self, path, **kwargs): - detail = kwargs.get("detail", False) - ret = self.fs.glob(self._join(path), **kwargs) - if detail: - return {self._relpath(path): info for path, info in ret.items()} - return self._relpath(ret) - - async def _du(self, path, *args, **kwargs): - total = kwargs.get("total", True) - ret = await self.fs._du(self._join(path), *args, **kwargs) - if total: - return ret - - return {self._relpath(path): size for path, size in ret.items()} - - def du(self, path, *args, **kwargs): - total = kwargs.get("total", True) - ret = self.fs.du(self._join(path), *args, **kwargs) - if total: - return ret - - return {self._relpath(path): size for path, size in ret.items()} - - async def _find(self, path, *args, **kwargs): - detail = kwargs.get("detail", False) - ret = await self.fs._find(self._join(path), *args, **kwargs) - if detail: - return {self._relpath(path): info for path, info in ret.items()} - return self._relpath(ret) - - def find(self, path, *args, **kwargs): - detail = kwargs.get("detail", False) - ret = self.fs.find(self._join(path), *args, **kwargs) - if detail: - return {self._relpath(path): info for path, info in ret.items()} - return self._relpath(ret) - - async def _expand_path(self, path, *args, **kwargs): - return self._relpath( - await self.fs._expand_path(self._join(path), *args, **kwargs) - ) - - def expand_path(self, path, *args, **kwargs): - return self._relpath(self.fs.expand_path(self._join(path), *args, **kwargs)) - - async def _mkdir(self, path, *args, **kwargs): - return await self.fs._mkdir(self._join(path), *args, **kwargs) - - def mkdir(self, path, *args, **kwargs): - return self.fs.mkdir(self._join(path), *args, **kwargs) - - async def _makedirs(self, path, *args, **kwargs): - return await self.fs._makedirs(self._join(path), *args, **kwargs) - - def makedirs(self, path, *args, **kwargs): - return self.fs.makedirs(self._join(path), *args, **kwargs) - - def rmdir(self, path): - return self.fs.rmdir(self._join(path)) - - def mv(self, path1, path2, **kwargs): - return self.fs.mv( - self._join(path1), - self._join(path2), - **kwargs, - ) - - def touch(self, path, **kwargs): - return self.fs.touch(self._join(path), **kwargs) - - def created(self, path): - return self.fs.created(self._join(path)) - - def modified(self, path): - return self.fs.modified(self._join(path)) - - def sign(self, path, *args, **kwargs): - return self.fs.sign(self._join(path), *args, **kwargs) - - def __repr__(self): - return f"{self.__class__.__qualname__}(path='{self.path}', fs={self.fs})" - - def open( - self, - path, - *args, - **kwargs, - ): - return self.fs.open( - self._join(path), - *args, - **kwargs, - ) - - async def open_async( - self, - path, - *args, - **kwargs, - ): - return await self.fs.open_async( - self._join(path), - *args, - **kwargs, - ) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/ftp.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/ftp.py deleted file mode 100644 index 5f20d4302c460e2460c6ab1bd99b07904fc765cb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/ftp.py +++ /dev/null @@ -1,442 +0,0 @@ -import os -import ssl -import uuid -from ftplib import FTP, FTP_TLS, Error, error_perm -from typing import Any - -from ..spec import AbstractBufferedFile, AbstractFileSystem -from ..utils import infer_storage_options, isfilelike - -SECURITY_PROTOCOL_MAP = { - "tls": ssl.PROTOCOL_TLS, - "sslv23": ssl.PROTOCOL_SSLv23, -} -for protocol in ["TLSv1", "TLSv1_1", "TLSv1_2"]: - if hasattr(ssl, f"PROTOCOL_{protocol}"): - SECURITY_PROTOCOL_MAP[protocol.lower()] = getattr(ssl, f"PROTOCOL_{protocol}") - - -class ImplicitFTPTLS(FTP_TLS): - """ - FTP_TLS subclass that automatically wraps sockets in SSL - to support implicit FTPS. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._sock = None - - @property - def sock(self): - """Return the socket.""" - return self._sock - - @sock.setter - def sock(self, value): - """When modifying the socket, ensure that it is ssl wrapped.""" - if value is not None and not isinstance(value, ssl.SSLSocket): - value = self.context.wrap_socket(value) - self._sock = value - - -class FTPFileSystem(AbstractFileSystem): - """A filesystem over classic FTP""" - - root_marker = "/" - cachable = False - protocol = "ftp" - - def __init__( - self, - host, - port=21, - username=None, - password=None, - acct=None, - block_size=None, - tempdir=None, - timeout=30, - encoding="utf-8", - tls=False, - **kwargs, - ): - """ - You can use _get_kwargs_from_urls to get some kwargs from - a reasonable FTP url. - - Authentication will be anonymous if username/password are not - given. - - Parameters - ---------- - host: str - The remote server name/ip to connect to - port: int - Port to connect with - username: str or None - If authenticating, the user's identifier - password: str of None - User's password on the server, if using - acct: str or None - Some servers also need an "account" string for auth - block_size: int or None - If given, the read-ahead or write buffer size. - tempdir: str - Directory on remote to put temporary files when in a transaction - timeout: int - Timeout of the ftp connection in seconds - encoding: str - Encoding to use for directories and filenames in FTP connection - tls: bool or str - Enable FTP-TLS for secure connections: - - False: Plain FTP (default) - - True: Explicit TLS (FTPS with AUTH TLS command) - - "tls": Auto-negotiate highest protocol - - "tlsv1": TLS v1.0 - - "tlsv1_1": TLS v1.1 - - "tlsv1_2": TLS v1.2 - """ - super().__init__(**kwargs) - self.host = host - self.port = port - self.tempdir = tempdir or "/tmp" - self.cred = username or "", password or "", acct or "" - self.timeout = timeout - self.encoding = encoding - if block_size is not None: - self.blocksize = block_size - else: - self.blocksize = 2**16 - self.tls = tls - self._connect() - if isinstance(self.tls, bool) and self.tls: - self.ftp.prot_p() - - def _connect(self): - security = None - if self.tls: - if isinstance(self.tls, str): - ftp_cls = ImplicitFTPTLS - security = SECURITY_PROTOCOL_MAP.get( - self.tls, - f"Not supported {self.tls} protocol", - ) - if isinstance(security, str): - raise ValueError(security) - else: - ftp_cls = FTP_TLS - else: - ftp_cls = FTP - self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding) - if security: - self.ftp.ssl_version = security - self.ftp.connect(self.host, self.port) - self.ftp.login(*self.cred) - - @classmethod - def _strip_protocol(cls, path): - return "/" + infer_storage_options(path)["path"].lstrip("/").rstrip("/") - - @staticmethod - def _get_kwargs_from_urls(urlpath): - out = infer_storage_options(urlpath) - out.pop("path", None) - out.pop("protocol", None) - return out - - def ls(self, path, detail=True, **kwargs): - path = self._strip_protocol(path) - out = [] - if path not in self.dircache: - try: - try: - out = [ - (fn, details) - for (fn, details) in self.ftp.mlsd(path) - if fn not in [".", ".."] - and details["type"] not in ["pdir", "cdir"] - ] - except error_perm: - out = _mlsd2(self.ftp, path) # Not platform independent - for fn, details in out: - details["name"] = "/".join( - ["" if path == "/" else path, fn.lstrip("/")] - ) - if details["type"] == "file": - details["size"] = int(details["size"]) - else: - details["size"] = 0 - if details["type"] == "dir": - details["type"] = "directory" - self.dircache[path] = out - except Error: - try: - info = self.info(path) - if info["type"] == "file": - out = [(path, info)] - except (Error, IndexError) as exc: - raise FileNotFoundError(path) from exc - files = self.dircache.get(path, out) - if not detail: - return sorted([fn for fn, details in files]) - return [details for fn, details in files] - - def info(self, path, **kwargs): - # implement with direct method - path = self._strip_protocol(path) - if path == "/": - # special case, since this dir has no real entry - return {"name": "/", "size": 0, "type": "directory"} - files = self.ls(self._parent(path).lstrip("/"), True) - try: - out = next(f for f in files if f["name"] == path) - except StopIteration as exc: - raise FileNotFoundError(path) from exc - return out - - def get_file(self, rpath, lpath, **kwargs): - if self.isdir(rpath): - if not os.path.exists(lpath): - os.mkdir(lpath) - return - if isfilelike(lpath): - outfile = lpath - else: - outfile = open(lpath, "wb") - - def cb(x): - outfile.write(x) - - self.ftp.retrbinary( - f"RETR {rpath}", - blocksize=self.blocksize, - callback=cb, - ) - if not isfilelike(lpath): - outfile.close() - - def cat_file(self, path, start=None, end=None, **kwargs): - if end is not None: - return super().cat_file(path, start, end, **kwargs) - out = [] - - def cb(x): - out.append(x) - - try: - self.ftp.retrbinary( - f"RETR {path}", - blocksize=self.blocksize, - rest=start, - callback=cb, - ) - except (Error, error_perm) as orig_exc: - raise FileNotFoundError(path) from orig_exc - return b"".join(out) - - def _open( - self, - path, - mode="rb", - block_size=None, - cache_options=None, - autocommit=True, - **kwargs, - ): - path = self._strip_protocol(path) - block_size = block_size or self.blocksize - return FTPFile( - self, - path, - mode=mode, - block_size=block_size, - tempdir=self.tempdir, - autocommit=autocommit, - cache_options=cache_options, - ) - - def _rm(self, path): - path = self._strip_protocol(path) - self.ftp.delete(path) - self.invalidate_cache(self._parent(path)) - - def rm(self, path, recursive=False, maxdepth=None): - paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) - for p in reversed(paths): - if self.isfile(p): - self.rm_file(p) - else: - self.rmdir(p) - - def mkdir(self, path: str, create_parents: bool = True, **kwargs: Any) -> None: - path = self._strip_protocol(path) - parent = self._parent(path) - if parent != self.root_marker and not self.exists(parent) and create_parents: - self.mkdir(parent, create_parents=create_parents) - - self.ftp.mkd(path) - self.invalidate_cache(self._parent(path)) - - def makedirs(self, path: str, exist_ok: bool = False) -> None: - path = self._strip_protocol(path) - if self.exists(path): - # NB: "/" does not "exist" as it has no directory entry - if not exist_ok: - raise FileExistsError(f"{path} exists without `exist_ok`") - # exists_ok=True -> no-op - else: - self.mkdir(path, create_parents=True) - - def rmdir(self, path): - path = self._strip_protocol(path) - self.ftp.rmd(path) - self.invalidate_cache(self._parent(path)) - - def mv(self, path1, path2, **kwargs): - path1 = self._strip_protocol(path1) - path2 = self._strip_protocol(path2) - self.ftp.rename(path1, path2) - self.invalidate_cache(self._parent(path1)) - self.invalidate_cache(self._parent(path2)) - - def __del__(self): - self.ftp.close() - - def invalidate_cache(self, path=None): - if path is None: - self.dircache.clear() - else: - self.dircache.pop(path, None) - super().invalidate_cache(path) - - -class TransferDone(Exception): - """Internal exception to break out of transfer""" - - pass - - -class FTPFile(AbstractBufferedFile): - """Interact with a remote FTP file with read/write buffering""" - - def __init__( - self, - fs, - path, - mode="rb", - block_size="default", - autocommit=True, - cache_type="readahead", - cache_options=None, - **kwargs, - ): - super().__init__( - fs, - path, - mode=mode, - block_size=block_size, - autocommit=autocommit, - cache_type=cache_type, - cache_options=cache_options, - **kwargs, - ) - if not autocommit: - self.target = self.path - self.path = "/".join([kwargs["tempdir"], str(uuid.uuid4())]) - - def commit(self): - self.fs.mv(self.path, self.target) - - def discard(self): - self.fs.rm(self.path) - - def _fetch_range(self, start, end): - """Get bytes between given byte limits - - Implemented by raising an exception in the fetch callback when the - number of bytes received reaches the requested amount. - - Will fail if the server does not respect the REST command on - retrieve requests. - """ - out = [] - total = [0] - - def callback(x): - total[0] += len(x) - if total[0] > end - start: - out.append(x[: (end - start) - total[0]]) - if end < self.size: - raise TransferDone - else: - out.append(x) - - if total[0] == end - start and end < self.size: - raise TransferDone - - try: - self.fs.ftp.retrbinary( - f"RETR {self.path}", - blocksize=self.blocksize, - rest=start, - callback=callback, - ) - except TransferDone: - try: - # stop transfer, we got enough bytes for this block - self.fs.ftp.abort() - self.fs.ftp.getmultiline() - except Error: - self.fs._connect() - - return b"".join(out) - - def _upload_chunk(self, final=False): - self.buffer.seek(0) - self.fs.ftp.storbinary( - f"STOR {self.path}", self.buffer, blocksize=self.blocksize, rest=self.offset - ) - return True - - -def _mlsd2(ftp, path="."): - """ - Fall back to using `dir` instead of `mlsd` if not supported. - - This parses a Linux style `ls -l` response to `dir`, but the response may - be platform dependent. - - Parameters - ---------- - ftp: ftplib.FTP - path: str - Expects to be given path, but defaults to ".". - """ - lines = [] - minfo = [] - ftp.dir(path, lines.append) - for line in lines: - split_line = line.split(maxsplit=8) - if len(split_line) < 9: - continue - name = split_line[8] - unix_mode = split_line[0] - if unix_mode[0] == "l" and " -> " in name: - # Symbolic link: " -> "; keep only the link name. - name = name.split(" -> ", 1)[0] - this = ( - name, - { - "modify": " ".join(split_line[5:8]), - "unix.owner": split_line[2], - "unix.group": split_line[3], - "unix.mode": unix_mode, - "size": split_line[4], - }, - ) - if this[1]["unix.mode"][0] == "d": - this[1]["type"] = "dir" - else: - this[1]["type"] = "file" - minfo.append(this) - return minfo diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/gist.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/gist.py deleted file mode 100644 index ad9ac0b6a1cdbcfba6188e2cdeab2350bb9aad0a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/gist.py +++ /dev/null @@ -1,241 +0,0 @@ -import requests - -from ..spec import AbstractFileSystem -from ..utils import infer_storage_options -from .memory import MemoryFile - - -class GistFileSystem(AbstractFileSystem): - """ - Interface to files in a single GitHub Gist. - - Provides read-only access to a gist's files. Gists do not contain - subdirectories, so file listing is straightforward. - - Parameters - ---------- - gist_id: str - The ID of the gist you want to access (the long hex value from the URL). - filenames: list[str] (optional) - If provided, only make a file system representing these files, and do not fetch - the list of all files for this gist. - sha: str (optional) - If provided, fetch a particular revision of the gist. If omitted, - the latest revision is used. - username: str (optional) - GitHub username for authentication. - token: str (optional) - GitHub personal access token (required if username is given), or. - timeout: (float, float) or float, optional - Connect and read timeouts for requests (default 60s each). - kwargs: dict - Stored on `self.request_kw` and passed to `requests.get` when fetching Gist - metadata or reading ("opening") a file. - """ - - protocol = "gist" - gist_url = "https://api.github.com/gists/{gist_id}" - gist_rev_url = "https://api.github.com/gists/{gist_id}/{sha}" - - def __init__( - self, - gist_id, - filenames=None, - sha=None, - username=None, - token=None, - timeout=None, - **kwargs, - ): - super().__init__() - self.gist_id = gist_id - self.filenames = filenames - self.sha = sha # revision of the gist (optional) - if username is not None and token is None: - raise ValueError("User auth requires a token") - self.username = username - self.token = token - self.request_kw = kwargs - # Default timeouts to 60s connect/read if none provided - self.timeout = timeout if timeout is not None else (60, 60) - - # We use a single-level "directory" cache, because a gist is essentially flat - self.dircache[""] = self._fetch_file_list() - - @property - def kw(self): - """Auth parameters passed to 'requests' if we have username/token.""" - kw = { - "headers": { - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - } - } - kw.update(self.request_kw) - if self.username and self.token: - kw["auth"] = (self.username, self.token) - elif self.token: - kw["headers"]["Authorization"] = f"Bearer {self.token}" - return kw - - def _fetch_gist_metadata(self): - """ - Fetch the JSON metadata for this gist (possibly for a specific revision). - """ - if self.sha: - url = self.gist_rev_url.format(gist_id=self.gist_id, sha=self.sha) - else: - url = self.gist_url.format(gist_id=self.gist_id) - - r = requests.get(url, timeout=self.timeout, **self.kw) - if r.status_code == 404: - raise FileNotFoundError( - f"Gist not found: {self.gist_id}@{self.sha or 'latest'}" - ) - r.raise_for_status() - return r.json() - - def _fetch_file_list(self): - """ - Returns a list of dicts describing each file in the gist. These get stored - in self.dircache[""]. - """ - meta = self._fetch_gist_metadata() - if self.filenames: - available_files = meta.get("files", {}) - files = {} - for fn in self.filenames: - if fn not in available_files: - raise FileNotFoundError(fn) - files[fn] = available_files[fn] - else: - files = meta.get("files", {}) - - out = [] - for fname, finfo in files.items(): - if finfo is None: - # Occasionally GitHub returns a file entry with null if it was deleted - continue - # Build a directory entry - out.append( - { - "name": fname, # file's name - "type": "file", # gists have no subdirectories - "size": finfo.get("size", 0), # file size in bytes - "raw_url": finfo.get("raw_url"), - } - ) - return out - - @classmethod - def _strip_protocol(cls, path): - """ - Remove 'gist://' from the path, if present. - """ - # The default infer_storage_options can handle gist://username:token@id/file - # or gist://id/file, but let's ensure we handle a normal usage too. - # We'll just strip the protocol prefix if it exists. - path = infer_storage_options(path).get("path", path) - return path.lstrip("/") - - @staticmethod - def _get_kwargs_from_urls(path): - """ - Parse 'gist://' style URLs into GistFileSystem constructor kwargs. - For example: - gist://:TOKEN@/file.txt - gist://username:TOKEN@/file.txt - """ - so = infer_storage_options(path) - out = {} - if "username" in so and so["username"]: - out["username"] = so["username"] - if "password" in so and so["password"]: - out["token"] = so["password"] - if "host" in so and so["host"]: - # We interpret 'host' as the gist ID - out["gist_id"] = so["host"] - - # Extract SHA and filename from path - if "path" in so and so["path"]: - path_parts = so["path"].rsplit("/", 2)[-2:] - if len(path_parts) == 2: - if path_parts[0]: # SHA present - out["sha"] = path_parts[0] - if path_parts[1]: # filename also present - out["filenames"] = [path_parts[1]] - - return out - - def ls(self, path="", detail=False, **kwargs): - """ - List files in the gist. Gists are single-level, so any 'path' is basically - the filename, or empty for all files. - - Parameters - ---------- - path : str, optional - The filename to list. If empty, returns all files in the gist. - detail : bool, default False - If True, return a list of dicts; if False, return a list of filenames. - """ - path = self._strip_protocol(path or "") - # If path is empty, return all - if path == "": - results = self.dircache[""] - else: - # We want just the single file with this name - all_files = self.dircache[""] - results = [f for f in all_files if f["name"] == path] - if not results: - raise FileNotFoundError(path) - if detail: - return results - else: - return sorted(f["name"] for f in results) - - def _open(self, path, mode="rb", block_size=None, **kwargs): - """ - Read a single file from the gist. - """ - if mode != "rb": - raise NotImplementedError("GitHub Gist FS is read-only (no write).") - - path = self._strip_protocol(path) - # Find the file entry in our dircache - matches = [f for f in self.dircache[""] if f["name"] == path] - if not matches: - raise FileNotFoundError(path) - finfo = matches[0] - - raw_url = finfo.get("raw_url") - if not raw_url: - raise FileNotFoundError(f"No raw_url for file: {path}") - - r = requests.get(raw_url, timeout=self.timeout, **self.kw) - if r.status_code == 404: - raise FileNotFoundError(path) - r.raise_for_status() - return MemoryFile(path, None, r.content) - - def cat(self, path, recursive=False, on_error="raise", **kwargs): - """ - Return {path: contents} for the given file or files. If 'recursive' is True, - and path is empty, returns all files in the gist. - """ - paths = self.expand_path(path, recursive=recursive) - out = {} - for p in paths: - try: - with self.open(p, "rb") as f: - out[p] = f.read() - except FileNotFoundError as e: - if on_error == "raise": - raise e - elif on_error == "omit": - pass # skip - else: - out[p] = e - if len(paths) == 1 and paths[0] == path: - return out[path] - return out diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/git.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/git.py deleted file mode 100644 index 808d293a1c991ea87d19a2129f3e56d9b813daaa..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/git.py +++ /dev/null @@ -1,114 +0,0 @@ -import os - -import pygit2 - -from fsspec.spec import AbstractFileSystem - -from .memory import MemoryFile - - -class GitFileSystem(AbstractFileSystem): - """Browse the files of a local git repo at any hash/tag/branch - - (experimental backend) - """ - - root_marker = "" - cachable = True - - def __init__(self, path=None, fo=None, ref=None, **kwargs): - """ - - Parameters - ---------- - path: str (optional) - Local location of the repo (uses current directory if not given). - May be deprecated in favour of ``fo``. When used with a higher - level function such as fsspec.open(), may be of the form - "git://[path-to-repo[:]][ref@]path/to/file" (but the actual - file path should not contain "@" or ":"). - fo: str (optional) - Same as ``path``, but passed as part of a chained URL. This one - takes precedence if both are given. - ref: str (optional) - Reference to work with, could be a hash, tag or branch name. Defaults - to current working tree. Note that ``ls`` and ``open`` also take hash, - so this becomes the default for those operations - kwargs - """ - super().__init__(**kwargs) - self.repo = pygit2.Repository(fo or path or os.getcwd()) - self.ref = ref or "master" - - @classmethod - def _strip_protocol(cls, path): - path = super()._strip_protocol(path).lstrip("/") - if ":" in path: - path = path.split(":", 1)[1] - if "@" in path: - path = path.split("@", 1)[1] - return path.lstrip("/") - - def _path_to_object(self, path, ref): - comm, ref = self.repo.resolve_refish(ref or self.ref) - parts = path.split("/") - tree = comm.tree - for part in parts: - if part and isinstance(tree, pygit2.Tree): - if part not in tree: - raise FileNotFoundError(path) - tree = tree[part] - return tree - - @staticmethod - def _get_kwargs_from_urls(path): - path = path.removeprefix("git://") - out = {} - if ":" in path: - out["path"], path = path.split(":", 1) - if "@" in path: - out["ref"], path = path.split("@", 1) - return out - - @staticmethod - def _object_to_info(obj, path=None): - # obj.name and obj.filemode are None for the root tree! - is_dir = isinstance(obj, pygit2.Tree) - return { - "type": "directory" if is_dir else "file", - "name": ( - "/".join([path, obj.name or ""]).lstrip("/") if path else obj.name - ), - "hex": str(obj.id), - "mode": "100644" if obj.filemode is None else f"{obj.filemode:o}", - "size": 0 if is_dir else obj.size, - } - - def ls(self, path, detail=True, ref=None, **kwargs): - tree = self._path_to_object(self._strip_protocol(path), ref) - return [ - GitFileSystem._object_to_info(obj, path) - if detail - else GitFileSystem._object_to_info(obj, path)["name"] - for obj in (tree if isinstance(tree, pygit2.Tree) else [tree]) - ] - - def info(self, path, ref=None, **kwargs): - tree = self._path_to_object(self._strip_protocol(path), ref) - return GitFileSystem._object_to_info(tree, path) - - def ukey(self, path, ref=None): - return self.info(path, ref=ref)["hex"] - - def _open( - self, - path, - mode="rb", - block_size=None, - autocommit=True, - cache_options=None, - ref=None, - **kwargs, - ): - obj = self._path_to_object(path, ref or self.ref) - return MemoryFile(data=obj.data) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/github.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/github.py deleted file mode 100644 index 3630f6db54413e2c396f6cc1b6b10cd379200043..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/github.py +++ /dev/null @@ -1,333 +0,0 @@ -import base64 -import re - -import requests - -from ..spec import AbstractFileSystem -from ..utils import infer_storage_options -from .memory import MemoryFile - - -class GithubFileSystem(AbstractFileSystem): - """Interface to files in github - - An instance of this class provides the files residing within a remote github - repository. You may specify a point in the repos history, by SHA, branch - or tag (default is current master). - - For files less than 1 MB in size, file content is returned directly in a - MemoryFile. For larger files, or for files tracked by git-lfs, file content - is returned as an HTTPFile wrapping the ``download_url`` provided by the - GitHub API. - - When using fsspec.open, allows URIs of the form: - - - "github://path/file", in which case you must specify org, repo and - may specify sha in the extra args - - 'github://org:repo@/precip/catalog.yml', where the org and repo are - part of the URI - - 'github://org:repo@sha/precip/catalog.yml', where the sha is also included - - ``sha`` can be the full or abbreviated hex of the commit you want to fetch - from, or a branch or tag name (so long as it doesn't contain special characters - like "/", "?", which would have to be HTTP-encoded). - - For authorised access, you must provide username and token, which can be made - at https://github.com/settings/tokens - """ - - url = "https://api.github.com/repos/{org}/{repo}/git/trees/{sha}" - content_url = "https://api.github.com/repos/{org}/{repo}/contents/{path}?ref={sha}" - protocol = "github" - timeout = (60, 60) # connect, read timeouts - - def __init__( - self, org, repo, sha=None, username=None, token=None, timeout=None, **kwargs - ): - super().__init__(**kwargs) - self.org = org - self.repo = repo - if (username is None) ^ (token is None): - raise ValueError("Auth required both username and token") - self.username = username - self.token = token - if timeout is not None: - self.timeout = timeout - if sha is None: - # look up default branch (not necessarily "master") - u = "https://api.github.com/repos/{org}/{repo}" - r = requests.get( - u.format(org=org, repo=repo), timeout=self.timeout, **self.kw - ) - r.raise_for_status() - sha = r.json()["default_branch"] - - self.root = sha - self.ls("") - try: - from .http import HTTPFileSystem - - self.http_fs = HTTPFileSystem(**kwargs) - except ImportError: - self.http_fs = None - - @property - def kw(self): - if self.username: - return {"auth": (self.username, self.token)} - return {} - - @classmethod - def repos(cls, org_or_user, is_org=True): - """List repo names for given org or user - - This may become the top level of the FS - - Parameters - ---------- - org_or_user: str - Name of the github org or user to query - is_org: bool (default True) - Whether the name is an organisation (True) or user (False) - - Returns - ------- - List of string - """ - r = requests.get( - f"https://api.github.com/{['users', 'orgs'][is_org]}/{org_or_user}/repos", - timeout=cls.timeout, - ) - r.raise_for_status() - return [repo["name"] for repo in r.json()] - - @property - def tags(self): - """Names of tags in the repo""" - r = requests.get( - f"https://api.github.com/repos/{self.org}/{self.repo}/tags", - timeout=self.timeout, - **self.kw, - ) - r.raise_for_status() - return [t["name"] for t in r.json()] - - @property - def branches(self): - """Names of branches in the repo""" - r = requests.get( - f"https://api.github.com/repos/{self.org}/{self.repo}/branches", - timeout=self.timeout, - **self.kw, - ) - r.raise_for_status() - return [t["name"] for t in r.json()] - - @property - def refs(self): - """Named references, tags and branches""" - return {"tags": self.tags, "branches": self.branches} - - def ls(self, path, detail=False, sha=None, _sha=None, **kwargs): - """List files at given path - - Parameters - ---------- - path: str - Location to list, relative to repo root - detail: bool - If True, returns list of dicts, one per file; if False, returns - list of full filenames only - sha: str (optional) - List at the given point in the repo history, branch or tag name or commit - SHA - _sha: str (optional) - List this specific tree object (used internally to descend into trees) - """ - path = self._strip_protocol(path) - if path == "": - _sha = sha or self.root - if _sha is None: - parts = path.rstrip("/").split("/") - so_far = "" - _sha = sha or self.root - for part in parts: - out = self.ls(so_far, True, sha=sha, _sha=_sha) - so_far += "/" + part if so_far else part - out = [o for o in out if o["name"] == so_far] - if not out: - raise FileNotFoundError(path) - out = out[0] - if out["type"] == "file": - if detail: - return [out] - else: - return path - _sha = out["sha"] - if path not in self.dircache or sha not in [self.root, None]: - r = requests.get( - self.url.format(org=self.org, repo=self.repo, sha=_sha), - timeout=self.timeout, - **self.kw, - ) - if r.status_code == 404: - raise FileNotFoundError(path) - r.raise_for_status() - types = {"blob": "file", "tree": "directory"} - out = [ - { - "name": path + "/" + f["path"] if path else f["path"], - "mode": f["mode"], - "type": types[f["type"]], - "size": f.get("size", 0), - "sha": f["sha"], - } - for f in r.json()["tree"] - if f["type"] in types - ] - if sha in [self.root, None]: - self.dircache[path] = out - else: - out = self.dircache[path] - if detail: - return out - else: - return sorted([f["name"] for f in out]) - - def invalidate_cache(self, path=None): - self.dircache.clear() - - @classmethod - def _strip_protocol(cls, path): - opts = infer_storage_options(path) - if "username" not in opts: - return super()._strip_protocol(path) - return opts["path"].lstrip("/") - - @staticmethod - def _get_kwargs_from_urls(path): - opts = infer_storage_options(path) - if "username" not in opts: - return {} - out = {"org": opts["username"], "repo": opts["password"]} - if opts["host"]: - out["sha"] = opts["host"] - return out - - def _open( - self, - path, - mode="rb", - block_size=None, - cache_options=None, - sha=None, - **kwargs, - ): - if mode != "rb": - raise NotImplementedError - - # construct a url to hit the GitHub API's repo contents API - url = self.content_url.format( - org=self.org, repo=self.repo, path=path, sha=sha or self.root - ) - - # make a request to this API, and parse the response as JSON - r = requests.get(url, timeout=self.timeout, **self.kw) - if r.status_code == 404: - raise FileNotFoundError(path) - r.raise_for_status() - content_json = r.json() - - # if the response's content key is not empty, try to parse it as base64 - if content_json["content"]: - content = base64.b64decode(content_json["content"]) - - # as long as the content does not start with the string - # "version https://git-lfs.github.com/" - # then it is probably not a git-lfs pointer and we can just return - # the content directly - if not content.startswith(b"version https://git-lfs.github.com/"): - return MemoryFile(None, None, content) - - # we land here if the content was not present in the first response - # (regular file over 1MB or git-lfs tracked file) - # in this case, we get let the HTTPFileSystem handle the download - if self.http_fs is None: - raise ImportError( - "Please install fsspec[http] to access github files >1 MB " - "or git-lfs tracked files." - ) - return self.http_fs.open( - content_json["download_url"], - mode=mode, - block_size=block_size, - cache_options=cache_options, - **kwargs, - ) - - def rm(self, path, recursive=False, maxdepth=None, message=None): - path = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) - for p in reversed(path): - self.rm_file(p, message=message) - - def rm_file(self, path, message=None, **kwargs): - """ - Remove a file from a specified branch using a given commit message. - - Since Github DELETE operation requires a branch name, and we can't reliably - determine whether the provided SHA refers to a branch, tag, or commit, we - assume it's a branch. If it's not, the user will encounter an error when - attempting to retrieve the file SHA or delete the file. - - Parameters - ---------- - path: str - The file's location relative to the repository root. - message: str, optional - The commit message for the deletion. - """ - - if not self.username: - raise ValueError("Authentication required") - - path = self._strip_protocol(path) - - # Attempt to get SHA from cache or Github API - sha = self._get_sha_from_cache(path) - if not sha: - url = self.content_url.format( - org=self.org, repo=self.repo, path=path.lstrip("/"), sha=self.root - ) - r = requests.get(url, timeout=self.timeout, **self.kw) - if r.status_code == 404: - raise FileNotFoundError(path) - r.raise_for_status() - sha = r.json()["sha"] - - # Delete the file - delete_url = self.content_url.format( - org=self.org, repo=self.repo, path=path, sha=self.root - ) - branch = self.root - data = { - "message": message or f"Delete {path}", - "sha": sha, - **({"branch": branch} if branch else {}), - } - - r = requests.delete(delete_url, json=data, timeout=self.timeout, **self.kw) - error_message = r.json().get("message", "") - if re.search(r"Branch .+ not found", error_message): - error = "Remove only works when the filesystem is initialised from a branch or default (None)" - raise ValueError(error) - r.raise_for_status() - - self.invalidate_cache(path) - - def _get_sha_from_cache(self, path): - for entries in self.dircache.values(): - for entry in entries: - entry_path = entry.get("name") - if entry_path and entry_path == path and "sha" in entry: - return entry["sha"] - return None diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/http.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/http.py deleted file mode 100644 index aff1a955c27091229513b988627de9e806dca8c5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/http.py +++ /dev/null @@ -1,902 +0,0 @@ -import asyncio -import io -import logging -import re -import weakref -from copy import copy -from urllib.parse import urlparse - -import aiohttp -import yarl - -from fsspec.asyn import AbstractAsyncStreamedFile, AsyncFileSystem, sync, sync_wrapper -from fsspec.callbacks import DEFAULT_CALLBACK -from fsspec.exceptions import FSTimeoutError -from fsspec.spec import AbstractBufferedFile -from fsspec.utils import ( - DEFAULT_BLOCK_SIZE, - glob_translate, - isfilelike, - nullcontext, - tokenize, -) - -from ..caching import AllBytes - -# https://stackoverflow.com/a/15926317/3821154 -ex = re.compile(r"""<(a|A)\s+(?:[^>]*?\s+)?(href|HREF)=["'](?P[^"']+)""") -ex2 = re.compile(r"""(?Phttp[s]?://[-a-zA-Z0-9@:%_+.~#?&/=]+)""") -logger = logging.getLogger("fsspec.http") - - -async def get_client(**kwargs): - return aiohttp.ClientSession(**kwargs) - - -class HTTPFileSystem(AsyncFileSystem): - """ - Simple File-System for fetching data via HTTP(S) - - ``ls()`` is implemented by loading the parent page and doing a regex - match on the result. If simple_link=True, anything of the form - "http(s)://server.com/stuff?thing=other"; otherwise only links within - HTML href tags will be used. - - URLs are passed unfiltered to aiohttp, so all addresses are accessible. Where URLs are - supplied by a user, the calling application may wish to filter to prevent scanning. - """ - - protocol = ("http", "https") - sep = "/" - - def __init__( - self, - simple_links=True, - block_size=None, - same_scheme=True, - size_policy=None, - cache_type="bytes", - cache_options=None, - asynchronous=False, - loop=None, - client_kwargs=None, - get_client=get_client, - encoded=False, - **storage_options, - ): - """ - NB: if this is called async, you must await set_client - - Parameters - ---------- - block_size: int - Blocks to read bytes; if 0, will default to raw requests file-like - objects instead of HTTPFile instances - simple_links: bool - If True, will consider both HTML tags and anything that looks - like a URL; if False, will consider only the former. - same_scheme: True - When doing ls/glob, if this is True, only consider paths that have - http/https matching the input URLs. - size_policy: this argument is deprecated - client_kwargs: dict - Passed to aiohttp.ClientSession, see - https://docs.aiohttp.org/en/stable/client_reference.html - For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}`` - get_client: Callable[..., aiohttp.ClientSession] - A callable, which takes keyword arguments and constructs - an aiohttp.ClientSession. Its state will be managed by - the HTTPFileSystem class. - storage_options: key-value - Any other parameters passed on to requests - cache_type, cache_options: defaults used in open() - """ - super().__init__(self, asynchronous=asynchronous, loop=loop, **storage_options) - self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE - self.simple_links = simple_links - self.same_schema = same_scheme - self.cache_type = cache_type - self.cache_options = cache_options - self.client_kwargs = client_kwargs or {} - self.get_client = get_client - self.encoded = encoded - self.kwargs = storage_options - self._session = None - - # Clean caching-related parameters from `storage_options` - # before propagating them as `request_options` through `self.kwargs`. - # TODO: Maybe rename `self.kwargs` to `self.request_options` to make - # it clearer. - request_options = copy(storage_options) - self.use_listings_cache = request_options.pop("use_listings_cache", False) - request_options.pop("listings_expiry_time", None) - request_options.pop("max_paths", None) - request_options.pop("skip_instance_cache", None) - self.kwargs = request_options - - @property - def fsid(self): - return "http" - - def encode_url(self, url): - return yarl.URL(url, encoded=self.encoded) - - @staticmethod - def close_session(loop, session): - if loop is not None and loop.is_running(): - try: - sync(loop, session.close, timeout=0.1) - return - except (TimeoutError, FSTimeoutError, NotImplementedError): - pass - connector = getattr(session, "_connector", None) - if connector is not None: - # close after loop is dead - connector._close() - - async def set_session(self): - if self._session is None: - self._session = await self.get_client(loop=self.loop, **self.client_kwargs) - if not self.asynchronous: - weakref.finalize(self, self.close_session, self.loop, self._session) - return self._session - - @classmethod - def _strip_protocol(cls, path): - """For HTTP, we always want to keep the full URL""" - return path - - @classmethod - def _parent(cls, path): - # override, since _strip_protocol is different for URLs - par = super()._parent(path) - if len(par) > 7: # "http://..." - return par - return "" - - async def _ls_real(self, url, detail=True, **kwargs): - # ignoring URL-encoded arguments - kw = self.kwargs.copy() - kw.update(kwargs) - logger.debug(url) - session = await self.set_session() - async with session.get(self.encode_url(url), **self.kwargs) as r: - self._raise_not_found_for_status(r, url) - - if "Content-Type" in r.headers: - mimetype = r.headers["Content-Type"].partition(";")[0] - else: - mimetype = None - - if mimetype in ("text/html", None): - try: - text = await r.text(errors="ignore") - if self.simple_links: - links = ex2.findall(text) + [u[2] for u in ex.findall(text)] - else: - links = [u[2] for u in ex.findall(text)] - except UnicodeDecodeError: - links = [] # binary, not HTML - else: - links = [] - - out = set() - parts = urlparse(url) - for l in links: - if isinstance(l, tuple): - l = l[1] - if l.startswith("/") and len(l) > 1: - # absolute URL on this server - l = f"{parts.scheme}://{parts.netloc}{l}" - if l.startswith("http"): - if self.same_schema and l.startswith(url.rstrip("/") + "/"): - out.add(l) - elif l.replace("https", "http").startswith( - url.replace("https", "http").rstrip("/") + "/" - ): - # allowed to cross http <-> https - out.add(l) - else: - if l not in ["..", "../"]: - # Ignore FTP-like "parent" - out.add("/".join([url.rstrip("/"), l.lstrip("/")])) - if not out and url.endswith("/"): - out = await self._ls_real(url.rstrip("/"), detail=False) - if detail: - return [ - { - "name": u, - "size": None, - "type": "directory" if u.endswith("/") else "file", - } - for u in out - ] - else: - return sorted(out) - - async def _ls(self, url, detail=True, **kwargs): - if self.use_listings_cache and url in self.dircache: - out = self.dircache[url] - else: - out = await self._ls_real(url, detail=detail, **kwargs) - self.dircache[url] = out - return out - - ls = sync_wrapper(_ls) - - def _raise_not_found_for_status(self, response, url): - """ - Raises FileNotFoundError for 404s, otherwise uses raise_for_status. - """ - if response.status == 404: - raise FileNotFoundError(url) - response.raise_for_status() - - async def _cat_file(self, url, start=None, end=None, **kwargs): - kw = self.kwargs.copy() - kw.update(kwargs) - logger.debug(url) - - if start is not None or end is not None: - if start == end: - return b"" - headers = kw.pop("headers", {}).copy() - - headers["Range"] = await self._process_limits(url, start, end) - kw["headers"] = headers - session = await self.set_session() - async with session.get(self.encode_url(url), **kw) as r: - out = await r.read() - self._raise_not_found_for_status(r, url) - return out - - async def _get_file( - self, rpath, lpath, chunk_size=5 * 2**20, callback=DEFAULT_CALLBACK, **kwargs - ): - kw = self.kwargs.copy() - kw.update(kwargs) - logger.debug(rpath) - session = await self.set_session() - async with session.get(self.encode_url(rpath), **kw) as r: - try: - size = int(r.headers["content-length"]) - except (ValueError, KeyError): - size = None - - callback.set_size(size) - self._raise_not_found_for_status(r, rpath) - if isfilelike(lpath): - outfile = lpath - else: - outfile = open(lpath, "wb") # noqa: ASYNC230 - - try: - chunk = True - while chunk: - chunk = await r.content.read(chunk_size) - outfile.write(chunk) - callback.relative_update(len(chunk)) - finally: - if not isfilelike(lpath): - outfile.close() - - async def _put_file( - self, - lpath, - rpath, - chunk_size=5 * 2**20, - callback=DEFAULT_CALLBACK, - method="post", - mode="overwrite", - **kwargs, - ): - if mode != "overwrite": - raise NotImplementedError("Exclusive write") - - async def gen_chunks(): - # Support passing arbitrary file-like objects - # and use them instead of streams. - if isinstance(lpath, io.IOBase): - context = nullcontext(lpath) - use_seek = False # might not support seeking - else: - context = open(lpath, "rb") # noqa: ASYNC230 - use_seek = True - - with context as f: - if use_seek: - callback.set_size(f.seek(0, 2)) - f.seek(0) - else: - callback.set_size(getattr(f, "size", None)) - - chunk = f.read(chunk_size) - while chunk: - yield chunk - callback.relative_update(len(chunk)) - chunk = f.read(chunk_size) - - kw = self.kwargs.copy() - kw.update(kwargs) - session = await self.set_session() - - method = method.lower() - if method not in ("post", "put"): - raise ValueError( - f"method has to be either 'post' or 'put', not: {method!r}" - ) - - meth = getattr(session, method) - async with meth(self.encode_url(rpath), data=gen_chunks(), **kw) as resp: - self._raise_not_found_for_status(resp, rpath) - - async def _exists(self, path, strict=False, **kwargs): - kw = self.kwargs.copy() - kw.update(kwargs) - try: - logger.debug(path) - session = await self.set_session() - r = await session.get(self.encode_url(path), **kw) - async with r: - if strict: - self._raise_not_found_for_status(r, path) - return r.status < 400 - except FileNotFoundError: - return False - except aiohttp.ClientError: - if strict: - raise - return False - - async def _isfile(self, path, **kwargs): - return await self._exists(path, **kwargs) - - def _open( - self, - path, - mode="rb", - block_size=None, - autocommit=None, # XXX: This differs from the base class. - cache_type=None, - cache_options=None, - size=None, - **kwargs, - ): - """Make a file-like object - - Parameters - ---------- - path: str - Full URL with protocol - mode: string - must be "rb" - block_size: int or None - Bytes to download in one request; use instance value if None. If - zero, will return a streaming Requests file-like instance. - kwargs: key-value - Any other parameters, passed to requests calls - """ - if mode != "rb": - raise NotImplementedError - block_size = block_size if block_size is not None else self.block_size - kw = self.kwargs.copy() - kw["asynchronous"] = self.asynchronous - kw.update(kwargs) - info = {} - size = size or info.update(self.info(path, **kwargs)) or info["size"] - session = sync(self.loop, self.set_session) - if block_size and size and info.get("partial", True): - return HTTPFile( - self, - path, - session=session, - block_size=block_size, - mode=mode, - size=size, - cache_type=cache_type or self.cache_type, - cache_options=cache_options or self.cache_options, - loop=self.loop, - **kw, - ) - else: - return HTTPStreamFile( - self, - path, - mode=mode, - loop=self.loop, - session=session, - **kw, - ) - - async def open_async(self, path, mode="rb", size=None, **kwargs): - session = await self.set_session() - if size is None: - try: - size = (await self._info(path, **kwargs))["size"] - except FileNotFoundError: - pass - return AsyncStreamFile( - self, - path, - loop=self.loop, - session=session, - size=size, - **kwargs, - ) - - def ukey(self, url): - """Unique identifier; assume HTTP files are static, unchanging""" - return tokenize(url, self.kwargs, self.protocol) - - async def _info(self, url, **kwargs): - """Get info of URL - - Tries to access location via HEAD, and then GET methods, but does - not fetch the data. - - It is possible that the server does not supply any size information, in - which case size will be given as None (and certain operations on the - corresponding file will not work). - """ - info = {} - session = await self.set_session() - - for policy in ["head", "get"]: - try: - info.update( - await _file_info( - self.encode_url(url), - size_policy=policy, - session=session, - **self.kwargs, - **kwargs, - ) - ) - if info.get("size") is not None: - break - except Exception as exc: - if policy == "get": - # If get failed, then raise a FileNotFoundError - raise FileNotFoundError(url) from exc - logger.debug("", exc_info=exc) - - return {"name": url, "size": None, **info, "type": "file"} - - async def _glob(self, path, maxdepth=None, **kwargs): - """ - Find files by glob-matching. - - This implementation is idntical to the one in AbstractFileSystem, - but "?" is not considered as a character for globbing, because it is - so common in URLs, often identifying the "query" part. - """ - if maxdepth is not None and maxdepth < 1: - raise ValueError("maxdepth must be at least 1") - import re - - ends_with_slash = path.endswith("/") # _strip_protocol strips trailing slash - path = self._strip_protocol(path) - append_slash_to_dirname = ends_with_slash or path.endswith(("/**", "/*")) - idx_star = path.find("*") if path.find("*") >= 0 else len(path) - idx_brace = path.find("[") if path.find("[") >= 0 else len(path) - - min_idx = min(idx_star, idx_brace) - - detail = kwargs.pop("detail", False) - - if not has_magic(path): - if await self._exists(path, **kwargs): - if not detail: - return [path] - else: - return {path: await self._info(path, **kwargs)} - else: - if not detail: - return [] # glob of non-existent returns empty - else: - return {} - elif "/" in path[:min_idx]: - min_idx = path[:min_idx].rindex("/") - root = path[: min_idx + 1] - depth = path[min_idx + 1 :].count("/") + 1 - else: - root = "" - depth = path[min_idx + 1 :].count("/") + 1 - - if "**" in path: - if maxdepth is not None: - idx_double_stars = path.find("**") - depth_double_stars = path[idx_double_stars:].count("/") + 1 - depth = depth - depth_double_stars + maxdepth - else: - depth = None - - allpaths = await self._find( - root, maxdepth=depth, withdirs=True, detail=True, **kwargs - ) - - pattern = glob_translate(path + ("/" if ends_with_slash else "")) - pattern = re.compile(pattern) - - out = { - ( - p.rstrip("/") - if not append_slash_to_dirname - and info["type"] == "directory" - and p.endswith("/") - else p - ): info - for p, info in sorted(allpaths.items()) - if pattern.match(p.rstrip("/")) - } - - if detail: - return out - else: - return list(out) - - async def _isdir(self, path): - # override, since all URLs are (also) files - try: - return bool(await self._ls(path)) - except (FileNotFoundError, ValueError): - return False - - async def _pipe_file(self, path, value, mode="overwrite", **kwargs): - """ - Write bytes to a remote file over HTTP. - - Parameters - ---------- - path : str - Target URL where the data should be written - value : bytes - Data to be written - mode : str - How to write to the file - 'overwrite' or 'append' - **kwargs : dict - Additional parameters to pass to the HTTP request - """ - url = self._strip_protocol(path) - headers = kwargs.pop("headers", {}) - headers["Content-Length"] = str(len(value)) - - session = await self.set_session() - - async with session.put( - self.encode_url(url), data=value, headers=headers, **kwargs - ) as r: - r.raise_for_status() - - -class HTTPFile(AbstractBufferedFile): - """ - A file-like object pointing to a remote HTTP(S) resource - - Supports only reading, with read-ahead of a predetermined block-size. - - In the case that the server does not supply the filesize, only reading of - the complete file in one go is supported. - - Parameters - ---------- - url: str - Full URL of the remote resource, including the protocol - session: aiohttp.ClientSession or None - All calls will be made within this session, to avoid restarting - connections where the server allows this - block_size: int or None - The amount of read-ahead to do, in bytes. Default is 5MB, or the value - configured for the FileSystem creating this file - size: None or int - If given, this is the size of the file in bytes, and we don't attempt - to call the server to find the value. - kwargs: all other key-values are passed to requests calls. - """ - - def __init__( - self, - fs, - url, - session=None, - block_size=None, - mode="rb", - cache_type="bytes", - cache_options=None, - size=None, - loop=None, - asynchronous=False, - **kwargs, - ): - if mode != "rb": - raise NotImplementedError("File mode not supported") - self.asynchronous = asynchronous - self.loop = loop - self.url = url - self.session = session - self.details = {"name": url, "size": size, "type": "file"} - super().__init__( - fs=fs, - path=url, - mode=mode, - block_size=block_size, - cache_type=cache_type, - cache_options=cache_options, - **kwargs, - ) - - def read(self, length=-1): - """Read bytes from file - - Parameters - ---------- - length: int - Read up to this many bytes. If negative, read all content to end of - file. If the server has not supplied the filesize, attempting to - read only part of the data will raise a ValueError. - """ - if ( - (length < 0 and self.loc == 0) # explicit read all - # but not when the size is known and fits into a block anyways - and not (self.size is not None and self.size <= self.blocksize) - ): - self._fetch_all() - if self.size is None: - if length < 0: - self._fetch_all() - else: - length = min(self.size - self.loc, length) - return super().read(length) - - async def async_fetch_all(self): - """Read whole file in one shot, without caching - - This is only called when position is still at zero, - and read() is called without a byte-count. - """ - logger.debug(f"Fetch all for {self}") - if not isinstance(self.cache, AllBytes): - r = await self.session.get(self.fs.encode_url(self.url), **self.kwargs) - async with r: - r.raise_for_status() - out = await r.read() - self.cache = AllBytes( - size=len(out), fetcher=None, blocksize=None, data=out - ) - self.size = len(out) - - _fetch_all = sync_wrapper(async_fetch_all) - - def _parse_content_range(self, headers): - """Parse the Content-Range header""" - s = headers.get("Content-Range", "") - m = re.match(r"bytes (\d+-\d+|\*)/(\d+|\*)", s) - if not m: - return None, None, None - - if m[1] == "*": - start = end = None - else: - start, end = [int(x) for x in m[1].split("-")] - total = None if m[2] == "*" else int(m[2]) - return start, end, total - - async def async_fetch_range(self, start, end): - """Download a block of data - - The expectation is that the server returns only the requested bytes, - with HTTP code 206. If this is not the case, we first check the headers, - and then stream the output - if the data size is bigger than we - requested, an exception is raised. - """ - logger.debug(f"Fetch range for {self}: {start}-{end}") - kwargs = self.kwargs.copy() - headers = kwargs.pop("headers", {}).copy() - headers["Range"] = f"bytes={start}-{end - 1}" - logger.debug(f"{self.url} : {headers['Range']}") - r = await self.session.get( - self.fs.encode_url(self.url), headers=headers, **kwargs - ) - async with r: - if r.status == 416: - # range request outside file - return b"" - r.raise_for_status() - - # If the server has handled the range request, it should reply - # with status 206 (partial content). But we'll guess that a suitable - # Content-Range header or a Content-Length no more than the - # requested range also mean we have got the desired range. - response_is_range = ( - r.status == 206 - or self._parse_content_range(r.headers)[0] == start - or int(r.headers.get("Content-Length", end + 1)) <= end - start - ) - - if response_is_range: - # partial content, as expected - out = await r.read() - elif start > 0: - raise ValueError( - "The HTTP server doesn't appear to support range requests. " - "Only reading this file from the beginning is supported. " - "Open with block_size=0 for a streaming file interface." - ) - else: - # Response is not a range, but we want the start of the file, - # so we can read the required amount anyway. - cl = 0 - out = [] - while True: - chunk = await r.content.read(2**20) - # data size unknown, let's read until we have enough - if chunk: - out.append(chunk) - cl += len(chunk) - if cl > end - start: - break - else: - break - out = b"".join(out)[: end - start] - return out - - _fetch_range = sync_wrapper(async_fetch_range) - - -magic_check = re.compile("([*[])") - - -def has_magic(s): - match = magic_check.search(s) - return match is not None - - -class HTTPStreamFile(AbstractBufferedFile): - def __init__(self, fs, url, mode="rb", loop=None, session=None, **kwargs): - self.asynchronous = kwargs.pop("asynchronous", False) - self.url = url - self.loop = loop - self.session = session - if mode != "rb": - raise ValueError - self.details = {"name": url, "size": None} - super().__init__(fs=fs, path=url, mode=mode, cache_type="none", **kwargs) - - async def cor(): - r = await self.session.get(self.fs.encode_url(url), **kwargs).__aenter__() - self.fs._raise_not_found_for_status(r, url) - return r - - self.r = sync(self.loop, cor) - self.loop = fs.loop - - def seek(self, loc, whence=0): - if loc == 0 and whence == 1: - return - if loc == self.loc and whence == 0: - return - raise ValueError("Cannot seek streaming HTTP file") - - async def _read(self, num=-1): - out = await self.r.content.read(num) - self.loc += len(out) - return out - - read = sync_wrapper(_read) - - async def _close(self): - self.r.close() - - def close(self): - asyncio.run_coroutine_threadsafe(self._close(), self.loop) - super().close() - - -class AsyncStreamFile(AbstractAsyncStreamedFile): - def __init__( - self, fs, url, mode="rb", loop=None, session=None, size=None, **kwargs - ): - self.url = url - self.session = session - self.r = None - if mode != "rb": - raise ValueError - self.details = {"name": url, "size": None} - self.kwargs = kwargs - super().__init__(fs=fs, path=url, mode=mode, cache_type="none") - self.size = size - - async def read(self, num=-1): - if self.r is None: - r = await self.session.get( - self.fs.encode_url(self.url), **self.kwargs - ).__aenter__() - self.fs._raise_not_found_for_status(r, self.url) - self.r = r - out = await self.r.content.read(num) - self.loc += len(out) - return out - - async def close(self): - if self.r is not None: - self.r.close() - self.r = None - await super().close() - - -async def get_range(session, url, start, end, file=None, **kwargs): - # explicit get a range when we know it must be safe - kwargs = kwargs.copy() - headers = kwargs.pop("headers", {}).copy() - headers["Range"] = f"bytes={start}-{end - 1}" - r = await session.get(url, headers=headers, **kwargs) - r.raise_for_status() - async with r: - out = await r.read() - if file: - with open(file, "r+b") as f: # noqa: ASYNC230 - f.seek(start) - f.write(out) - else: - return out - - -async def _file_info(url, session, size_policy="head", **kwargs): - """Call HEAD on the server to get details about the file (size/checksum etc.) - - Default operation is to explicitly allow redirects and use encoding - 'identity' (no compression) to get the true size of the target. - """ - logger.debug("Retrieve file size for %s", url) - kwargs = kwargs.copy() - ar = kwargs.pop("allow_redirects", True) - head = kwargs.get("headers", {}).copy() - head["Accept-Encoding"] = "identity" - kwargs["headers"] = head - - info = {} - if size_policy == "head": - r = await session.head(url, allow_redirects=ar, **kwargs) - elif size_policy == "get": - r = await session.get(url, allow_redirects=ar, **kwargs) - else: - raise TypeError(f'size_policy must be "head" or "get", got {size_policy}') - async with r: - r.raise_for_status() - - if "Content-Length" in r.headers: - # Some servers may choose to ignore Accept-Encoding and return - # compressed content, in which case the returned size is unreliable. - if "Content-Encoding" not in r.headers or r.headers["Content-Encoding"] in [ - "identity", - "", - ]: - info["size"] = int(r.headers["Content-Length"]) - elif "Content-Range" in r.headers: - info["size"] = int(r.headers["Content-Range"].split("/")[1]) - - if "Content-Type" in r.headers: - info["mimetype"] = r.headers["Content-Type"].partition(";")[0] - - if r.headers.get("Accept-Ranges") == "none": - # Some servers may explicitly discourage partial content requests, but - # the lack of "Accept-Ranges" does not always indicate they would fail - info["partial"] = False - - info["url"] = str(r.url) - - for checksum_field in ["ETag", "Content-MD5", "Digest", "Last-Modified"]: - if r.headers.get(checksum_field): - info[checksum_field] = r.headers[checksum_field] - - return info - - -async def _file_size(url, session=None, *args, **kwargs): - if session is None: - session = await get_client() - info = await _file_info(url, session=session, *args, **kwargs) - return info.get("size") - - -file_size = sync_wrapper(_file_size) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/http_sync.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/http_sync.py deleted file mode 100644 index a67ea3ea5fee9e6b51f7f3f66773e8cf65735e52..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/http_sync.py +++ /dev/null @@ -1,937 +0,0 @@ -"""This file is largely copied from http.py""" - -import io -import logging -import re -import urllib.error -import urllib.parse -from copy import copy -from json import dumps, loads -from urllib.parse import urlparse - -try: - import yarl -except (ImportError, ModuleNotFoundError, OSError): - yarl = False - -from fsspec.callbacks import _DEFAULT_CALLBACK -from fsspec.registry import register_implementation -from fsspec.spec import AbstractBufferedFile, AbstractFileSystem -from fsspec.utils import DEFAULT_BLOCK_SIZE, isfilelike, nullcontext, tokenize - -from ..caching import AllBytes - -# https://stackoverflow.com/a/15926317/3821154 -ex = re.compile(r"""<(a|A)\s+(?:[^>]*?\s+)?(href|HREF)=["'](?P[^"']+)""") -ex2 = re.compile(r"""(?Phttp[s]?://[-a-zA-Z0-9@:%_+.~#?&/=]+)""") -logger = logging.getLogger("fsspec.http") - - -class JsHttpException(urllib.error.HTTPError): ... - - -class StreamIO(io.BytesIO): - # fake class, so you can set attributes on it - # will eventually actually stream - ... - - -class ResponseProxy: - """Looks like a requests response""" - - def __init__(self, req, stream=False): - self.request = req - self.stream = stream - self._data = None - self._headers = None - - @property - def raw(self): - if self._data is None: - b = self.request.response.to_bytes() - if self.stream: - self._data = StreamIO(b) - else: - self._data = b - return self._data - - def close(self): - if hasattr(self, "_data"): - del self._data - - @property - def headers(self): - if self._headers is None: - self._headers = dict( - [ - _.split(": ") - for _ in self.request.getAllResponseHeaders().strip().split("\r\n") - ] - ) - return self._headers - - @property - def status_code(self): - return int(self.request.status) - - def raise_for_status(self): - if not self.ok: - raise JsHttpException( - self.url, self.status_code, self.reason, self.headers, None - ) - - def iter_content(self, chunksize, *_, **__): - while True: - out = self.raw.read(chunksize) - if out: - yield out - else: - break - - @property - def reason(self): - return self.request.statusText - - @property - def ok(self): - return self.status_code < 400 - - @property - def url(self): - return self.request.response.responseURL - - @property - def text(self): - # TODO: encoding from headers - return self.content.decode() - - @property - def content(self): - self.stream = False - return self.raw - - def json(self): - return loads(self.text) - - -class RequestsSessionShim: - def __init__(self): - self.headers = {} - - def request( - self, - method, - url, - params=None, - data=None, - headers=None, - cookies=None, - files=None, - auth=None, - timeout=None, - allow_redirects=None, - proxies=None, - hooks=None, - stream=None, - verify=None, - cert=None, - json=None, - ): - from js import Blob, XMLHttpRequest - - logger.debug("JS request: %s %s", method, url) - - if cert or verify or proxies or files or cookies or hooks: - raise NotImplementedError - if data and json: - raise ValueError("Use json= or data=, not both") - req = XMLHttpRequest.new() - extra = auth if auth else () - if params: - url = f"{url}?{urllib.parse.urlencode(params)}" - req.open(method, url, False, *extra) - if timeout: - req.timeout = timeout - if headers: - for k, v in headers.items(): - req.setRequestHeader(k, v) - - req.setRequestHeader("Accept", "application/octet-stream") - req.responseType = "arraybuffer" - if json: - blob = Blob.new([dumps(data)], {type: "application/json"}) - req.send(blob) - elif data: - if isinstance(data, io.IOBase): - data = data.read() - blob = Blob.new([data], {type: "application/octet-stream"}) - req.send(blob) - else: - req.send(None) - return ResponseProxy(req, stream=stream) - - def get(self, url, **kwargs): - return self.request("GET", url, **kwargs) - - def head(self, url, **kwargs): - return self.request("HEAD", url, **kwargs) - - def post(self, url, **kwargs): - return self.request("POST}", url, **kwargs) - - def put(self, url, **kwargs): - return self.request("PUT", url, **kwargs) - - def patch(self, url, **kwargs): - return self.request("PATCH", url, **kwargs) - - def delete(self, url, **kwargs): - return self.request("DELETE", url, **kwargs) - - -class HTTPFileSystem(AbstractFileSystem): - """ - Simple File-System for fetching data via HTTP(S) - - This is the BLOCKING version of the normal HTTPFileSystem. It uses - requests in normal python and the JS runtime in pyodide. - - ***This implementation is extremely experimental, do not use unless - you are testing pyodide/pyscript integration*** - """ - - protocol = ("http", "https", "sync-http", "sync-https") - sep = "/" - - def __init__( - self, - simple_links=True, - block_size=None, - same_scheme=True, - cache_type="readahead", - cache_options=None, - client_kwargs=None, - encoded=False, - **storage_options, - ): - """ - - Parameters - ---------- - block_size: int - Blocks to read bytes; if 0, will default to raw requests file-like - objects instead of HTTPFile instances - simple_links: bool - If True, will consider both HTML tags and anything that looks - like a URL; if False, will consider only the former. - same_scheme: True - When doing ls/glob, if this is True, only consider paths that have - http/https matching the input URLs. - size_policy: this argument is deprecated - client_kwargs: dict - Passed to aiohttp.ClientSession, see - https://docs.aiohttp.org/en/stable/client_reference.html - For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}`` - storage_options: key-value - Any other parameters passed on to requests - cache_type, cache_options: defaults used in open - """ - super().__init__(self, **storage_options) - self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE - self.simple_links = simple_links - self.same_schema = same_scheme - self.cache_type = cache_type - self.cache_options = cache_options - self.client_kwargs = client_kwargs or {} - self.encoded = encoded - self.kwargs = storage_options - - try: - import js # noqa: F401 - - logger.debug("Starting JS session") - self.session = RequestsSessionShim() - self.js = True - except Exception as e: - import requests - - logger.debug("Starting cpython session because of: %s", e) - self.session = requests.Session(**(client_kwargs or {})) - self.js = False - - request_options = copy(storage_options) - self.use_listings_cache = request_options.pop("use_listings_cache", False) - request_options.pop("listings_expiry_time", None) - request_options.pop("max_paths", None) - request_options.pop("skip_instance_cache", None) - self.kwargs = request_options - - @property - def fsid(self): - return "sync-http" - - def encode_url(self, url): - if yarl: - return yarl.URL(url, encoded=self.encoded) - return url - - @classmethod - def _strip_protocol(cls, path: str) -> str: - """For HTTP, we always want to keep the full URL""" - path = path.replace("sync-http://", "http://").replace( - "sync-https://", "https://" - ) - return path - - @classmethod - def _parent(cls, path): - # override, since _strip_protocol is different for URLs - par = super()._parent(path) - if len(par) > 7: # "http://..." - return par - return "" - - def _ls_real(self, url, detail=True, **kwargs): - # ignoring URL-encoded arguments - kw = self.kwargs.copy() - kw.update(kwargs) - logger.debug(url) - r = self.session.get(self.encode_url(url), **self.kwargs) - self._raise_not_found_for_status(r, url) - text = r.text - if self.simple_links: - links = ex2.findall(text) + [u[2] for u in ex.findall(text)] - else: - links = [u[2] for u in ex.findall(text)] - out = set() - parts = urlparse(url) - for l in links: - if isinstance(l, tuple): - l = l[1] - if l.startswith("/") and len(l) > 1: - # absolute URL on this server - l = parts.scheme + "://" + parts.netloc + l - if l.startswith("http"): - if self.same_schema and l.startswith(url.rstrip("/") + "/"): - out.add(l) - elif l.replace("https", "http").startswith( - url.replace("https", "http").rstrip("/") + "/" - ): - # allowed to cross http <-> https - out.add(l) - else: - if l not in ["..", "../"]: - # Ignore FTP-like "parent" - out.add("/".join([url.rstrip("/"), l.lstrip("/")])) - if not out and url.endswith("/"): - out = self._ls_real(url.rstrip("/"), detail=False) - if detail: - return [ - { - "name": u, - "size": None, - "type": "directory" if u.endswith("/") else "file", - } - for u in out - ] - else: - return sorted(out) - - def ls(self, url, detail=True, **kwargs): - if self.use_listings_cache and url in self.dircache: - out = self.dircache[url] - else: - out = self._ls_real(url, detail=detail, **kwargs) - self.dircache[url] = out - return out - - def _raise_not_found_for_status(self, response, url): - """ - Raises FileNotFoundError for 404s, otherwise uses raise_for_status. - """ - if response.status_code == 404: - raise FileNotFoundError(url) - response.raise_for_status() - - def cat_file(self, url, start=None, end=None, **kwargs): - kw = self.kwargs.copy() - kw.update(kwargs) - logger.debug(url) - - if start is not None or end is not None: - if start == end: - return b"" - headers = kw.pop("headers", {}).copy() - - headers["Range"] = self._process_limits(url, start, end) - kw["headers"] = headers - r = self.session.get(self.encode_url(url), **kw) - self._raise_not_found_for_status(r, url) - return r.content - - def get_file( - self, rpath, lpath, chunk_size=5 * 2**20, callback=_DEFAULT_CALLBACK, **kwargs - ): - kw = self.kwargs.copy() - kw.update(kwargs) - logger.debug(rpath) - r = self.session.get(self.encode_url(rpath), **kw) - try: - size = int( - r.headers.get("content-length", None) - or r.headers.get("Content-Length", None) - ) - except (ValueError, KeyError, TypeError): - size = None - - callback.set_size(size) - self._raise_not_found_for_status(r, rpath) - if not isfilelike(lpath): - lpath = open(lpath, "wb") - for chunk in r.iter_content(chunk_size, decode_unicode=False): - lpath.write(chunk) - callback.relative_update(len(chunk)) - - def put_file( - self, - lpath, - rpath, - chunk_size=5 * 2**20, - callback=_DEFAULT_CALLBACK, - method="post", - **kwargs, - ): - def gen_chunks(): - # Support passing arbitrary file-like objects - # and use them instead of streams. - if isinstance(lpath, io.IOBase): - context = nullcontext(lpath) - use_seek = False # might not support seeking - else: - context = open(lpath, "rb") - use_seek = True - - with context as f: - if use_seek: - callback.set_size(f.seek(0, 2)) - f.seek(0) - else: - callback.set_size(getattr(f, "size", None)) - - chunk = f.read(chunk_size) - while chunk: - yield chunk - callback.relative_update(len(chunk)) - chunk = f.read(chunk_size) - - kw = self.kwargs.copy() - kw.update(kwargs) - - method = method.lower() - if method not in ("post", "put"): - raise ValueError( - f"method has to be either 'post' or 'put', not: {method!r}" - ) - - meth = getattr(self.session, method) - resp = meth(rpath, data=gen_chunks(), **kw) - self._raise_not_found_for_status(resp, rpath) - - def _process_limits(self, url, start, end): - """Helper for "Range"-based _cat_file""" - size = None - suff = False - if start is not None and start < 0: - # if start is negative and end None, end is the "suffix length" - if end is None: - end = -start - start = "" - suff = True - else: - size = size or self.info(url)["size"] - start = size + start - elif start is None: - start = 0 - if not suff: - if end is not None and end < 0: - if start is not None: - size = size or self.info(url)["size"] - end = size + end - elif end is None: - end = "" - if isinstance(end, int): - end -= 1 # bytes range is inclusive - return f"bytes={start}-{end}" - - def exists(self, path, strict=False, **kwargs): - kw = self.kwargs.copy() - kw.update(kwargs) - try: - logger.debug(path) - r = self.session.get(self.encode_url(path), **kw) - if strict: - self._raise_not_found_for_status(r, path) - return r.status_code < 400 - except FileNotFoundError: - return False - except Exception: - if strict: - raise - return False - - def isfile(self, path, **kwargs): - return self.exists(path, **kwargs) - - def _open( - self, - path, - mode="rb", - block_size=None, - autocommit=None, # XXX: This differs from the base class. - cache_type=None, - cache_options=None, - size=None, - **kwargs, - ): - """Make a file-like object - - Parameters - ---------- - path: str - Full URL with protocol - mode: string - must be "rb" - block_size: int or None - Bytes to download in one request; use instance value if None. If - zero, will return a streaming Requests file-like instance. - kwargs: key-value - Any other parameters, passed to requests calls - """ - if mode != "rb": - raise NotImplementedError - block_size = block_size if block_size is not None else self.block_size - kw = self.kwargs.copy() - kw.update(kwargs) - size = size or self.info(path, **kwargs)["size"] - if block_size and size: - return HTTPFile( - self, - path, - session=self.session, - block_size=block_size, - mode=mode, - size=size, - cache_type=cache_type or self.cache_type, - cache_options=cache_options or self.cache_options, - **kw, - ) - else: - return HTTPStreamFile( - self, - path, - mode=mode, - session=self.session, - **kw, - ) - - def ukey(self, url): - """Unique identifier; assume HTTP files are static, unchanging""" - return tokenize(url, self.kwargs, self.protocol) - - def info(self, url, **kwargs): - """Get info of URL - - Tries to access location via HEAD, and then GET methods, but does - not fetch the data. - - It is possible that the server does not supply any size information, in - which case size will be given as None (and certain operations on the - corresponding file will not work). - """ - info = {} - for policy in ["head", "get"]: - try: - info.update( - _file_info( - self.encode_url(url), - size_policy=policy, - session=self.session, - **self.kwargs, - **kwargs, - ) - ) - if info.get("size") is not None: - break - except Exception as exc: - if policy == "get": - # If get failed, then raise a FileNotFoundError - raise FileNotFoundError(url) from exc - logger.debug(str(exc)) - - return {"name": url, "size": None, **info, "type": "file"} - - def glob(self, path, maxdepth=None, **kwargs): - """ - Find files by glob-matching. - - This implementation is idntical to the one in AbstractFileSystem, - but "?" is not considered as a character for globbing, because it is - so common in URLs, often identifying the "query" part. - """ - import re - - ends = path.endswith("/") - path = self._strip_protocol(path) - indstar = path.find("*") if path.find("*") >= 0 else len(path) - indbrace = path.find("[") if path.find("[") >= 0 else len(path) - - ind = min(indstar, indbrace) - - detail = kwargs.pop("detail", False) - - if not has_magic(path): - root = path - depth = 1 - if ends: - path += "/*" - elif self.exists(path): - if not detail: - return [path] - else: - return {path: self.info(path)} - else: - if not detail: - return [] # glob of non-existent returns empty - else: - return {} - elif "/" in path[:ind]: - ind2 = path[:ind].rindex("/") - root = path[: ind2 + 1] - depth = None if "**" in path else path[ind2 + 1 :].count("/") + 1 - else: - root = "" - depth = None if "**" in path else path[ind + 1 :].count("/") + 1 - - allpaths = self.find( - root, maxdepth=maxdepth or depth, withdirs=True, detail=True, **kwargs - ) - # Escape characters special to python regex, leaving our supported - # special characters in place. - # See https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html - # for shell globbing details. - pattern = ( - "^" - + ( - path.replace("\\", r"\\") - .replace(".", r"\.") - .replace("+", r"\+") - .replace("//", "/") - .replace("(", r"\(") - .replace(")", r"\)") - .replace("|", r"\|") - .replace("^", r"\^") - .replace("$", r"\$") - .replace("{", r"\{") - .replace("}", r"\}") - .rstrip("/") - ) - + "$" - ) - pattern = re.sub("[*]{2}", "=PLACEHOLDER=", pattern) - pattern = re.sub("[*]", "[^/]*", pattern) - pattern = re.compile(pattern.replace("=PLACEHOLDER=", ".*")) - out = { - p: allpaths[p] - for p in sorted(allpaths) - if pattern.match(p.replace("//", "/").rstrip("/")) - } - if detail: - return out - else: - return list(out) - - def isdir(self, path): - # override, since all URLs are (also) files - try: - return bool(self.ls(path)) - except (FileNotFoundError, ValueError): - return False - - -class HTTPFile(AbstractBufferedFile): - """ - A file-like object pointing to a remove HTTP(S) resource - - Supports only reading, with read-ahead of a predermined block-size. - - In the case that the server does not supply the filesize, only reading of - the complete file in one go is supported. - - Parameters - ---------- - url: str - Full URL of the remote resource, including the protocol - session: requests.Session or None - All calls will be made within this session, to avoid restarting - connections where the server allows this - block_size: int or None - The amount of read-ahead to do, in bytes. Default is 5MB, or the value - configured for the FileSystem creating this file - size: None or int - If given, this is the size of the file in bytes, and we don't attempt - to call the server to find the value. - kwargs: all other key-values are passed to requests calls. - """ - - def __init__( - self, - fs, - url, - session=None, - block_size=None, - mode="rb", - cache_type="bytes", - cache_options=None, - size=None, - **kwargs, - ): - if mode != "rb": - raise NotImplementedError("File mode not supported") - self.url = url - self.session = session - self.details = {"name": url, "size": size, "type": "file"} - super().__init__( - fs=fs, - path=url, - mode=mode, - block_size=block_size, - cache_type=cache_type, - cache_options=cache_options, - **kwargs, - ) - - def read(self, length=-1): - """Read bytes from file - - Parameters - ---------- - length: int - Read up to this many bytes. If negative, read all content to end of - file. If the server has not supplied the filesize, attempting to - read only part of the data will raise a ValueError. - """ - if ( - (length < 0 and self.loc == 0) # explicit read all - # but not when the size is known and fits into a block anyways - and not (self.size is not None and self.size <= self.blocksize) - ): - self._fetch_all() - if self.size is None: - if length < 0: - self._fetch_all() - else: - length = min(self.size - self.loc, length) - return super().read(length) - - def _fetch_all(self): - """Read whole file in one shot, without caching - - This is only called when position is still at zero, - and read() is called without a byte-count. - """ - logger.debug(f"Fetch all for {self}") - if not isinstance(self.cache, AllBytes): - r = self.session.get(self.fs.encode_url(self.url), **self.kwargs) - r.raise_for_status() - out = r.content - self.cache = AllBytes(size=len(out), fetcher=None, blocksize=None, data=out) - self.size = len(out) - - def _parse_content_range(self, headers): - """Parse the Content-Range header""" - s = headers.get("Content-Range", "") - m = re.match(r"bytes (\d+-\d+|\*)/(\d+|\*)", s) - if not m: - return None, None, None - - if m[1] == "*": - start = end = None - else: - start, end = [int(x) for x in m[1].split("-")] - total = None if m[2] == "*" else int(m[2]) - return start, end, total - - def _fetch_range(self, start, end): - """Download a block of data - - The expectation is that the server returns only the requested bytes, - with HTTP code 206. If this is not the case, we first check the headers, - and then stream the output - if the data size is bigger than we - requested, an exception is raised. - """ - logger.debug(f"Fetch range for {self}: {start}-{end}") - kwargs = self.kwargs.copy() - headers = kwargs.pop("headers", {}).copy() - headers["Range"] = f"bytes={start}-{end - 1}" - logger.debug("%s : %s", self.url, headers["Range"]) - r = self.session.get(self.fs.encode_url(self.url), headers=headers, **kwargs) - if r.status_code == 416: - # range request outside file - return b"" - r.raise_for_status() - - # If the server has handled the range request, it should reply - # with status 206 (partial content). But we'll guess that a suitable - # Content-Range header or a Content-Length no more than the - # requested range also mean we have got the desired range. - cl = r.headers.get("Content-Length", r.headers.get("content-length", end + 1)) - response_is_range = ( - r.status_code == 206 - or self._parse_content_range(r.headers)[0] == start - or int(cl) <= end - start - ) - - if response_is_range: - # partial content, as expected - out = r.content - elif start > 0: - raise ValueError( - "The HTTP server doesn't appear to support range requests. " - "Only reading this file from the beginning is supported. " - "Open with block_size=0 for a streaming file interface." - ) - else: - # Response is not a range, but we want the start of the file, - # so we can read the required amount anyway. - cl = 0 - out = [] - for chunk in r.iter_content(2**20, False): - out.append(chunk) - cl += len(chunk) - out = b"".join(out)[: end - start] - return out - - -magic_check = re.compile("([*[])") - - -def has_magic(s): - match = magic_check.search(s) - return match is not None - - -class HTTPStreamFile(AbstractBufferedFile): - def __init__(self, fs, url, mode="rb", session=None, **kwargs): - self.url = url - self.session = session - if mode != "rb": - raise ValueError - self.details = {"name": url, "size": None} - super().__init__(fs=fs, path=url, mode=mode, cache_type="readahead", **kwargs) - - r = self.session.get(self.fs.encode_url(url), stream=True, **kwargs) - self.fs._raise_not_found_for_status(r, url) - self.it = r.iter_content(1024, False) - self.leftover = b"" - - self.r = r - - def seek(self, *args, **kwargs): - raise ValueError("Cannot seek streaming HTTP file") - - def read(self, num=-1): - bufs = [self.leftover] - leng = len(self.leftover) - while leng < num or num < 0: - try: - out = self.it.__next__() - except StopIteration: - break - if out: - bufs.append(out) - else: - break - leng += len(out) - out = b"".join(bufs) - if num >= 0: - self.leftover = out[num:] - out = out[:num] - else: - self.leftover = b"" - self.loc += len(out) - return out - - def close(self): - self.r.close() - self.closed = True - - -def get_range(session, url, start, end, **kwargs): - # explicit get a range when we know it must be safe - kwargs = kwargs.copy() - headers = kwargs.pop("headers", {}).copy() - headers["Range"] = f"bytes={start}-{end - 1}" - r = session.get(url, headers=headers, **kwargs) - r.raise_for_status() - return r.content - - -def _file_info(url, session, size_policy="head", **kwargs): - """Call HEAD on the server to get details about the file (size/checksum etc.) - - Default operation is to explicitly allow redirects and use encoding - 'identity' (no compression) to get the true size of the target. - """ - logger.debug("Retrieve file size for %s", url) - kwargs = kwargs.copy() - ar = kwargs.pop("allow_redirects", True) - head = kwargs.get("headers", {}).copy() - # TODO: not allowed in JS - # head["Accept-Encoding"] = "identity" - kwargs["headers"] = head - - info = {} - if size_policy == "head": - r = session.head(url, allow_redirects=ar, **kwargs) - elif size_policy == "get": - r = session.get(url, allow_redirects=ar, **kwargs) - else: - raise TypeError(f'size_policy must be "head" or "get", got {size_policy}') - r.raise_for_status() - - # TODO: - # recognise lack of 'Accept-Ranges', - # or 'Accept-Ranges': 'none' (not 'bytes') - # to mean streaming only, no random access => return None - if "Content-Length" in r.headers: - info["size"] = int(r.headers["Content-Length"]) - elif "Content-Range" in r.headers: - info["size"] = int(r.headers["Content-Range"].split("/")[1]) - elif "content-length" in r.headers: - info["size"] = int(r.headers["content-length"]) - elif "content-range" in r.headers: - info["size"] = int(r.headers["content-range"].split("/")[1]) - - for checksum_field in ["ETag", "Content-MD5", "Digest"]: - if r.headers.get(checksum_field): - info[checksum_field] = r.headers[checksum_field] - - return info - - -# importing this is enough to register it -def register(): - register_implementation("http", HTTPFileSystem, clobber=True) - register_implementation("https", HTTPFileSystem, clobber=True) - register_implementation("sync-http", HTTPFileSystem, clobber=True) - register_implementation("sync-https", HTTPFileSystem, clobber=True) - - -register() - - -def unregister(): - from fsspec.implementations.http import HTTPFileSystem - - register_implementation("http", HTTPFileSystem, clobber=True) - register_implementation("https", HTTPFileSystem, clobber=True) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/jupyter.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/jupyter.py deleted file mode 100644 index e5571ed56582170051f3b7cd903093eed4c65244..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/jupyter.py +++ /dev/null @@ -1,129 +0,0 @@ -import base64 -import io -import re - -import requests - -import fsspec - - -class JupyterFileSystem(fsspec.AbstractFileSystem): - """View of the files as seen by a Jupyter server (notebook or lab)""" - - protocol = ("jupyter", "jlab") - - def __init__(self, url, tok=None, **kwargs): - """ - - Parameters - ---------- - url : str - Base URL of the server, like "http://127.0.0.1:8888". May include - token in the string, which is given by the process when starting up - tok : str - If the token is obtained separately, can be given here - kwargs - """ - if "?" in url: - if tok is None: - try: - tok = re.findall("token=([a-z0-9]+)", url)[0] - except IndexError as e: - raise ValueError("Could not determine token") from e - url = url.split("?", 1)[0] - self.url = url.rstrip("/") + "/api/contents" - self.session = requests.Session() - if tok: - self.session.headers["Authorization"] = f"token {tok}" - - super().__init__(**kwargs) - - def ls(self, path, detail=True, **kwargs): - path = self._strip_protocol(path) - r = self.session.get(f"{self.url}/{path}") - if r.status_code == 404: - raise FileNotFoundError(path) - r.raise_for_status() - out = r.json() - - if out["type"] == "directory": - out = out["content"] - else: - out = [out] - for o in out: - o["name"] = o.pop("path") - o.pop("content") - if o["type"] == "notebook": - o["type"] = "file" - if detail: - return out - return [o["name"] for o in out] - - def cat_file(self, path, start=None, end=None, **kwargs): - path = self._strip_protocol(path) - r = self.session.get(f"{self.url}/{path}") - if r.status_code == 404: - raise FileNotFoundError(path) - r.raise_for_status() - out = r.json() - if out["format"] == "text": - # data should be binary - b = out["content"].encode() - else: - b = base64.b64decode(out["content"]) - return b[start:end] - - def pipe_file(self, path, value, **_): - path = self._strip_protocol(path) - json = { - "name": path.rsplit("/", 1)[-1], - "path": path, - "size": len(value), - "content": base64.b64encode(value).decode(), - "format": "base64", - "type": "file", - } - self.session.put(f"{self.url}/{path}", json=json) - - def mkdir(self, path, create_parents=True, **kwargs): - path = self._strip_protocol(path) - if create_parents and "/" in path: - self.mkdir(path.rsplit("/", 1)[0], True) - json = { - "name": path.rsplit("/", 1)[-1], - "path": path, - "size": None, - "content": None, - "type": "directory", - } - self.session.put(f"{self.url}/{path}", json=json) - - def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs): - if path1 == path2: - return - self.session.patch(f"{self.url}/{path1}", json={"path": path2}) - - def _rm(self, path): - path = self._strip_protocol(path) - self.session.delete(f"{self.url}/{path}") - - def _open(self, path, mode="rb", **kwargs): - path = self._strip_protocol(path) - if mode == "rb": - data = self.cat_file(path) - return io.BytesIO(data) - else: - return SimpleFileWriter(self, path, mode="wb") - - -class SimpleFileWriter(fsspec.spec.AbstractBufferedFile): - def _upload_chunk(self, final=False): - """Never uploads a chunk until file is done - - Not suitable for large files - """ - if final is False: - return False - self.buffer.seek(0) - data = self.buffer.read() - self.fs.pipe_file(self.path, data) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/libarchive.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/libarchive.py deleted file mode 100644 index 6f8e750002df72865d611b48022e6634f9572614..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/libarchive.py +++ /dev/null @@ -1,213 +0,0 @@ -from contextlib import contextmanager -from ctypes import ( - CFUNCTYPE, - POINTER, - c_int, - c_longlong, - c_void_p, - cast, - create_string_buffer, -) - -import libarchive -import libarchive.ffi as ffi - -from fsspec import open_files -from fsspec.archive import AbstractArchiveFileSystem -from fsspec.implementations.memory import MemoryFile -from fsspec.utils import DEFAULT_BLOCK_SIZE - -# Libarchive requires seekable files or memory only for certain archive -# types. However, since we read the directory first to cache the contents -# and also allow random access to any file, the file-like object needs -# to be seekable no matter what. - -# Seek call-backs (not provided in the libarchive python wrapper) -SEEK_CALLBACK = CFUNCTYPE(c_longlong, c_int, c_void_p, c_longlong, c_int) -read_set_seek_callback = ffi.ffi( - "read_set_seek_callback", [ffi.c_archive_p, SEEK_CALLBACK], c_int, ffi.check_int -) -new_api = hasattr(ffi, "NO_OPEN_CB") - - -@contextmanager -def custom_reader(file, format_name="all", filter_name="all", block_size=ffi.page_size): - """Read an archive from a seekable file-like object. - - The `file` object must support the standard `readinto` and 'seek' methods. - """ - buf = create_string_buffer(block_size) - buf_p = cast(buf, c_void_p) - - def read_func(archive_p, context, ptrptr): - # readinto the buffer, returns number of bytes read - length = file.readinto(buf) - # write the address of the buffer into the pointer - ptrptr = cast(ptrptr, POINTER(c_void_p)) - ptrptr[0] = buf_p - # tell libarchive how much data was written into the buffer - return length - - def seek_func(archive_p, context, offset, whence): - file.seek(offset, whence) - # tell libarchvie the current position - return file.tell() - - read_cb = ffi.READ_CALLBACK(read_func) - seek_cb = SEEK_CALLBACK(seek_func) - - if new_api: - open_cb = ffi.NO_OPEN_CB - close_cb = ffi.NO_CLOSE_CB - else: - open_cb = libarchive.read.OPEN_CALLBACK(ffi.VOID_CB) - close_cb = libarchive.read.CLOSE_CALLBACK(ffi.VOID_CB) - - with libarchive.read.new_archive_read(format_name, filter_name) as archive_p: - read_set_seek_callback(archive_p, seek_cb) - ffi.read_open(archive_p, None, open_cb, read_cb, close_cb) - yield libarchive.read.ArchiveRead(archive_p) - - -class LibArchiveFileSystem(AbstractArchiveFileSystem): - """Compressed archives as a file-system (read-only) - - Supports the following formats: - tar, pax , cpio, ISO9660, zip, mtree, shar, ar, raw, xar, lha/lzh, rar - Microsoft CAB, 7-Zip, WARC - - See the libarchive documentation for further restrictions. - https://www.libarchive.org/ - - Keeps file object open while instance lives. It only works in seekable - file-like objects. In case the filesystem does not support this kind of - file object, it is recommended to cache locally. - - This class is pickleable, but not necessarily thread-safe (depends on the - platform). See libarchive documentation for details. - """ - - root_marker = "" - protocol = "libarchive" - cachable = False - - def __init__( - self, - fo="", - mode="r", - target_protocol=None, - target_options=None, - block_size=DEFAULT_BLOCK_SIZE, - **kwargs, - ): - """ - Parameters - ---------- - fo: str or file-like - Contains ZIP, and must exist. If a str, will fetch file using - :meth:`~fsspec.open_files`, which must return one file exactly. - mode: str - Currently, only 'r' accepted - target_protocol: str (optional) - If ``fo`` is a string, this value can be used to override the - FS protocol inferred from a URL - target_options: dict (optional) - Kwargs passed when instantiating the target FS, if ``fo`` is - a string. - """ - super().__init__(self, **kwargs) - if mode != "r": - raise ValueError("Only read from archive files accepted") - if isinstance(fo, str): - files = open_files(fo, protocol=target_protocol, **(target_options or {})) - if len(files) != 1: - raise ValueError( - f'Path "{fo}" did not resolve to exactly one file: "{files}"' - ) - fo = files[0] - self.of = fo - self.fo = fo.__enter__() # the whole instance is a context - self.block_size = block_size - self.dir_cache = None - - @contextmanager - def _open_archive(self): - self.fo.seek(0) - with custom_reader(self.fo, block_size=self.block_size) as arc: - yield arc - - @classmethod - def _strip_protocol(cls, path): - # file paths are always relative to the archive root - return super()._strip_protocol(path).lstrip("/") - - def _get_dirs(self): - fields = { - "name": "pathname", - "size": "size", - "created": "ctime", - "mode": "mode", - "uid": "uid", - "gid": "gid", - "mtime": "mtime", - } - - if self.dir_cache is not None: - return - - self.dir_cache = {} - list_names = [] - with self._open_archive() as arc: - for entry in arc: - if not entry.isdir and not entry.isfile: - # Skip symbolic links, fifo entries, etc. - continue - self.dir_cache.update( - { - dirname: {"name": dirname, "size": 0, "type": "directory"} - for dirname in self._all_dirnames(set(entry.name)) - } - ) - f = {key: getattr(entry, fields[key]) for key in fields} - f["type"] = "directory" if entry.isdir else "file" - list_names.append(entry.name) - - self.dir_cache[f["name"]] = f - # libarchive does not seem to return an entry for the directories (at least - # not in all formats), so get the directories names from the files names - self.dir_cache.update( - { - dirname: {"name": dirname, "size": 0, "type": "directory"} - for dirname in self._all_dirnames(list_names) - } - ) - - def _open( - self, - path, - mode="rb", - block_size=None, - autocommit=True, - cache_options=None, - **kwargs, - ): - path = self._strip_protocol(path) - if mode != "rb": - raise NotImplementedError - - data = b"" - with self._open_archive() as arc: - for entry in arc: - if entry.pathname != path: - continue - - if entry.size == 0: - # empty file, so there are no blocks - break - - for block in entry.get_blocks(entry.size): - data = block - break - else: - raise ValueError - return MemoryFile(fs=self, path=path, data=data) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/local.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/local.py deleted file mode 100644 index fd7531e7e9744a66a978d71b91ff93a2444d7510..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/local.py +++ /dev/null @@ -1,517 +0,0 @@ -import datetime -import io -import logging -import os -import os.path as osp -import shutil -import stat -import tempfile -from functools import lru_cache - -from fsspec import AbstractFileSystem -from fsspec.compression import compr -from fsspec.core import get_compression -from fsspec.utils import isfilelike, stringify_path - -logger = logging.getLogger("fsspec.local") - - -class LocalFileSystem(AbstractFileSystem): - """Interface to files on local storage - - Parameters - ---------- - auto_mkdir: bool - Whether, when opening a file, the directory containing it should - be created (if it doesn't already exist). This is assumed by pyarrow - code. - """ - - root_marker = "/" - protocol = "file", "local" - local_file = True - - def __init__(self, auto_mkdir=False, **kwargs): - super().__init__(**kwargs) - self.auto_mkdir = auto_mkdir - - @property - def fsid(self): - return "local" - - def mkdir(self, path, create_parents=True, **kwargs): - path = self._strip_protocol(path) - if self.exists(path): - raise FileExistsError(path) - if create_parents: - self.makedirs(path, exist_ok=True) - else: - os.mkdir(path, **kwargs) - - def makedirs(self, path, exist_ok=False): - path = self._strip_protocol(path) - os.makedirs(path, exist_ok=exist_ok) - - def rmdir(self, path): - path = self._strip_protocol(path) - os.rmdir(path) - - def ls(self, path, detail=False, **kwargs): - path = self._strip_protocol(path) - path_info = self.info(path) - infos = [] - if path_info["type"] == "directory": - with os.scandir(path) as it: - for f in it: - try: - # Only get the info if requested since it is a bit expensive (the stat call inside) - # The strip_protocol is also used in info() and calls make_path_posix to always return posix paths - info = self.info(f) if detail else self._strip_protocol(f.path) - infos.append(info) - except FileNotFoundError: - pass - else: - infos = [path_info] if detail else [path_info["name"]] - - return infos - - def info(self, path, **kwargs): - if isinstance(path, os.DirEntry): - # scandir DirEntry - out = path.stat(follow_symlinks=False) - link = path.is_symlink() - if path.is_dir(follow_symlinks=False): - t = "directory" - elif path.is_file(follow_symlinks=False): - t = "file" - else: - t = "other" - - size = out.st_size - if link: - try: - out2 = path.stat(follow_symlinks=True) - size = out2.st_size - except OSError: - size = 0 - path = self._strip_protocol(path.path) - else: - # str or path-like - path = self._strip_protocol(path) - out = os.stat(path, follow_symlinks=False) - link = stat.S_ISLNK(out.st_mode) - if link: - out = os.stat(path, follow_symlinks=True) - size = out.st_size - if stat.S_ISDIR(out.st_mode): - t = "directory" - elif stat.S_ISREG(out.st_mode): - t = "file" - else: - t = "other" - - # Check for the 'st_birthtime' attribute, which is not always present; fallback to st_ctime - created_time = getattr(out, "st_birthtime", out.st_ctime) - - result = { - "name": path, - "size": size, - "type": t, - "created": created_time, - "islink": link, - } - for field in ["mode", "uid", "gid", "mtime", "ino", "nlink"]: - result[field] = getattr(out, f"st_{field}") - if link: - result["destination"] = os.readlink(path) - return result - - def lexists(self, path, **kwargs): - return osp.lexists(path) - - def cp_file(self, path1, path2, **kwargs): - path1 = self._strip_protocol(path1) - path2 = self._strip_protocol(path2) - if self.auto_mkdir: - self.makedirs(self._parent(path2), exist_ok=True) - if self.isfile(path1): - shutil.copyfile(path1, path2) - elif self.isdir(path1): - self.mkdirs(path2, exist_ok=True) - else: - raise FileNotFoundError(path1) - - def isfile(self, path): - path = self._strip_protocol(path) - return os.path.isfile(path) - - def isdir(self, path): - path = self._strip_protocol(path) - return os.path.isdir(path) - - def get_file(self, path1, path2, callback=None, **kwargs): - if isfilelike(path2): - with open(path1, "rb") as f: - shutil.copyfileobj(f, path2) - else: - return self.cp_file(path1, path2, **kwargs) - - def put_file(self, path1, path2, callback=None, **kwargs): - return self.cp_file(path1, path2, **kwargs) - - def mv(self, path1, path2, recursive: bool = True, **kwargs): - """Move files/directories - For the specific case of local, all ops on directories are recursive and - the recursive= kwarg is ignored. - """ - path1 = self._strip_protocol(path1) - path2 = self._strip_protocol(path2) - - if self.auto_mkdir: - self.makedirs(self._parent(path2), exist_ok=True) - - shutil.move(path1, path2) - - def link(self, src, dst, **kwargs): - src = self._strip_protocol(src) - dst = self._strip_protocol(dst) - os.link(src, dst, **kwargs) - - def symlink(self, src, dst, **kwargs): - src = self._strip_protocol(src) - dst = self._strip_protocol(dst) - os.symlink(src, dst, **kwargs) - - def islink(self, path) -> bool: - return os.path.islink(self._strip_protocol(path)) - - def rm_file(self, path): - os.remove(self._strip_protocol(path)) - - def rm(self, path, recursive=False, maxdepth=None): - if not isinstance(path, list): - path = [path] - - for p in path: - p = self._strip_protocol(p) - if self.isdir(p) and not self.islink(p): - if not recursive: - raise ValueError("Cannot delete directory, set recursive=True") - if osp.abspath(p) == os.getcwd(): - raise ValueError("Cannot delete current working directory") - shutil.rmtree(p) - else: - os.remove(p) - - def unstrip_protocol(self, name): - protocol = self.protocol if isinstance(self.protocol, str) else self.protocol[0] - name = self._strip_protocol(name) # normalise for local/win/... - return f"{protocol}://{name}" - - def _open(self, path, mode="rb", block_size=None, **kwargs): - path = self._strip_protocol(path) - if self.auto_mkdir and ("w" in mode or "x" in mode or "a" in mode): - self.makedirs(self._parent(path), exist_ok=True) - return LocalFileOpener(path, mode, fs=self, **kwargs) - - def touch(self, path, truncate=True, **kwargs): - path = self._strip_protocol(path) - if self.auto_mkdir: - self.makedirs(self._parent(path), exist_ok=True) - if self.exists(path): - os.utime(path, None) - else: - open(path, "a").close() - if truncate: - os.truncate(path, 0) - - def created(self, path): - info = self.info(path=path) - return datetime.datetime.fromtimestamp( - info["created"], tz=datetime.timezone.utc - ) - - def modified(self, path): - info = self.info(path=path) - return datetime.datetime.fromtimestamp(info["mtime"], tz=datetime.timezone.utc) - - @classmethod - def _parent(cls, path): - path = cls._strip_protocol(path) - if os.sep == "/": - # posix native - return path.rsplit("/", 1)[0] or "/" - else: - # NT - path_ = path.rsplit("/", 1)[0] - if len(path_) <= 3: - if path_[1:2] == ":": - # nt root (something like c:/) - return path_[0] + ":/" - # More cases may be required here - return path_ - - @classmethod - def _strip_protocol(cls, path): - path = stringify_path(path) - protos = (cls.protocol,) if isinstance(cls.protocol, str) else cls.protocol - prefixes = (protocol + sep for protocol in protos for sep in ("://", ":")) - for prefix in prefixes: - if path.startswith(prefix): - path = path.removeprefix(prefix) - break - - path = make_path_posix(path) - if os.sep != "/": - # This code-path is a stripped down version of - # > drive, path = ntpath.splitdrive(path) - if path[1:2] == ":": - # Absolute drive-letter path, e.g. X:\Windows - # Relative path with drive, e.g. X:Windows - drive, path = path[:2], path[2:] - elif path[:2] == "//": - # UNC drives, e.g. \\server\share or \\?\UNC\server\share - # Device drives, e.g. \\.\device or \\?\device - if (index1 := path.find("/", 2)) == -1 or ( - index2 := path.find("/", index1 + 1) - ) == -1: - drive, path = path, "" - else: - drive, path = path[:index2], path[index2:] - else: - # Relative path, e.g. Windows - drive = "" - - path = path.rstrip("/") or cls.root_marker - return drive + path - - else: - return path.rstrip("/") or cls.root_marker - - def _isfilestore(self): - # Inheriting from DaskFileSystem makes this False (S3, etc. were) - # the original motivation. But we are a posix-like file system. - # See https://github.com/dask/dask/issues/5526 - return True - - def chmod(self, path, mode): - path = stringify_path(path) - return os.chmod(path, mode) - - -def make_path_posix(path): - """Make path generic and absolute for current OS""" - if not isinstance(path, str): - if isinstance(path, (list, set, tuple)): - return type(path)(make_path_posix(p) for p in path) - else: - path = stringify_path(path) - if not isinstance(path, str): - raise TypeError(f"could not convert {path!r} to string") - if os.sep == "/": - # Native posix - if path.startswith("/"): - # most common fast case for posix - return path - elif path.startswith("~"): - return osp.expanduser(path) - elif path.startswith("./"): - path = path[2:] - elif path == ".": - path = "" - return f"{os.getcwd()}/{path}" - else: - # NT handling - if path[0:1] == "/" and path[2:3] == ":": - # path is like "/c:/local/path" - path = path[1:] - if path[1:2] == ":": - # windows full path like "C:\\local\\path" - if len(path) <= 3: - # nt root (something like c:/) - return path[0] + ":/" - path = path.replace("\\", "/") - return path - elif path[0:1] == "~": - return make_path_posix(osp.expanduser(path)) - elif path.startswith(("\\\\", "//")): - # windows UNC/DFS-style paths - return "//" + path[2:].replace("\\", "/") - elif path.startswith(("\\", "/")): - # windows relative path with root - path = path.replace("\\", "/") - return f"{osp.splitdrive(os.getcwd())[0]}{path}" - else: - path = path.replace("\\", "/") - if path.startswith("./"): - path = path[2:] - elif path == ".": - path = "" - return f"{make_path_posix(os.getcwd())}/{path}" - - -def trailing_sep(path): - """Return True if the path ends with a path separator. - - A forward slash is always considered a path separator, even on Operating - Systems that normally use a backslash. - """ - # TODO: if all incoming paths were posix-compliant then separator would - # always be a forward slash, simplifying this function. - # See https://github.com/fsspec/filesystem_spec/pull/1250 - return path.endswith(os.sep) or (os.altsep is not None and path.endswith(os.altsep)) - - -@lru_cache(maxsize=1) -def get_umask(mask: int = 0o666) -> int: - """Get the current umask. - - Follows https://stackoverflow.com/a/44130549 to get the umask. - Temporarily sets the umask to the given value, and then resets it to the - original value. - """ - value = os.umask(mask) - os.umask(value) - return value - - -class LocalFileOpener(io.IOBase): - def __init__( - self, path, mode, autocommit=True, fs=None, compression=None, **kwargs - ): - logger.debug("open file: %s", path) - self.path = path - self.mode = mode - self.fs = fs - self.f = None - self.autocommit = autocommit - self.compression = get_compression(path, compression) - self.blocksize = io.DEFAULT_BUFFER_SIZE - self._open() - - def _open(self): - if self.f is None or self.f.closed: - if self.autocommit or "w" not in self.mode: - self.f = open(self.path, mode=self.mode) - if self.compression: - compress = compr[self.compression] - self.f = compress(self.f, mode=self.mode) - else: - # TODO: check if path is writable? - i, name = tempfile.mkstemp() - os.close(i) # we want normal open and normal buffered file - self.temp = name - self.f = open(name, mode=self.mode) - if "w" not in self.mode: - self.size = self.f.seek(0, 2) - self.f.seek(0) - self.f.size = self.size - - def _fetch_range(self, start, end): - # probably only used by cached FS - if "r" not in self.mode: - raise ValueError - self._open() - self.f.seek(start) - return self.f.read(end - start) - - def __setstate__(self, state): - self.f = None - loc = state.pop("loc", None) - self.__dict__.update(state) - if "r" in state["mode"]: - self.f = None - self._open() - self.f.seek(loc) - - def __getstate__(self): - d = self.__dict__.copy() - d.pop("f") - if "r" in self.mode: - d["loc"] = self.f.tell() - else: - if not self.f.closed: - raise ValueError("Cannot serialise open write-mode local file") - return d - - def commit(self): - if self.autocommit: - raise RuntimeError("Can only commit if not already set to autocommit") - try: - shutil.move(self.temp, self.path) - except PermissionError as e: - # shutil.move raises PermissionError if os.rename - # and the default copy2 fallback with shutil.copystats fail. - # The file should be there nonetheless, but without copied permissions. - # If it doesn't exist, there was no permission to create the file. - if not os.path.exists(self.path): - raise e - else: - # If PermissionError is not raised, permissions can be set. - try: - mask = 0o666 - os.chmod(self.path, mask & ~get_umask(mask)) - except RuntimeError: - pass - - def discard(self): - if self.autocommit: - raise RuntimeError("Cannot discard if set to autocommit") - os.remove(self.temp) - - def readable(self) -> bool: - return True - - def writable(self) -> bool: - return "r" not in self.mode - - def read(self, *args, **kwargs): - return self.f.read(*args, **kwargs) - - def write(self, *args, **kwargs): - return self.f.write(*args, **kwargs) - - def tell(self, *args, **kwargs): - return self.f.tell(*args, **kwargs) - - def seek(self, *args, **kwargs): - return self.f.seek(*args, **kwargs) - - def seekable(self, *args, **kwargs): - return self.f.seekable(*args, **kwargs) - - def readline(self, *args, **kwargs): - return self.f.readline(*args, **kwargs) - - def readlines(self, *args, **kwargs): - return self.f.readlines(*args, **kwargs) - - def close(self): - return self.f.close() - - def truncate(self, size=None) -> int: - return self.f.truncate(size) - - @property - def closed(self): - return self.f.closed - - def fileno(self): - return self.raw.fileno() - - def flush(self) -> None: - self.f.flush() - - def __iter__(self): - return self.f.__iter__() - - def __getattr__(self, item): - return getattr(self.f, item) - - def __enter__(self): - self._incontext = True - return self - - def __exit__(self, exc_type, exc_value, traceback): - self._incontext = False - self.f.__exit__(exc_type, exc_value, traceback) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/memory.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/memory.py deleted file mode 100644 index 378eb1e15a21d3ff8b7820f562c805ee37289492..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/memory.py +++ /dev/null @@ -1,393 +0,0 @@ -from __future__ import annotations - -import logging -from datetime import datetime, timezone -from errno import ENOTEMPTY -from io import BytesIO -from pathlib import PurePath, PureWindowsPath -from typing import Any, ClassVar - -from fsspec import AbstractFileSystem -from fsspec.implementations.local import LocalFileSystem -from fsspec.utils import stringify_path - -logger = logging.getLogger("fsspec.memoryfs") - - -class MemoryFileSystem(AbstractFileSystem): - """A filesystem based on a dict of BytesIO objects - - This is a global filesystem so instances of this class all point to the same - in memory filesystem. - """ - - store: ClassVar[dict[str, Any]] = {} # global, do not overwrite! - pseudo_dirs = [""] # global, do not overwrite! - protocol = "memory" - root_marker = "/" - - @classmethod - def _strip_protocol(cls, path): - if isinstance(path, PurePath): - if isinstance(path, PureWindowsPath): - return LocalFileSystem._strip_protocol(path) - else: - path = stringify_path(path) - - path = path.removeprefix("memory://") - if "::" in path or "://" in path: - return path.rstrip("/") - path = path.lstrip("/").rstrip("/") - return "/" + path if path else "" - - def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): - # The base implementation calls ls() once per directory, and each ls() - # scans the whole (global) store, giving O(n_dirs * n_entries) behaviour - # for a tree. Since the store is a flat mapping of every path, the same - # result can be produced with a single pass over it. - if maxdepth is not None and maxdepth < 1: - raise ValueError("maxdepth must be at least 1") - path = self._strip_protocol(path) - if path in self.store: - # path is itself a file - if not detail: - return [path] - filelike = self.store[path] - return { - path: { - "name": path, - "size": filelike.size, - "type": "file", - "created": filelike.created.timestamp(), - } - } - - # Uniform prefix so that the search root "" (the filesystem root) and a - # nested path are handled the same way; rel depth is rel.count("/") + 1. - prefix = path + "/" if path else "/" - out = {} - dirs = {} - - def add_ancestor_dirs(name): - # Register every directory implied between ``path`` and ``name`` that - # is within maxdepth, mirroring how walk() surfaces implied dirs. - idx = name.rfind("/") - while idx > len(path): - parent = name[:idx] - if parent in dirs: - break - rel = parent[len(prefix) :] - if maxdepth is None or rel.count("/") + 1 <= maxdepth: - dirs[parent] = {"name": parent, "size": 0, "type": "directory"} - idx = parent.rfind("/") - - # `store` is shared by every MemoryFileSystem instance, so iterate a - # snapshot: a concurrent create/delete would otherwise raise - # "dictionary changed size during iteration". ls() does the same. - for name, filelike in tuple(self.store.items()): - if not name.startswith(prefix): - continue - rel = name[len(prefix) :] - if withdirs: - add_ancestor_dirs(name) - if maxdepth is not None and rel.count("/") + 1 > maxdepth: - continue - out[name] = { - "name": name, - "size": filelike.size, - "type": "file", - "created": filelike.created.timestamp(), - } - - if withdirs: - # Explicitly-created (possibly empty) directories live in pseudo_dirs. - for pdir in self.pseudo_dirs: - if pdir and pdir.startswith(prefix): - add_ancestor_dirs(pdir + "/") - out.update(dirs) - # Mirror the base find(): include the search root itself when it is - # a directory (needed for posix glob compliance). - if path != "" and self.isdir(path): - out[path] = self.info(path) - - names = sorted(out) - return {name: out[name] for name in names} if detail else names - - def ls(self, path, detail=True, **kwargs): - path = self._strip_protocol(path) - if path in self.store: - # there is a key with this exact name - if not detail: - return [path] - return [ - { - "name": path, - "size": self.store[path].size, - "type": "file", - "created": self.store[path].created.timestamp(), - } - ] - paths = set() - starter = path + "/" - out = [] - for p2 in tuple(self.store): - if p2.startswith(starter): - if "/" not in p2[len(starter) :]: - # exact child - out.append( - { - "name": p2, - "size": self.store[p2].size, - "type": "file", - "created": self.store[p2].created.timestamp(), - } - ) - elif len(p2) > len(starter): - # implied child directory - ppath = starter + p2[len(starter) :].split("/", 1)[0] - if ppath not in paths: - out = out or [] - out.append( - { - "name": ppath, - "size": 0, - "type": "directory", - } - ) - paths.add(ppath) - for p2 in self.pseudo_dirs: - if p2.startswith(starter): - if "/" not in p2[len(starter) :]: - # exact child pdir - if p2 not in paths: - out.append({"name": p2, "size": 0, "type": "directory"}) - paths.add(p2) - else: - # directory implied by deeper pdir - ppath = starter + p2[len(starter) :].split("/", 1)[0] - if ppath not in paths: - out.append({"name": ppath, "size": 0, "type": "directory"}) - paths.add(ppath) - if not out: - if path in self.pseudo_dirs: - # empty dir - return [] - raise FileNotFoundError(path) - if detail: - return out - return sorted([f["name"] for f in out]) - - def mkdir(self, path, create_parents=True, **kwargs): - path = self._strip_protocol(path) - if path in self.store or path in self.pseudo_dirs: - raise FileExistsError(path) - if self._parent(path).strip("/") and self.isfile(self._parent(path)): - raise NotADirectoryError(self._parent(path)) - if create_parents and self._parent(path).strip("/"): - try: - self.mkdir(self._parent(path), create_parents, **kwargs) - except FileExistsError: - pass - if path and path not in self.pseudo_dirs: - self.pseudo_dirs.append(path) - - def makedirs(self, path, exist_ok=False): - try: - self.mkdir(path, create_parents=True) - except FileExistsError: - if not exist_ok: - raise - - def pipe_file(self, path, value, mode="overwrite", **kwargs): - """Set the bytes of given file - - Avoids copies of the data if possible - """ - mode = "xb" if mode == "create" else "wb" - self.open(path, mode=mode, data=value) - - def rmdir(self, path): - path = self._strip_protocol(path) - if path == "": - # silently avoid deleting FS root - return - if path in self.pseudo_dirs: - if not self.ls(path): - self.pseudo_dirs.remove(path) - else: - raise OSError(ENOTEMPTY, "Directory not empty", path) - else: - raise FileNotFoundError(path) - - def info(self, path, **kwargs): - logger.debug("info: %s", path) - path = self._strip_protocol(path) - if path in self.pseudo_dirs or any( - p.startswith(path + "/") for p in list(self.store) + self.pseudo_dirs - ): - return { - "name": path, - "size": 0, - "type": "directory", - } - elif path in self.store: - filelike = self.store[path] - return { - "name": path, - "size": filelike.size, - "type": "file", - "created": getattr(filelike, "created", None), - } - else: - raise FileNotFoundError(path) - - def _open( - self, - path, - mode="rb", - block_size=None, - autocommit=True, - cache_options=None, - **kwargs, - ): - path = self._strip_protocol(path) - if "x" in mode and self.exists(path): - raise FileExistsError - if path in self.pseudo_dirs: - raise IsADirectoryError(path) - parent = path - while len(parent) > 1: - parent = self._parent(parent) - if self.isfile(parent): - raise FileExistsError(parent) - if mode in ["rb", "ab", "r+b", "a+b"]: - if path in self.store: - f = self.store[path] - if "a" in mode: - # position at the end of file - f.seek(0, 2) - else: - # position at the beginning of file - f.seek(0) - return f - elif "a" in mode: - # append modes create the file if it does not exist, matching - # builtin open() and LocalFileSystem - m = MemoryFile(self, path, kwargs.get("data")) - if not self._intrans: - m.commit() - # position at the end of file, like the existing-file path above - m.seek(0, 2) - return m - else: - raise FileNotFoundError(path) - elif mode in {"wb", "w+b", "xb", "x+b"}: - if "x" in mode and self.exists(path): - raise FileExistsError - m = MemoryFile(self, path, kwargs.get("data")) - if not self._intrans: - m.commit() - return m - else: - name = self.__class__.__name__ - raise ValueError(f"unsupported file mode for {name}: {mode!r}") - - def cp_file(self, path1, path2, **kwargs): - path1 = self._strip_protocol(path1) - path2 = self._strip_protocol(path2) - if self.isfile(path1): - self.store[path2] = MemoryFile( - self, path2, self.store[path1].getvalue() - ) # implicit copy - elif self.isdir(path1): - if path2 not in self.pseudo_dirs: - self.pseudo_dirs.append(path2) - else: - raise FileNotFoundError(path1) - - def cat_file(self, path, start=None, end=None, **kwargs): - logger.debug("cat: %s", path) - path = self._strip_protocol(path) - try: - return bytes(self.store[path].getbuffer()[start:end]) - except KeyError as e: - raise FileNotFoundError(path) from e - - def _rm(self, path): - path = self._strip_protocol(path) - try: - del self.store[path] - except KeyError as e: - raise FileNotFoundError(path) from e - - def modified(self, path): - path = self._strip_protocol(path) - try: - return self.store[path].modified - except KeyError as e: - raise FileNotFoundError(path) from e - - def created(self, path): - path = self._strip_protocol(path) - try: - return self.store[path].created - except KeyError as e: - raise FileNotFoundError(path) from e - - def isfile(self, path): - path = self._strip_protocol(path) - return path in self.store - - def rm(self, path, recursive=False, maxdepth=None): - if isinstance(path, str): - path = self._strip_protocol(path) - else: - path = [self._strip_protocol(p) for p in path] - paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) - for p in reversed(paths): - if self.isfile(p): - self.rm_file(p) - # If the expanded path doesn't exist, it is only because the expanded - # path was a directory that does not exist in self.pseudo_dirs. This - # is possible if you directly create files without making the - # directories first. - elif not self.exists(p): - continue - else: - self.rmdir(p) - - -class MemoryFile(BytesIO): - """A BytesIO which can't close and works as a context manager - - Can initialise with data. Each path should only be active once at any moment. - - No need to provide fs, path if auto-committing (default) - """ - - def __init__(self, fs=None, path=None, data=None): - logger.debug("open file %s", path) - self.fs = fs - self.path = path - self.created = datetime.now(tz=timezone.utc) - self.modified = datetime.now(tz=timezone.utc) - if data: - super().__init__(data) - self.seek(0) - - @property - def size(self): - return self.getbuffer().nbytes - - def __enter__(self): - return self - - def close(self): - pass - - def discard(self): - pass - - def commit(self): - self.fs.store[self.path] = self - self.modified = datetime.now(tz=timezone.utc) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/reference.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/reference.py deleted file mode 100644 index 1b5e1b1b6bc938f1cbc1c595e761e4d062aea31d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/reference.py +++ /dev/null @@ -1,1339 +0,0 @@ -import base64 -import collections -import io -import itertools -import logging -import math -import os -from functools import lru_cache -from itertools import chain -from typing import TYPE_CHECKING, Literal - -import fsspec.core -from fsspec.spec import AbstractBufferedFile - -try: - import ujson as json -except ImportError: - if not TYPE_CHECKING: - import json - -from fsspec.asyn import AsyncFileSystem -from fsspec.callbacks import DEFAULT_CALLBACK -from fsspec.core import filesystem, open, split_protocol -from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper -from fsspec.utils import ( - isfilelike, - merge_offset_ranges, - other_paths, -) - -logger = logging.getLogger("fsspec.reference") - - -class ReferenceNotReachable(RuntimeError): - def __init__(self, reference, target, *args): - super().__init__(*args) - self.reference = reference - self.target = target - - def __str__(self): - return f'Reference "{self.reference}" failed to fetch target {self.target}' - - -def _first(d): - return next(iter(d.values())) - - -def _prot_in_references(path, references): - ref = references.get(path) - if isinstance(ref, (list, tuple)) and isinstance(ref[0], str): - return split_protocol(ref[0])[0] if ref[0] else ref[0] - - -def _protocol_groups(paths, references): - if isinstance(paths, str): - return {_prot_in_references(paths, references): [paths]} - out = {} - for path in paths: - protocol = _prot_in_references(path, references) - out.setdefault(protocol, []).append(path) - return out - - -class RefsValuesView(collections.abc.ValuesView): - def __iter__(self): - for val in self._mapping.zmetadata.values(): - yield json.dumps(val).encode() - yield from self._mapping._items.values() - for field in self._mapping.listdir(): - chunk_sizes = self._mapping._get_chunk_sizes(field) - if len(chunk_sizes) == 0: - yield self._mapping[field + "/0"] - continue - yield from self._mapping._generate_all_records(field) - - -class RefsItemsView(collections.abc.ItemsView): - def __iter__(self): - return zip(self._mapping.keys(), self._mapping.values()) - - -def ravel_multi_index(idx, sizes): - val = 0 - mult = 1 - for i, s in zip(idx[::-1], sizes[::-1]): - val += i * mult - mult *= s - return val - - -class LazyReferenceMapper(collections.abc.MutableMapping): - """This interface can be used to read/write references from Parquet stores. - It is not intended for other types of references. - It can be used with Kerchunk's MultiZarrToZarr method to combine - references into a parquet store. - Examples of this use-case can be found here: - https://fsspec.github.io/kerchunk/advanced.html?highlight=parquet#parquet-storage""" - - # import is class level to prevent numpy dep requirement for fsspec - @property - def np(self): - import numpy as np - - return np - - @property - def pd(self): - import pandas as pd - - return pd - - def __init__( - self, - root, - fs=None, - out_root=None, - cache_size=128, - categorical_threshold=10, - engine: Literal["fastparquet", "pyarrow"] = "fastparquet", - ): - """ - - This instance will be writable, storing changes in memory until full partitions - are accumulated or .flush() is called. - - To create an empty lazy store, use .create() - - Parameters - ---------- - root : str - Root of parquet store - fs : fsspec.AbstractFileSystem - fsspec filesystem object, default is local filesystem. - cache_size : int, default=128 - Maximum size of LRU cache, where cache_size*record_size denotes - the total number of references that can be loaded in memory at once. - categorical_threshold : int - Encode urls as pandas.Categorical to reduce memory footprint if the ratio - of the number of unique urls to total number of refs for each variable - is greater than or equal to this number. (default 10) - engine: Literal["fastparquet","pyarrow"] - Engine choice for reading parquet files. (default is "fastparquet") - """ - - self.root = root - self.chunk_sizes = {} - self.cat_thresh = categorical_threshold - self.engine = engine - self.cache_size = cache_size - self.url = self.root + "/{field}/refs.{record}.parq" - # TODO: derive fs from `root` - self.fs = fsspec.filesystem("file") if fs is None else fs - self.out_root = self.fs.unstrip_protocol(out_root or self.root) - - from importlib.util import find_spec - - if self.engine == "pyarrow" and find_spec("pyarrow") is None: - raise ImportError("engine choice `pyarrow` is not installed.") - - # Apply `lru_cache` decorator manually per instance. - # This way `self` reference is not held on class level. - # WARNING: However, this means that self and its members are not reflected - # in the cache key, so we expect they won't be mutated once a value is cached. - self.listdir = lru_cache()(self.listdir) - self._key_to_record = lru_cache(maxsize=4096)(self._key_to_record) - - def __getattr__(self, item): - if item in ("_items", "record_size", "zmetadata"): - self.setup() - # avoid possible recursion if setup fails somehow - return self.__dict__[item] - raise AttributeError(item) - - def setup(self): - self._items = {} - self._items[".zmetadata"] = self.fs.cat_file( - "/".join([self.root, ".zmetadata"]) - ) - met = json.loads(self._items[".zmetadata"]) - self.record_size = met["record_size"] - self.zmetadata = met["metadata"] - - # Define function to open and decompress refs - @lru_cache(maxsize=self.cache_size) - def open_refs(field, record): - """cached parquet file loader""" - path = self.url.format(field=field, record=record) - data = io.BytesIO(self.fs.cat_file(path)) - try: - df = self.pd.read_parquet(data, engine=self.engine) - refs = {c: df[c].to_numpy(copy=True) for c in df.columns} - except OSError: - refs = None - return refs - - self.open_refs = open_refs - - @staticmethod - def create(root, storage_options=None, fs=None, record_size=10000, **kwargs): - """Make empty parquet reference set - - First deletes the contents of the given directory, if it exists. - - Parameters - ---------- - root: str - Directory to contain the output; will be created - storage_options: dict | None - For making the filesystem to use for writing is fs is None - fs: FileSystem | None - Filesystem for writing - record_size: int - Number of references per parquet file - kwargs: passed to __init__ - - Returns - ------- - LazyReferenceMapper instance - """ - met = {"metadata": {}, "record_size": record_size} - if fs is None: - fs, root = fsspec.core.url_to_fs(root, **(storage_options or {})) - if fs.exists(root): - fs.rm(root, recursive=True) - fs.makedirs(root, exist_ok=True) - fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode()) - return LazyReferenceMapper(root, fs, **kwargs) - - def listdir(self): - """List top-level directories""" - dirs = (p.rsplit("/", 1)[0] for p in self.zmetadata if not p.startswith(".z")) - return set(dirs) - - def ls(self, path="", detail=True): - """Shortcut file listings""" - path = path.rstrip("/") - pathdash = path + "/" if path else "" - dirnames = self.listdir() - dirs = [ - d - for d in dirnames - if d.startswith(pathdash) and "/" not in d.lstrip(pathdash) - ] - if dirs: - others = { - f - for f in chain( - [".zmetadata"], - (name for name in self.zmetadata), - (name for name in self._items), - ) - if f.startswith(pathdash) and "/" not in f.lstrip(pathdash) - } - if detail is False: - others.update(dirs) - return sorted(others) - dirinfo = [{"name": name, "type": "directory", "size": 0} for name in dirs] - fileinfo = [ - { - "name": name, - "type": "file", - "size": len( - json.dumps(self.zmetadata[name]) - if name in self.zmetadata - else self._items[name] - ), - } - for name in others - ] - return sorted(dirinfo + fileinfo, key=lambda s: s["name"]) - field = path - others = set( - [name for name in self.zmetadata if name.startswith(f"{path}/")] - + [name for name in self._items if name.startswith(f"{path}/")] - ) - fileinfo = [ - { - "name": name, - "type": "file", - "size": len( - json.dumps(self.zmetadata[name]) - if name in self.zmetadata - else self._items[name] - ), - } - for name in others - ] - keys = self._keys_in_field(field) - - if detail is False: - return list(others) + list(keys) - recs = self._generate_all_records(field) - recinfo = [ - {"name": name, "type": "file", "size": rec[-1]} - for name, rec in zip(keys, recs) - if rec[0] # filters out path==None, deleted/missing - ] - return fileinfo + recinfo - - def _load_one_key(self, key): - """Get the reference for one key - - Returns bytes, one-element list or three-element list. - """ - if key in self._items: - return self._items[key] - elif key in self.zmetadata: - return json.dumps(self.zmetadata[key]).encode() - elif "/" not in key or self._is_meta(key): - raise KeyError(key) - field, _ = key.rsplit("/", 1) - record, ri, chunk_size = self._key_to_record(key) - maybe = self._items.get((field, record), {}).get(ri, False) - if maybe is None: - # explicitly deleted - raise KeyError - elif maybe: - return maybe - elif chunk_size == 0: - return b"" - - # Chunk keys can be loaded from row group and cached in LRU cache - try: - refs = self.open_refs(field, record) - except (ValueError, TypeError, FileNotFoundError) as exc: - raise KeyError(key) from exc - columns = ["path", "offset", "size", "raw"] - selection = [refs[c][ri] if c in refs else None for c in columns] - raw = selection[-1] - if raw is not None: - return raw - if selection[0] is None: - raise KeyError("This reference does not exist or has been deleted") - if selection[1:3] == [0, 0]: - # URL only - return selection[:1] - # URL, offset, size - return selection[:3] - - def _key_to_record(self, key): - """Details needed to construct a reference for one key""" - field, chunk = key.rsplit("/", 1) - chunk_sizes = self._get_chunk_sizes(field) - if len(chunk_sizes) == 0: - return 0, 0, 0 - chunk_idx = [int(c) for c in chunk.split(".")] - chunk_number = ravel_multi_index(chunk_idx, chunk_sizes) - record = chunk_number // self.record_size - ri = chunk_number % self.record_size - return record, ri, len(chunk_sizes) - - def _get_chunk_sizes(self, field): - """The number of chunks along each axis for a given field""" - if field not in self.chunk_sizes: - zarray = self.zmetadata[f"{field}/.zarray"] - size_ratio = [ - math.ceil(s / c) for s, c in zip(zarray["shape"], zarray["chunks"]) - ] - self.chunk_sizes[field] = size_ratio or [1] - return self.chunk_sizes[field] - - def _generate_record(self, field, record): - """The references for a given parquet file of a given field""" - refs = self.open_refs(field, record) - it = iter(zip(*refs.values())) - if len(refs) == 3: - # All urls - return (list(t) for t in it) - elif len(refs) == 1: - # All raws - return refs["raw"] - else: - # Mix of urls and raws - return (list(t[:3]) if not t[3] else t[3] for t in it) - - def _generate_all_records(self, field): - """Load all the references within a field by iterating over the parquet files""" - nrec = 1 - for ch in self._get_chunk_sizes(field): - nrec *= ch - nrec = math.ceil(nrec / self.record_size) - for record in range(nrec): - yield from self._generate_record(field, record) - - def values(self): - return RefsValuesView(self) - - def items(self): - return RefsItemsView(self) - - def __hash__(self): - return id(self) - - def __getitem__(self, key): - return self._load_one_key(key) - - def __setitem__(self, key, value): - if "/" in key and not self._is_meta(key): - field, chunk = key.rsplit("/", 1) - record, i, _ = self._key_to_record(key) - subdict = self._items.setdefault((field, record), {}) - subdict[i] = value - if len(subdict) == self.record_size: - self.write(field, record) - else: - # metadata or top-level - if hasattr(value, "to_bytes"): - val = value.to_bytes().decode() - elif isinstance(value, bytes): - val = value.decode() - else: - val = value - self._items[key] = val - new_value = json.loads(val) - self.zmetadata[key] = {**self.zmetadata.get(key, {}), **new_value} - - @staticmethod - def _is_meta(key): - return key.startswith(".z") or "/.z" in key - - def __delitem__(self, key): - if key in self._items: - del self._items[key] - elif key in self.zmetadata: - del self.zmetadata[key] - else: - if "/" in key and not self._is_meta(key): - field, _ = key.rsplit("/", 1) - record, i, _ = self._key_to_record(key) - subdict = self._items.setdefault((field, record), {}) - subdict[i] = None - if len(subdict) == self.record_size: - self.write(field, record) - else: - # metadata or top-level - self._items[key] = None - - def write(self, field, record, base_url=None, storage_options=None): - # extra requirements if writing - import kerchunk.df - import numpy as np - import pandas as pd - - partition = self._items[(field, record)] - original = False - if len(partition) < self.record_size: - try: - original = self.open_refs(field, record) - except OSError: - pass - - if original: - paths = original["path"] - offsets = original["offset"] - sizes = original["size"] - raws = original["raw"] - else: - paths = np.full(self.record_size, np.nan, dtype="O") - offsets = np.zeros(self.record_size, dtype="int64") - sizes = np.zeros(self.record_size, dtype="int64") - raws = np.full(self.record_size, np.nan, dtype="O") - for j, data in partition.items(): - if isinstance(data, list): - if ( - str(paths.dtype) == "category" - and data[0] not in paths.dtype.categories - ): - paths = paths.add_categories(data[0]) - paths[j] = data[0] - if len(data) > 1: - offsets[j] = data[1] - sizes[j] = data[2] - elif data is None: - # delete - paths[j] = None - offsets[j] = 0 - sizes[j] = 0 - raws[j] = None - else: - # this is the only call into kerchunk, could remove - raws[j] = kerchunk.df._proc_raw(data) - # TODO: only save needed columns - df = pd.DataFrame( - { - "path": paths, - "offset": offsets, - "size": sizes, - "raw": raws, - }, - copy=False, - ) - if df.path.count() / (df.path.nunique() or 1) > self.cat_thresh: - df["path"] = df["path"].astype("category") - object_encoding = {"raw": "bytes", "path": "utf8"} - has_nulls = ["path", "raw"] - - fn = f"{base_url or self.out_root}/{field}/refs.{record}.parq" - self.fs.mkdirs(f"{base_url or self.out_root}/{field}", exist_ok=True) - - if self.engine == "pyarrow": - df_backend_kwargs = {"write_statistics": False} - elif self.engine == "fastparquet": - df_backend_kwargs = { - "stats": False, - "object_encoding": object_encoding, - "has_nulls": has_nulls, - } - else: - raise NotImplementedError(f"{self.engine} not supported") - df.to_parquet( - fn, - engine=self.engine, - storage_options=storage_options - or getattr(self.fs, "storage_options", None), - compression="zstd", - index=False, - **df_backend_kwargs, - ) - - partition.clear() - self._items.pop((field, record)) - - def flush(self, base_url=None, storage_options=None): - """Output any modified or deleted keys - - Parameters - ---------- - base_url: str - Location of the output - """ - - # write what we have so far and clear sub chunks - for thing in list(self._items): - if isinstance(thing, tuple): - field, record = thing - self.write( - field, - record, - base_url=base_url, - storage_options=storage_options, - ) - - # gather .zmetadata from self._items and write that too - for k in list(self._items): - if k != ".zmetadata" and ".z" in k: - self.zmetadata[k] = json.loads(self._items.pop(k)) - met = {"metadata": self.zmetadata, "record_size": self.record_size} - self._items.clear() - self._items[".zmetadata"] = json.dumps(met).encode() - self.fs.pipe( - "/".join([base_url or self.out_root, ".zmetadata"]), - self._items[".zmetadata"], - ) - - # TODO: only clear those that we wrote to? - self.open_refs.cache_clear() - - def __len__(self): - # Caveat: This counts expected references, not actual - but is fast - count = 0 - for field in self.listdir(): - if field.startswith("."): - count += 1 - else: - count += math.prod(self._get_chunk_sizes(field)) - count += len(self.zmetadata) # all metadata keys - # any other files not in reference partitions - count += sum(1 for _ in self._items if not isinstance(_, tuple)) - return count - - def __iter__(self): - # Caveat: returns only existing keys, so the number of these does not - # match len(self) - metas = set(self.zmetadata) - metas.update(self._items) - for bit in metas: - if isinstance(bit, str): - yield bit - for field in self.listdir(): - for k in self._keys_in_field(field): - if k in self: - yield k - - def __contains__(self, item): - try: - self._load_one_key(item) - return True - except KeyError: - return False - - def _keys_in_field(self, field): - """List key names in given field - - Produces strings like "field/x.y" appropriate from the chunking of the array - """ - chunk_sizes = self._get_chunk_sizes(field) - if len(chunk_sizes) == 0: - yield field + "/0" - return - inds = itertools.product(*(range(i) for i in chunk_sizes)) - for ind in inds: - yield field + "/" + ".".join([str(c) for c in ind]) - - -class ReferenceFileSystem(AsyncFileSystem): - """View byte ranges of some other file as a file system - Initial version: single file system target, which must support - async, and must allow start and end args in _cat_file. Later versions - may allow multiple arbitrary URLs for the targets. - This FileSystem is read-only. It is designed to be used with async - targets (for now). We do not get original file details from the target FS. - Configuration is by passing a dict of references at init, or a URL to - a JSON file containing the same; this dict - can also contain concrete data for some set of paths. - Reference dict format: - {path0: bytes_data, path1: (target_url, offset, size)} - https://github.com/fsspec/kerchunk/blob/main/README.md - - simple_references: if True (default), no jinja interpreting is done, - which is the safe option. - """ - - protocol = "reference" - cachable = False - - def __init__( - self, - fo, - target=None, - ref_storage_args=None, - target_protocol=None, - target_options=None, - remote_protocol=None, - remote_options=None, - fs=None, - template_overrides=None, - simple_templates=True, - max_gap=64_000, - max_block=256_000_000, - cache_size=128, - **kwargs, - ): - """ - Parameters - ---------- - fo : dict or str - The set of references to use for this instance, with a structure as above. - If str referencing a JSON file, will use fsspec.open, in conjunction - with target_options and target_protocol to open and parse JSON at this - location. If a directory, then assume references are a set of parquet - files to be loaded lazily. - target : str - For any references having target_url as None, this is the default file - target to use - ref_storage_args : dict - If references is a str, use these kwargs for loading the JSON file. - Deprecated: use target_options instead. - target_protocol : str - Used for loading the reference file, if it is a path. If None, protocol - will be derived from the given path - target_options : dict - Extra FS options for loading the reference file ``fo``, if given as a path - remote_protocol : str - The protocol of the filesystem on which the references will be evaluated - (unless fs is provided). If not given, will be derived from the first - URL that has a protocol in the templates or in the references, in that - order. - remote_options : dict - kwargs to go with remote_protocol - fs : AbstractFileSystem | dict(str, (AbstractFileSystem | dict)) - Directly provide a file system(s): - - a single filesystem instance - - a dict of protocol:filesystem, where each value is either a filesystem - instance, or a dict of kwargs that can be used to create in - instance for the given protocol - - If this is given, remote_options and remote_protocol are ignored. - template_overrides : dict - Swap out any templates in the references file with these - useful for - testing. - simple_templates: bool - Whether templates can be processed with simple replace (True) or if - jinja is needed (False, much slower). All reference sets produced by - ``kerchunk`` are simple in this sense, but the spec allows for complex. - max_gap, max_block: int - For merging multiple concurrent requests to the same remote file. - Neighboring byte ranges will only be merged when their - inter-range gap is <= ``max_gap``. Default is 64KB. Set to 0 - to only merge when it requires no extra bytes. Pass a negative - number to disable merging, appropriate for local target files. - Neighboring byte ranges will only be merged when the size of - the aggregated range is <= ``max_block``. Default is 256MB. - cache_size : int - Maximum size of LRU cache, where cache_size*record_size denotes - the total number of references that can be loaded in memory at once. - Only used for lazily loaded references. - kwargs : passed to parent class - """ - super().__init__(**kwargs) - self.target = target - self.template_overrides = template_overrides - self.simple_templates = simple_templates - self.templates = {} - self.fss = {} - self._dircache = {} - self.max_gap = max_gap - self.max_block = max_block - if isinstance(fo, str): - dic = dict( - **(ref_storage_args or target_options or {}), protocol=target_protocol - ) - ref_fs, fo2 = fsspec.core.url_to_fs(fo, **dic) - if ".json" not in fo2 and ( - fo.endswith(("parq", "parquet", "/")) or ref_fs.isdir(fo2) - ): - # Lazy parquet refs - logger.info("Open lazy reference dict from URL %s", fo) - self.references = LazyReferenceMapper( - fo2, - fs=ref_fs, - cache_size=cache_size, - ) - else: - # text JSON - with fsspec.open(fo, "rb", **dic) as f: - logger.info("Read reference from URL %s", fo) - text = json.load(f) - self._process_references(text, template_overrides) - else: - # dictionaries - self._process_references(fo, template_overrides) - if isinstance(fs, dict): - self.fss = { - k: ( - fsspec.filesystem(k.split(":", 1)[0], **opts) - if isinstance(opts, dict) - else opts - ) - for k, opts in fs.items() - } - if None not in self.fss: - self.fss[None] = filesystem("file") - return - if fs is not None: - # single remote FS - remote_protocol = ( - fs.protocol[0] if isinstance(fs.protocol, tuple) else fs.protocol - ) - self.fss[remote_protocol] = fs - - if remote_protocol is None: - # get single protocol from any templates - for ref in self.templates.values(): - if callable(ref): - ref = ref() - protocol, _ = fsspec.core.split_protocol(ref) - if protocol and protocol not in self.fss: - fs = filesystem(protocol, **(remote_options or {})) - self.fss[protocol] = fs - if remote_protocol is None: - # get single protocol from references - # TODO: warning here, since this can be very expensive? - for ref in self.references.values(): - if callable(ref): - ref = ref() - if isinstance(ref, list) and ref[0]: - protocol, _ = fsspec.core.split_protocol(ref[0]) - if protocol not in self.fss: - fs = filesystem(protocol, **(remote_options or {})) - self.fss[protocol] = fs - # only use first remote URL - break - - if remote_protocol and remote_protocol not in self.fss: - fs = filesystem(remote_protocol, **(remote_options or {})) - self.fss[remote_protocol] = fs - - self.fss[None] = fs or filesystem("file") # default one - # Wrap any non-async filesystems to ensure async methods are available below - for k, f in self.fss.items(): - if not f.async_impl: - self.fss[k] = AsyncFileSystemWrapper(f, asynchronous=self.asynchronous) - elif self.asynchronous ^ f.asynchronous: - raise ValueError( - "Reference-FS's target filesystem must have same value " - "of asynchronous" - ) - - def _cat_common(self, path, start=None, end=None): - path = self._strip_protocol(path) - logger.debug(f"cat: {path}") - try: - part = self.references[path] - except KeyError as exc: - raise FileNotFoundError(path) from exc - if isinstance(part, str): - part = part.encode() - if hasattr(part, "to_bytes"): - part = part.to_bytes() - if isinstance(part, bytes): - logger.debug(f"Reference: {path}, type bytes") - if part.startswith(b"base64:"): - part = base64.b64decode(part[7:]) - return part, None, None - - if len(part) == 1: - logger.debug(f"Reference: {path}, whole file => {part}") - url = part[0] - start1, end1 = start, end - else: - url, start0, size = part - logger.debug(f"Reference: {path} => {url}, offset {start0}, size {size}") - end0 = start0 + size - - if start is not None: - if start >= 0: - start1 = start0 + start - else: - start1 = end0 + start - else: - start1 = start0 - if end is not None: - if end >= 0: - end1 = start0 + end - else: - end1 = end0 + end - else: - end1 = end0 - if url is None: - url = self.target - return url, start1, end1 - - async def _cat_file(self, path, start=None, end=None, **kwargs): - part_or_url, start0, end0 = self._cat_common(path, start=start, end=end) - if isinstance(part_or_url, bytes): - return part_or_url[start:end] - protocol, _ = split_protocol(part_or_url) - try: - return await self.fss[protocol]._cat_file( - part_or_url, start=start0, end=end0 - ) - except Exception as e: - raise ReferenceNotReachable(path, part_or_url) from e - - def cat_file(self, path, start=None, end=None, **kwargs): - part_or_url, start0, end0 = self._cat_common(path, start=start, end=end) - if isinstance(part_or_url, bytes): - return part_or_url[start:end] - protocol, _ = split_protocol(part_or_url) - try: - return self.fss[protocol].cat_file(part_or_url, start=start0, end=end0) - except Exception as e: - raise ReferenceNotReachable(path, part_or_url) from e - - def pipe_file(self, path, value, **_): - """Temporarily add binary data or reference as a file""" - self.references[path] = value - - async def _get_file(self, rpath, lpath, **kwargs): - if self.isdir(rpath): - return os.makedirs(lpath, exist_ok=True) - data = await self._cat_file(rpath) - with open(lpath, "wb") as f: - f.write(data) - - def get_file(self, rpath, lpath, callback=DEFAULT_CALLBACK, **kwargs): - if self.isdir(rpath): - return os.makedirs(lpath, exist_ok=True) - data = self.cat_file(rpath, **kwargs) - callback.set_size(len(data)) - if isfilelike(lpath): - lpath.write(data) - else: - with open(lpath, "wb") as f: - f.write(data) - callback.absolute_update(len(data)) - - def get(self, rpath, lpath, recursive=False, **kwargs): - if recursive: - # trigger directory build - self.ls("") - rpath = self.expand_path(rpath, recursive=recursive) - fs = fsspec.filesystem("file", auto_mkdir=True) - targets = other_paths(rpath, lpath) - if recursive: - data = self.cat([r for r in rpath if not self.isdir(r)]) - else: - data = self.cat(rpath) - for remote, local in zip(rpath, targets): - if remote in data: - fs.pipe_file(local, data[remote]) - - def cat(self, path, recursive=False, on_error="raise", **kwargs): - if isinstance(path, str) and recursive: - raise NotImplementedError - if isinstance(path, list) and (recursive or any("*" in p for p in path)): - raise NotImplementedError - # TODO: if references is lazy, pre-fetch all paths in batch before access - proto_dict = _protocol_groups(path, self.references) - out = {} - for proto, paths in proto_dict.items(): - fs = self.fss[proto] - urls, starts, ends, valid_paths = [], [], [], [] - for p in paths: - # find references or label not-found. Early exit if any not - # found and on_error is "raise" - try: - u, s, e = self._cat_common(p) - if not isinstance(u, (bytes, str)): - # nan/None from parquet - continue - except FileNotFoundError as err: - if on_error == "raise": - raise - if on_error != "omit": - out[p] = err - else: - urls.append(u) - starts.append(s) - ends.append(e) - valid_paths.append(p) - - # process references into form for merging - urls2 = [] - starts2 = [] - ends2 = [] - paths2 = [] - whole_files = set() - for u, s, e, p in zip(urls, starts, ends, valid_paths): - if isinstance(u, bytes): - # data - out[p] = u - elif s is None: - # whole file - limits are None, None, but no further - # entries take for this file - whole_files.add(u) - urls2.append(u) - starts2.append(s) - ends2.append(e) - paths2.append(p) - for u, s, e, p in zip(urls, starts, ends, valid_paths): - # second run to account for files that are to be loaded whole - if s is not None and u not in whole_files: - urls2.append(u) - starts2.append(s) - ends2.append(e) - paths2.append(p) - - # merge and fetch consolidated ranges - new_paths, new_starts, new_ends = merge_offset_ranges( - list(urls2), - list(starts2), - list(ends2), - sort=True, - max_gap=self.max_gap, - max_block=self.max_block, - ) - bytes_out = fs.cat_ranges(new_paths, new_starts, new_ends) - - # unbundle from merged bytes - simple approach - for u, s, e, p in zip(urls, starts, ends, valid_paths): - if p in out: - continue # was bytes, already handled - for np, ns, ne, b in zip(new_paths, new_starts, new_ends, bytes_out): - if np == u and (ns is None or ne is None): - if isinstance(b, Exception): - out[p] = b - else: - out[p] = b[s:e] - elif np == u and s >= ns and e <= ne: - if isinstance(b, Exception): - out[p] = b - else: - out[p] = b[s - ns : (e - ne) or None] - - for k, v in out.copy().items(): - # these were valid references, but fetch failed, so transform exc - if isinstance(v, Exception) and k in self.references: - ex = out[k] - new_ex = ReferenceNotReachable(k, self.references[k]) - new_ex.__cause__ = ex - if on_error == "raise": - raise new_ex - elif on_error != "omit": - out[k] = new_ex - - if len(out) == 1 and isinstance(path, str) and "*" not in path: - return _first(out) - return out - - def _process_references(self, references, template_overrides=None): - vers = references.get("version", None) - if vers is None: - self._process_references0(references) - elif vers == 1: - self._process_references1(references, template_overrides=template_overrides) - else: - raise ValueError(f"Unknown reference spec version: {vers}") - # TODO: we make dircache by iterating over all entries, but for Spec >= 1, - # can replace with programmatic. Is it even needed for mapper interface? - - def _process_references0(self, references): - """Make reference dict for Spec Version 0""" - if isinstance(references, dict): - # do not do this for lazy/parquet backend, which will not make dicts, - # but must remain writable in the original object - references = { - key: json.dumps(val) if isinstance(val, dict) else val - for key, val in references.items() - } - self.references = references - - def _process_references1(self, references, template_overrides=None): - if not self.simple_templates or self.templates: - import jinja2.sandbox - self.references = {} - self._process_templates(references.get("templates", {})) - - @lru_cache(1000) - def _render_jinja(u): - return ( - jinja2.sandbox.SandboxedEnvironment() - .from_string(u) - .render(**self.templates) - ) - - for k, v in references.get("refs", {}).items(): - if isinstance(v, str): - if v.startswith("base64:"): - self.references[k] = base64.b64decode(v[7:]) - self.references[k] = v - elif isinstance(v, dict): - self.references[k] = json.dumps(v) - elif self.templates: - u = v[0] - if "{{" in u: - if self.simple_templates: - u = ( - u.replace("{{", "{") - .replace("}}", "}") - .format(**self.templates) - ) - else: - u = _render_jinja(u) - self.references[k] = [u] if len(v) == 1 else [u, v[1], v[2]] - else: - self.references[k] = v - self.references.update(self._process_gen(references.get("gen", []))) - - def _process_templates(self, tmp): - self.templates = {} - if self.template_overrides is not None: - tmp.update(self.template_overrides) - for k, v in tmp.items(): - if "{{" in v: - import jinja2.sandbox - - self.templates[k] = ( - lambda temp=v, **kwargs: jinja2.sandbox.SandboxedEnvironment() - .from_string(temp) - .render(**kwargs) - ) - else: - self.templates[k] = v - - def _process_gen(self, gens): - out = {} - if self.simple_templates: - return out - for gen in gens: - dimension = { - k: ( - v - if isinstance(v, list) - else range(v.get("start", 0), v["stop"], v.get("step", 1)) - ) - for k, v in gen["dimensions"].items() - } - products = ( - dict(zip(dimension.keys(), values)) - for values in itertools.product(*dimension.values()) - ) - for pr in products: - import jinja2.sandbox - - key = ( - jinja2.sandbox.SandboxedEnvironment() - .from_string(gen["key"]) - .render(**pr, **self.templates) - ) - url = ( - jinja2.sandbox.SandboxedEnvironment() - .from_string(gen["url"]) - .render(**pr, **self.templates) - ) - if ("offset" in gen) and ("length" in gen): - offset = int( - jinja2.sandbox.SandboxedEnvironment() - .from_string(gen["offset"]) - .render(**pr, **self.templates) - ) - length = int( - jinja2.sandbox.SandboxedEnvironment() - .from_string(gen["length"]) - .render(**pr, **self.templates) - ) - out[key] = [url, offset, length] - elif ("offset" in gen) ^ ("length" in gen): - raise ValueError( - "Both 'offset' and 'length' are required for a " - "reference generator entry if either is provided." - ) - else: - out[key] = [url] - return out - - def _dircache_from_items(self): - self.dircache = {"": []} - it = self.references.items() - for path, part in it: - if isinstance(part, (bytes, str)) or hasattr(part, "to_bytes"): - size = len(part) - elif len(part) == 1: - size = None - else: - _, _, size = part - par = path.rsplit("/", 1)[0] if "/" in path else "" - par0 = par - subdirs = [par0] - while par0 and par0 not in self.dircache: - # collect parent directories - par0 = self._parent(par0) - subdirs.append(par0) - - subdirs.reverse() - for parent, child in zip(subdirs, subdirs[1:]): - # register newly discovered directories - assert child not in self.dircache - assert parent in self.dircache - self.dircache[parent].append( - {"name": child, "type": "directory", "size": 0} - ) - self.dircache[child] = [] - - self.dircache[par].append({"name": path, "type": "file", "size": size}) - - def _open(self, path, mode="rb", block_size=None, cache_options=None, **kwargs): - part_or_url, start0, end0 = self._cat_common(path) - # This logic is kept outside `ReferenceFile` to avoid unnecessary redirection. - # That does mean `_cat_common` gets called twice if it eventually reaches `ReferenceFile`. - if isinstance(part_or_url, bytes): - return io.BytesIO(part_or_url[start0:end0]) - - protocol, _ = split_protocol(part_or_url) - if start0 is None and end0 is None: - return self.fss[protocol]._open( - part_or_url, - mode, - block_size=block_size, - cache_options=cache_options, - **kwargs, - ) - - return ReferenceFile( - self, - path, - mode, - block_size=block_size, - cache_options=cache_options, - **kwargs, - ) - - def ls(self, path, detail=True, **kwargs): - logger.debug("list %s", path) - path = self._strip_protocol(path) - if isinstance(self.references, LazyReferenceMapper): - try: - return self.references.ls(path, detail) - except KeyError: - pass - raise FileNotFoundError(f"'{path}' is not a known key") - if not self.dircache: - self._dircache_from_items() - out = self._ls_from_cache(path) - if out is None: - raise FileNotFoundError(path) - if detail: - return out - return [o["name"] for o in out] - - def exists(self, path, **kwargs): # overwrite auto-sync version - return self.isdir(path) or self.isfile(path) - - def isdir(self, path): # overwrite auto-sync version - if self.dircache: - return path in self.dircache - elif isinstance(self.references, LazyReferenceMapper): - return path in self.references.listdir() - else: - # this may be faster than building dircache for single calls, but - # by looping will be slow for many calls; could cache it? - return any(_.startswith(f"{path}/") for _ in self.references) - - def isfile(self, path): # overwrite auto-sync version - return path in self.references - - async def _ls(self, path, detail=True, **kwargs): # calls fast sync code - return self.ls(path, detail, **kwargs) - - def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): - if withdirs: - return super().find( - path, maxdepth=maxdepth, withdirs=withdirs, detail=detail, **kwargs - ) - if path: - path = self._strip_protocol(path) - r = sorted(k for k in self.references if k.startswith(path)) - else: - r = sorted(self.references) - if detail: - if not self.dircache: - self._dircache_from_items() - return {k: self._ls_from_cache(k)[0] for k in r} - else: - return r - - def info(self, path, **kwargs): - out = self.references.get(path) - if out is not None: - if isinstance(out, (str, bytes)): - # decode base64 here - return {"name": path, "type": "file", "size": len(out)} - elif len(out) > 1: - return {"name": path, "type": "file", "size": out[2]} - else: - out0 = [{"name": path, "type": "file", "size": None}] - else: - out = self.ls(path, True) - out0 = [o for o in out if o["name"] == path] - if not out0: - return {"name": path, "type": "directory", "size": 0} - if out0[0]["size"] is None: - # if this is a whole remote file, update size using remote FS - prot, _ = split_protocol(self.references[path][0]) - out0[0]["size"] = self.fss[prot].size(self.references[path][0]) - return out0[0] - - async def _info(self, path, **kwargs): # calls fast sync code - return self.info(path) - - async def _rm_file(self, path, **kwargs): - self.references.pop( - path, None - ) # ignores FileNotFound, just as well for directories - self.dircache.clear() # this is a bit heavy handed - - async def _pipe_file(self, path, data, mode="overwrite", **kwargs): - if mode == "create" and self.exists(path): - raise FileExistsError - # can be str or bytes - self.references[path] = data - self.dircache.clear() # this is a bit heavy handed - - async def _put_file(self, lpath, rpath, mode="overwrite", **kwargs): - # puts binary - if mode == "create" and self.exists(rpath): - raise FileExistsError - with open(lpath, "rb") as f: - self.references[rpath] = f.read() - self.dircache.clear() # this is a bit heavy handed - - def save_json(self, url, **storage_options): - """Write modified references into new location""" - out = {} - for k, v in self.references.items(): - if isinstance(v, bytes): - try: - out[k] = v.decode("ascii") - except UnicodeDecodeError: - out[k] = (b"base64:" + base64.b64encode(v)).decode() - else: - out[k] = v - with fsspec.open(url, "wb", **storage_options) as f: - f.write(json.dumps({"version": 1, "refs": out}).encode()) - - -class ReferenceFile(AbstractBufferedFile): - def __init__( - self, - fs, - path, - mode="rb", - block_size="default", - autocommit=True, - cache_type="readahead", - cache_options=None, - size=None, - **kwargs, - ): - super().__init__( - fs, - path, - mode=mode, - block_size=block_size, - autocommit=autocommit, - size=size, - cache_type=cache_type, - cache_options=cache_options, - **kwargs, - ) - part_or_url, self.start, self.end = self.fs._cat_common(self.path) - protocol, _ = split_protocol(part_or_url) - self.src_fs = self.fs.fss[protocol] - self.src_path = part_or_url - self._f = None - - @property - def f(self): - if self._f is None or self._f.closed: - self._f = self.src_fs._open( - self.src_path, - mode=self.mode, - block_size=self.blocksize, - autocommit=self.autocommit, - cache_type="none", - **self.kwargs, - ) - return self._f - - def close(self): - if self._f is not None: - self._f.close() - return super().close() - - def _fetch_range(self, start, end): - start = start + self.start - end = min(end + self.start, self.end) - self.f.seek(start) - return self.f.read(end - start) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/sftp.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/sftp.py deleted file mode 100644 index 7c347963d692d50390b131225a56477b328f7a3c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/sftp.py +++ /dev/null @@ -1,187 +0,0 @@ -import datetime -import logging -import os -import types -import uuid -from stat import S_ISDIR, S_ISLNK - -import paramiko - -from .. import AbstractFileSystem -from ..utils import infer_storage_options - -logger = logging.getLogger("fsspec.sftp") - - -class SFTPFileSystem(AbstractFileSystem): - """Files over SFTP/SSH - - Peer-to-peer filesystem over SSH using paramiko. - - Note: if using this with the ``open`` or ``open_files``, with full URLs, - there is no way to tell if a path is relative, so all paths are assumed - to be absolute. - """ - - protocol = "sftp", "ssh" - - def __init__(self, host, **ssh_kwargs): - """ - - Parameters - ---------- - host: str - Hostname or IP as a string - temppath: str - Location on the server to put files, when within a transaction - ssh_kwargs: dict - Parameters passed on to connection. See details in - https://docs.paramiko.org/en/3.3/api/client.html#paramiko.client.SSHClient.connect - May include port, username, password... - """ - if self._cached: - return - super().__init__(**ssh_kwargs) - self.temppath = ssh_kwargs.pop("temppath", "/tmp") # remote temp directory - self.host = host - self.ssh_kwargs = ssh_kwargs - self._connect() - - def _connect(self): - logger.debug("Connecting to SFTP server %s", self.host) - self.client = paramiko.SSHClient() - self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - self.client.connect(self.host, **self.ssh_kwargs) - self.ftp = self.client.open_sftp() - - @classmethod - def _strip_protocol(cls, path): - return infer_storage_options(path)["path"] - - @staticmethod - def _get_kwargs_from_urls(urlpath): - out = infer_storage_options(urlpath) - out.pop("path", None) - out.pop("protocol", None) - return out - - def mkdir(self, path, create_parents=True, mode=511): - path = self._strip_protocol(path) - logger.debug("Creating folder %s", path) - if self.exists(path): - raise FileExistsError(f"File exists: {path}") - - if create_parents: - self.makedirs(path) - else: - self.ftp.mkdir(path, mode) - - def makedirs(self, path, exist_ok=False, mode=511): - if self.exists(path) and not exist_ok: - raise FileExistsError(f"File exists: {path}") - - parts = path.split("/") - new_path = "/" if path[:1] == "/" else "" - - for part in parts: - if part: - new_path = f"{new_path}/{part}" if new_path else part - if not self.exists(new_path): - self.ftp.mkdir(new_path, mode) - - def rmdir(self, path): - path = self._strip_protocol(path) - logger.debug("Removing folder %s", path) - self.ftp.rmdir(path) - - def info(self, path): - path = self._strip_protocol(path) - stat = self._decode_stat(self.ftp.stat(path)) - stat["name"] = path - return stat - - @staticmethod - def _decode_stat(stat, parent_path=None): - if S_ISDIR(stat.st_mode): - t = "directory" - elif S_ISLNK(stat.st_mode): - t = "link" - else: - t = "file" - out = { - "name": "", - "size": stat.st_size, - "type": t, - "uid": stat.st_uid, - "gid": stat.st_gid, - "time": datetime.datetime.fromtimestamp( - stat.st_atime, tz=datetime.timezone.utc - ), - "mtime": datetime.datetime.fromtimestamp( - stat.st_mtime, tz=datetime.timezone.utc - ), - } - if parent_path: - out["name"] = "/".join([parent_path.rstrip("/"), stat.filename]) - return out - - def ls(self, path, detail=False): - path = self._strip_protocol(path) - logger.debug("Listing folder %s", path) - stats = [self._decode_stat(stat, path) for stat in self.ftp.listdir_iter(path)] - if detail: - return stats - else: - paths = [stat["name"] for stat in stats] - return sorted(paths) - - def put_file(self, lpath, rpath, callback=None, **kwargs): - self.mkdirs(self._parent(os.fspath(rpath)), exist_ok=True) - logger.debug("Put file %s into %s", lpath, rpath) - self.ftp.put(lpath, rpath) - - def get_file(self, rpath, lpath, **kwargs): - if self.isdir(rpath): - os.makedirs(lpath, exist_ok=True) - else: - self.ftp.get(self._strip_protocol(rpath), lpath) - - def _open(self, path, mode="rb", block_size=None, **kwargs): - """ - block_size: int or None - If 0, no buffering, if 1, line buffering, if >1, buffer that many - bytes, if None use default from paramiko. - """ - logger.debug("Opening file %s", path) - if kwargs.get("autocommit", True) is False: - # writes to temporary file, move on commit - path2 = "/".join([self.temppath, str(uuid.uuid4())]) - f = self.ftp.open(path2, mode, bufsize=block_size if block_size else -1) - f.temppath = path2 - f.targetpath = path - f.fs = self - f.commit = types.MethodType(commit_a_file, f) - f.discard = types.MethodType(discard_a_file, f) - else: - f = self.ftp.open(path, mode, bufsize=block_size if block_size else -1) - return f - - def _rm(self, path): - if self.isdir(path): - self.ftp.rmdir(path) - else: - self.ftp.remove(path) - - def mv(self, old, new): - new = self._strip_protocol(new) - old = self._strip_protocol(old) - logger.debug("Renaming %s into %s", old, new) - self.ftp.posix_rename(old, new) - - -def commit_a_file(self): - self.fs.mv(self.temppath, self.targetpath) - - -def discard_a_file(self): - self.fs._rm(self.temppath) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/smb.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/smb.py deleted file mode 100644 index db6b3f5c3702de90cf121ccca49f3ca2b580df9f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/smb.py +++ /dev/null @@ -1,416 +0,0 @@ -""" -This module contains SMBFileSystem class responsible for handling access to -Windows Samba network shares by using package smbprotocol -""" - -import datetime -import re -import uuid -from stat import S_ISDIR, S_ISLNK - -import smbclient -import smbprotocol.exceptions - -from .. import AbstractFileSystem -from ..utils import infer_storage_options - -# ! pylint: disable=bad-continuation - - -class SMBFileSystem(AbstractFileSystem): - """Allow reading and writing to Windows and Samba network shares. - - When using `fsspec.open()` for getting a file-like object the URI - should be specified as this format: - ``smb://workgroup;user:password@server:port/share/folder/file.csv``. - - Example:: - - >>> import fsspec - >>> with fsspec.open( - ... 'smb://myuser:mypassword@myserver.com/' 'share/folder/file.csv' - ... ) as smbfile: - ... df = pd.read_csv(smbfile, sep='|', header=None) - - Note that you need to pass in a valid hostname or IP address for the host - component of the URL. Do not use the Windows/NetBIOS machine name for the - host component. - - The first component of the path in the URL points to the name of the shared - folder. Subsequent path components will point to the directory/folder/file. - - The URL components ``workgroup`` , ``user``, ``password`` and ``port`` may be - optional. - - .. note:: - - For working this source require `smbprotocol`_ to be installed, e.g.:: - - $ pip install smbprotocol - # or - # pip install smbprotocol[kerberos] - - .. _smbprotocol: https://github.com/jborean93/smbprotocol#requirements - - Note: if using this with the ``open`` or ``open_files``, with full URLs, - there is no way to tell if a path is relative, so all paths are assumed - to be absolute. - """ - - protocol = "smb" - - # pylint: disable=too-many-arguments - def __init__( - self, - host, - port=None, - username=None, - password=None, - timeout=60, - encrypt=None, - share_access=None, - register_session_retries=4, - register_session_retry_wait=1, - register_session_retry_factor=10, - auto_mkdir=False, - **kwargs, - ): - """ - You can use _get_kwargs_from_urls to get some kwargs from - a reasonable SMB url. - - Authentication will be anonymous or integrated if username/password are not - given. - - Parameters - ---------- - host: str - The remote server name/ip to connect to - port: int or None - Port to connect with. Usually 445, sometimes 139. - username: str or None - Username to connect with. Required if Kerberos auth is not being used. - password: str or None - User's password on the server, if using username - timeout: int - Connection timeout in seconds - encrypt: bool - Whether to force encryption or not, once this has been set to True - the session cannot be changed back to False. - share_access: str or None - Specifies the default access applied to file open operations - performed with this file system object. - This affects whether other processes can concurrently open a handle - to the same file. - - - None (the default): exclusively locks the file until closed. - - 'r': Allow other handles to be opened with read access. - - 'w': Allow other handles to be opened with write access. - - 'd': Allow other handles to be opened with delete access. - register_session_retries: int - Number of retries to register a session with the server. Retries are not performed - for authentication errors, as they are considered as invalid credentials and not network - issues. If set to negative value, no register attempts will be performed. - register_session_retry_wait: int - Time in seconds to wait between each retry. Number must be non-negative. - register_session_retry_factor: int - Base factor for the wait time between each retry. The wait time - is calculated using exponential function. For factor=1 all wait times - will be equal to `register_session_retry_wait`. For any number of retries, - the last wait time will be equal to `register_session_retry_wait` and for retries>1 - the first wait time will be equal to `register_session_retry_wait / factor`. - Number must be equal to or greater than 1. Optimal factor is 10. - auto_mkdir: bool - Whether, when opening a file, the directory containing it should - be created (if it doesn't already exist). This is assumed by pyarrow - and zarr-python code. - """ - super().__init__(**kwargs) - self.host = host - self.port = port - self.username = username - self.password = password - self.timeout = timeout - self.encrypt = encrypt - self.temppath = kwargs.pop("temppath", "") - self.share_access = share_access - self.register_session_retries = register_session_retries - if register_session_retry_wait < 0: - raise ValueError( - "register_session_retry_wait must be a non-negative integer" - ) - self.register_session_retry_wait = register_session_retry_wait - if register_session_retry_factor < 1: - raise ValueError( - "register_session_retry_factor must be a positive " - "integer equal to or greater than 1" - ) - self.register_session_retry_factor = register_session_retry_factor - self.auto_mkdir = auto_mkdir - self._connect() - - @property - def _port(self): - return 445 if self.port is None else self.port - - def _connect(self): - import time - - if self.register_session_retries <= -1: - return - - retried_errors = [] - - wait_time = self.register_session_retry_wait - n_waits = ( - self.register_session_retries - 1 - ) # -1 = No wait time after the last retry - factor = self.register_session_retry_factor - - # Generate wait times for each retry attempt. - # Wait times are calculated using exponential function. For factor=1 all wait times - # will be equal to `wait`. For any number of retries the last wait time will be - # equal to `wait` and for retries>2 the first wait time will be equal to `wait / factor`. - wait_times = iter( - factor ** (n / n_waits - 1) * wait_time for n in range(0, n_waits + 1) - ) - - for attempt in range(self.register_session_retries + 1): - try: - smbclient.register_session( - self.host, - username=self.username, - password=self.password, - port=self._port, - encrypt=self.encrypt, - connection_timeout=self.timeout, - ) - return - except ( - smbprotocol.exceptions.SMBAuthenticationError, - smbprotocol.exceptions.LogonFailure, - ): - # These exceptions should not be repeated, as they clearly indicate - # that the credentials are invalid and not a network issue. - raise - except ValueError as exc: - if re.findall(r"\[Errno -\d+]", str(exc)): - # This exception is raised by the smbprotocol.transport:Tcp.connect - # and originates from socket.gaierror (OSError). These exceptions might - # be raised due to network instability. We will retry to connect. - retried_errors.append(exc) - else: - # All another ValueError exceptions should be raised, as they are not - # related to network issues. - raise - except Exception as exc: - # Save the exception and retry to connect. This except might be dropped - # in the future, once all exceptions suited for retry are identified. - retried_errors.append(exc) - - if attempt < self.register_session_retries: - time.sleep(next(wait_times)) - - # Raise last exception to inform user about the connection issues. - # Note: Should we use ExceptionGroup to raise all exceptions? - raise retried_errors[-1] - - @classmethod - def _strip_protocol(cls, path): - return infer_storage_options(path)["path"] - - @staticmethod - def _get_kwargs_from_urls(path): - # smb://workgroup;user:password@host:port/share/folder/file.csv - out = infer_storage_options(path) - out.pop("path", None) - out.pop("protocol", None) - return out - - def mkdir(self, path, create_parents=True, **kwargs): - wpath = _as_unc_path(self.host, path) - if create_parents: - smbclient.makedirs(wpath, exist_ok=False, port=self._port, **kwargs) - else: - smbclient.mkdir(wpath, port=self._port, **kwargs) - - def makedirs(self, path, exist_ok=False): - if _share_has_path(path): - wpath = _as_unc_path(self.host, path) - smbclient.makedirs(wpath, exist_ok=exist_ok, port=self._port) - - def rmdir(self, path): - if _share_has_path(path): - wpath = _as_unc_path(self.host, path) - smbclient.rmdir(wpath, port=self._port) - - def info(self, path, **kwargs): - wpath = _as_unc_path(self.host, path) - stats = smbclient.stat(wpath, port=self._port, **kwargs) - if S_ISDIR(stats.st_mode): - stype = "directory" - elif S_ISLNK(stats.st_mode): - stype = "link" - else: - stype = "file" - res = { - "name": path + "/" if stype == "directory" else path, - "size": stats.st_size, - "type": stype, - "uid": stats.st_uid, - "gid": stats.st_gid, - "time": stats.st_atime, - "mtime": stats.st_mtime, - } - return res - - def created(self, path): - """Return the created timestamp of a file as a datetime.datetime""" - wpath = _as_unc_path(self.host, path) - stats = smbclient.stat(wpath, port=self._port) - return datetime.datetime.fromtimestamp(stats.st_ctime, tz=datetime.timezone.utc) - - def modified(self, path): - """Return the modified timestamp of a file as a datetime.datetime""" - wpath = _as_unc_path(self.host, path) - stats = smbclient.stat(wpath, port=self._port) - return datetime.datetime.fromtimestamp(stats.st_mtime, tz=datetime.timezone.utc) - - def ls(self, path, detail=True, **kwargs): - unc = _as_unc_path(self.host, path) - listed = smbclient.listdir(unc, port=self._port, **kwargs) - dirs = ["/".join([path.rstrip("/"), p]) for p in listed] - if detail: - dirs = [self.info(d) for d in dirs] - return dirs - - # pylint: disable=too-many-arguments - def _open( - self, - path, - mode="rb", - block_size=-1, - autocommit=True, - cache_options=None, - **kwargs, - ): - """ - block_size: int or None - If 0, no buffering, 1, line buffering, >1, buffer that many bytes - - Notes - ----- - By specifying 'share_access' in 'kwargs' it is possible to override the - default shared access setting applied in the constructor of this object. - """ - if self.auto_mkdir and "w" in mode: - self.makedirs(self._parent(path), exist_ok=True) - bls = block_size if block_size is not None and block_size >= 0 else -1 - wpath = _as_unc_path(self.host, path) - share_access = kwargs.pop("share_access", self.share_access) - if "w" in mode and autocommit is False: - temp = _as_temp_path(self.host, path, self.temppath) - return SMBFileOpener( - wpath, temp, mode, port=self._port, block_size=bls, **kwargs - ) - return smbclient.open_file( - wpath, - mode, - buffering=bls, - share_access=share_access, - port=self._port, - **kwargs, - ) - - def copy(self, path1, path2, **kwargs): - """Copy within two locations in the same filesystem""" - wpath1 = _as_unc_path(self.host, path1) - wpath2 = _as_unc_path(self.host, path2) - if self.auto_mkdir: - self.makedirs(self._parent(path2), exist_ok=True) - smbclient.copyfile(wpath1, wpath2, port=self._port, **kwargs) - - def _rm(self, path): - if _share_has_path(path): - wpath = _as_unc_path(self.host, path) - stats = smbclient.stat(wpath, port=self._port) - if S_ISDIR(stats.st_mode): - smbclient.rmdir(wpath, port=self._port) - else: - smbclient.remove(wpath, port=self._port) - - def mv(self, path1, path2, recursive=None, maxdepth=None, **kwargs): - wpath1 = _as_unc_path(self.host, path1) - wpath2 = _as_unc_path(self.host, path2) - smbclient.rename(wpath1, wpath2, port=self._port, **kwargs) - - -def _as_unc_path(host, path): - rpath = path.replace("/", "\\") - unc = f"\\\\{host}{rpath}" - return unc - - -def _as_temp_path(host, path, temppath): - share = path.split("/")[1] - temp_file = f"/{share}{temppath}/{uuid.uuid4()}" - unc = _as_unc_path(host, temp_file) - return unc - - -def _share_has_path(path): - parts = path.count("/") - if path.endswith("/"): - return parts > 2 - return parts > 1 - - -class SMBFileOpener: - """writes to remote temporary file, move on commit""" - - def __init__(self, path, temp, mode, port=445, block_size=-1, **kwargs): - self.path = path - self.temp = temp - self.mode = mode - self.block_size = block_size - self.kwargs = kwargs - self.smbfile = None - self._incontext = False - self.port = port - self._open() - - def _open(self): - if self.smbfile is None or self.smbfile.closed: - self.smbfile = smbclient.open_file( - self.temp, - self.mode, - port=self.port, - buffering=self.block_size, - **self.kwargs, - ) - - def commit(self): - """Move temp file to definitive on success.""" - # TODO: use transaction support in SMB protocol - smbclient.replace(self.temp, self.path, port=self.port) - - def discard(self): - """Remove the temp file on failure.""" - smbclient.remove(self.temp, port=self.port) - - def __fspath__(self): - return self.path - - def __iter__(self): - return self.smbfile.__iter__() - - def __getattr__(self, item): - return getattr(self.smbfile, item) - - def __enter__(self): - self._incontext = True - return self.smbfile.__enter__() - - def __exit__(self, exc_type, exc_value, traceback): - self._incontext = False - self.smbfile.__exit__(exc_type, exc_value, traceback) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/tar.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/tar.py deleted file mode 100644 index e3ecaba691ace8dd4109c06dd0ab8a3b02e34634..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/tar.py +++ /dev/null @@ -1,139 +0,0 @@ -import logging -import re -import tarfile - -import fsspec -from fsspec.archive import AbstractArchiveFileSystem -from fsspec.compression import compr -from fsspec.utils import infer_compression - -typemap = {b"0": "file", b"5": "directory"} - -logger = logging.getLogger("tar") - - -class TarFileSystem(AbstractArchiveFileSystem): - """Compressed Tar archives as a file-system (read-only) - - Supports the following formats: - tar.gz, tar.bz2, tar.xz - """ - - root_marker = "" - protocol = "tar" - cachable = False - - def __init__( - self, - fo="", - index_store=None, - target_options=None, - target_protocol=None, - compression=None, - **kwargs, - ): - super().__init__(**kwargs) - target_options = target_options or {} - - if isinstance(fo, str): - self.of = fsspec.open(fo, protocol=target_protocol, **target_options) - fo = self.of.open() # keep the reference - - # Try to infer compression. - if compression is None: - name = None - - # Try different ways to get hold of the filename. `fo` might either - # be a `fsspec.LocalFileOpener`, an `io.BufferedReader` or an - # `fsspec.AbstractFileSystem` instance. - try: - # Amended io.BufferedReader or similar. - # This uses a "protocol extension" where original filenames are - # propagated to archive-like filesystems in order to let them - # infer the right compression appropriately. - if hasattr(fo, "original"): - name = fo.original - - # fsspec.LocalFileOpener - elif hasattr(fo, "path"): - name = fo.path - - # io.BufferedReader - elif hasattr(fo, "name"): - name = fo.name - - # fsspec.AbstractFileSystem - elif hasattr(fo, "info"): - name = fo.info()["name"] - - except Exception as ex: - logger.warning( - f"Unable to determine file name, not inferring compression: {ex}" - ) - - if name is not None: - compression = infer_compression(name) - logger.info(f"Inferred compression {compression} from file name {name}") - - if compression is not None: - # TODO: tarfile already implements compression with modes like "'r:gz'", - # but then would seek to offset in the file work? - fo = compr[compression](fo) - - self._fo_ref = fo - self.fo = fo # the whole instance is a context - self.tar = tarfile.TarFile(fileobj=self.fo) - self.dir_cache = None - - self.index_store = index_store - self.index = None - self._index() - - def _index(self): - # TODO: load and set saved index, if exists - out = {} - for ti in self.tar: - info = ti.get_info() - info["type"] = typemap.get(info["type"], "file") - orig_name = info["name"].rstrip("/") - # Collapse duplicate slashes in the filesystem-facing name. - name = re.sub("/+", "/", orig_name) - info["name"] = name - out[name] = (info, ti.offset_data, orig_name) - - self.index = out - # TODO: save index to self.index_store here, if set - - def _get_dirs(self): - if self.dir_cache is not None: - return - - # This enables ls to get directories as children as well as files - self.dir_cache = { - dirname: {"name": dirname, "size": 0, "type": "directory"} - for dirname in self._all_dirnames(self.index) - } - self.dir_cache.update( - {info["name"]: info for info, _, _ in self.index.values()} - ) - - def _open(self, path, mode="rb", **kwargs): - if mode != "rb": - raise ValueError("Read-only filesystem implementation") - # Accept paths containing the archive's duplicate slashes too. - path = re.sub("/+", "/", path) - details, _, orig_name = self.index[path] - if details["type"] != "file": - raise ValueError("Can only handle regular files") - return self.tar.extractfile(orig_name) - - def close(self): - """Commits any write changes to the file. Done on ``del`` too.""" - self.tar.close() - - def __del__(self): - if hasattr(self, "tar"): - self.close() - del self.tar - if hasattr(self, "of") and hasattr(self.of, "__exit__"): - self.of.__exit__(None, None, None) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/webhdfs.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/webhdfs.py deleted file mode 100644 index e3048b6a3638cf34e400804a3057521e720b6e81..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/webhdfs.py +++ /dev/null @@ -1,503 +0,0 @@ -# https://hadoop.apache.org/docs/r1.0.4/webhdfs.html - -import logging -import os -import secrets -import shutil -import tempfile -import uuid -from contextlib import suppress -from datetime import datetime -from urllib.parse import quote - -import requests - -from ..spec import AbstractBufferedFile, AbstractFileSystem -from ..utils import infer_storage_options, tokenize - -logger = logging.getLogger("webhdfs") - - -class WebHDFS(AbstractFileSystem): - """ - Interface to HDFS over HTTP using the WebHDFS API. Supports also HttpFS gateways. - - Four auth mechanisms are supported: - - insecure: no auth is done, and the user is assumed to be whoever they - say they are (parameter ``user``), or a predefined value such as - "dr.who" if not given - spnego: when kerberos authentication is enabled, auth is negotiated by - requests_kerberos https://github.com/requests/requests-kerberos . - This establishes a session based on existing kinit login and/or - specified principal/password; parameters are passed with ``kerb_kwargs`` - token: uses an existing Hadoop delegation token from another secured - service. Indeed, this client can also generate such tokens when - not insecure. Note that tokens expire, but can be renewed (by a - previously specified user) and may allow for proxying. - basic-auth: used when both parameter ``user`` and parameter ``password`` - are provided. - - """ - - tempdir = str(tempfile.gettempdir()) - protocol = "webhdfs", "webHDFS" - - def __init__( - self, - host, - port=50070, - kerberos=False, - token=None, - user=None, - password=None, - proxy_to=None, - kerb_kwargs=None, - data_proxy=None, - use_https=False, - session_cert=None, - session_verify=True, - **kwargs, - ): - """ - Parameters - ---------- - host: str - Name-node address - port: int - Port for webHDFS - kerberos: bool - Whether to authenticate with kerberos for this connection - token: str or None - If given, use this token on every call to authenticate. A user - and user-proxy may be encoded in the token and should not be also - given - user: str or None - If given, assert the user name to connect with - password: str or None - If given, assert the password to use for basic auth. If password - is provided, user must be provided also - proxy_to: str or None - If given, the user has the authority to proxy, and this value is - the user in who's name actions are taken - kerb_kwargs: dict - Any extra arguments for HTTPKerberosAuth, see - ``_ - data_proxy: dict, callable or None - If given, map data-node addresses. This can be necessary if the - HDFS cluster is behind a proxy, running on Docker or otherwise has - a mismatch between the host-names given by the name-node and the - address by which to refer to them from the client. If a dict, - maps host names ``host->data_proxy[host]``; if a callable, full - URLs are passed, and function must conform to - ``url->data_proxy(url)``. - use_https: bool - Whether to connect to the Name-node using HTTPS instead of HTTP - session_cert: str or Tuple[str, str] or None - Path to a certificate file, or tuple of (cert, key) files to use - for the requests.Session - session_verify: str, bool or None - Path to a certificate file to use for verifying the requests.Session. - kwargs - """ - if self._cached: - return - super().__init__(**kwargs) - self.url = f"{'https' if use_https else 'http'}://{host}:{port}/webhdfs/v1" - self.kerb = kerberos - self.kerb_kwargs = kerb_kwargs or {} - self.pars = {} - self.proxy = data_proxy or {} - if token is not None: - if user is not None or proxy_to is not None: - raise ValueError( - "If passing a delegation token, must not set " - "user or proxy_to, as these are encoded in the" - " token" - ) - self.pars["delegation"] = token - self.user = user - self.password = password - - if password is not None: - if user is None: - raise ValueError( - "If passing a password, the user must also be" - "set in order to set up the basic-auth" - ) - else: - if user is not None: - self.pars["user.name"] = user - - if proxy_to is not None: - self.pars["doas"] = proxy_to - if kerberos and user is not None: - raise ValueError( - "If using Kerberos auth, do not specify the " - "user, this is handled by kinit." - ) - - self.session_cert = session_cert - self.session_verify = session_verify - - self._connect() - - self._fsid = f"webhdfs_{tokenize(host, port)}" - - @property - def fsid(self): - return self._fsid - - def _connect(self): - self.session = requests.Session() - - if self.session_cert: - self.session.cert = self.session_cert - - self.session.verify = self.session_verify - - if self.kerb: - from requests_kerberos import HTTPKerberosAuth - - self.session.auth = HTTPKerberosAuth(**self.kerb_kwargs) - - if self.user is not None and self.password is not None: - from requests.auth import HTTPBasicAuth - - self.session.auth = HTTPBasicAuth(self.user, self.password) - - def _call(self, op, method="get", path=None, data=None, redirect=True, **kwargs): - path = self._strip_protocol(path) if path is not None else "" - url = self._apply_proxy(self.url + quote(path, safe="/=")) - args = kwargs.copy() - args.update(self.pars) - args["op"] = op.upper() - logger.debug("sending %s with %s", url, method) - out = self.session.request( - method=method.upper(), - url=url, - params=args, - data=data, - allow_redirects=redirect, - ) - if out.status_code in [400, 401, 403, 404, 500]: - try: - err = out.json() - msg = err["RemoteException"]["message"] - exp = err["RemoteException"]["exception"] - except (ValueError, KeyError): - pass - else: - if exp in ["IllegalArgumentException", "UnsupportedOperationException"]: - raise ValueError(msg) - elif exp in ["SecurityException", "AccessControlException"]: - raise PermissionError(msg) - elif exp in ["FileNotFoundException"]: - raise FileNotFoundError(msg) - else: - raise RuntimeError(msg) - out.raise_for_status() - return out - - def _open( - self, - path, - mode="rb", - block_size=None, - autocommit=True, - replication=None, - permissions=None, - **kwargs, - ): - """ - - Parameters - ---------- - path: str - File location - mode: str - 'rb', 'wb', etc. - block_size: int - Client buffer size for read-ahead or write buffer - autocommit: bool - If False, writes to temporary file that only gets put in final - location upon commit - replication: int - Number of copies of file on the cluster, write mode only - permissions: str or int - posix permissions, write mode only - kwargs - - Returns - ------- - WebHDFile instance - """ - block_size = block_size or self.blocksize - return WebHDFile( - self, - path, - mode=mode, - block_size=block_size, - tempdir=self.tempdir, - autocommit=autocommit, - replication=replication, - permissions=permissions, - ) - - @staticmethod - def _process_info(info): - info["type"] = info["type"].lower() - info["size"] = info["length"] - return info - - @classmethod - def _strip_protocol(cls, path): - return infer_storage_options(path)["path"] - - @staticmethod - def _get_kwargs_from_urls(urlpath): - out = infer_storage_options(urlpath) - out.pop("path", None) - out.pop("protocol", None) - if "username" in out: - out["user"] = out.pop("username") - return out - - def info(self, path): - out = self._call("GETFILESTATUS", path=path) - info = out.json()["FileStatus"] - info["name"] = path - return self._process_info(info) - - def created(self, path): - """Return the created timestamp of a file as a datetime.datetime""" - # The API does not provide creation time, so we use modification time - info = self.info(path) - mtime = info.get("modificationTime", None) - if mtime is not None: - return datetime.fromtimestamp(mtime / 1000) - raise RuntimeError("Could not retrieve creation time (modification time).") - - def modified(self, path): - """Return the modified timestamp of a file as a datetime.datetime""" - info = self.info(path) - mtime = info.get("modificationTime", None) - if mtime is not None: - return datetime.fromtimestamp(mtime / 1000) - raise RuntimeError("Could not retrieve modification time.") - - def ls(self, path, detail=False, **kwargs): - out = self._call("LISTSTATUS", path=path) - infos = out.json()["FileStatuses"]["FileStatus"] - for info in infos: - self._process_info(info) - info["name"] = path.rstrip("/") + "/" + info["pathSuffix"] - if detail: - return sorted(infos, key=lambda i: i["name"]) - else: - return sorted(info["name"] for info in infos) - - def content_summary(self, path): - """Total numbers of files, directories and bytes under path""" - out = self._call("GETCONTENTSUMMARY", path=path) - return out.json()["ContentSummary"] - - def ukey(self, path): - """Checksum info of file, giving method and result""" - out = self._call("GETFILECHECKSUM", path=path, redirect=False) - if "Location" in out.headers: - location = self._apply_proxy(out.headers["Location"]) - out2 = self.session.get(location) - out2.raise_for_status() - return out2.json()["FileChecksum"] - else: - out.raise_for_status() - return out.json()["FileChecksum"] - - def home_directory(self): - """Get user's home directory""" - out = self._call("GETHOMEDIRECTORY") - return out.json()["Path"] - - def get_delegation_token(self, renewer=None): - """Retrieve token which can give the same authority to other uses - - Parameters - ---------- - renewer: str or None - User who may use this token; if None, will be current user - """ - if renewer: - out = self._call("GETDELEGATIONTOKEN", renewer=renewer) - else: - out = self._call("GETDELEGATIONTOKEN") - t = out.json()["Token"] - if t is None: - raise ValueError("No token available for this user/security context") - return t["urlString"] - - def renew_delegation_token(self, token): - """Make token live longer. Returns new expiry time""" - out = self._call("RENEWDELEGATIONTOKEN", method="put", token=token) - return out.json()["long"] - - def cancel_delegation_token(self, token): - """Stop the token from being useful""" - self._call("CANCELDELEGATIONTOKEN", method="put", token=token) - - def chmod(self, path, mod): - """Set the permission at path - - Parameters - ---------- - path: str - location to set (file or directory) - mod: str or int - posix epresentation or permission, give as oct string, e.g, '777' - or 0o777 - """ - self._call("SETPERMISSION", method="put", path=path, permission=mod) - - def chown(self, path, owner=None, group=None): - """Change owning user and/or group""" - kwargs = {} - if owner is not None: - kwargs["owner"] = owner - if group is not None: - kwargs["group"] = group - self._call("SETOWNER", method="put", path=path, **kwargs) - - def set_replication(self, path, replication): - """ - Set file replication factor - - Parameters - ---------- - path: str - File location (not for directories) - replication: int - Number of copies of file on the cluster. Should be smaller than - number of data nodes; normally 3 on most systems. - """ - self._call("SETREPLICATION", path=path, method="put", replication=replication) - - def mkdir(self, path, **kwargs): - self._call("MKDIRS", method="put", path=path) - - def makedirs(self, path, exist_ok=False): - if exist_ok is False and self.exists(path): - raise FileExistsError(path) - self.mkdir(path) - - def mv(self, path1, path2, **kwargs): - self._call("RENAME", method="put", path=path1, destination=path2) - - def rm(self, path, recursive=False, **kwargs): - self._call( - "DELETE", - method="delete", - path=path, - recursive="true" if recursive else "false", - ) - - def rm_file(self, path, **kwargs): - self.rm(path) - - def cp_file(self, lpath, rpath, **kwargs): - with self.open(lpath) as lstream: - tmp_fname = "/".join([self._parent(rpath), f".tmp.{secrets.token_hex(16)}"]) - # Perform an atomic copy (stream to a temporary file and - # move it to the actual destination). - try: - with self.open(tmp_fname, "wb") as rstream: - shutil.copyfileobj(lstream, rstream) - self.mv(tmp_fname, rpath) - except BaseException: - with suppress(FileNotFoundError): - self.rm(tmp_fname) - raise - - def _apply_proxy(self, location): - if self.proxy and callable(self.proxy): - location = self.proxy(location) - elif self.proxy: - # as a dict - for k, v in self.proxy.items(): - location = location.replace(k, v, 1) - return location - - -class WebHDFile(AbstractBufferedFile): - """A file living in HDFS over webHDFS""" - - def __init__(self, fs, path, **kwargs): - super().__init__(fs, path, **kwargs) - kwargs = kwargs.copy() - if kwargs.get("permissions", None) is None: - kwargs.pop("permissions", None) - if kwargs.get("replication", None) is None: - kwargs.pop("replication", None) - self.permissions = kwargs.pop("permissions", 511) - tempdir = kwargs.pop("tempdir") - if kwargs.pop("autocommit", False) is False: - self.target = self.path - self.path = os.path.join(tempdir, str(uuid.uuid4())) - - def _upload_chunk(self, final=False): - """Write one part of a multi-block file upload - - Parameters - ========== - final: bool - This is the last block, so should complete file, if - self.autocommit is True. - """ - out = self.fs.session.post( - self.location, - data=self.buffer.getvalue(), - headers={"content-type": "application/octet-stream"}, - ) - out.raise_for_status() - return True - - def _initiate_upload(self): - """Create remote file/upload""" - kwargs = self.kwargs.copy() - if "a" in self.mode: - op, method = "APPEND", "POST" - else: - op, method = "CREATE", "PUT" - kwargs["overwrite"] = "true" - out = self.fs._call(op, method, self.path, redirect=False, **kwargs) - location = self.fs._apply_proxy(out.headers["Location"]) - if "w" in self.mode: - # create empty file to append to - out2 = self.fs.session.put( - location, headers={"content-type": "application/octet-stream"} - ) - out2.raise_for_status() - # after creating empty file, change location to append to - out2 = self.fs._call("APPEND", "POST", self.path, redirect=False, **kwargs) - self.location = self.fs._apply_proxy(out2.headers["Location"]) - - def _fetch_range(self, start, end): - start = max(start, 0) - end = min(self.size, end) - if start >= end or start >= self.size: - return b"" - out = self.fs._call( - "OPEN", path=self.path, offset=start, length=end - start, redirect=False - ) - out.raise_for_status() - if "Location" in out.headers: - location = out.headers["Location"] - out2 = self.fs.session.get(self.fs._apply_proxy(location)) - return out2.content - else: - return out.content - - def commit(self): - self.fs.mv(self.path, self.target) - - def discard(self): - self.fs.rm(self.path) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/zip.py b/bundle/python-cpu/Lib/site-packages/fsspec/implementations/zip.py deleted file mode 100644 index 485307a34f8daba70eb30bfda713c95893bbdb0b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/implementations/zip.py +++ /dev/null @@ -1,183 +0,0 @@ -import os -import zipfile - -import fsspec -from fsspec.archive import AbstractArchiveFileSystem - - -class ZipFileSystem(AbstractArchiveFileSystem): - """Read/Write contents of ZIP archive as a file-system - - Keeps file object open while instance lives. - - This class is pickleable, but not necessarily thread-safe - """ - - root_marker = "" - protocol = "zip" - cachable = False - - def __init__( - self, - fo="", - mode="r", - target_protocol=None, - target_options=None, - compression=zipfile.ZIP_STORED, - allowZip64=True, - compresslevel=None, - **kwargs, - ): - """ - Parameters - ---------- - fo: str or file-like - Contains ZIP, and must exist. If a str, will fetch file using - :meth:`~fsspec.open_files`, which must return one file exactly. - mode: str - Accept: "r", "w", "a" - target_protocol: str (optional) - If ``fo`` is a string, this value can be used to override the - FS protocol inferred from a URL - target_options: dict (optional) - Kwargs passed when instantiating the target FS, if ``fo`` is - a string. - compression, allowZip64, compresslevel: passed to ZipFile - Only relevant when creating a ZIP - """ - super().__init__(self, **kwargs) - if mode not in set("rwa"): - raise ValueError(f"mode '{mode}' no understood") - self.mode = mode - if isinstance(fo, (str, os.PathLike)): - if mode == "a": - m = "r+b" - else: - m = mode + "b" - fo = fsspec.open( - fo, mode=m, protocol=target_protocol, **(target_options or {}) - ) - self.force_zip_64 = allowZip64 - self.of = fo - self.fo = fo.__enter__() # the whole instance is a context - self.zip = zipfile.ZipFile( - self.fo, - mode=mode, - compression=compression, - allowZip64=allowZip64, - compresslevel=compresslevel, - ) - self.dir_cache = None - - @classmethod - def _strip_protocol(cls, path): - # zip file paths are always relative to the archive root - return super()._strip_protocol(path).lstrip("/") - - def __del__(self): - if hasattr(self, "zip"): - self.close() - del self.zip - if hasattr(self, "of") and hasattr(self.of, "__exit__"): - self.of.__exit__(None, None, None) - - def close(self): - """Commits any write changes to the file. Done on ``del`` too.""" - self.zip.close() - - def _get_dirs(self): - if self.dir_cache is None or self.mode in set("wa"): - # when writing, dir_cache is always in the ZipFile's attributes, - # not read from the file. - files = self.zip.infolist() - self.dir_cache = { - dirname.rstrip("/"): { - "name": dirname.rstrip("/"), - "size": 0, - "type": "directory", - } - for dirname in self._all_dirnames(self.zip.namelist()) - } - for z in files: - f = {s: getattr(z, s, None) for s in zipfile.ZipInfo.__slots__} - f.update( - { - "name": z.filename.rstrip("/"), - "size": z.file_size, - "type": ("directory" if z.is_dir() else "file"), - } - ) - self.dir_cache[f["name"]] = f - - def pipe_file(self, path, value, **kwargs): - # override upstream, because we know the exact file size in this case - self.zip.writestr(path, value, **kwargs) - - def _open( - self, - path, - mode="rb", - block_size=None, - autocommit=True, - cache_options=None, - **kwargs, - ): - path = self._strip_protocol(path) - if "r" in mode and self.mode in set("wa"): - if self.exists(path): - raise OSError("ZipFS can only be open for reading or writing, not both") - raise FileNotFoundError(path) - if "r" in self.mode and "w" in mode: - raise OSError("ZipFS can only be open for reading or writing, not both") - out = self.zip.open(path, mode.strip("b"), force_zip64=self.force_zip_64) - if "r" in mode: - info = self.info(path) - out.size = info["size"] - out.name = info["name"] - return out - - def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): - if maxdepth is not None and maxdepth < 1: - raise ValueError("maxdepth must be at least 1") - - def to_parts(_path: str): - return list(filter(None, _path.replace("\\", "/").split("/"))) - - if not isinstance(path, str): - path = str(path) - - # Remove the leading slash, as the zip file paths are always - # given without a leading slash - path = path.lstrip("/") - path_parts = to_parts(path) - path_depth = len(path_parts) - - self._get_dirs() - - result = {} - # To match posix find, if an exact file name is given, we should - # return only that file - if path in self.dir_cache and self.dir_cache[path]["type"] == "file": - result[path] = self.dir_cache[path] - return result if detail else [path] - - for file_path, file_info in self.dir_cache.items(): - if len(file_parts := to_parts(file_path)) < path_depth or any( - a != b for a, b in zip(path_parts, file_parts) - ): - # skip parent folders and mismatching paths - continue - - if file_info["type"] == "directory": - if withdirs and file_path not in result: - result[file_path.strip("/")] = file_info - continue - - if file_path not in result: - result[file_path] = file_info if detail else None - - if maxdepth: - result = { - k: v for k, v in result.items() if k.count("/") < maxdepth + path_depth - } - return result if detail else sorted(result) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/json.py b/bundle/python-cpu/Lib/site-packages/fsspec/json.py deleted file mode 100644 index 5c53a24913d0b28f4b53a163b97ff8f58abeb031..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/json.py +++ /dev/null @@ -1,112 +0,0 @@ -import json -from collections.abc import Callable, Mapping, Sequence -from contextlib import suppress -from pathlib import PurePath -from typing import Any, ClassVar - -from .registry import _import_class, get_filesystem_class -from .spec import AbstractFileSystem - - -class FilesystemJSONEncoder(json.JSONEncoder): - include_password: ClassVar[bool] = True - - def default(self, o: Any) -> Any: - if isinstance(o, AbstractFileSystem): - return o.to_dict(include_password=self.include_password) - if isinstance(o, PurePath): - cls = type(o) - return {"cls": f"{cls.__module__}.{cls.__name__}", "str": str(o)} - - return super().default(o) - - def make_serializable(self, obj: Any) -> Any: - """ - Recursively converts an object so that it can be JSON serialized via - :func:`json.dumps` and :func:`json.dump`, without actually calling - said functions. - """ - if isinstance(obj, (str, int, float, bool)): - return obj - if isinstance(obj, Mapping): - return {k: self.make_serializable(v) for k, v in obj.items()} - if isinstance(obj, Sequence): - return [self.make_serializable(v) for v in obj] - - return self.default(obj) - - -class FilesystemJSONDecoder(json.JSONDecoder): - def __init__( - self, - *, - object_hook: Callable[[dict[str, Any]], Any] | None = None, - parse_float: Callable[[str], Any] | None = None, - parse_int: Callable[[str], Any] | None = None, - parse_constant: Callable[[str], Any] | None = None, - strict: bool = True, - object_pairs_hook: Callable[[list[tuple[str, Any]]], Any] | None = None, - ) -> None: - self.original_object_hook = object_hook - - super().__init__( - object_hook=self.custom_object_hook, - parse_float=parse_float, - parse_int=parse_int, - parse_constant=parse_constant, - strict=strict, - object_pairs_hook=object_pairs_hook, - ) - - @classmethod - def try_resolve_path_cls(cls, dct: dict[str, Any]): - with suppress(Exception): - fqp = dct["cls"] - - path_cls = _import_class(fqp) - - if issubclass(path_cls, PurePath): - return path_cls - - return None - - @classmethod - def try_resolve_fs_cls(cls, dct: dict[str, Any]): - with suppress(Exception): - if "cls" in dct: - try: - fs_cls = _import_class(dct["cls"]) - if issubclass(fs_cls, AbstractFileSystem): - return fs_cls - except Exception: - if "protocol" in dct: # Fallback if cls cannot be imported - return get_filesystem_class(dct["protocol"]) - - raise - - return None - - def custom_object_hook(self, dct: dict[str, Any]): - if "cls" in dct: - if (obj_cls := self.try_resolve_fs_cls(dct)) is not None: - return AbstractFileSystem.from_dict(dct) - if (obj_cls := self.try_resolve_path_cls(dct)) is not None: - return obj_cls(dct["str"]) - - if self.original_object_hook is not None: - return self.original_object_hook(dct) - - return dct - - def unmake_serializable(self, obj: Any) -> Any: - """ - Inverse function of :meth:`FilesystemJSONEncoder.make_serializable`. - """ - if isinstance(obj, dict): - obj = self.custom_object_hook(obj) - if isinstance(obj, dict): - return {k: self.unmake_serializable(v) for k, v in obj.items()} - if isinstance(obj, (list, tuple)): - return [self.unmake_serializable(v) for v in obj] - - return obj diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/mapping.py b/bundle/python-cpu/Lib/site-packages/fsspec/mapping.py deleted file mode 100644 index 752eef35273b13eded7297e2e801b58e436a25b1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/mapping.py +++ /dev/null @@ -1,251 +0,0 @@ -import array -import logging -import posixpath -import warnings -from collections.abc import MutableMapping -from functools import cached_property - -from fsspec.core import url_to_fs - -logger = logging.getLogger("fsspec.mapping") - - -class FSMap(MutableMapping): - """Wrap a FileSystem instance as a mutable wrapping. - - The keys of the mapping become files under the given root, and the - values (which must be bytes) the contents of those files. - - Parameters - ---------- - root: string - prefix for all the files - fs: FileSystem instance - check: bool (=True) - performs a touch at the location, to check for write access. - - Examples - -------- - >>> fs = FileSystem(**parameters) # doctest: +SKIP - >>> d = FSMap('my-data/path/', fs) # doctest: +SKIP - or, more likely - >>> d = fs.get_mapper('my-data/path/') - - >>> d['loc1'] = b'Hello World' # doctest: +SKIP - >>> list(d.keys()) # doctest: +SKIP - ['loc1'] - >>> d['loc1'] # doctest: +SKIP - b'Hello World' - """ - - def __init__(self, root, fs, check=False, create=False, missing_exceptions=None): - self.fs = fs - self.root = fs._strip_protocol(root) - self._root_key_to_str = fs._strip_protocol(posixpath.join(root, "x"))[:-1] - if missing_exceptions is None: - missing_exceptions = ( - FileNotFoundError, - IsADirectoryError, - NotADirectoryError, - ) - self.missing_exceptions = missing_exceptions - self.check = check - self.create = create - if create: - if not self.fs.exists(root): - self.fs.mkdir(root) - if check: - if not self.fs.exists(root): - raise ValueError( - f"Path {root} does not exist. Create " - f" with the ``create=True`` keyword" - ) - self.fs.touch(root + "/a") - self.fs.rm(root + "/a") - - @cached_property - def dirfs(self): - """dirfs instance that can be used with the same keys as the mapper""" - from .implementations.dirfs import DirFileSystem - - return DirFileSystem(path=self._root_key_to_str, fs=self.fs) - - def clear(self): - """Remove all keys below root - empties out mapping""" - logger.info("Clear mapping at %s", self.root) - try: - self.fs.rm(self.root, True) - self.fs.mkdir(self.root) - except: # noqa: E722 - pass - - def getitems(self, keys, on_error="raise"): - """Fetch multiple items from the store - - If the backend is async-able, this might proceed concurrently - - Parameters - ---------- - keys: list(str) - They keys to be fetched - on_error : "raise", "omit", "return" - If raise, an underlying exception will be raised (converted to KeyError - if the type is in self.missing_exceptions); if omit, keys with exception - will simply not be included in the output; if "return", all keys are - included in the output, but the value will be bytes or an exception - instance. - - Returns - ------- - dict(key, bytes|exception) - """ - keys2 = [self._key_to_str(k) for k in keys] - oe = on_error if on_error == "raise" else "return" - try: - out = self.fs.cat(keys2, on_error=oe) - if isinstance(out, bytes): - out = {keys2[0]: out} - except self.missing_exceptions as e: - raise KeyError from e - out = { - k: (KeyError() if isinstance(v, self.missing_exceptions) else v) - for k, v in out.items() - } - return { - key: out[k2] if on_error == "raise" else out.get(k2, KeyError(k2)) - for key, k2 in zip(keys, keys2) - if on_error == "return" or not isinstance(out[k2], BaseException) - } - - def setitems(self, values_dict): - """Set the values of multiple items in the store - - Parameters - ---------- - values_dict: dict(str, bytes) - """ - values = {self._key_to_str(k): maybe_convert(v) for k, v in values_dict.items()} - self.fs.pipe(values) - - def delitems(self, keys): - """Remove multiple keys from the store""" - self.fs.rm([self._key_to_str(k) for k in keys]) - - def _key_to_str(self, key): - """Generate full path for the key""" - if not isinstance(key, str): - # raise TypeError("key must be of type `str`, got `{type(key).__name__}`" - warnings.warn( - "from fsspec 2023.5 onward FSMap non-str keys will raise TypeError", - DeprecationWarning, - ) - if isinstance(key, list): - key = tuple(key) - key = str(key) - return f"{self._root_key_to_str}{key}".rstrip("/") - - def _str_to_key(self, s): - """Strip path of to leave key name""" - return s[len(self.root) :].lstrip("/") - - def __getitem__(self, key, default=None): - """Retrieve data""" - k = self._key_to_str(key) - try: - result = self.fs.cat(k) - except self.missing_exceptions as exc: - if default is not None: - return default - raise KeyError(key) from exc - return result - - def pop(self, key, default=None): - """Pop data""" - result = self.__getitem__(key, default) - try: - del self[key] - except KeyError: - pass - return result - - def __setitem__(self, key, value): - """Store value in key""" - key = self._key_to_str(key) - self.fs.mkdirs(self.fs._parent(key), exist_ok=True) - self.fs.pipe_file(key, maybe_convert(value)) - - def __iter__(self): - return (self._str_to_key(x) for x in self.fs.find(self.root)) - - def __len__(self): - return len(self.fs.find(self.root)) - - def __delitem__(self, key): - """Remove key""" - try: - self.fs.rm(self._key_to_str(key)) - except Exception as exc: - raise KeyError from exc - - def __contains__(self, key): - """Does key exist in mapping?""" - path = self._key_to_str(key) - return self.fs.isfile(path) - - def __reduce__(self): - return FSMap, (self.root, self.fs, False, False, self.missing_exceptions) - - -def maybe_convert(value): - if isinstance(value, array.array) or hasattr(value, "__array__"): - # bytes-like things - if hasattr(value, "dtype") and value.dtype.kind in "Mm": - # The buffer interface doesn't support datetime64/timdelta64 numpy - # arrays - value = value.view("int64") - value = bytes(memoryview(value)) - return value - - -def get_mapper( - url="", - check=False, - create=False, - missing_exceptions=None, - alternate_root=None, - **kwargs, -): - """Create key-value interface for given URL and options - - The URL will be of the form "protocol://location" and point to the root - of the mapper required. All keys will be file-names below this location, - and their values the contents of each key. - - Also accepts compound URLs like zip::s3://bucket/file.zip , see ``fsspec.open``. - - Parameters - ---------- - url: str - Root URL of mapping - check: bool - Whether to attempt to read from the location before instantiation, to - check that the mapping does exist - create: bool - Whether to make the directory corresponding to the root before - instantiating - missing_exceptions: None or tuple - If given, these exception types will be regarded as missing keys and - return KeyError when trying to read data. By default, you get - (FileNotFoundError, IsADirectoryError, NotADirectoryError) - alternate_root: None or str - In cases of complex URLs, the parser may fail to pick the correct part - for the mapper root, so this arg can override - - Returns - ------- - ``FSMap`` instance, the dict-like key-value store. - """ - # Removing protocol here - could defer to each open() on the backend - fs, urlpath = url_to_fs(url, **kwargs) - root = alternate_root if alternate_root is not None else urlpath - return FSMap(root, fs, check, create, missing_exceptions=missing_exceptions) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/parquet.py b/bundle/python-cpu/Lib/site-packages/fsspec/parquet.py deleted file mode 100644 index e6e03fb14f9ec9224804a2ccb77bb84fe3390952..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/parquet.py +++ /dev/null @@ -1,572 +0,0 @@ -import io -import json -import warnings - -import fsspec - -from .core import url_to_fs -from .spec import AbstractBufferedFile -from .utils import merge_offset_ranges - -# Parquet-Specific Utilities for fsspec -# -# Most of the functions defined in this module are NOT -# intended for public consumption. The only exception -# to this is `open_parquet_file`, which should be used -# place of `fs.open()` to open parquet-formatted files -# on remote file systems. - - -class AlreadyBufferedFile(AbstractBufferedFile): - def _fetch_range(self, start, end): - raise NotImplementedError - - -def open_parquet_files( - path: list[str], - fs: None | fsspec.AbstractFileSystem = None, - metadata=None, - columns: None | list[str] = None, - row_groups: None | list[int] = None, - storage_options: None | dict = None, - engine: str = "auto", - max_gap: int = 64_000, - max_block: int = 256_000_000, - footer_sample_size: int = 1_000_000, - filters: None | list[list[list[str]]] = None, - **kwargs, -): - """ - Return a file-like object for a single Parquet file. - - The specified parquet `engine` will be used to parse the - footer metadata, and determine the required byte ranges - from the file. The target path will then be opened with - the "parts" (`KnownPartsOfAFile`) caching strategy. - - Note that this method is intended for usage with remote - file systems, and is unlikely to improve parquet-read - performance on local file systems. - - Parameters - ---------- - path: str - Target file path. - metadata: Any, optional - Parquet metadata object. Object type must be supported - by the backend parquet engine. For now, only the "fastparquet" - engine supports an explicit `ParquetFile` metadata object. - If a metadata object is supplied, the remote footer metadata - will not need to be transferred into local memory. - fs: AbstractFileSystem, optional - Filesystem object to use for opening the file. If nothing is - specified, an `AbstractFileSystem` object will be inferred. - engine : str, default "auto" - Parquet engine to use for metadata parsing. Allowed options - include "fastparquet", "pyarrow", and "auto". The specified - engine must be installed in the current environment. If - "auto" is specified, and both engines are installed, - "fastparquet" will take precedence over "pyarrow". - columns: list, optional - List of all column names that may be read from the file. - row_groups : list, optional - List of all row-groups that may be read from the file. This - may be a list of row-group indices (integers), or it may be - a list of `RowGroup` metadata objects (if the "fastparquet" - engine is used). - storage_options : dict, optional - Used to generate an `AbstractFileSystem` object if `fs` was - not specified. - max_gap : int, optional - Neighboring byte ranges will only be merged when their - inter-range gap is <= `max_gap`. Default is 64KB. - max_block : int, optional - Neighboring byte ranges will only be merged when the size of - the aggregated range is <= `max_block`. Default is 256MB. - footer_sample_size : int, optional - Number of bytes to read from the end of the path to look - for the footer metadata. If the sampled bytes do not contain - the footer, a second read request will be required, and - performance will suffer. Default is 1MB. - filters : list[list], optional - List of filters to apply to prevent reading row groups, of the - same format as accepted by the loading engines. Ignored if - ``row_groups`` is specified. - **kwargs : - Optional key-word arguments to pass to `fs.open` - """ - - # Make sure we have an `AbstractFileSystem` object - # to work with - if fs is None: - path0 = path - if isinstance(path, (list, tuple)): - path = path[0] - fs, path = url_to_fs(path, **(storage_options or {})) - else: - path0 = path - - # For now, `columns == []` not supported, is the same - # as all columns - if columns is not None and len(columns) == 0: - columns = None - - # Set the engine - engine = _set_engine(engine) - - if isinstance(path0, (list, tuple)): - paths = path0 - elif "*" in path: - paths = fs.glob(path) - elif path0.endswith("/"): # or fs.isdir(path): - paths = [ - _ - for _ in fs.find(path, withdirs=False, detail=False) - if _.endswith((".parquet", ".parq")) - ] - else: - paths = [path] - - data = _get_parquet_byte_ranges( - paths, - fs, - metadata=metadata, - columns=columns, - row_groups=row_groups, - engine=engine, - max_gap=max_gap, - max_block=max_block, - footer_sample_size=footer_sample_size, - filters=filters, - ) - - # Call self.open with "parts" caching - options = kwargs.pop("cache_options", {}).copy() - return [ - AlreadyBufferedFile( - fs=None, - path=fn, - mode="rb", - cache_type="parts", - cache_options={ - **options, - "data": ranges, - }, - size=max(_[1] for _ in ranges), - **kwargs, - ) - for fn, ranges in data.items() - ] - - -def open_parquet_file(*args, **kwargs): - """Create files tailed to reading specific parts of parquet files - - Please see ``open_parquet_files`` for details of the arguments. The - difference is, this function always returns a single ``AlreadyBufferedFile``, - whereas `open_parquet_files`` always returns a list of files, even if - there are one or zero matching parquet files. - """ - return open_parquet_files(*args, **kwargs)[0] - - -def _get_parquet_byte_ranges( - paths, - fs, - metadata=None, - columns=None, - row_groups=None, - max_gap=64_000, - max_block=256_000_000, - footer_sample_size=1_000_000, - engine="auto", - filters=None, -): - """Get a dictionary of the known byte ranges needed - to read a specific column/row-group selection from a - Parquet dataset. Each value in the output dictionary - is intended for use as the `data` argument for the - `KnownPartsOfAFile` caching strategy of a single path. - """ - - # Set engine if necessary - if isinstance(engine, str): - engine = _set_engine(engine) - - # Pass to a specialized function if metadata is defined - if metadata is not None: - # Use the provided parquet metadata object - # to avoid transferring/parsing footer metadata - return _get_parquet_byte_ranges_from_metadata( - metadata, - fs, - engine, - columns=columns, - row_groups=row_groups, - max_gap=max_gap, - max_block=max_block, - filters=filters, - ) - - # Populate global paths, starts, & ends - if columns is None and row_groups is None and filters is None: - # We are NOT selecting specific columns or row-groups. - # - # We can avoid sampling the footers, and just transfer - # all file data with cat_ranges - result = {path: {(0, len(data)): data} for path, data in fs.cat(paths).items()} - else: - # We ARE selecting specific columns or row-groups. - # - # Get file sizes asynchronously - file_sizes = fs.sizes(paths) - data_paths = [] - data_starts = [] - data_ends = [] - # Gather file footers. - # We just take the last `footer_sample_size` bytes of each - # file (or the entire file if it is smaller than that) - footer_starts = [ - max(0, file_size - footer_sample_size) for file_size in file_sizes - ] - footer_samples = fs.cat_ranges(paths, footer_starts, file_sizes) - - # Check our footer samples and re-sample if necessary. - large_footer = [] - for i, path in enumerate(paths): - footer_size = int.from_bytes(footer_samples[i][-8:-4], "little") - real_footer_start = file_sizes[i] - (footer_size + 8) - if real_footer_start < footer_starts[i]: - large_footer.append((i, real_footer_start)) - if large_footer: - warnings.warn( - f"Not enough data was used to sample the parquet footer. " - f"Try setting footer_sample_size >= {large_footer}." - ) - path0 = [paths[i] for i, _ in large_footer] - starts = [_[1] for _ in large_footer] - ends = [file_sizes[i] - footer_sample_size for i, _ in large_footer] - data = fs.cat_ranges(path0, starts, ends) - for i, (path, start, block) in enumerate(zip(path0, starts, data)): - footer_samples[i] = block + footer_samples[i] - footer_starts[i] = start - result = { - path: {(start, size): data} - for path, start, size, data in zip( - paths, footer_starts, file_sizes, footer_samples - ) - } - - # Calculate required byte ranges for each path - for i, path in enumerate(paths): - # Use "engine" to collect data byte ranges - path_data_starts, path_data_ends = engine._parquet_byte_ranges( - columns, - row_groups=row_groups, - footer=footer_samples[i], - footer_start=footer_starts[i], - filters=filters, - ) - - data_paths += [path] * len(path_data_starts) - data_starts += path_data_starts - data_ends += path_data_ends - - # Merge adjacent offset ranges - data_paths, data_starts, data_ends = merge_offset_ranges( - data_paths, - data_starts, - data_ends, - max_gap=max_gap, - max_block=max_block, - sort=True, - ) - - # Transfer the data byte-ranges into local memory - _transfer_ranges(fs, result, data_paths, data_starts, data_ends) - - # Add b"PAR1" to headers - _add_header_magic(result) - - return result - - -def _get_parquet_byte_ranges_from_metadata( - metadata, - fs, - engine, - columns=None, - row_groups=None, - max_gap=64_000, - max_block=256_000_000, - filters=None, -): - """Simplified version of `_get_parquet_byte_ranges` for - the case that an engine-specific `metadata` object is - provided, and the remote footer metadata does not need to - be transferred before calculating the required byte ranges. - """ - - # Use "engine" to collect data byte ranges - data_paths, data_starts, data_ends = engine._parquet_byte_ranges( - columns, row_groups=row_groups, metadata=metadata, filters=filters - ) - - # Merge adjacent offset ranges - data_paths, data_starts, data_ends = merge_offset_ranges( - data_paths, - data_starts, - data_ends, - max_gap=max_gap, - max_block=max_block, - sort=False, # Should be sorted - ) - - # Transfer the data byte-ranges into local memory - result = {fn: {} for fn in list(set(data_paths))} - _transfer_ranges(fs, result, data_paths, data_starts, data_ends) - - # Add b"PAR1" to header - _add_header_magic(result) - - return result - - -def _transfer_ranges(fs, blocks, paths, starts, ends): - # Use cat_ranges to gather the data byte_ranges - ranges = (paths, starts, ends) - for path, start, stop, data in zip(*ranges, fs.cat_ranges(*ranges)): - blocks[path][(start, stop)] = data - - -def _add_header_magic(data): - # Add b"PAR1" to file headers - for path in list(data): - add_magic = True - for k in data[path]: - if k[0] == 0 and k[1] >= 4: - add_magic = False - break - if add_magic: - data[path][(0, 4)] = b"PAR1" - - -def _set_engine(engine_str): - # Define a list of parquet engines to try - if engine_str == "auto": - try_engines = ("fastparquet", "pyarrow") - elif not isinstance(engine_str, str): - raise ValueError( - "Failed to set parquet engine! " - "Please pass 'fastparquet', 'pyarrow', or 'auto'" - ) - elif engine_str not in ("fastparquet", "pyarrow"): - raise ValueError(f"{engine_str} engine not supported by `fsspec.parquet`") - else: - try_engines = [engine_str] - - # Try importing the engines in `try_engines`, - # and choose the first one that succeeds - for engine in try_engines: - try: - if engine == "fastparquet": - return FastparquetEngine() - elif engine == "pyarrow": - return PyarrowEngine() - except ImportError: - pass - - # Raise an error if a supported parquet engine - # was not found - raise ImportError( - f"The following parquet engines are not installed " - f"in your python environment: {try_engines}." - f"Please install 'fastparquert' or 'pyarrow' to " - f"utilize the `fsspec.parquet` module." - ) - - -class FastparquetEngine: - # The purpose of the FastparquetEngine class is - # to check if fastparquet can be imported (on initialization) - # and to define a `_parquet_byte_ranges` method. In the - # future, this class may also be used to define other - # methods/logic that are specific to fastparquet. - - def __init__(self): - import fastparquet as fp - - self.fp = fp - - def _parquet_byte_ranges( - self, - columns, - row_groups=None, - metadata=None, - footer=None, - footer_start=None, - filters=None, - ): - # Initialize offset ranges and define ParqetFile metadata - pf = metadata - data_paths, data_starts, data_ends = [], [], [] - if filters and row_groups: - raise ValueError("filters and row_groups cannot be used together") - if pf is None: - pf = self.fp.ParquetFile(io.BytesIO(footer)) - - # Convert columns to a set and add any index columns - # specified in the pandas metadata (just in case) - column_set = None if columns is None else {c.split(".", 1)[0] for c in columns} - if column_set is not None and hasattr(pf, "pandas_metadata"): - md_index = [ - ind - for ind in pf.pandas_metadata.get("index_columns", []) - # Ignore RangeIndex information - if not isinstance(ind, dict) - ] - column_set |= set(md_index) - - # Check if row_groups is a list of integers - # or a list of row-group metadata - if filters: - from fastparquet.api import filter_row_groups - - row_group_indices = None - row_groups = filter_row_groups(pf, filters) - elif row_groups and not isinstance(row_groups[0], int): - # Input row_groups contains row-group metadata - row_group_indices = None - else: - # Input row_groups contains row-group indices - row_group_indices = row_groups - row_groups = pf.row_groups - if column_set is not None: - column_set = [ - _ if isinstance(_, list) else _.split(".") for _ in column_set - ] - - # Loop through column chunks to add required byte ranges - for r, row_group in enumerate(row_groups): - # Skip this row-group if we are targeting - # specific row-groups - if row_group_indices is None or r in row_group_indices: - # Find the target parquet-file path for `row_group` - fn = pf.row_group_filename(row_group) - - for column in row_group.columns: - name = column.meta_data.path_in_schema - # Skip this column if we are targeting specific columns - if column_set is None or _cmp(name, column_set): - file_offset0 = column.meta_data.dictionary_page_offset - if file_offset0 is None: - file_offset0 = column.meta_data.data_page_offset - num_bytes = column.meta_data.total_compressed_size - if footer_start is None or file_offset0 < footer_start: - data_paths.append(fn) - data_starts.append(file_offset0) - data_ends.append( - min( - file_offset0 + num_bytes, - footer_start or (file_offset0 + num_bytes), - ) - ) - - if metadata: - # The metadata in this call may map to multiple - # file paths. Need to include `data_paths` - return data_paths, data_starts, data_ends - return data_starts, data_ends - - -class PyarrowEngine: - # The purpose of the PyarrowEngine class is - # to check if pyarrow can be imported (on initialization) - # and to define a `_parquet_byte_ranges` method. In the - # future, this class may also be used to define other - # methods/logic that are specific to pyarrow. - - def __init__(self): - import pyarrow.parquet as pq - - self.pq = pq - - def _parquet_byte_ranges( - self, - columns, - row_groups=None, - metadata=None, - footer=None, - footer_start=None, - filters=None, - ): - if metadata is not None: - raise ValueError("metadata input not supported for PyarrowEngine") - if filters: - # there must be a way! - raise NotImplementedError - - data_starts, data_ends = [], [] - md = self.pq.ParquetFile(io.BytesIO(footer)).metadata - - # Convert columns to a set and add any index columns - # specified in the pandas metadata (just in case) - column_set = None if columns is None else set(columns) - if column_set is not None: - schema = md.schema.to_arrow_schema() - has_pandas_metadata = ( - schema.metadata is not None and b"pandas" in schema.metadata - ) - if has_pandas_metadata: - md_index = [ - ind - for ind in json.loads( - schema.metadata[b"pandas"].decode("utf8") - ).get("index_columns", []) - # Ignore RangeIndex information - if not isinstance(ind, dict) - ] - column_set |= set(md_index) - if column_set is not None: - column_set = [ - _[:1] if isinstance(_, list) else _.split(".")[:1] for _ in column_set - ] - - # Loop through column chunks to add required byte ranges - for r in range(md.num_row_groups): - # Skip this row-group if we are targeting - # specific row-groups - if row_groups is None or r in row_groups: - row_group = md.row_group(r) - for c in range(row_group.num_columns): - column = row_group.column(c) - name = column.path_in_schema.split(".") - # Skip this column if we are targeting specific columns - if column_set is None or _cmp(name, column_set): - meta = column.to_dict() - # Any offset could be the first one - file_offset0 = min( - _ - for _ in [ - meta.get("dictionary_page_offset"), - meta.get("data_page_offset"), - meta.get("index_page_offset"), - ] - if _ is not None - ) - if file_offset0 < footer_start: - data_starts.append(file_offset0) - data_ends.append( - min( - meta["total_compressed_size"] + file_offset0, - footer_start, - ) - ) - - data_starts.append(footer_start) - data_ends.append(footer_start + len(footer)) - return data_starts, data_ends - - -def _cmp(name, column_set): - return any(all(a == b for a, b in zip(name, _)) for _ in column_set) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/registry.py b/bundle/python-cpu/Lib/site-packages/fsspec/registry.py deleted file mode 100644 index 305d1e8908504fb6c38e94708bd7f20dba722705..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/registry.py +++ /dev/null @@ -1,337 +0,0 @@ -from __future__ import annotations - -import importlib -import types -import warnings - -__all__ = ["registry", "get_filesystem_class", "default"] - -# internal, mutable -_registry: dict[str, type] = {} - -# external, immutable -registry = types.MappingProxyType(_registry) -default = "file" - - -def register_implementation(name, cls, clobber=False, errtxt=None): - """Add implementation class to the registry - - Parameters - ---------- - name: str - Protocol name to associate with the class - cls: class or str - if a class: fsspec-compliant implementation class (normally inherits from - ``fsspec.AbstractFileSystem``, gets added straight to the registry. If a - str, the full path to an implementation class like package.module.class, - which gets added to known_implementations, - so the import is deferred until the filesystem is actually used. - clobber: bool (optional) - Whether to overwrite a protocol with the same name; if False, will raise - instead. - errtxt: str (optional) - If given, then a failure to import the given class will result in this - text being given. - """ - if isinstance(cls, str): - if name in known_implementations and clobber is False: - if cls != known_implementations[name]["class"]: - raise ValueError( - f"Name ({name}) already in the known_implementations and clobber " - f"is False" - ) - else: - known_implementations[name] = { - "class": cls, - "err": errtxt or f"{cls} import failed for protocol {name}", - } - - else: - if name in registry and clobber is False: - if _registry[name] is not cls: - raise ValueError( - f"Name ({name}) already in the registry and clobber is False" - ) - else: - _registry[name] = cls - - -# protocols mapped to the class which implements them. This dict can be -# updated with register_implementation -known_implementations = { - "abfs": { - "class": "adlfs.AzureBlobFileSystem", - "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage", - }, - "adl": { - "class": "adlfs.AzureDatalakeFileSystem", - "err": ( - "Azure Data Lake Storage Gen1 is retired and no longer supported. Please " - "install adlfs and use the `az://` protocol to access Azure Blob Storage " - "and Azure Data Lake Storage Gen2 instead." - ), - }, - "arrow_hdfs": { - "class": "fsspec.implementations.arrow.HadoopFileSystem", - "err": "pyarrow and local java libraries required for HDFS", - }, - "async_wrapper": { - "class": "fsspec.implementations.asyn_wrapper.AsyncFileSystemWrapper", - }, - "asynclocal": { - "class": "morefs.asyn_local.AsyncLocalFileSystem", - "err": "Install 'morefs[asynclocalfs]' to use AsyncLocalFileSystem", - }, - "asyncwrapper": { - "class": "fsspec.implementations.asyn_wrapper.AsyncFileSystemWrapper", - }, - "az": { - "class": "adlfs.AzureBlobFileSystem", - "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage", - }, - "blockcache": {"class": "fsspec.implementations.cached.CachingFileSystem"}, - "box": { - "class": "boxfs.BoxFileSystem", - "err": "Please install boxfs to access BoxFileSystem", - }, - "cached": {"class": "fsspec.implementations.cached.CachingFileSystem"}, - "dask": { - "class": "fsspec.implementations.dask.DaskWorkerFileSystem", - "err": "Install dask distributed to access worker file system", - }, - "data": {"class": "fsspec.implementations.data.DataFileSystem"}, - "dbfs": { - "class": "fsspec.implementations.dbfs.DatabricksFileSystem", - "err": "Install the requests package to use the DatabricksFileSystem", - }, - "dir": {"class": "fsspec.implementations.dirfs.DirFileSystem"}, - "dropbox": { - "class": "dropboxdrivefs.DropboxDriveFileSystem", - "err": ( - 'DropboxFileSystem requires "dropboxdrivefs","requests" and "' - '"dropbox" to be installed' - ), - }, - "dvc": { - "class": "dvc.api.DVCFileSystem", - "err": "Install dvc to access DVCFileSystem", - }, - "file": {"class": "fsspec.implementations.local.LocalFileSystem"}, - "filecache": {"class": "fsspec.implementations.cached.WholeFileCacheFileSystem"}, - "ftp": {"class": "fsspec.implementations.ftp.FTPFileSystem"}, - "gcs": { - "class": "gcsfs.GCSFileSystem", - "err": "Please install gcsfs to access Google Storage", - }, - "gdrive": { - "class": "gdrive_fsspec.GoogleDriveFileSystem", - "err": "Please install gdrive_fs for access to Google Drive", - }, - "generic": {"class": "fsspec.generic.GenericFileSystem"}, - "gist": { - "class": "fsspec.implementations.gist.GistFileSystem", - "err": "Install the requests package to use the gist FS", - }, - "git": { - "class": "fsspec.implementations.git.GitFileSystem", - "err": "Install pygit2 to browse local git repos", - }, - "github": { - "class": "fsspec.implementations.github.GithubFileSystem", - "err": "Install the requests package to use the github FS", - }, - "gs": { - "class": "gcsfs.GCSFileSystem", - "err": "Please install gcsfs to access Google Storage", - }, - "hdfs": { - "class": "fsspec.implementations.arrow.HadoopFileSystem", - "err": "pyarrow and local java libraries required for HDFS", - }, - "hf": { - "class": "huggingface_hub.HfFileSystem", - "err": "Install huggingface_hub to access HfFileSystem", - }, - "http": { - "class": "fsspec.implementations.http.HTTPFileSystem", - "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', - }, - "https": { - "class": "fsspec.implementations.http.HTTPFileSystem", - "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', - }, - "jlab": { - "class": "fsspec.implementations.jupyter.JupyterFileSystem", - "err": "Jupyter FS requires requests to be installed", - }, - "jupyter": { - "class": "fsspec.implementations.jupyter.JupyterFileSystem", - "err": "Jupyter FS requires requests to be installed", - }, - "lakefs": { - "class": "lakefs_spec.LakeFSFileSystem", - "err": "Please install lakefs-spec to access LakeFSFileSystem", - }, - "libarchive": { - "class": "fsspec.implementations.libarchive.LibArchiveFileSystem", - "err": "LibArchive requires to be installed", - }, - "local": {"class": "fsspec.implementations.local.LocalFileSystem"}, - "memory": {"class": "fsspec.implementations.memory.MemoryFileSystem"}, - "oci": { - "class": "ocifs.OCIFileSystem", - "err": "Install ocifs to access OCI Object Storage", - }, - "ocilake": { - "class": "ocifs.OCIFileSystem", - "err": "Install ocifs to access OCI Data Lake", - }, - "oss": { - "class": "ossfs.OSSFileSystem", - "err": "Install ossfs to access Alibaba Object Storage System", - }, - "pyscript": { - "class": "pyscript_fsspec_client.client.PyscriptFileSystem", - "err": "This only runs in a pyscript context", - }, - "reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"}, - "root": { - "class": "fsspec_xrootd.XRootDFileSystem", - "err": ( - "Install fsspec-xrootd to access xrootd storage system. " - "Note: 'root' is the protocol name for xrootd storage systems, " - "not referring to root directories" - ), - }, - "s3": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, - "s3a": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, - "sftp": { - "class": "fsspec.implementations.sftp.SFTPFileSystem", - "err": 'SFTPFileSystem requires "paramiko" to be installed', - }, - "simplecache": {"class": "fsspec.implementations.cached.SimpleCacheFileSystem"}, - "smb": { - "class": "fsspec.implementations.smb.SMBFileSystem", - "err": 'SMB requires "smbprotocol" or "smbprotocol[kerberos]" installed', - }, - "ssh": { - "class": "fsspec.implementations.sftp.SFTPFileSystem", - "err": 'SFTPFileSystem requires "paramiko" to be installed', - }, - "tar": {"class": "fsspec.implementations.tar.TarFileSystem"}, - "tos": { - "class": "tosfs.TosFileSystem", - "err": "Install tosfs to access ByteDance volcano engine Tinder Object Storage", - }, - "tosfs": { - "class": "tosfs.TosFileSystem", - "err": "Install tosfs to access ByteDance volcano engine Tinder Object Storage", - }, - "wandb": {"class": "wandbfs.WandbFS", "err": "Install wandbfs to access wandb"}, - "webdav": { - "class": "webdav4.fsspec.WebdavFileSystem", - "err": "Install webdav4 to access WebDAV", - }, - "webhdfs": { - "class": "fsspec.implementations.webhdfs.WebHDFS", - "err": 'webHDFS access requires "requests" to be installed', - }, - "zip": {"class": "fsspec.implementations.zip.ZipFileSystem"}, -} - -assert list(known_implementations) == sorted(known_implementations), ( - "Not in alphabetical order" -) - - -def get_filesystem_class(protocol): - """Fetch named protocol implementation from the registry - - The dict ``known_implementations`` maps protocol names to the locations - of classes implementing the corresponding file-system. When used for the - first time, appropriate imports will happen and the class will be placed in - the registry. All subsequent calls will fetch directly from the registry. - - Some protocol implementations require additional dependencies, and so the - import may fail. In this case, the string in the "err" field of the - ``known_implementations`` will be given as the error message. - """ - if not protocol: - protocol = default - - if protocol not in registry: - if protocol not in known_implementations: - raise ValueError(f"Protocol not known: {protocol}") - bit = known_implementations[protocol] - try: - register_implementation(protocol, _import_class(bit["class"])) - except ImportError as e: - raise ImportError(bit.get("err")) from e - cls = registry[protocol] - if getattr(cls, "protocol", None) in ("abstract", None): - cls.protocol = protocol - - return cls - - -s3_msg = """Your installed version of s3fs is very old and known to cause -severe performance issues, see also https://github.com/dask/dask/issues/10276 - -To fix, you should specify a lower version bound on s3fs, or -update the current installation. -""" - - -def _import_class(fqp: str): - """Take a fully-qualified path and return the imported class or identifier. - - ``fqp`` is of the form "package.module.klass" or - "package.module:subobject.klass". - - Warnings - -------- - This can import arbitrary modules. Make sure you haven't installed any modules - that may execute malicious code at import time. - """ - if ":" in fqp: - mod, name = fqp.rsplit(":", 1) - else: - mod, name = fqp.rsplit(".", 1) - - is_s3 = mod == "s3fs" - mod = importlib.import_module(mod) - if is_s3 and mod.__version__.split(".") < ["0", "5"]: - warnings.warn(s3_msg) - for part in name.split("."): - mod = getattr(mod, part) - - if not isinstance(mod, type): - raise TypeError(f"{fqp} is not a class") - - return mod - - -def filesystem(protocol, **storage_options): - """Instantiate filesystems for given protocol and arguments - - ``storage_options`` are specific to the protocol being chosen, and are - passed directly to the class. - """ - if protocol == "arrow_hdfs": - warnings.warn( - "The 'arrow_hdfs' protocol has been deprecated and will be " - "removed in the future. Specify it as 'hdfs'.", - DeprecationWarning, - ) - - cls = get_filesystem_class(protocol) - return cls(**storage_options) - - -def available_protocols(): - """Return a list of the implemented protocols. - - Note that any given protocol may require extra packages to be importable. - """ - return list(known_implementations) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/spec.py b/bundle/python-cpu/Lib/site-packages/fsspec/spec.py deleted file mode 100644 index 94d286b04fb55b0140a003193ab2409532fc674d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/spec.py +++ /dev/null @@ -1,2342 +0,0 @@ -from __future__ import annotations - -import io -import json -import logging -import os -import threading -import warnings -import weakref -from errno import ESPIPE -from glob import has_magic -from hashlib import sha256 -from typing import Any, ClassVar - -from .callbacks import DEFAULT_CALLBACK -from .config import apply_config, conf -from .dircache import DirCache -from .transaction import Transaction -from .utils import ( - _unstrip_protocol, - glob_translate, - isfilelike, - other_paths, - read_block, - stringify_path, - tokenize, -) - -logger = logging.getLogger("fsspec") - - -def make_instance(cls, args, kwargs): - return cls(*args, **kwargs) - - -FORK_AVAILABLE = hasattr(os, "register_at_fork") - - -if FORK_AVAILABLE: - _registered_classes = weakref.WeakSet() - - def _reset_instances_lock(): - for cls in _registered_classes: - cls._instantiation_lock = threading.RLock() - cls._cache.clear() - cls._pid = os.getpid() - - os.register_at_fork(after_in_child=_reset_instances_lock) - - -class _Cached(type): - """ - Metaclass for caching file system instances. - - Notes - ----- - Instances are cached according to - - * The values of the class attributes listed in `_extra_tokenize_attributes` - * The arguments passed to ``__init__``. - - This creates an additional reference to the filesystem, which prevents the - filesystem from being garbage collected when all *user* references go away. - A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also* - be made for a filesystem instance to be garbage collected. - """ - - def __init__(cls, *args, **kwargs): - super().__init__(*args, **kwargs) - - # Note: we intentionally create a reference here, to avoid garbage - # collecting instances when all other references are gone. To really - # delete a FileSystem, the cache must be cleared. - if conf.get("weakref_instance_cache"): # pragma: no cover - # debug option for analysing fork/spawn conditions - cls._cache = weakref.WeakValueDictionary() - else: - cls._cache = {} - cls._pid = os.getpid() - cls._instantiation_lock = threading.RLock() - - if FORK_AVAILABLE: - _registered_classes.add(cls) - - def _check_instance_cache(cls, token): - inst = cls._cache.get(token) - if inst is not None: - cls._latest = token - return inst - - def __call__(cls, *args, **kwargs): - kwargs = apply_config(cls, kwargs) - extra_tokens = tuple( - getattr(cls, attr, None) for attr in cls._extra_tokenize_attributes - ) - strip_tokenize_options = { - k: kwargs.pop(k) for k in cls._strip_tokenize_options if k in kwargs - } - pid = os.getpid() - - if getattr(cls, "async_impl", False) and not kwargs.get("asynchronous", False): - token = tokenize(cls, pid, *args, *extra_tokens, **kwargs) - else: - token = tokenize( - cls, pid, threading.get_ident(), *args, *extra_tokens, **kwargs - ) - skip = kwargs.pop("skip_instance_cache", False) - - if pid != cls._pid: - with cls._instantiation_lock: - if pid != cls._pid: - cls._cache.clear() - cls._pid = pid - - if not skip and cls.cachable: - inst = cls._check_instance_cache(token) - if inst is not None: - return inst - - with cls._instantiation_lock: - # protect against the race condition that a new instance was created - # and inserted into the cache since the initial check just above - inst = cls._check_instance_cache(token) - if inst is not None: - return inst - - obj = super().__call__(*args, **kwargs, **strip_tokenize_options) - # Setting _fs_token here causes some static linters to complain. - obj._fs_token_ = token - obj.storage_args = args - obj.storage_options = kwargs - if obj.async_impl and obj.mirror_sync_methods: - from .asyn import mirror_sync_methods - - mirror_sync_methods(obj) - - if cls.cachable and not skip: - with cls._instantiation_lock: - # another thread may have created the instance while we were calling - # super().__call__(), so we check again. - inst = cls._check_instance_cache(token) - if inst is not None: - return inst - - cls._latest = token - cls._cache[token] = obj - return obj - - -class AbstractFileSystem(metaclass=_Cached): - """ - An abstract super-class for pythonic file-systems - - Implementations are expected to be compatible with or, better, subclass - from here. - """ - - cachable = True # this class can be cached, instances reused - _cached = False - blocksize = 2**22 - sep = "/" - protocol: ClassVar[str | tuple[str, ...]] = "abstract" - _latest = None - async_impl = False - mirror_sync_methods = False - root_marker = "" # For some FSs, may require leading '/' or other character - transaction_type = Transaction - - #: Extra *class attributes* that should be considered when hashing. - _extra_tokenize_attributes = () - #: *storage options* that should not be considered when hashing. - _strip_tokenize_options = () - - # Set by _Cached metaclass - storage_args: tuple[Any, ...] - storage_options: dict[str, Any] - - def __init__(self, *args, **storage_options): - """Create and configure file-system instance - - Instances may be cachable, so if similar enough arguments are seen - a new instance is not required. The token attribute exists to allow - implementations to cache instances if they wish. - - A reasonable default should be provided if there are no arguments. - - Subclasses should call this method. - - Parameters - ---------- - use_listings_cache, listings_expiry_time, max_paths: - passed to ``DirCache``, if the implementation supports - directory listing caching. Pass use_listings_cache=False - to disable such caching. - skip_instance_cache: bool - If this is a cachable implementation, pass True here to force - creating a new instance even if a matching instance exists, and prevent - storing this instance. - asynchronous: bool - loop: asyncio-compatible IOLoop or None - """ - if self._cached: - # reusing instance, don't change - return - self._cached = True - self._intrans = False - self._transaction = None - self._invalidated_caches_in_transaction = [] - self.dircache = DirCache(**storage_options) - - if storage_options.pop("add_docs", None): - warnings.warn("add_docs is no longer supported.", FutureWarning) - - if storage_options.pop("add_aliases", None): - warnings.warn("add_aliases has been removed.", FutureWarning) - # This is set in _Cached - self._fs_token_ = None - - @property - def fsid(self): - """Persistent filesystem id that can be used to compare filesystems - across sessions. - """ - raise NotImplementedError - - @property - def _fs_token(self): - return self._fs_token_ - - def __dask_tokenize__(self): - return self._fs_token - - def __hash__(self): - return int(self._fs_token, 16) - - def __eq__(self, other): - return isinstance(other, type(self)) and self._fs_token == other._fs_token - - def __reduce__(self): - return make_instance, (type(self), self.storage_args, self.storage_options) - - @classmethod - def _strip_protocol(cls, path): - """Turn path from fully-qualified to file-system-specific - - May require FS-specific handling, e.g., for relative paths or links. - """ - if isinstance(path, list): - return [cls._strip_protocol(p) for p in path] - path = stringify_path(path) - protos = (cls.protocol,) if isinstance(cls.protocol, str) else cls.protocol - for protocol in protos: - if path.startswith(protocol + "://"): - path = path[len(protocol) + 3 :] - elif path.startswith(protocol + "::"): - path = path[len(protocol) + 2 :] - path = path.rstrip("/") - # use of root_marker to make minimum required path, e.g., "/" - return path or cls.root_marker - - def unstrip_protocol(self, name: str) -> str: - """Format FS-specific path to generic, including protocol""" - protos = (self.protocol,) if isinstance(self.protocol, str) else self.protocol - for protocol in protos: - if name.startswith(f"{protocol}://"): - return name - return f"{protos[0]}://{name}" - - @staticmethod - def _get_kwargs_from_urls(path): - """If kwargs can be encoded in the paths, extract them here - - This should happen before instantiation of the class; incoming paths - then should be amended to strip the options in methods. - - Examples may look like an sftp path "sftp://user@host:/my/path", where - the user and host should become kwargs and later get stripped. - """ - # by default, nothing happens - return {} - - @classmethod - def current(cls): - """Return the most recently instantiated FileSystem - - If no instance has been created, then create one with defaults - """ - inst = cls._cache.get(cls._latest) - if inst is not None: - return inst - return cls() - - @property - def transaction(self): - """A context within which files are committed together upon exit - - Requires the file class to implement `.commit()` and `.discard()` - for the normal and exception cases. - """ - if self._transaction is None: - self._transaction = self.transaction_type(self) - return self._transaction - - def start_transaction(self): - """Begin write transaction for deferring files, non-context version""" - self._intrans = True - self._transaction = self.transaction_type(self) - return self.transaction - - def end_transaction(self): - """Finish write transaction, non-context version""" - self.transaction.complete() - self._transaction = None - # The invalid cache must be cleared after the transaction is completed. - for path in self._invalidated_caches_in_transaction: - self.invalidate_cache(path) - self._invalidated_caches_in_transaction.clear() - - def invalidate_cache(self, path=None): - """ - Discard any cached directory information - - Parameters - ---------- - path: string or None - If None, clear all listings cached else listings at or under given - path. - """ - # Not necessary to implement invalidation mechanism, may have no cache. - # But if have, you should call this method of parent class from your - # subclass to ensure expiring caches after transacations correctly. - # See the implementation of FTPFileSystem in ftp.py - if self._intrans: - self._invalidated_caches_in_transaction.append(path) - - def mkdir(self, path, create_parents=True, **kwargs): - """ - Create directory entry at path - - For systems that don't have true directories, may create an for - this instance only and not touch the real filesystem - - Parameters - ---------- - path: str - location - create_parents: bool - if True, this is equivalent to ``makedirs`` - kwargs: - may be permissions, etc. - """ - pass # not necessary to implement, may not have directories - - def makedirs(self, path, exist_ok=False): - """Recursively make directories - - Creates directory at path and any intervening required directories. - Raises exception if, for instance, the path already exists but is a - file. - - Parameters - ---------- - path: str - leaf directory name - exist_ok: bool (False) - If False, will error if the target already exists - """ - pass # not necessary to implement, may not have directories - - def rmdir(self, path): - """Remove a directory, if empty""" - pass # not necessary to implement, may not have directories - - def ls(self, path, detail=True, **kwargs): - """List objects at path. - - This should include subdirectories and files at that location. The - difference between a file and a directory must be clear when details - are requested. - - The specific keys, or perhaps a FileInfo class, or similar, is TBD, - but must be consistent across implementations. - Must include: - - - full path to the entry (without protocol) - - size of the entry, in bytes. If the value cannot be determined, will - be ``None``. - - type of entry, "file", "directory" or other - - Additional information - may be present, appropriate to the file-system, e.g., generation, - checksum, etc. - - May use refresh=True|False to allow use of self._ls_from_cache to - check for a saved listing and avoid calling the backend. This would be - common where listing may be expensive. - - Parameters - ---------- - path: str - detail: bool - if True, gives a list of dictionaries, where each is the same as - the result of ``info(path)``. If False, gives a list of paths - (str). - kwargs: may have additional backend-specific options, such as version - information - - Returns - ------- - List of strings if detail is False, or list of directory information - dicts if detail is True. - """ - raise NotImplementedError - - def _ls_from_cache(self, path): - """Check cache for listing - - Returns listing, if found (may be empty list for a directly that exists - but contains nothing), None if not in cache. - """ - parent = self._parent(path) - try: - return self.dircache[path.rstrip("/")] - except KeyError: - pass - try: - files = [ - f - for f in self.dircache[parent] - if f["name"] == path - or (f["name"] == path.rstrip("/") and f["type"] == "directory") - ] - if len(files) == 0: - # parent dir was listed but did not contain this file - raise FileNotFoundError(path) - return files - except KeyError: - pass - - def walk(self, path, maxdepth=None, topdown=True, on_error="omit", **kwargs): - """Return all files under the given path. - - List all files, recursing into subdirectories; output is iterator-style, - like ``os.walk()``. For a simple list of files, ``find()`` is available. - - When topdown is True, the caller can modify the dirnames list in-place (perhaps - using del or slice assignment), and walk() will - only recurse into the subdirectories whose names remain in dirnames; - this can be used to prune the search, impose a specific order of visiting, - or even to inform walk() about directories the caller creates or renames before - it resumes walk() again. - Modifying dirnames when topdown is False has no effect. (see os.walk) - - Note that the "files" outputted will include anything that is not - a directory, such as links. - - Parameters - ---------- - path: str - Root to recurse into - maxdepth: int - Maximum recursion depth. None means limitless, but not recommended - on link-based file-systems. - topdown: bool (True) - Whether to walk the directory tree from the top downwards or from - the bottom upwards. - on_error: "omit", "raise", a callable - if omit (default), path with exception will simply be empty; - If raise, an underlying exception will be raised; - if callable, it will be called with a single OSError instance as argument - kwargs: passed to ``ls`` - """ - if maxdepth is not None and maxdepth < 1: - raise ValueError("maxdepth must be at least 1") - - path = self._strip_protocol(path) - full_dirs = {} - dirs = {} - files = {} - - detail = kwargs.pop("detail", False) - try: - listing = self.ls(path, detail=True, **kwargs) - except (FileNotFoundError, OSError) as e: - if on_error == "raise": - raise - if callable(on_error): - on_error(e) - return - - for info in listing: - # each info name must be at least [path]/part , but here - # we check also for names like [path]/part/ - pathname = info["name"].rstrip("/") - name = pathname.rsplit("/", 1)[-1] - if info["type"] == "directory" and pathname != path: - # do not include "self" path - full_dirs[name] = pathname - dirs[name] = info - elif pathname == path: - # file-like with same name as give path - files[""] = info - else: - files[name] = info - - if not detail: - dirs = list(dirs) - files = list(files) - - if topdown: - # Yield before recursion if walking top down - yield path, dirs, files - - if maxdepth is not None: - maxdepth -= 1 - if maxdepth < 1: - if not topdown: - yield path, dirs, files - return - - for d in dirs: - yield from self.walk( - full_dirs[d], - maxdepth=maxdepth, - detail=detail, - topdown=topdown, - **kwargs, - ) - - if not topdown: - # Yield after recursion if walking bottom up - yield path, dirs, files - - def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): - """List all files below path. - - Like posix ``find`` command without conditions - - Parameters - ---------- - path : str - maxdepth: int or None - If not None, the maximum number of levels to descend - withdirs: bool - Whether to include directory paths in the output. This is True - when used by glob, but users usually only want files. - kwargs are passed to ``ls``. - """ - # TODO: allow equivalent of -name parameter - path = self._strip_protocol(path) - out = {} - - # Add the root directory if withdirs is requested - # This is needed for posix glob compliance - if withdirs and path != "" and self.isdir(path): - out[path] = self.info(path) - - for _, dirs, files in self.walk(path, maxdepth, detail=True, **kwargs): - if withdirs: - files.update(dirs) - out.update({info["name"]: info for name, info in files.items()}) - if not out and self.isfile(path): - # walk works on directories, but find should also return [path] - # when path happens to be a file - out[path] = {} - names = sorted(out) - if not detail: - return names - else: - return {name: out[name] for name in names} - - def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs): - """Space used by files and optionally directories within a path - - Directory size does not include the size of its contents. - - Parameters - ---------- - path: str - total: bool - Whether to sum all the file sizes - maxdepth: int or None - Maximum number of directory levels to descend, None for unlimited. - withdirs: bool - Whether to include directory paths in the output. - kwargs: passed to ``find`` - - Returns - ------- - Dict of {path: size} if total=False, or int otherwise, where numbers - refer to bytes used. - """ - sizes = {} - if withdirs and self.isdir(path): - # Include top-level directory in output - info = self.info(path) - sizes[info["name"]] = info["size"] - for f in self.find(path, maxdepth=maxdepth, withdirs=withdirs, **kwargs): - info = self.info(f) - sizes[info["name"]] = info["size"] - if total: - return sum(sizes.values()) - else: - return sizes - - def glob(self, path, maxdepth=None, **kwargs): - """Find files by glob-matching. - - Pattern matching capabilities for finding files that match the given pattern. - - Parameters - ---------- - path: str - The glob pattern to match against - maxdepth: int or None - Maximum depth for ``'**'`` patterns. Applied on the first ``'**'`` found. - Must be at least 1 if provided. - kwargs: - Additional arguments passed to ``find`` (e.g., detail=True) - - Returns - ------- - List of matched paths, or dict of paths and their info if detail=True - - Notes - ----- - Supported patterns: - - '*': Matches any sequence of characters within a single directory level - - ``'**'``: Matches any number of directory levels (must be an entire path component) - - '?': Matches exactly one character - - '[abc]': Matches any character in the set - - '[a-z]': Matches any character in the range - - '[!abc]': Matches any character NOT in the set - - Special behaviors: - - If the path ends with '/', only folders are returned - - Consecutive '*' characters are compressed into a single '*' - - Empty set '[]' or negated empty negated set '[!]' never match anything - - Special characters in character classes are escaped properly - - Limitations: - - ``'**'`` must be a complete path component (e.g., ``'a/**/b'``, not ``'a**b'``) - - No brace expansion ('{a,b}.txt') - - No extended glob patterns ('+(pattern)', '!(pattern)') - """ - if maxdepth is not None and maxdepth < 1: - raise ValueError("maxdepth must be at least 1") - - import re - - seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,) - ends_with_sep = path.endswith(seps) # _strip_protocol strips trailing slash - path = self._strip_protocol(path) - append_slash_to_dirname = ends_with_sep or path.endswith( - tuple(sep + "**" for sep in seps) - ) - idx_star = path.find("*") if path.find("*") >= 0 else len(path) - idx_qmark = path.find("?") if path.find("?") >= 0 else len(path) - idx_brace = path.find("[") if path.find("[") >= 0 else len(path) - - min_idx = min(idx_star, idx_qmark, idx_brace) - - detail = kwargs.pop("detail", False) - withdirs = kwargs.pop("withdirs", True) - - if not has_magic(path): - if self.exists(path, **kwargs): - if not detail: - return [path] - else: - return {path: self.info(path, **kwargs)} - else: - if not detail: - return [] # glob of non-existent returns empty - else: - return {} - elif "/" in path[:min_idx]: - min_idx = path[:min_idx].rindex("/") - root = path[: min_idx + 1] - depth = path[min_idx + 1 :].count("/") + 1 - else: - root = "" - depth = path[min_idx + 1 :].count("/") + 1 - - if "**" in path: - if maxdepth is not None: - idx_double_stars = path.find("**") - depth_double_stars = path[idx_double_stars:].count("/") + 1 - depth = depth - depth_double_stars + maxdepth - else: - depth = None - - allpaths = self.find( - root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs - ) - - pattern = glob_translate(path + ("/" if ends_with_sep else "")) - pattern = re.compile(pattern) - - out = { - p: info - for p, info in sorted(allpaths.items()) - if pattern.match( - p + "/" - if append_slash_to_dirname and info["type"] == "directory" - else p - ) - } - - if detail: - return out - else: - return list(out) - - def exists(self, path, **kwargs): - """Is there a file at the given path""" - try: - self.info(path, **kwargs) - return True - except: # noqa: E722 - # any exception allowed bar FileNotFoundError? - return False - - def lexists(self, path, **kwargs): - """If there is a file at the given path (including - broken links)""" - return self.exists(path) - - def info(self, path, **kwargs): - """Give details of entry at path - - Returns a single dictionary, with exactly the same information as ``ls`` - would with ``detail=True``. - - The default implementation calls ls and could be overridden by a - shortcut. kwargs are passed on to ```ls()``. - - Some file systems might not be able to measure the file's size, in - which case, the returned dict will include ``'size': None``. - - Returns - ------- - dict with keys: name (full path in the FS), size (in bytes), type (file, - directory, or something else) and other FS-specific keys. - """ - path = self._strip_protocol(path) - out = self.ls(self._parent(path), detail=True, **kwargs) - out = [o for o in out if o["name"].rstrip("/") == path] - if out: - return out[0] - out = self.ls(path, detail=True, **kwargs) - path = path.rstrip("/") - out1 = [o for o in out if o["name"].rstrip("/") == path] - if len(out1) == 1: - if "size" not in out1[0]: - out1[0]["size"] = None - return out1[0] - elif len(out1) > 1 or out: - return {"name": path, "size": 0, "type": "directory"} - else: - raise FileNotFoundError(path) - - def checksum(self, path): - """Unique value for current version of file - - If the checksum is the same from one moment to another, the contents - are guaranteed to be the same. If the checksum changes, the contents - *might* have changed. - - This should normally be overridden; default will probably capture - creation/modification timestamp (which would be good) or maybe - access timestamp (which would be bad) - """ - return int(tokenize(self.info(path)), 16) - - def size(self, path): - """Size in bytes of file""" - return self.info(path).get("size", None) - - def sizes(self, paths): - """Size in bytes of each file in a list of paths""" - return [self.size(p) for p in paths] - - def isdir(self, path): - """Is this entry directory-like?""" - try: - return self.info(path)["type"] == "directory" - except OSError: - return False - - def isfile(self, path): - """Is this entry file-like?""" - try: - return self.info(path)["type"] == "file" - except: # noqa: E722 - return False - - def read_text(self, path, encoding=None, errors=None, newline=None, **kwargs): - """Get the contents of the file as a string. - - Parameters - ---------- - path: str - URL of file on this filesystems - encoding, errors, newline: same as `open`. - """ - with self.open( - path, - mode="r", - encoding=encoding, - errors=errors, - newline=newline, - **kwargs, - ) as f: - return f.read() - - def write_text( - self, path, value, encoding=None, errors=None, newline=None, **kwargs - ): - """Write the text to the given file. - - An existing file will be overwritten. - - Parameters - ---------- - path: str - URL of file on this filesystems - value: str - Text to write. - encoding, errors, newline: same as `open`. - """ - with self.open( - path, - mode="w", - encoding=encoding, - errors=errors, - newline=newline, - **kwargs, - ) as f: - return f.write(value) - - def cat_file(self, path, start=None, end=None, **kwargs): - """Get the content of a file - - Parameters - ---------- - path: URL of file on this filesystems - start, end: int - Bytes limits of the read. If negative, backwards from end, - like usual python slices. Either can be None for start or - end of file, respectively - kwargs: passed to ``open()``. - """ - # explicitly set buffering off? - with self.open(path, "rb", **kwargs) as f: - if start is not None: - if start >= 0: - f.seek(start) - else: - f.seek(max(0, f.size + start)) - if end is not None: - if end < 0: - end = f.size + end - return f.read(end - f.tell()) - return f.read() - - def pipe_file(self, path, value, mode="overwrite", **kwargs): - """Set the bytes of given file""" - if mode == "create" and self.exists(path): - # non-atomic but simple way; or could use "xb" in open(), which is likely - # not as well supported - raise FileExistsError - with self.open(path, "wb", **kwargs) as f: - f.write(value) - - def pipe(self, path, value=None, **kwargs): - """Put value into path - - (counterpart to ``cat``) - - Parameters - ---------- - path: string or dict(str, bytes) - If a string, a single remote location to put ``value`` bytes; if a dict, - a mapping of {path: bytesvalue}. - value: bytes, optional - If using a single path, these are the bytes to put there. Ignored if - ``path`` is a dict - """ - if isinstance(path, str): - self.pipe_file(self._strip_protocol(path), value, **kwargs) - elif isinstance(path, dict): - for k, v in path.items(): - self.pipe_file(self._strip_protocol(k), v, **kwargs) - else: - raise ValueError("path must be str or dict") - - def cat_ranges( - self, paths, starts, ends, max_gap=None, on_error="return", **kwargs - ): - """Get the contents of byte ranges from one or more files - - Parameters - ---------- - paths: list - A list of of filepaths on this filesystems - starts, ends: int or list - Bytes limits of the read. If using a single int, the same value will be - used to read all the specified files. - """ - if max_gap is not None: - raise NotImplementedError - if not isinstance(paths, list): - raise TypeError - if not isinstance(starts, list): - starts = [starts] * len(paths) - if not isinstance(ends, list): - ends = [ends] * len(paths) - if len(starts) != len(paths) or len(ends) != len(paths): - raise ValueError - out = [] - for p, s, e in zip(paths, starts, ends): - try: - out.append(self.cat_file(p, s, e, **kwargs)) - except Exception as e: - if on_error == "return": - out.append(e) - else: - raise - return out - - def cat(self, path, recursive=False, on_error="raise", **kwargs): - """Fetch (potentially multiple) paths' contents - - Parameters - ---------- - recursive: bool - If True, assume the path(s) are directories, and get all the - contained files - on_error : "raise", "omit", "return" - If raise, an underlying exception will be raised (converted to KeyError - if the type is in self.missing_exceptions); if omit, keys with exception - will simply not be included in the output; if "return", all keys are - included in the output, but the value will be bytes or an exception - instance. - kwargs: passed to cat_file - - Returns - ------- - dict of {path: contents} if there are multiple paths - or the path has been otherwise expanded - """ - paths = self.expand_path(path, recursive=recursive, **kwargs) - if ( - len(paths) > 1 - or isinstance(path, list) - or paths[0] != self._strip_protocol(path) - ): - out = {} - for path in paths: - try: - out[path] = self.cat_file(path, **kwargs) - except Exception as e: - if on_error == "raise": - raise - if on_error == "return": - out[path] = e - return out - else: - return self.cat_file(paths[0], **kwargs) - - def get_file(self, rpath, lpath, callback=DEFAULT_CALLBACK, outfile=None, **kwargs): - """Copy single remote file to local""" - from .implementations.local import LocalFileSystem - - if isfilelike(lpath): - outfile = lpath - elif self.isdir(rpath): - os.makedirs(lpath, exist_ok=True) - return None - - fs = LocalFileSystem(auto_mkdir=True) - fs.makedirs(fs._parent(lpath), exist_ok=True) - - with self.open(rpath, "rb", **kwargs) as f1: - if outfile is None: - outfile = open(lpath, "wb") - - try: - callback.set_size(getattr(f1, "size", None)) - data = True - while data: - data = f1.read(self.blocksize) - segment_len = outfile.write(data) - if segment_len is None: - segment_len = len(data) - callback.relative_update(segment_len) - finally: - if not isfilelike(lpath): - outfile.close() - - def get( - self, - rpath, - lpath, - recursive=False, - callback=DEFAULT_CALLBACK, - maxdepth=None, - **kwargs, - ): - """Copy file(s) to local. - - Copies a specific file or tree of files (if recursive=True). If lpath - ends with a "/", it will be assumed to be a directory, and target files - will go within. Can submit a list of paths, which may be glob-patterns - and will be expanded. - - Calls get_file for each source. - """ - if isinstance(lpath, list) and isinstance(rpath, list): - # No need to expand paths when both source and destination - # are provided as lists - rpaths = rpath - lpaths = lpath - else: - from .implementations.local import ( - LocalFileSystem, - make_path_posix, - trailing_sep, - ) - - source_is_str = isinstance(rpath, str) - rpaths = self.expand_path( - rpath, recursive=recursive, maxdepth=maxdepth, **kwargs - ) - if source_is_str and (not recursive or maxdepth is not None): - # Non-recursive glob does not copy directories - rpaths = [p for p in rpaths if not (trailing_sep(p) or self.isdir(p))] - if not rpaths: - return - - if isinstance(lpath, str): - lpath = make_path_posix(lpath) - - source_is_file = len(rpaths) == 1 - dest_is_dir = isinstance(lpath, str) and ( - trailing_sep(lpath) or LocalFileSystem().isdir(lpath) - ) - - exists = source_is_str and ( - (has_magic(rpath) and source_is_file) - or (not has_magic(rpath) and dest_is_dir and not trailing_sep(rpath)) - ) - lpaths = other_paths( - rpaths, - lpath, - exists=exists, - flatten=not source_is_str, - ) - - callback.set_size(len(lpaths)) - for lpath, rpath in callback.wrap(zip(lpaths, rpaths)): - with callback.branched(rpath, lpath) as child: - self.get_file(rpath, lpath, callback=child, **kwargs) - - def put_file( - self, lpath, rpath, callback=DEFAULT_CALLBACK, mode="overwrite", **kwargs - ): - """Copy single file to remote""" - if mode == "create" and self.exists(rpath): - raise FileExistsError - if os.path.isdir(lpath): - self.makedirs(rpath, exist_ok=True) - return None - - with open(lpath, "rb") as f1: - size = f1.seek(0, 2) - callback.set_size(size) - f1.seek(0) - - self.mkdirs(self._parent(os.fspath(rpath)), exist_ok=True) - with self.open(rpath, "wb", **kwargs) as f2: - while f1.tell() < size: - data = f1.read(self.blocksize) - segment_len = f2.write(data) - if segment_len is None: - segment_len = len(data) - callback.relative_update(segment_len) - - def put( - self, - lpath, - rpath, - recursive=False, - callback=DEFAULT_CALLBACK, - maxdepth=None, - **kwargs, - ): - """Copy file(s) from local. - - Copies a specific file or tree of files (if recursive=True). If rpath - ends with a "/", it will be assumed to be a directory, and target files - will go within. - - Calls put_file for each source. - """ - if isinstance(lpath, list) and isinstance(rpath, list): - # No need to expand paths when both source and destination - # are provided as lists - rpaths = rpath - lpaths = lpath - else: - from .implementations.local import ( - LocalFileSystem, - make_path_posix, - trailing_sep, - ) - - source_is_str = isinstance(lpath, str) - if source_is_str: - lpath = make_path_posix(lpath) - fs = LocalFileSystem() - lpaths = fs.expand_path( - lpath, recursive=recursive, maxdepth=maxdepth, **kwargs - ) - if source_is_str and (not recursive or maxdepth is not None): - # Non-recursive glob does not copy directories - lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))] - if not lpaths: - return - - source_is_file = len(lpaths) == 1 - dest_is_dir = isinstance(rpath, str) and ( - trailing_sep(rpath) or self.isdir(rpath) - ) - - rpath = ( - self._strip_protocol(rpath) - if isinstance(rpath, str) - else [self._strip_protocol(p) for p in rpath] - ) - exists = source_is_str and ( - (has_magic(lpath) and source_is_file) - or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath)) - ) - rpaths = other_paths( - lpaths, - rpath, - exists=exists, - flatten=not source_is_str, - ) - - callback.set_size(len(rpaths)) - for lpath, rpath in callback.wrap(zip(lpaths, rpaths)): - with callback.branched(lpath, rpath) as child: - self.put_file(lpath, rpath, callback=child, **kwargs) - - def head(self, path, size=1024): - """Get the first ``size`` bytes from file""" - with self.open(path, "rb") as f: - return f.read(size) - - def tail(self, path, size=1024): - """Get the last ``size`` bytes from file""" - with self.open(path, "rb") as f: - f.seek(max(-size, -f.size), 2) - return f.read() - - def cp_file(self, path1, path2, **kwargs): - raise NotImplementedError - - def copy( - self, path1, path2, recursive=False, maxdepth=None, on_error=None, **kwargs - ): - """Copy within two locations in the filesystem - - on_error : "raise", "ignore" - If raise, any not-found exceptions will be raised; if ignore any - not-found exceptions will cause the path to be skipped; defaults to - raise unless recursive is true, where the default is ignore - """ - if on_error is None and recursive: - on_error = "ignore" - elif on_error is None: - on_error = "raise" - - if isinstance(path1, list) and isinstance(path2, list): - # No need to expand paths when both source and destination - # are provided as lists - paths1 = path1 - paths2 = path2 - else: - from .implementations.local import trailing_sep - - source_is_str = isinstance(path1, str) - paths1 = self.expand_path( - path1, recursive=recursive, maxdepth=maxdepth, **kwargs - ) - if source_is_str and (not recursive or maxdepth is not None): - # Non-recursive glob does not copy directories - paths1 = [p for p in paths1 if not (trailing_sep(p) or self.isdir(p))] - if not paths1: - return - - source_is_file = len(paths1) == 1 - dest_is_dir = isinstance(path2, str) and ( - trailing_sep(path2) or self.isdir(path2) - ) - - exists = source_is_str and ( - (has_magic(path1) and source_is_file) - or (not has_magic(path1) and dest_is_dir and not trailing_sep(path1)) - ) - paths2 = other_paths( - paths1, - path2, - exists=exists, - flatten=not source_is_str, - ) - - for p1, p2 in zip(paths1, paths2): - try: - self.cp_file(p1, p2, **kwargs) - except FileNotFoundError: - if on_error == "raise": - raise - - def expand_path( - self, path, recursive=False, maxdepth=None, assume_literal=False, **kwargs - ): - """Turn one or more globs or directories into a list of all matching paths - to files or directories. - - kwargs are passed to ``glob`` or ``find``, which may in turn call ``ls`` - """ - - if maxdepth is not None and maxdepth < 1: - raise ValueError("maxdepth must be at least 1") - - if isinstance(path, (str, os.PathLike)): - out = self.expand_path([path], recursive, maxdepth, **kwargs) - else: - out = set() - path = [self._strip_protocol(p) for p in path] - for p in path: - if not assume_literal and has_magic(p): - bit = set(self.glob(p, maxdepth=maxdepth, **kwargs)) - out |= bit - if recursive: - # glob call above expanded one depth so if maxdepth is defined - # then decrement it in expand_path call below. If it is zero - # after decrementing then avoid expand_path call. - if maxdepth is not None and maxdepth <= 1: - continue - out |= set( - self.expand_path( - list(bit), - recursive=recursive, - maxdepth=maxdepth - 1 if maxdepth is not None else None, - assume_literal=True, - **kwargs, - ) - ) - continue - elif recursive: - rec = set( - self.find( - p, maxdepth=maxdepth, withdirs=True, detail=False, **kwargs - ) - ) - out |= rec - if p not in out and (recursive is False or self.exists(p)): - # should only check once, for the root - out.add(p) - if not out: - raise FileNotFoundError(path) - return sorted(out) - - def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs): - """Move file(s) from one location to another""" - if path1 == path2: - logger.debug("%s mv: The paths are the same, so no files were moved.", self) - else: - # explicitly raise exception to prevent data corruption - self.copy( - path1, path2, recursive=recursive, maxdepth=maxdepth, on_error="raise" - ) - self.rm(path1, recursive=recursive) - - def rm_file(self, path): - """Delete a file""" - self._rm(path) - - def _rm(self, path): - """Delete one file""" - # this is the old name for the method, prefer rm_file - raise NotImplementedError - - def rm(self, path, recursive=False, maxdepth=None): - """Delete files. - - Parameters - ---------- - path: str or list of str - File(s) to delete. - recursive: bool - If file(s) are directories, recursively delete contents and then - also remove the directory - maxdepth: int or None - Depth to pass to walk for finding files to delete, if recursive. - If None, there will be no limit and infinite recursion may be - possible. - """ - path = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) - for p in reversed(path): - self.rm_file(p) - - @classmethod - def _parent(cls, path): - path = cls._strip_protocol(path) - if "/" in path: - parent = path.rsplit("/", 1)[0].lstrip(cls.root_marker) - return cls.root_marker + parent - else: - return cls.root_marker - - def _open( - self, - path, - mode="rb", - block_size=None, - autocommit=True, - cache_options=None, - **kwargs, - ): - """Return raw bytes-mode file-like from the file-system""" - return AbstractBufferedFile( - self, - path, - mode, - block_size, - autocommit, - cache_options=cache_options, - **kwargs, - ) - - def open( - self, - path, - mode="rb", - block_size=None, - cache_options=None, - compression=None, - **kwargs, - ): - """ - Return a file-like object from the filesystem - - The resultant instance must function correctly in a context ``with`` - block. - - Parameters - ---------- - path: str - Target file - mode: str like 'rb', 'w' - See builtin ``open()`` - Mode "x" (exclusive write) may be implemented by the backend. Even if - it is, whether it is checked up front or on commit, and whether it is - atomic is implementation-dependent. - block_size: int - Some indication of buffering - this is a value in bytes - cache_options : dict, optional - Extra arguments to pass through to the cache. - compression: string or None - If given, open file using compression codec. Can either be a compression - name (a key in ``fsspec.compression.compr``) or "infer" to guess the - compression from the filename suffix. - encoding, errors, newline: passed on to TextIOWrapper for text mode - """ - import io - - path = self._strip_protocol(path) - if "b" not in mode: - mode = mode.replace("t", "") + "b" - - text_kwargs = { - k: kwargs.pop(k) - for k in ["encoding", "errors", "newline"] - if k in kwargs - } - return io.TextIOWrapper( - self.open( - path, - mode, - block_size=block_size, - cache_options=cache_options, - compression=compression, - **kwargs, - ), - **text_kwargs, - ) - else: - ac = kwargs.pop("autocommit", not self._intrans) - f = self._open( - path, - mode=mode, - block_size=block_size, - autocommit=ac, - cache_options=cache_options, - **kwargs, - ) - if compression is not None: - from fsspec.compression import compr - from fsspec.core import get_compression - - compression = get_compression(path, compression) - compress = compr[compression] - f = compress(f, mode=mode[0]) - - if not ac and "r" not in mode: - self.transaction.files.append(f) - return f - - def touch(self, path, truncate=True, **kwargs): - """Create empty file, or update timestamp - - Parameters - ---------- - path: str - file location - truncate: bool - If True, always set file size to 0; if False, update timestamp and - leave file unchanged, if backend allows this - """ - if truncate or not self.exists(path): - with self.open(path, "wb", **kwargs): - pass - else: - raise NotImplementedError # update timestamp, if possible - - def ukey(self, path): - """Hash of file properties, to tell if it has changed""" - return sha256(str(self.info(path)).encode()).hexdigest() - - def read_block(self, fn, offset, length, delimiter=None): - """Read a block of bytes from - - Starting at ``offset`` of the file, read ``length`` bytes. If - ``delimiter`` is set then we ensure that the read starts and stops at - delimiter boundaries that follow the locations ``offset`` and ``offset - + length``. If ``offset`` is zero then we start at zero. The - bytestring returned WILL include the end delimiter string. - - If offset+length is beyond the eof, reads to eof. - - Parameters - ---------- - fn: string - Path to filename - offset: int - Byte offset to start read - length: int - Number of bytes to read. If None, read to end. - delimiter: bytes (optional) - Ensure reading starts and stops at delimiter bytestring - - Examples - -------- - >>> fs.read_block('data/file.csv', 0, 13) # doctest: +SKIP - b'Alice, 100\\nBo' - >>> fs.read_block('data/file.csv', 0, 13, delimiter=b'\\n') # doctest: +SKIP - b'Alice, 100\\nBob, 200\\n' - - Use ``length=None`` to read to the end of the file. - >>> fs.read_block('data/file.csv', 0, None, delimiter=b'\\n') # doctest: +SKIP - b'Alice, 100\\nBob, 200\\nCharlie, 300' - - See Also - -------- - :func:`fsspec.utils.read_block` - """ - with self.open(fn, "rb") as f: - size = f.size - if length is None: - length = size - if size is not None and offset + length > size: - length = size - offset - return read_block(f, offset, length, delimiter) - - def to_json(self, *, include_password: bool = True) -> str: - """ - JSON representation of this filesystem instance. - - Parameters - ---------- - include_password: bool, default True - Whether to include the password (if any) in the output. - - Returns - ------- - JSON string with keys ``cls`` (the python location of this class), - protocol (text name of this class's protocol, first one in case of - multiple), ``args`` (positional args, usually empty), and all other - keyword arguments as their own keys. - - Warnings - -------- - Serialized filesystems may contain sensitive information which have been - passed to the constructor, such as passwords and tokens. Make sure you - store and send them in a secure environment! - """ - from .json import FilesystemJSONEncoder - - return json.dumps( - self, - cls=type( - "_FilesystemJSONEncoder", - (FilesystemJSONEncoder,), - {"include_password": include_password}, - ), - ) - - @staticmethod - def from_json(blob: str) -> AbstractFileSystem: - """ - Recreate a filesystem instance from JSON representation. - - See ``.to_json()`` for the expected structure of the input. - - Parameters - ---------- - blob: str - - Returns - ------- - file system instance, not necessarily of this particular class. - - Warnings - -------- - This can import arbitrary modules (as determined by the ``cls`` key). - Make sure you haven't installed any modules that may execute malicious code - at import time. - """ - from .json import FilesystemJSONDecoder - - return json.loads(blob, cls=FilesystemJSONDecoder) - - def to_dict(self, *, include_password: bool = True) -> dict[str, Any]: - """ - JSON-serializable dictionary representation of this filesystem instance. - - Parameters - ---------- - include_password: bool, default True - Whether to include the password (if any) in the output. - - Returns - ------- - Dictionary with keys ``cls`` (the python location of this class), - protocol (text name of this class's protocol, first one in case of - multiple), ``args`` (positional args, usually empty), and all other - keyword arguments as their own keys. - - Warnings - -------- - Serialized filesystems may contain sensitive information which have been - passed to the constructor, such as passwords and tokens. Make sure you - store and send them in a secure environment! - """ - from .json import FilesystemJSONEncoder - - json_encoder = FilesystemJSONEncoder() - - cls = type(self) - proto = self.protocol - - storage_options = dict(self.storage_options) - if not include_password: - storage_options.pop("password", None) - - return dict( - cls=f"{cls.__module__}:{cls.__name__}", - protocol=proto[0] if isinstance(proto, (tuple, list)) else proto, - args=json_encoder.make_serializable(self.storage_args), - **json_encoder.make_serializable(storage_options), - ) - - @staticmethod - def from_dict(dct: dict[str, Any]) -> AbstractFileSystem: - """ - Recreate a filesystem instance from dictionary representation. - - See ``.to_dict()`` for the expected structure of the input. - - Parameters - ---------- - dct: Dict[str, Any] - - Returns - ------- - file system instance, not necessarily of this particular class. - - Warnings - -------- - This can import arbitrary modules (as determined by the ``cls`` key). - Make sure you haven't installed any modules that may execute malicious code - at import time. - """ - from .json import FilesystemJSONDecoder - - json_decoder = FilesystemJSONDecoder() - - dct = dict(dct) # Defensive copy - - cls = FilesystemJSONDecoder.try_resolve_fs_cls(dct) - if cls is None: - raise ValueError("Not a serialized AbstractFileSystem") - - dct.pop("cls", None) - dct.pop("protocol", None) - - return cls( - *json_decoder.unmake_serializable(dct.pop("args", ())), - **json_decoder.unmake_serializable(dct), - ) - - def _get_pyarrow_filesystem(self): - """ - Make a version of the FS instance which will be acceptable to pyarrow - """ - # all instances already also derive from pyarrow - return self - - def get_mapper(self, root="", check=False, create=False, missing_exceptions=None): - """Create key/value store based on this file-system - - Makes a MutableMapping interface to the FS at the given root path. - See ``fsspec.mapping.FSMap`` for further details. - """ - from .mapping import FSMap - - return FSMap( - root, - self, - check=check, - create=create, - missing_exceptions=missing_exceptions, - ) - - @classmethod - def clear_instance_cache(cls): - """ - Clear the cache of filesystem instances. - - Notes - ----- - Unless overridden by setting the ``cachable`` class attribute to False, - the filesystem class stores a reference to newly created instances. This - prevents Python's normal rules around garbage collection from working, - since the instances refcount will not drop to zero until - ``clear_instance_cache`` is called. - """ - cls._cache.clear() - - def created(self, path): - """Return the created timestamp of a file as a datetime.datetime""" - raise NotImplementedError - - def modified(self, path): - """Return the modified timestamp of a file as a datetime.datetime""" - raise NotImplementedError - - def tree( - self, - path: str = "/", - recursion_limit: int = 2, - max_display: int = 25, - display_size: bool = False, - prefix: str = "", - is_last: bool = True, - first: bool = True, - indent_size: int = 4, - ) -> str: - """ - Return a tree-like structure of the filesystem starting from the given path as a string. - - Parameters - ---------- - path: Root path to start traversal from - recursion_limit: Maximum depth of directory traversal - max_display: Maximum number of items to display per directory - display_size: Whether to display file sizes - prefix: Current line prefix for visual tree structure - is_last: Whether current item is last in its level - first: Whether this is the first call (displays root path) - indent_size: Number of spaces by indent - - Returns - ------- - str: A string representing the tree structure. - - Example - ------- - >>> from fsspec import filesystem - - >>> fs = filesystem('ftp', host='test.rebex.net', user='demo', password='password') - >>> tree = fs.tree(display_size=True, recursion_limit=3, indent_size=8, max_display=10) - >>> print(tree) - """ - - def format_bytes(n: int) -> str: - """Format bytes as text.""" - for prefix, k in ( - ("P", 2**50), - ("T", 2**40), - ("G", 2**30), - ("M", 2**20), - ("k", 2**10), - ): - if n >= 0.9 * k: - return f"{n / k:.2f} {prefix}b" - return f"{n}B" - - result = [] - - if first: - result.append(path) - - if recursion_limit: - indent = " " * indent_size - contents = self.ls(path, detail=True) - contents.sort( - key=lambda x: (x.get("type") != "directory", x.get("name", "")) - ) - - if max_display is not None and len(contents) > max_display: - displayed_contents = contents[:max_display] - remaining_count = len(contents) - max_display - else: - displayed_contents = contents - remaining_count = 0 - - for i, item in enumerate(displayed_contents): - is_last_item = (i == len(displayed_contents) - 1) and ( - remaining_count == 0 - ) - - branch = ( - "└" + ("─" * (indent_size - 2)) - if is_last_item - else "├" + ("─" * (indent_size - 2)) - ) - branch += " " - new_prefix = prefix + ( - indent if is_last_item else "│" + " " * (indent_size - 1) - ) - - name = os.path.basename(item.get("name", "")) - - if display_size and item.get("type") == "directory": - sub_contents = self.ls(item.get("name", ""), detail=True) - num_files = sum( - 1 for sub_item in sub_contents if sub_item.get("type") == "file" - ) - num_folders = sum( - 1 - for sub_item in sub_contents - if sub_item.get("type") == "directory" - ) - - if num_files == 0 and num_folders == 0: - size = " (empty folder)" - elif num_files == 0: - size = f" ({num_folders} subfolder{'s' if num_folders > 1 else ''})" - elif num_folders == 0: - size = f" ({num_files} file{'s' if num_files > 1 else ''})" - else: - size = f" ({num_files} file{'s' if num_files > 1 else ''}, {num_folders} subfolder{'s' if num_folders > 1 else ''})" - elif display_size and item.get("type") == "file": - size = f" ({format_bytes(item.get('size', 0))})" - else: - size = "" - - result.append(f"{prefix}{branch}{name}{size}") - - if item.get("type") == "directory" and recursion_limit > 0: - result.append( - self.tree( - path=item.get("name", ""), - recursion_limit=recursion_limit - 1, - max_display=max_display, - display_size=display_size, - prefix=new_prefix, - is_last=is_last_item, - first=False, - indent_size=indent_size, - ) - ) - - if remaining_count > 0: - more_message = f"{remaining_count} more item(s) not displayed." - result.append( - f"{prefix}{'└' + ('─' * (indent_size - 2))} {more_message}" - ) - - return "\n".join(_ for _ in result if _) - - # ------------------------------------------------------------------------ - # Aliases - - def read_bytes(self, path, start=None, end=None, **kwargs): - """Alias of `AbstractFileSystem.cat_file`.""" - return self.cat_file(path, start=start, end=end, **kwargs) - - def write_bytes(self, path, value, **kwargs): - """Alias of `AbstractFileSystem.pipe_file`.""" - self.pipe_file(path, value, **kwargs) - - def makedir(self, path, create_parents=True, **kwargs): - """Alias of `AbstractFileSystem.mkdir`.""" - return self.mkdir(path, create_parents=create_parents, **kwargs) - - def mkdirs(self, path, exist_ok=False): - """Alias of `AbstractFileSystem.makedirs`.""" - return self.makedirs(path, exist_ok=exist_ok) - - def listdir(self, path, detail=True, **kwargs): - """Alias of `AbstractFileSystem.ls`.""" - return self.ls(path, detail=detail, **kwargs) - - def cp(self, path1, path2, **kwargs): - """Alias of `AbstractFileSystem.copy`.""" - return self.copy(path1, path2, **kwargs) - - def move(self, path1, path2, **kwargs): - """Alias of `AbstractFileSystem.mv`.""" - return self.mv(path1, path2, **kwargs) - - def stat(self, path, **kwargs): - """Alias of `AbstractFileSystem.info`.""" - return self.info(path, **kwargs) - - def disk_usage(self, path, total=True, maxdepth=None, **kwargs): - """Alias of `AbstractFileSystem.du`.""" - return self.du(path, total=total, maxdepth=maxdepth, **kwargs) - - def rename(self, path1, path2, **kwargs): - """Alias of `AbstractFileSystem.mv`.""" - return self.mv(path1, path2, **kwargs) - - def delete(self, path, recursive=False, maxdepth=None): - """Alias of `AbstractFileSystem.rm`.""" - return self.rm(path, recursive=recursive, maxdepth=maxdepth) - - def upload(self, lpath, rpath, recursive=False, **kwargs): - """Alias of `AbstractFileSystem.put`.""" - return self.put(lpath, rpath, recursive=recursive, **kwargs) - - def download(self, rpath, lpath, recursive=False, **kwargs): - """Alias of `AbstractFileSystem.get`.""" - return self.get(rpath, lpath, recursive=recursive, **kwargs) - - def sign(self, path, expiration=100, **kwargs): - """Create a signed URL representing the given path - - Some implementations allow temporary URLs to be generated, as a - way of delegating credentials. - - Parameters - ---------- - path : str - The path on the filesystem - expiration : int - Number of seconds to enable the URL for (if supported) - - Returns - ------- - URL : str - The signed URL - - Raises - ------ - NotImplementedError : if method is not implemented for a filesystem - """ - raise NotImplementedError("Sign is not implemented for this filesystem") - - def _isfilestore(self): - # Originally inherited from pyarrow DaskFileSystem. Keeping this - # here for backwards compatibility as long as pyarrow uses its - # legacy fsspec-compatible filesystems and thus accepts fsspec - # filesystems as well - return False - - -class AbstractBufferedFile(io.IOBase): - """Convenient class to derive from to provide buffering - - In the case that the backend does not provide a pythonic file-like object - already, this class contains much of the logic to build one. The only - methods that need to be overridden are ``_upload_chunk``, - ``_initiate_upload`` and ``_fetch_range``. - """ - - DEFAULT_BLOCK_SIZE = 5 * 2**20 - _details = None - - def __init__( - self, - fs, - path, - mode="rb", - block_size="default", - autocommit=True, - cache_type="readahead", - cache_options=None, - size=None, - **kwargs, - ): - """ - Template for files with buffered reading and writing - - Parameters - ---------- - fs: instance of FileSystem - path: str - location in file-system - mode: str - Normal file modes. Currently only 'wb', 'ab' or 'rb'. Some file - systems may be read-only, and some may not support append. - block_size: int - Buffer size for reading or writing, 'default' for class default - autocommit: bool - Whether to write to final destination; may only impact what - happens when file is being closed. - cache_type: {"readahead", "none", "mmap", "bytes"}, default "readahead" - Caching policy in read mode. See the definitions in ``core``. - cache_options : dict - Additional options passed to the constructor for the cache specified - by `cache_type`. - size: int - If given and in read mode, suppressed having to look up the file size - kwargs: - Gets stored as self.kwargs - """ - from .core import caches - - self.path = path - self.fs = fs - self.mode = mode - self.blocksize = ( - self.DEFAULT_BLOCK_SIZE if block_size in ["default", None] else block_size - ) - self.loc = 0 - self.autocommit = autocommit - self.end = None - self.start = None - self.closed = False - - if cache_options is None: - cache_options = {} - - if "trim" in kwargs: - warnings.warn( - "Passing 'trim' to control the cache behavior has been deprecated. " - "Specify it within the 'cache_options' argument instead.", - FutureWarning, - ) - cache_options["trim"] = kwargs.pop("trim") - - self.kwargs = kwargs - - if mode not in {"ab", "rb", "wb", "xb"}: - raise NotImplementedError("File mode not supported") - if mode == "rb": - if size is not None: - self.size = size - else: - self.size = self.details["size"] - self.cache = caches[cache_type]( - self.blocksize, self._fetch_range, self.size, **cache_options - ) - else: - self.buffer = io.BytesIO() - self.offset = None - self.forced = False - self.location = None - - @property - def details(self): - if self._details is None: - self._details = self.fs.info(self.path) - return self._details - - @details.setter - def details(self, value): - self._details = value - self.size = value["size"] - - @property - def full_name(self): - return _unstrip_protocol(self.path, self.fs) - - @property - def closed(self): - # get around this attr being read-only in IOBase - # use getattr here, since this can be called during del - return getattr(self, "_closed", True) - - @closed.setter - def closed(self, c): - self._closed = c - - def __hash__(self): - if "w" in self.mode: - return id(self) - else: - return int(tokenize(self.details), 16) - - def __eq__(self, other): - """Files are equal if they have the same checksum, only in read mode""" - if self is other: - return True - return ( - isinstance(other, type(self)) - and self.mode == "rb" - and other.mode == "rb" - and hash(self) == hash(other) - ) - - def commit(self): - """Move from temp to final destination""" - - def discard(self): - """Throw away temporary file""" - - def info(self): - """File information about this path""" - if self.readable(): - return self.details - else: - raise ValueError("Info not available while writing") - - def tell(self): - """Current file location""" - return self.loc - - def seek(self, loc, whence=0): - """Set current file location - - Parameters - ---------- - loc: int - byte location - whence: {0, 1, 2} - from start of file, current location or end of file, resp. - """ - loc = int(loc) - if not self.mode == "rb": - raise OSError(ESPIPE, "Seek only available in read mode") - if whence == 0: - nloc = loc - elif whence == 1: - nloc = self.loc + loc - elif whence == 2: - nloc = self.size + loc - else: - raise ValueError(f"invalid whence ({whence}, should be 0, 1 or 2)") - if nloc < 0: - raise ValueError("Seek before start of file") - self.loc = nloc - return self.loc - - def write(self, data): - """ - Write data to buffer. - - Buffer only sent on flush() or if buffer is greater than - or equal to blocksize. - - Parameters - ---------- - data: bytes - Set of bytes to be written. - """ - if not self.writable(): - raise ValueError("File not in write mode") - if self.closed: - raise ValueError("I/O operation on closed file.") - if self.forced: - raise ValueError("This file has been force-flushed, can only close") - out = self.buffer.write(data) - self.loc += out - if self.buffer.tell() >= self.blocksize: - self.flush() - return out - - def flush(self, force=False): - """ - Write buffered data to backend store. - - Writes the current buffer, if it is larger than the block-size, or if - the file is being closed. - - Parameters - ---------- - force: bool - When closing, write the last block even if it is smaller than - blocks are allowed to be. Disallows further writing to this file. - """ - - if self.closed: - raise ValueError("Flush on closed file") - if force and self.forced: - raise ValueError("Force flush cannot be called more than once") - if force: - self.forced = True - - if self.readable(): - # no-op to flush on read-mode - return - - if not force and self.buffer.tell() < self.blocksize: - # Defer write on small block - return - - if self.offset is None: - # Initialize a multipart upload - self.offset = 0 - try: - self._initiate_upload() - except Exception: - self.closed = True - raise - - if self._upload_chunk(final=force) is not False: - self.offset += self.buffer.seek(0, 2) - self.buffer = io.BytesIO() - - def _upload_chunk(self, final=False): - """Write one part of a multi-block file upload - - Parameters - ========== - final: bool - This is the last block, so should complete file, if - self.autocommit is True. - """ - # may not yet have been initialized, may need to call _initialize_upload - - def _initiate_upload(self): - """Create remote file/upload""" - pass - - def _fetch_range(self, start, end): - """Get the specified set of bytes from remote""" - return self.fs.cat_file(self.path, start=start, end=end) - - def read(self, length=-1): - """ - Return data from cache, or fetch pieces as necessary - - Parameters - ---------- - length: int (-1) - Number of bytes to read; if <0, all remaining bytes. - """ - length = -1 if length is None else int(length) - if self.mode != "rb": - raise ValueError("File not in read mode") - if length < 0: - length = self.size - self.loc - if self.closed: - raise ValueError("I/O operation on closed file.") - if length == 0: - # don't even bother calling fetch - return b"" - out = self.cache._fetch(self.loc, self.loc + length) - - logger.debug( - "%s read: %i - %i %s", - self, - self.loc, - self.loc + length, - self.cache._log_stats(), - ) - self.loc += len(out) - return out - - def readinto(self, b): - """mirrors builtin file's readinto method - - https://docs.python.org/3/library/io.html#io.RawIOBase.readinto - """ - out = memoryview(b).cast("B") - data = self.read(out.nbytes) - out[: len(data)] = data - return len(data) - - def readuntil(self, char=b"\n", blocks=None): - """Return data between current position and first occurrence of char - - char is included in the output, except if the end of the tile is - encountered first. - - Parameters - ---------- - char: bytes - Thing to find - blocks: None or int - How much to read in each go. Defaults to file blocksize - which may - mean a new read on every call. - """ - out = [] - while True: - start = self.tell() - part = self.read(blocks or self.blocksize) - if len(part) == 0: - break - found = part.find(char) - if found > -1: - out.append(part[: found + len(char)]) - self.seek(start + found + len(char)) - break - out.append(part) - return b"".join(out) - - def readline(self): - """Read until and including the first occurrence of newline character - - Note that, because of character encoding, this is not necessarily a - true line ending. - """ - return self.readuntil(b"\n") - - def __next__(self): - out = self.readline() - if out: - return out - raise StopIteration - - def __iter__(self): - return self - - def readlines(self): - """Return all data, split by the newline character, including the newline character""" - data = self.read() - lines = data.split(b"\n") - out = [l + b"\n" for l in lines[:-1]] - if data.endswith(b"\n"): - return out - else: - return out + [lines[-1]] - # return list(self) ??? - - def readinto1(self, b): - return self.readinto(b) - - def close(self): - """Close file - - Finalizes writes, discards cache - """ - if getattr(self, "_unclosable", False): - return - if self.closed: - return - try: - if self.mode == "rb": - cache = getattr(self, "cache", None) - if cache is not None: - close = getattr(cache, "close", None) - if callable(close): - close() - self.cache = None - else: - if not getattr(self, "forced", True): - self.flush(force=True) - - if self.fs is not None: - self.fs.invalidate_cache(self.path) - self.fs.invalidate_cache(self.fs._parent(self.path)) - finally: - self.closed = True - - def readable(self): - """Whether opened for reading""" - return "r" in self.mode and not self.closed - - def seekable(self): - """Whether is seekable (only in read mode)""" - return self.readable() - - def writable(self): - """Whether opened for writing""" - return self.mode in {"wb", "ab", "xb"} and not self.closed - - def __reduce__(self): - if self.mode != "rb": - raise RuntimeError("Pickling a writeable file is not supported") - - return reopen, ( - self.fs, - self.path, - self.mode, - self.blocksize, - self.loc, - self.size, - self.autocommit, - self.cache.name if self.cache else "none", - self.kwargs, - ) - - def __del__(self): - if not self.closed: - self.close() - - def __str__(self): - return f"" - - __repr__ = __str__ - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - -def reopen(fs, path, mode, blocksize, loc, size, autocommit, cache_type, kwargs): - file = fs.open( - path, - mode=mode, - block_size=blocksize, - autocommit=autocommit, - cache_type=cache_type, - size=size, - **kwargs, - ) - if loc > 0: - file.seek(loc) - return file diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/__init__.py b/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/__init__.py deleted file mode 100644 index f5414dfd4239b27fd9a7c3946cf82d889d45106e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/__init__.py +++ /dev/null @@ -1,290 +0,0 @@ -import os -from hashlib import md5 - -import pytest - -from fsspec.implementations.local import LocalFileSystem -from fsspec.tests.abstract.copy import AbstractCopyTests # noqa: F401 -from fsspec.tests.abstract.get import AbstractGetTests # noqa: F401 -from fsspec.tests.abstract.open import AbstractOpenTests # noqa: F401 -from fsspec.tests.abstract.pipe import AbstractPipeTests # noqa: F401 -from fsspec.tests.abstract.put import AbstractPutTests # noqa: F401 - - -class BaseAbstractFixtures: - """ - Abstract base class containing fixtures that are used by but never need to - be overridden in derived filesystem-specific classes to run the abstract - tests on such filesystems. - """ - - @pytest.fixture - def fs_bulk_operations_scenario_0(self, fs, fs_join, fs_path): - """ - Scenario on remote filesystem that is used for many cp/get/put tests. - - Cleans up at the end of each test it which it is used. - """ - source = self._bulk_operations_scenario_0(fs, fs_join, fs_path) - yield source - fs.rm(source, recursive=True) - - @pytest.fixture - def fs_glob_edge_cases_files(self, fs, fs_join, fs_path): - """ - Scenario on remote filesystem that is used for glob edge cases cp/get/put tests. - - Cleans up at the end of each test it which it is used. - """ - source = self._glob_edge_cases_files(fs, fs_join, fs_path) - yield source - fs.rm(source, recursive=True) - - @pytest.fixture - def fs_dir_and_file_with_same_name_prefix(self, fs, fs_join, fs_path): - """ - Scenario on remote filesystem that is used to check cp/get/put on directory - and file with the same name prefixes. - - Cleans up at the end of each test it which it is used. - """ - source = self._dir_and_file_with_same_name_prefix(fs, fs_join, fs_path) - yield source - fs.rm(source, recursive=True) - - @pytest.fixture - def fs_10_files_with_hashed_names(self, fs, fs_join, fs_path): - """ - Scenario on remote filesystem that is used to check cp/get/put files order - when source and destination are lists. - - Cleans up at the end of each test it which it is used. - """ - source = self._10_files_with_hashed_names(fs, fs_join, fs_path) - yield source - fs.rm(source, recursive=True) - - @pytest.fixture - def fs_target(self, fs, fs_join, fs_path): - """ - Return name of remote directory that does not yet exist to copy into. - - Cleans up at the end of each test it which it is used. - """ - target = fs_join(fs_path, "target") - yield target - if fs.exists(target): - fs.rm(target, recursive=True) - - @pytest.fixture - def local_bulk_operations_scenario_0(self, local_fs, local_join, local_path): - """ - Scenario on local filesystem that is used for many cp/get/put tests. - - Cleans up at the end of each test it which it is used. - """ - source = self._bulk_operations_scenario_0(local_fs, local_join, local_path) - yield source - local_fs.rm(source, recursive=True) - - @pytest.fixture - def local_glob_edge_cases_files(self, local_fs, local_join, local_path): - """ - Scenario on local filesystem that is used for glob edge cases cp/get/put tests. - - Cleans up at the end of each test it which it is used. - """ - source = self._glob_edge_cases_files(local_fs, local_join, local_path) - yield source - local_fs.rm(source, recursive=True) - - @pytest.fixture - def local_dir_and_file_with_same_name_prefix( - self, local_fs, local_join, local_path - ): - """ - Scenario on local filesystem that is used to check cp/get/put on directory - and file with the same name prefixes. - - Cleans up at the end of each test it which it is used. - """ - source = self._dir_and_file_with_same_name_prefix( - local_fs, local_join, local_path - ) - yield source - local_fs.rm(source, recursive=True) - - @pytest.fixture - def local_10_files_with_hashed_names(self, local_fs, local_join, local_path): - """ - Scenario on local filesystem that is used to check cp/get/put files order - when source and destination are lists. - - Cleans up at the end of each test it which it is used. - """ - source = self._10_files_with_hashed_names(local_fs, local_join, local_path) - yield source - local_fs.rm(source, recursive=True) - - @pytest.fixture - def local_target(self, local_fs, local_join, local_path): - """ - Return name of local directory that does not yet exist to copy into. - - Cleans up at the end of each test it which it is used. - """ - target = local_join(local_path, "target") - yield target - if local_fs.exists(target): - local_fs.rm(target, recursive=True) - - def _glob_edge_cases_files(self, some_fs, some_join, some_path): - """ - Scenario that is used for glob edge cases cp/get/put tests. - Creates the following directory and file structure: - - 📁 source - ├── 📄 file1 - ├── 📄 file2 - ├── 📁 subdir0 - │ ├── 📄 subfile1 - │ ├── 📄 subfile2 - │ └── 📁 nesteddir - │ └── 📄 nestedfile - └── 📁 subdir1 - ├── 📄 subfile1 - ├── 📄 subfile2 - └── 📁 nesteddir - └── 📄 nestedfile - """ - source = some_join(some_path, "source") - some_fs.touch(some_join(source, "file1")) - some_fs.touch(some_join(source, "file2")) - - for subdir_idx in range(2): - subdir = some_join(source, f"subdir{subdir_idx}") - nesteddir = some_join(subdir, "nesteddir") - some_fs.makedirs(nesteddir) - some_fs.touch(some_join(subdir, "subfile1")) - some_fs.touch(some_join(subdir, "subfile2")) - some_fs.touch(some_join(nesteddir, "nestedfile")) - - return source - - def _bulk_operations_scenario_0(self, some_fs, some_join, some_path): - """ - Scenario that is used for many cp/get/put tests. Creates the following - directory and file structure: - - 📁 source - ├── 📄 file1 - ├── 📄 file2 - └── 📁 subdir - ├── 📄 subfile1 - ├── 📄 subfile2 - └── 📁 nesteddir - └── 📄 nestedfile - """ - source = some_join(some_path, "source") - subdir = some_join(source, "subdir") - nesteddir = some_join(subdir, "nesteddir") - some_fs.makedirs(nesteddir) - some_fs.touch(some_join(source, "file1")) - some_fs.touch(some_join(source, "file2")) - some_fs.touch(some_join(subdir, "subfile1")) - some_fs.touch(some_join(subdir, "subfile2")) - some_fs.touch(some_join(nesteddir, "nestedfile")) - return source - - def _dir_and_file_with_same_name_prefix(self, some_fs, some_join, some_path): - """ - Scenario that is used to check cp/get/put on directory and file with - the same name prefixes. Creates the following directory and file structure: - - 📁 source - ├── 📄 subdir.txt - └── 📁 subdir - └── 📄 subfile.txt - """ - source = some_join(some_path, "source") - subdir = some_join(source, "subdir") - file = some_join(source, "subdir.txt") - subfile = some_join(subdir, "subfile.txt") - some_fs.makedirs(subdir) - some_fs.touch(file) - some_fs.touch(subfile) - return source - - def _10_files_with_hashed_names(self, some_fs, some_join, some_path): - """ - Scenario that is used to check cp/get/put files order when source and - destination are lists. Creates the following directory and file structure: - - 📁 source - └── 📄 {hashed([0-9])}.txt - """ - source = some_join(some_path, "source") - for i in range(10): - hashed_i = md5(str(i).encode("utf-8")).hexdigest() - path = some_join(source, f"{hashed_i}.txt") - some_fs.pipe(path=path, value=f"{i}".encode()) - return source - - -class AbstractFixtures(BaseAbstractFixtures): - """ - Abstract base class containing fixtures that may be overridden in derived - filesystem-specific classes to run the abstract tests on such filesystems. - - For any particular filesystem some of these fixtures must be overridden, - such as ``fs`` and ``fs_path``, and others may be overridden if the - default functions here are not appropriate, such as ``fs_join``. - """ - - @pytest.fixture - def fs(self): - raise NotImplementedError("This function must be overridden in derived classes") - - @pytest.fixture - def fs_join(self): - """ - Return a function that joins its arguments together into a path. - - Most fsspec implementations join paths in a platform-dependent way, - but some will override this to always use a forward slash. - """ - return os.path.join - - @pytest.fixture - def fs_path(self): - raise NotImplementedError("This function must be overridden in derived classes") - - @pytest.fixture(scope="class") - @classmethod - def local_fs(cls): - # Maybe need an option for auto_mkdir=False? This is only relevant - # for certain implementations. - return LocalFileSystem(auto_mkdir=True) - - @pytest.fixture - def local_join(self): - """ - Return a function that joins its arguments together into a path, on - the local filesystem. - """ - return os.path.join - - @pytest.fixture - def local_path(self, tmpdir): - return tmpdir - - @pytest.fixture - def supports_empty_directories(self): - """ - Return whether this implementation supports empty directories. - """ - return True - - @pytest.fixture - def fs_sanitize_path(self): - return lambda x: x diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/common.py b/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/common.py deleted file mode 100644 index 22e7c4140404ab2a8928689721419cf05c2760b9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/common.py +++ /dev/null @@ -1,175 +0,0 @@ -GLOB_EDGE_CASES_TESTS = { - "argnames": ("path", "recursive", "maxdepth", "expected"), - "argvalues": [ - ("fil?1", False, None, ["file1"]), - ("fil?1", True, None, ["file1"]), - ("file[1-2]", False, None, ["file1", "file2"]), - ("file[1-2]", True, None, ["file1", "file2"]), - ("*", False, None, ["file1", "file2"]), - ( - "*", - True, - None, - [ - "file1", - "file2", - "subdir0/subfile1", - "subdir0/subfile2", - "subdir0/nesteddir/nestedfile", - "subdir1/subfile1", - "subdir1/subfile2", - "subdir1/nesteddir/nestedfile", - ], - ), - ("*", True, 1, ["file1", "file2"]), - ( - "*", - True, - 2, - [ - "file1", - "file2", - "subdir0/subfile1", - "subdir0/subfile2", - "subdir1/subfile1", - "subdir1/subfile2", - ], - ), - ("*1", False, None, ["file1"]), - ( - "*1", - True, - None, - [ - "file1", - "subdir1/subfile1", - "subdir1/subfile2", - "subdir1/nesteddir/nestedfile", - ], - ), - ("*1", True, 2, ["file1", "subdir1/subfile1", "subdir1/subfile2"]), - ( - "**", - False, - None, - [ - "file1", - "file2", - "subdir0/subfile1", - "subdir0/subfile2", - "subdir0/nesteddir/nestedfile", - "subdir1/subfile1", - "subdir1/subfile2", - "subdir1/nesteddir/nestedfile", - ], - ), - ( - "**", - True, - None, - [ - "file1", - "file2", - "subdir0/subfile1", - "subdir0/subfile2", - "subdir0/nesteddir/nestedfile", - "subdir1/subfile1", - "subdir1/subfile2", - "subdir1/nesteddir/nestedfile", - ], - ), - ("**", True, 1, ["file1", "file2"]), - ( - "**", - True, - 2, - [ - "file1", - "file2", - "subdir0/subfile1", - "subdir0/subfile2", - "subdir0/nesteddir/nestedfile", - "subdir1/subfile1", - "subdir1/subfile2", - "subdir1/nesteddir/nestedfile", - ], - ), - ( - "**", - False, - 2, - [ - "file1", - "file2", - "subdir0/subfile1", - "subdir0/subfile2", - "subdir1/subfile1", - "subdir1/subfile2", - ], - ), - ("**/*1", False, None, ["file1", "subdir0/subfile1", "subdir1/subfile1"]), - ( - "**/*1", - True, - None, - [ - "file1", - "subdir0/subfile1", - "subdir1/subfile1", - "subdir1/subfile2", - "subdir1/nesteddir/nestedfile", - ], - ), - ("**/*1", True, 1, ["file1"]), - ( - "**/*1", - True, - 2, - ["file1", "subdir0/subfile1", "subdir1/subfile1", "subdir1/subfile2"], - ), - ("**/*1", False, 2, ["file1", "subdir0/subfile1", "subdir1/subfile1"]), - ("**/subdir0", False, None, []), - ("**/subdir0", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]), - ("**/subdir0/nested*", False, 2, []), - ("**/subdir0/nested*", True, 2, ["nestedfile"]), - ("subdir[1-2]", False, None, []), - ("subdir[1-2]", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]), - ("subdir[1-2]", True, 2, ["subfile1", "subfile2"]), - ("subdir[0-1]", False, None, []), - ( - "subdir[0-1]", - True, - None, - [ - "subdir0/subfile1", - "subdir0/subfile2", - "subdir0/nesteddir/nestedfile", - "subdir1/subfile1", - "subdir1/subfile2", - "subdir1/nesteddir/nestedfile", - ], - ), - ( - "subdir[0-1]/*fil[e]*", - False, - None, - [ - "subdir0/subfile1", - "subdir0/subfile2", - "subdir1/subfile1", - "subdir1/subfile2", - ], - ), - ( - "subdir[0-1]/*fil[e]*", - True, - None, - [ - "subdir0/subfile1", - "subdir0/subfile2", - "subdir1/subfile1", - "subdir1/subfile2", - ], - ), - ], -} diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/copy.py b/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/copy.py deleted file mode 100644 index e39e57e5f7d52bfda8ab5e2398b04cc2303630a0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/copy.py +++ /dev/null @@ -1,557 +0,0 @@ -from hashlib import md5 -from itertools import product - -import pytest - -from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS - - -class AbstractCopyTests: - def test_copy_file_to_existing_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - fs_target, - supports_empty_directories, - ): - # Copy scenario 1a - source = fs_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - if not supports_empty_directories: - # Force target directory to exist by adding a dummy file - fs.touch(fs_join(target, "dummy")) - assert fs.isdir(target) - - target_file2 = fs_join(target, "file2") - target_subfile1 = fs_join(target, "subfile1") - - # Copy from source directory - fs.cp(fs_join(source, "file2"), target) - assert fs.isfile(target_file2) - - # Copy from sub directory - fs.cp(fs_join(source, "subdir", "subfile1"), target) - assert fs.isfile(target_subfile1) - - # Remove copied files - fs.rm([target_file2, target_subfile1]) - assert not fs.exists(target_file2) - assert not fs.exists(target_subfile1) - - # Repeat with trailing slash on target - fs.cp(fs_join(source, "file2"), target + "/") - assert fs.isdir(target) - assert fs.isfile(target_file2) - - fs.cp(fs_join(source, "subdir", "subfile1"), target + "/") - assert fs.isfile(target_subfile1) - - def test_copy_file_to_new_directory( - self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target - ): - # Copy scenario 1b - source = fs_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - - fs.cp( - fs_join(source, "subdir", "subfile1"), fs_join(target, "newdir/") - ) # Note trailing slash - assert fs.isdir(target) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - - def test_copy_file_to_file_in_existing_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - fs_target, - supports_empty_directories, - ): - # Copy scenario 1c - source = fs_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - if not supports_empty_directories: - # Force target directory to exist by adding a dummy file - fs.touch(fs_join(target, "dummy")) - assert fs.isdir(target) - - fs.cp(fs_join(source, "subdir", "subfile1"), fs_join(target, "newfile")) - assert fs.isfile(fs_join(target, "newfile")) - - def test_copy_file_to_file_in_new_directory( - self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target - ): - # Copy scenario 1d - source = fs_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - - fs.cp( - fs_join(source, "subdir", "subfile1"), fs_join(target, "newdir", "newfile") - ) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "newfile")) - - def test_copy_directory_to_existing_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - fs_target, - supports_empty_directories, - ): - # Copy scenario 1e - source = fs_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - if not supports_empty_directories: - # Force target directory to exist by adding a dummy file - dummy = fs_join(target, "dummy") - fs.touch(dummy) - assert fs.isdir(target) - - for source_slash, target_slash in zip([False, True], [False, True]): - s = fs_join(source, "subdir") - if source_slash: - s += "/" - t = target + "/" if target_slash else target - - # Without recursive does nothing - fs.cp(s, t) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - # With recursive - fs.cp(s, t, recursive=True) - if source_slash: - assert fs.isfile(fs_join(target, "subfile1")) - assert fs.isfile(fs_join(target, "subfile2")) - assert fs.isdir(fs_join(target, "nesteddir")) - assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm( - [ - fs_join(target, "subfile1"), - fs_join(target, "subfile2"), - fs_join(target, "nesteddir"), - ], - recursive=True, - ) - else: - assert fs.isdir(fs_join(target, "subdir")) - assert fs.isfile(fs_join(target, "subdir", "subfile1")) - assert fs.isfile(fs_join(target, "subdir", "subfile2")) - assert fs.isdir(fs_join(target, "subdir", "nesteddir")) - assert fs.isfile(fs_join(target, "subdir", "nesteddir", "nestedfile")) - - fs.rm(fs_join(target, "subdir"), recursive=True) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - # Limit recursive by maxdepth - fs.cp(s, t, recursive=True, maxdepth=1) - if source_slash: - assert fs.isfile(fs_join(target, "subfile1")) - assert fs.isfile(fs_join(target, "subfile2")) - assert not fs.exists(fs_join(target, "nesteddir")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm( - [ - fs_join(target, "subfile1"), - fs_join(target, "subfile2"), - ], - recursive=True, - ) - else: - assert fs.isdir(fs_join(target, "subdir")) - assert fs.isfile(fs_join(target, "subdir", "subfile1")) - assert fs.isfile(fs_join(target, "subdir", "subfile2")) - assert not fs.exists(fs_join(target, "subdir", "nesteddir")) - - fs.rm(fs_join(target, "subdir"), recursive=True) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - def test_copy_directory_to_new_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - fs_target, - supports_empty_directories, - ): - # Copy scenario 1f - source = fs_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - - for source_slash, target_slash in zip([False, True], [False, True]): - s = fs_join(source, "subdir") - if source_slash: - s += "/" - t = fs_join(target, "newdir") - if target_slash: - t += "/" - - # Without recursive does nothing - fs.cp(s, t) - if supports_empty_directories: - assert fs.ls(target) == [] - else: - with pytest.raises(FileNotFoundError): - fs.ls(target) - - # With recursive - fs.cp(s, t, recursive=True) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - assert fs.isfile(fs_join(target, "newdir", "subfile2")) - assert fs.isdir(fs_join(target, "newdir", "nesteddir")) - assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm(fs_join(target, "newdir"), recursive=True) - assert not fs.exists(fs_join(target, "newdir")) - - # Limit recursive by maxdepth - fs.cp(s, t, recursive=True, maxdepth=1) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - assert fs.isfile(fs_join(target, "newdir", "subfile2")) - assert not fs.exists(fs_join(target, "newdir", "nesteddir")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm(fs_join(target, "newdir"), recursive=True) - assert not fs.exists(fs_join(target, "newdir")) - - def test_copy_glob_to_existing_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - fs_target, - supports_empty_directories, - ): - # Copy scenario 1g - source = fs_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - if not supports_empty_directories: - # Force target directory to exist by adding a dummy file - dummy = fs_join(target, "dummy") - fs.touch(dummy) - assert fs.isdir(target) - - for target_slash in [False, True]: - t = target + "/" if target_slash else target - - # Without recursive - fs.cp(fs_join(source, "subdir", "*"), t) - assert fs.isfile(fs_join(target, "subfile1")) - assert fs.isfile(fs_join(target, "subfile2")) - assert not fs.isdir(fs_join(target, "nesteddir")) - assert not fs.exists(fs_join(target, "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm( - [ - fs_join(target, "subfile1"), - fs_join(target, "subfile2"), - ], - recursive=True, - ) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - # With recursive - for glob, recursive in zip(["*", "**"], [True, False]): - fs.cp(fs_join(source, "subdir", glob), t, recursive=recursive) - assert fs.isfile(fs_join(target, "subfile1")) - assert fs.isfile(fs_join(target, "subfile2")) - assert fs.isdir(fs_join(target, "nesteddir")) - assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm( - [ - fs_join(target, "subfile1"), - fs_join(target, "subfile2"), - fs_join(target, "nesteddir"), - ], - recursive=True, - ) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - # Limit recursive by maxdepth - fs.cp( - fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 - ) - assert fs.isfile(fs_join(target, "subfile1")) - assert fs.isfile(fs_join(target, "subfile2")) - assert not fs.exists(fs_join(target, "nesteddir")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm( - [ - fs_join(target, "subfile1"), - fs_join(target, "subfile2"), - ], - recursive=True, - ) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - def test_copy_glob_to_new_directory( - self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target - ): - # Copy scenario 1h - source = fs_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - - for target_slash in [False, True]: - t = fs_join(target, "newdir") - if target_slash: - t += "/" - - # Without recursive - fs.cp(fs_join(source, "subdir", "*"), t) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - assert fs.isfile(fs_join(target, "newdir", "subfile2")) - assert not fs.exists(fs_join(target, "newdir", "nesteddir")) - assert not fs.exists(fs_join(target, "newdir", "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - assert not fs.exists(fs_join(target, "newdir", "subdir")) - - fs.rm(fs_join(target, "newdir"), recursive=True) - assert not fs.exists(fs_join(target, "newdir")) - - # With recursive - for glob, recursive in zip(["*", "**"], [True, False]): - fs.cp(fs_join(source, "subdir", glob), t, recursive=recursive) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - assert fs.isfile(fs_join(target, "newdir", "subfile2")) - assert fs.isdir(fs_join(target, "newdir", "nesteddir")) - assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - assert not fs.exists(fs_join(target, "newdir", "subdir")) - - fs.rm(fs_join(target, "newdir"), recursive=True) - assert not fs.exists(fs_join(target, "newdir")) - - # Limit recursive by maxdepth - fs.cp( - fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 - ) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - assert fs.isfile(fs_join(target, "newdir", "subfile2")) - assert not fs.exists(fs_join(target, "newdir", "nesteddir")) - assert not fs.exists(fs_join(target, "subdir")) - assert not fs.exists(fs_join(target, "newdir", "subdir")) - - fs.rm(fs_join(target, "newdir"), recursive=True) - assert not fs.exists(fs_join(target, "newdir")) - - @pytest.mark.parametrize( - GLOB_EDGE_CASES_TESTS["argnames"], - GLOB_EDGE_CASES_TESTS["argvalues"], - ) - def test_copy_glob_edge_cases( - self, - path, - recursive, - maxdepth, - expected, - fs, - fs_join, - fs_glob_edge_cases_files, - fs_target, - fs_sanitize_path, - ): - # Copy scenario 1g - source = fs_glob_edge_cases_files - - target = fs_target - - for new_dir, target_slash in product([True, False], [True, False]): - fs.mkdir(target) - - t = fs_join(target, "newdir") if new_dir else target - t = t + "/" if target_slash else t - - fs.copy(fs_join(source, path), t, recursive=recursive, maxdepth=maxdepth) - - output = fs.find(target) - if new_dir: - prefixed_expected = [ - fs_sanitize_path(fs_join(target, "newdir", p)) for p in expected - ] - else: - prefixed_expected = [ - fs_sanitize_path(fs_join(target, p)) for p in expected - ] - assert sorted(output) == sorted(prefixed_expected) - - try: - fs.rm(target, recursive=True) - except FileNotFoundError: - pass - - def test_copy_list_of_files_to_existing_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - fs_target, - supports_empty_directories, - ): - # Copy scenario 2a - source = fs_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - if not supports_empty_directories: - # Force target directory to exist by adding a dummy file - dummy = fs_join(target, "dummy") - fs.touch(dummy) - assert fs.isdir(target) - - source_files = [ - fs_join(source, "file1"), - fs_join(source, "file2"), - fs_join(source, "subdir", "subfile1"), - ] - - for target_slash in [False, True]: - t = target + "/" if target_slash else target - - fs.cp(source_files, t) - assert fs.isfile(fs_join(target, "file1")) - assert fs.isfile(fs_join(target, "file2")) - assert fs.isfile(fs_join(target, "subfile1")) - - fs.rm( - [ - fs_join(target, "file1"), - fs_join(target, "file2"), - fs_join(target, "subfile1"), - ], - recursive=True, - ) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - def test_copy_list_of_files_to_new_directory( - self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target - ): - # Copy scenario 2b - source = fs_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - - source_files = [ - fs_join(source, "file1"), - fs_join(source, "file2"), - fs_join(source, "subdir", "subfile1"), - ] - - fs.cp(source_files, fs_join(target, "newdir") + "/") # Note trailing slash - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "file1")) - assert fs.isfile(fs_join(target, "newdir", "file2")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - - def test_copy_two_files_new_directory( - self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target - ): - # This is a duplicate of test_copy_list_of_files_to_new_directory and - # can eventually be removed. - source = fs_bulk_operations_scenario_0 - - target = fs_target - assert not fs.exists(target) - fs.cp([fs_join(source, "file1"), fs_join(source, "file2")], target) - - assert fs.isdir(target) - assert fs.isfile(fs_join(target, "file1")) - assert fs.isfile(fs_join(target, "file2")) - - def test_copy_directory_without_files_with_same_name_prefix( - self, - fs, - fs_join, - fs_target, - fs_dir_and_file_with_same_name_prefix, - supports_empty_directories, - ): - # Create the test dirs - source = fs_dir_and_file_with_same_name_prefix - target = fs_target - - # Test without glob - fs.cp(fs_join(source, "subdir"), target, recursive=True) - - assert fs.isfile(fs_join(target, "subfile.txt")) - assert not fs.isfile(fs_join(target, "subdir.txt")) - - fs.rm([fs_join(target, "subfile.txt")]) - if supports_empty_directories: - assert fs.ls(target) == [] - else: - assert not fs.exists(target) - - # Test with glob - fs.cp(fs_join(source, "subdir*"), target, recursive=True) - - assert fs.isdir(fs_join(target, "subdir")) - assert fs.isfile(fs_join(target, "subdir", "subfile.txt")) - assert fs.isfile(fs_join(target, "subdir.txt")) - - def test_copy_with_source_and_destination_as_list( - self, fs, fs_target, fs_join, fs_10_files_with_hashed_names - ): - # Create the test dir - source = fs_10_files_with_hashed_names - target = fs_target - - # Create list of files for source and destination - source_files = [] - destination_files = [] - for i in range(10): - hashed_i = md5(str(i).encode("utf-8")).hexdigest() - source_files.append(fs_join(source, f"{hashed_i}.txt")) - destination_files.append(fs_join(target, f"{hashed_i}.txt")) - - # Copy and assert order was kept - fs.copy(path1=source_files, path2=destination_files) - - for i in range(10): - file_content = fs.cat(destination_files[i]).decode("utf-8") - assert file_content == str(i) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/get.py b/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/get.py deleted file mode 100644 index 851ab81ee581e74cac41c64c83ef0af75826d6b0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/get.py +++ /dev/null @@ -1,587 +0,0 @@ -from hashlib import md5 -from itertools import product - -import pytest - -from fsspec.implementations.local import make_path_posix -from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS - - -class AbstractGetTests: - def test_get_file_to_existing_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - local_fs, - local_join, - local_target, - ): - # Copy scenario 1a - source = fs_bulk_operations_scenario_0 - - target = local_target - local_fs.mkdir(target) - assert local_fs.isdir(target) - - target_file2 = local_join(target, "file2") - target_subfile1 = local_join(target, "subfile1") - - # Copy from source directory - fs.get(fs_join(source, "file2"), target) - assert local_fs.isfile(target_file2) - - # Copy from sub directory - fs.get(fs_join(source, "subdir", "subfile1"), target) - assert local_fs.isfile(target_subfile1) - - # Remove copied files - local_fs.rm([target_file2, target_subfile1]) - assert not local_fs.exists(target_file2) - assert not local_fs.exists(target_subfile1) - - # Repeat with trailing slash on target - fs.get(fs_join(source, "file2"), target + "/") - assert local_fs.isdir(target) - assert local_fs.isfile(target_file2) - - fs.get(fs_join(source, "subdir", "subfile1"), target + "/") - assert local_fs.isfile(target_subfile1) - - def test_get_file_to_new_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - local_fs, - local_join, - local_target, - ): - # Copy scenario 1b - source = fs_bulk_operations_scenario_0 - - target = local_target - local_fs.mkdir(target) - - fs.get( - fs_join(source, "subdir", "subfile1"), local_join(target, "newdir/") - ) # Note trailing slash - - assert local_fs.isdir(target) - assert local_fs.isdir(local_join(target, "newdir")) - assert local_fs.isfile(local_join(target, "newdir", "subfile1")) - - def test_get_file_to_file_in_existing_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - local_fs, - local_join, - local_target, - ): - # Copy scenario 1c - source = fs_bulk_operations_scenario_0 - - target = local_target - local_fs.mkdir(target) - - fs.get(fs_join(source, "subdir", "subfile1"), local_join(target, "newfile")) - assert local_fs.isfile(local_join(target, "newfile")) - - def test_get_file_to_file_in_new_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - local_fs, - local_join, - local_target, - ): - # Copy scenario 1d - source = fs_bulk_operations_scenario_0 - - target = local_target - local_fs.mkdir(target) - - fs.get( - fs_join(source, "subdir", "subfile1"), - local_join(target, "newdir", "newfile"), - ) - assert local_fs.isdir(local_join(target, "newdir")) - assert local_fs.isfile(local_join(target, "newdir", "newfile")) - - def test_get_directory_to_existing_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - local_fs, - local_join, - local_target, - ): - # Copy scenario 1e - source = fs_bulk_operations_scenario_0 - - target = local_target - local_fs.mkdir(target) - assert local_fs.isdir(target) - - for source_slash, target_slash in zip([False, True], [False, True]): - s = fs_join(source, "subdir") - if source_slash: - s += "/" - t = target + "/" if target_slash else target - - # Without recursive does nothing - fs.get(s, t) - assert local_fs.ls(target) == [] - - # With recursive - fs.get(s, t, recursive=True) - if source_slash: - assert local_fs.isfile(local_join(target, "subfile1")) - assert local_fs.isfile(local_join(target, "subfile2")) - assert local_fs.isdir(local_join(target, "nesteddir")) - assert local_fs.isfile(local_join(target, "nesteddir", "nestedfile")) - assert not local_fs.exists(local_join(target, "subdir")) - - local_fs.rm( - [ - local_join(target, "subfile1"), - local_join(target, "subfile2"), - local_join(target, "nesteddir"), - ], - recursive=True, - ) - else: - assert local_fs.isdir(local_join(target, "subdir")) - assert local_fs.isfile(local_join(target, "subdir", "subfile1")) - assert local_fs.isfile(local_join(target, "subdir", "subfile2")) - assert local_fs.isdir(local_join(target, "subdir", "nesteddir")) - assert local_fs.isfile( - local_join(target, "subdir", "nesteddir", "nestedfile") - ) - - local_fs.rm(local_join(target, "subdir"), recursive=True) - assert local_fs.ls(target) == [] - - # Limit recursive by maxdepth - fs.get(s, t, recursive=True, maxdepth=1) - if source_slash: - assert local_fs.isfile(local_join(target, "subfile1")) - assert local_fs.isfile(local_join(target, "subfile2")) - assert not local_fs.exists(local_join(target, "nesteddir")) - assert not local_fs.exists(local_join(target, "subdir")) - - local_fs.rm( - [ - local_join(target, "subfile1"), - local_join(target, "subfile2"), - ], - recursive=True, - ) - else: - assert local_fs.isdir(local_join(target, "subdir")) - assert local_fs.isfile(local_join(target, "subdir", "subfile1")) - assert local_fs.isfile(local_join(target, "subdir", "subfile2")) - assert not local_fs.exists(local_join(target, "subdir", "nesteddir")) - - local_fs.rm(local_join(target, "subdir"), recursive=True) - assert local_fs.ls(target) == [] - - def test_get_directory_to_new_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - local_fs, - local_join, - local_target, - ): - # Copy scenario 1f - source = fs_bulk_operations_scenario_0 - - target = local_target - local_fs.mkdir(target) - - for source_slash, target_slash in zip([False, True], [False, True]): - s = fs_join(source, "subdir") - if source_slash: - s += "/" - t = local_join(target, "newdir") - if target_slash: - t += "/" - - # Without recursive does nothing - fs.get(s, t) - assert local_fs.ls(target) == [] - - # With recursive - fs.get(s, t, recursive=True) - assert local_fs.isdir(local_join(target, "newdir")) - assert local_fs.isfile(local_join(target, "newdir", "subfile1")) - assert local_fs.isfile(local_join(target, "newdir", "subfile2")) - assert local_fs.isdir(local_join(target, "newdir", "nesteddir")) - assert local_fs.isfile( - local_join(target, "newdir", "nesteddir", "nestedfile") - ) - assert not local_fs.exists(local_join(target, "subdir")) - - local_fs.rm(local_join(target, "newdir"), recursive=True) - assert local_fs.ls(target) == [] - - # Limit recursive by maxdepth - fs.get(s, t, recursive=True, maxdepth=1) - assert local_fs.isdir(local_join(target, "newdir")) - assert local_fs.isfile(local_join(target, "newdir", "subfile1")) - assert local_fs.isfile(local_join(target, "newdir", "subfile2")) - assert not local_fs.exists(local_join(target, "newdir", "nesteddir")) - assert not local_fs.exists(local_join(target, "subdir")) - - local_fs.rm(local_join(target, "newdir"), recursive=True) - assert not local_fs.exists(local_join(target, "newdir")) - - def test_get_glob_to_existing_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - local_fs, - local_join, - local_target, - ): - # Copy scenario 1g - source = fs_bulk_operations_scenario_0 - - target = local_target - local_fs.mkdir(target) - - for target_slash in [False, True]: - t = target + "/" if target_slash else target - - # Without recursive - fs.get(fs_join(source, "subdir", "*"), t) - assert local_fs.isfile(local_join(target, "subfile1")) - assert local_fs.isfile(local_join(target, "subfile2")) - assert not local_fs.isdir(local_join(target, "nesteddir")) - assert not local_fs.exists(local_join(target, "nesteddir", "nestedfile")) - assert not local_fs.exists(local_join(target, "subdir")) - - local_fs.rm( - [ - local_join(target, "subfile1"), - local_join(target, "subfile2"), - ], - recursive=True, - ) - assert local_fs.ls(target) == [] - - # With recursive - for glob, recursive in zip(["*", "**"], [True, False]): - fs.get(fs_join(source, "subdir", glob), t, recursive=recursive) - assert local_fs.isfile(local_join(target, "subfile1")) - assert local_fs.isfile(local_join(target, "subfile2")) - assert local_fs.isdir(local_join(target, "nesteddir")) - assert local_fs.isfile(local_join(target, "nesteddir", "nestedfile")) - assert not local_fs.exists(local_join(target, "subdir")) - - local_fs.rm( - [ - local_join(target, "subfile1"), - local_join(target, "subfile2"), - local_join(target, "nesteddir"), - ], - recursive=True, - ) - assert local_fs.ls(target) == [] - - # Limit recursive by maxdepth - fs.get( - fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 - ) - assert local_fs.isfile(local_join(target, "subfile1")) - assert local_fs.isfile(local_join(target, "subfile2")) - assert not local_fs.exists(local_join(target, "nesteddir")) - assert not local_fs.exists(local_join(target, "subdir")) - - local_fs.rm( - [ - local_join(target, "subfile1"), - local_join(target, "subfile2"), - ], - recursive=True, - ) - assert local_fs.ls(target) == [] - - def test_get_glob_to_new_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - local_fs, - local_join, - local_target, - ): - # Copy scenario 1h - source = fs_bulk_operations_scenario_0 - - target = local_target - local_fs.mkdir(target) - - for target_slash in [False, True]: - t = fs_join(target, "newdir") - if target_slash: - t += "/" - - # Without recursive - fs.get(fs_join(source, "subdir", "*"), t) - assert local_fs.isdir(local_join(target, "newdir")) - assert local_fs.isfile(local_join(target, "newdir", "subfile1")) - assert local_fs.isfile(local_join(target, "newdir", "subfile2")) - assert not local_fs.exists(local_join(target, "newdir", "nesteddir")) - assert not local_fs.exists( - local_join(target, "newdir", "nesteddir", "nestedfile") - ) - assert not local_fs.exists(local_join(target, "subdir")) - assert not local_fs.exists(local_join(target, "newdir", "subdir")) - - local_fs.rm(local_join(target, "newdir"), recursive=True) - assert local_fs.ls(target) == [] - - # With recursive - for glob, recursive in zip(["*", "**"], [True, False]): - fs.get(fs_join(source, "subdir", glob), t, recursive=recursive) - assert local_fs.isdir(local_join(target, "newdir")) - assert local_fs.isfile(local_join(target, "newdir", "subfile1")) - assert local_fs.isfile(local_join(target, "newdir", "subfile2")) - assert local_fs.isdir(local_join(target, "newdir", "nesteddir")) - assert local_fs.isfile( - local_join(target, "newdir", "nesteddir", "nestedfile") - ) - assert not local_fs.exists(local_join(target, "subdir")) - assert not local_fs.exists(local_join(target, "newdir", "subdir")) - - local_fs.rm(local_join(target, "newdir"), recursive=True) - assert not local_fs.exists(local_join(target, "newdir")) - - # Limit recursive by maxdepth - fs.get( - fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 - ) - assert local_fs.isdir(local_join(target, "newdir")) - assert local_fs.isfile(local_join(target, "newdir", "subfile1")) - assert local_fs.isfile(local_join(target, "newdir", "subfile2")) - assert not local_fs.exists(local_join(target, "newdir", "nesteddir")) - assert not local_fs.exists(local_join(target, "subdir")) - assert not local_fs.exists(local_join(target, "newdir", "subdir")) - - local_fs.rm(local_fs.ls(target, detail=False), recursive=True) - assert not local_fs.exists(local_join(target, "newdir")) - - @pytest.mark.parametrize( - GLOB_EDGE_CASES_TESTS["argnames"], - GLOB_EDGE_CASES_TESTS["argvalues"], - ) - def test_get_glob_edge_cases( - self, - path, - recursive, - maxdepth, - expected, - fs, - fs_join, - fs_glob_edge_cases_files, - local_fs, - local_join, - local_target, - ): - # Copy scenario 1g - source = fs_glob_edge_cases_files - - target = local_target - - for new_dir, target_slash in product([True, False], [True, False]): - local_fs.mkdir(target) - - t = local_join(target, "newdir") if new_dir else target - t = t + "/" if target_slash else t - - fs.get(fs_join(source, path), t, recursive=recursive, maxdepth=maxdepth) - - output = local_fs.find(target) - if new_dir: - prefixed_expected = [ - make_path_posix(local_join(target, "newdir", p)) for p in expected - ] - else: - prefixed_expected = [ - make_path_posix(local_join(target, p)) for p in expected - ] - assert sorted(output) == sorted(prefixed_expected) - - try: - local_fs.rm(target, recursive=True) - except FileNotFoundError: - pass - - def test_get_list_of_files_to_existing_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - local_fs, - local_join, - local_target, - ): - # Copy scenario 2a - source = fs_bulk_operations_scenario_0 - - target = local_target - local_fs.mkdir(target) - - source_files = [ - fs_join(source, "file1"), - fs_join(source, "file2"), - fs_join(source, "subdir", "subfile1"), - ] - - for target_slash in [False, True]: - t = target + "/" if target_slash else target - - fs.get(source_files, t) - assert local_fs.isfile(local_join(target, "file1")) - assert local_fs.isfile(local_join(target, "file2")) - assert local_fs.isfile(local_join(target, "subfile1")) - - local_fs.rm( - [ - local_join(target, "file1"), - local_join(target, "file2"), - local_join(target, "subfile1"), - ], - recursive=True, - ) - assert local_fs.ls(target) == [] - - def test_get_list_of_files_to_new_directory( - self, - fs, - fs_join, - fs_bulk_operations_scenario_0, - local_fs, - local_join, - local_target, - ): - # Copy scenario 2b - source = fs_bulk_operations_scenario_0 - - target = local_target - local_fs.mkdir(target) - - source_files = [ - fs_join(source, "file1"), - fs_join(source, "file2"), - fs_join(source, "subdir", "subfile1"), - ] - - fs.get(source_files, local_join(target, "newdir") + "/") # Note trailing slash - assert local_fs.isdir(local_join(target, "newdir")) - assert local_fs.isfile(local_join(target, "newdir", "file1")) - assert local_fs.isfile(local_join(target, "newdir", "file2")) - assert local_fs.isfile(local_join(target, "newdir", "subfile1")) - - def test_get_directory_recursive( - self, fs, fs_join, fs_path, local_fs, local_join, local_target - ): - # https://github.com/fsspec/filesystem_spec/issues/1062 - # Recursive cp/get/put of source directory into non-existent target directory. - src = fs_join(fs_path, "src") - src_file = fs_join(src, "file") - fs.mkdir(src) - fs.touch(src_file) - - target = local_target - - # get without slash - assert not local_fs.exists(target) - for loop in range(2): - fs.get(src, target, recursive=True) - assert local_fs.isdir(target) - - if loop == 0: - assert local_fs.isfile(local_join(target, "file")) - assert not local_fs.exists(local_join(target, "src")) - else: - assert local_fs.isfile(local_join(target, "file")) - assert local_fs.isdir(local_join(target, "src")) - assert local_fs.isfile(local_join(target, "src", "file")) - - local_fs.rm(target, recursive=True) - - # get with slash - assert not local_fs.exists(target) - for loop in range(2): - fs.get(src + "/", target, recursive=True) - assert local_fs.isdir(target) - assert local_fs.isfile(local_join(target, "file")) - assert not local_fs.exists(local_join(target, "src")) - - def test_get_directory_without_files_with_same_name_prefix( - self, - fs, - fs_join, - local_fs, - local_join, - local_target, - fs_dir_and_file_with_same_name_prefix, - ): - # Create the test dirs - source = fs_dir_and_file_with_same_name_prefix - target = local_target - - # Test without glob - fs.get(fs_join(source, "subdir"), target, recursive=True) - - assert local_fs.isfile(local_join(target, "subfile.txt")) - assert not local_fs.isfile(local_join(target, "subdir.txt")) - - local_fs.rm([local_join(target, "subfile.txt")]) - assert local_fs.ls(target) == [] - - # Test with glob - fs.get(fs_join(source, "subdir*"), target, recursive=True) - - assert local_fs.isdir(local_join(target, "subdir")) - assert local_fs.isfile(local_join(target, "subdir", "subfile.txt")) - assert local_fs.isfile(local_join(target, "subdir.txt")) - - def test_get_with_source_and_destination_as_list( - self, - fs, - fs_join, - local_fs, - local_join, - local_target, - fs_10_files_with_hashed_names, - ): - # Create the test dir - source = fs_10_files_with_hashed_names - target = local_target - - # Create list of files for source and destination - source_files = [] - destination_files = [] - for i in range(10): - hashed_i = md5(str(i).encode("utf-8")).hexdigest() - source_files.append(fs_join(source, f"{hashed_i}.txt")) - destination_files.append( - make_path_posix(local_join(target, f"{hashed_i}.txt")) - ) - - # Copy and assert order was kept - fs.get(rpath=source_files, lpath=destination_files) - - for i in range(10): - file_content = local_fs.cat(destination_files[i]).decode("utf-8") - assert file_content == str(i) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/mv.py b/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/mv.py deleted file mode 100644 index 39f6caa3de815e024fa84de2acecc986c823ed29..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/mv.py +++ /dev/null @@ -1,57 +0,0 @@ -import os - -import pytest - -import fsspec - - -def test_move_raises_error_with_tmpdir(tmpdir): - # Create a file in the temporary directory - source = tmpdir.join("source_file.txt") - source.write("content") - - # Define a destination that simulates a protected or invalid path - destination = tmpdir.join("non_existent_directory/destination_file.txt") - - # Instantiate the filesystem (assuming the local file system interface) - fs = fsspec.filesystem("file") - - # Use the actual file paths as string - with pytest.raises(FileNotFoundError): - fs.mv(str(source), str(destination)) - - -@pytest.mark.parametrize("recursive", (True, False)) -def test_move_raises_error_with_tmpdir_permission(recursive, tmpdir): - # Create a file in the temporary directory - source = tmpdir.join("source_file.txt") - source.write("content") - - # Create a protected directory (non-writable) - protected_dir = tmpdir.mkdir("protected_directory") - protected_path = str(protected_dir) - - # Set the directory to read-only - if os.name == "nt": - os.system(f'icacls "{protected_path}" /deny Everyone:(W)') - else: - os.chmod(protected_path, 0o555) # Sets the directory to read-only - - # Define a destination inside the protected directory - destination = protected_dir.join("destination_file.txt") - - # Instantiate the filesystem (assuming the local file system interface) - fs = fsspec.filesystem("file") - - # Try to move the file to the read-only directory, expecting a permission error - with pytest.raises(PermissionError): - fs.mv(str(source), str(destination), recursive=recursive) - - # Assert the file was not created in the destination - assert not os.path.exists(destination) - - # Cleanup: Restore permissions so the directory can be cleaned up - if os.name == "nt": - os.system(f'icacls "{protected_path}" /remove:d Everyone') - else: - os.chmod(protected_path, 0o755) # Restore write permission for cleanup diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/open.py b/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/open.py deleted file mode 100644 index bb75ea852276fb8d834345883813b8e27a0ae24c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/open.py +++ /dev/null @@ -1,11 +0,0 @@ -import pytest - - -class AbstractOpenTests: - def test_open_exclusive(self, fs, fs_target): - with fs.open(fs_target, "wb") as f: - f.write(b"data") - with fs.open(fs_target, "rb") as f: - assert f.read() == b"data" - with pytest.raises(FileExistsError): - fs.open(fs_target, "xb") diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/pipe.py b/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/pipe.py deleted file mode 100644 index 8ecca96e9d23ff268a253c48269d5cca451ea270..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/pipe.py +++ /dev/null @@ -1,11 +0,0 @@ -import pytest - - -class AbstractPipeTests: - def test_pipe_exclusive(self, fs, fs_target): - fs.pipe_file(fs_target, b"data") - assert fs.cat_file(fs_target) == b"data" - with pytest.raises(FileExistsError): - fs.pipe_file(fs_target, b"data", mode="create") - fs.pipe_file(fs_target, b"new data", mode="overwrite") - assert fs.cat_file(fs_target) == b"new data" diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/put.py b/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/put.py deleted file mode 100644 index 9fc349977f0384d9fc86126498be5c6ad99a21d3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/tests/abstract/put.py +++ /dev/null @@ -1,591 +0,0 @@ -from hashlib import md5 -from itertools import product - -import pytest - -from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS - - -class AbstractPutTests: - def test_put_file_to_existing_directory( - self, - fs, - fs_join, - fs_target, - local_join, - local_bulk_operations_scenario_0, - supports_empty_directories, - ): - # Copy scenario 1a - source = local_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - if not supports_empty_directories: - # Force target directory to exist by adding a dummy file - fs.touch(fs_join(target, "dummy")) - assert fs.isdir(target) - - target_file2 = fs_join(target, "file2") - target_subfile1 = fs_join(target, "subfile1") - - # Copy from source directory - fs.put(local_join(source, "file2"), target) - assert fs.isfile(target_file2) - - # Copy from sub directory - fs.put(local_join(source, "subdir", "subfile1"), target) - assert fs.isfile(target_subfile1) - - # Remove copied files - fs.rm([target_file2, target_subfile1]) - assert not fs.exists(target_file2) - assert not fs.exists(target_subfile1) - - # Repeat with trailing slash on target - fs.put(local_join(source, "file2"), target + "/") - assert fs.isdir(target) - assert fs.isfile(target_file2) - - fs.put(local_join(source, "subdir", "subfile1"), target + "/") - assert fs.isfile(target_subfile1) - - def test_put_file_to_new_directory( - self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 - ): - # Copy scenario 1b - source = local_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - - fs.put( - local_join(source, "subdir", "subfile1"), fs_join(target, "newdir/") - ) # Note trailing slash - assert fs.isdir(target) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - - def test_put_file_to_file_in_existing_directory( - self, - fs, - fs_join, - fs_target, - local_join, - supports_empty_directories, - local_bulk_operations_scenario_0, - ): - # Copy scenario 1c - source = local_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - if not supports_empty_directories: - # Force target directory to exist by adding a dummy file - fs.touch(fs_join(target, "dummy")) - assert fs.isdir(target) - - fs.put(local_join(source, "subdir", "subfile1"), fs_join(target, "newfile")) - assert fs.isfile(fs_join(target, "newfile")) - - def test_put_file_to_file_in_new_directory( - self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 - ): - # Copy scenario 1d - source = local_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - - fs.put( - local_join(source, "subdir", "subfile1"), - fs_join(target, "newdir", "newfile"), - ) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "newfile")) - - def test_put_directory_to_existing_directory( - self, - fs, - fs_join, - fs_target, - local_bulk_operations_scenario_0, - supports_empty_directories, - ): - # Copy scenario 1e - source = local_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - if not supports_empty_directories: - # Force target directory to exist by adding a dummy file - dummy = fs_join(target, "dummy") - fs.touch(dummy) - assert fs.isdir(target) - - for source_slash, target_slash in zip([False, True], [False, True]): - s = fs_join(source, "subdir") - if source_slash: - s += "/" - t = target + "/" if target_slash else target - - # Without recursive does nothing - fs.put(s, t) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - # With recursive - fs.put(s, t, recursive=True) - if source_slash: - assert fs.isfile(fs_join(target, "subfile1")) - assert fs.isfile(fs_join(target, "subfile2")) - assert fs.isdir(fs_join(target, "nesteddir")) - assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm( - [ - fs_join(target, "subfile1"), - fs_join(target, "subfile2"), - fs_join(target, "nesteddir"), - ], - recursive=True, - ) - else: - assert fs.isdir(fs_join(target, "subdir")) - assert fs.isfile(fs_join(target, "subdir", "subfile1")) - assert fs.isfile(fs_join(target, "subdir", "subfile2")) - assert fs.isdir(fs_join(target, "subdir", "nesteddir")) - assert fs.isfile(fs_join(target, "subdir", "nesteddir", "nestedfile")) - - fs.rm(fs_join(target, "subdir"), recursive=True) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - # Limit recursive by maxdepth - fs.put(s, t, recursive=True, maxdepth=1) - if source_slash: - assert fs.isfile(fs_join(target, "subfile1")) - assert fs.isfile(fs_join(target, "subfile2")) - assert not fs.exists(fs_join(target, "nesteddir")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm( - [ - fs_join(target, "subfile1"), - fs_join(target, "subfile2"), - ], - recursive=True, - ) - else: - assert fs.isdir(fs_join(target, "subdir")) - assert fs.isfile(fs_join(target, "subdir", "subfile1")) - assert fs.isfile(fs_join(target, "subdir", "subfile2")) - assert not fs.exists(fs_join(target, "subdir", "nesteddir")) - - fs.rm(fs_join(target, "subdir"), recursive=True) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - def test_put_directory_to_new_directory( - self, - fs, - fs_join, - fs_target, - local_bulk_operations_scenario_0, - supports_empty_directories, - ): - # Copy scenario 1f - source = local_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - - for source_slash, target_slash in zip([False, True], [False, True]): - s = fs_join(source, "subdir") - if source_slash: - s += "/" - t = fs_join(target, "newdir") - if target_slash: - t += "/" - - # Without recursive does nothing - fs.put(s, t) - if supports_empty_directories: - assert fs.ls(target) == [] - else: - with pytest.raises(FileNotFoundError): - fs.ls(target) - - # With recursive - fs.put(s, t, recursive=True) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - assert fs.isfile(fs_join(target, "newdir", "subfile2")) - assert fs.isdir(fs_join(target, "newdir", "nesteddir")) - assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm(fs_join(target, "newdir"), recursive=True) - assert not fs.exists(fs_join(target, "newdir")) - - # Limit recursive by maxdepth - fs.put(s, t, recursive=True, maxdepth=1) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - assert fs.isfile(fs_join(target, "newdir", "subfile2")) - assert not fs.exists(fs_join(target, "newdir", "nesteddir")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm(fs_join(target, "newdir"), recursive=True) - assert not fs.exists(fs_join(target, "newdir")) - - def test_put_glob_to_existing_directory( - self, - fs, - fs_join, - fs_target, - local_join, - supports_empty_directories, - local_bulk_operations_scenario_0, - ): - # Copy scenario 1g - source = local_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - if not supports_empty_directories: - # Force target directory to exist by adding a dummy file - dummy = fs_join(target, "dummy") - fs.touch(dummy) - assert fs.isdir(target) - - for target_slash in [False, True]: - t = target + "/" if target_slash else target - - # Without recursive - fs.put(local_join(source, "subdir", "*"), t) - assert fs.isfile(fs_join(target, "subfile1")) - assert fs.isfile(fs_join(target, "subfile2")) - assert not fs.isdir(fs_join(target, "nesteddir")) - assert not fs.exists(fs_join(target, "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm( - [ - fs_join(target, "subfile1"), - fs_join(target, "subfile2"), - ], - recursive=True, - ) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - # With recursive - for glob, recursive in zip(["*", "**"], [True, False]): - fs.put(local_join(source, "subdir", glob), t, recursive=recursive) - assert fs.isfile(fs_join(target, "subfile1")) - assert fs.isfile(fs_join(target, "subfile2")) - assert fs.isdir(fs_join(target, "nesteddir")) - assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm( - [ - fs_join(target, "subfile1"), - fs_join(target, "subfile2"), - fs_join(target, "nesteddir"), - ], - recursive=True, - ) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - # Limit recursive by maxdepth - fs.put( - local_join(source, "subdir", glob), - t, - recursive=recursive, - maxdepth=1, - ) - assert fs.isfile(fs_join(target, "subfile1")) - assert fs.isfile(fs_join(target, "subfile2")) - assert not fs.exists(fs_join(target, "nesteddir")) - assert not fs.exists(fs_join(target, "subdir")) - - fs.rm( - [ - fs_join(target, "subfile1"), - fs_join(target, "subfile2"), - ], - recursive=True, - ) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - def test_put_glob_to_new_directory( - self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 - ): - # Copy scenario 1h - source = local_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - - for target_slash in [False, True]: - t = fs_join(target, "newdir") - if target_slash: - t += "/" - - # Without recursive - fs.put(local_join(source, "subdir", "*"), t) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - assert fs.isfile(fs_join(target, "newdir", "subfile2")) - assert not fs.exists(fs_join(target, "newdir", "nesteddir")) - assert not fs.exists(fs_join(target, "newdir", "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - assert not fs.exists(fs_join(target, "newdir", "subdir")) - - fs.rm(fs_join(target, "newdir"), recursive=True) - assert not fs.exists(fs_join(target, "newdir")) - - # With recursive - for glob, recursive in zip(["*", "**"], [True, False]): - fs.put(local_join(source, "subdir", glob), t, recursive=recursive) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - assert fs.isfile(fs_join(target, "newdir", "subfile2")) - assert fs.isdir(fs_join(target, "newdir", "nesteddir")) - assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) - assert not fs.exists(fs_join(target, "subdir")) - assert not fs.exists(fs_join(target, "newdir", "subdir")) - - fs.rm(fs_join(target, "newdir"), recursive=True) - assert not fs.exists(fs_join(target, "newdir")) - - # Limit recursive by maxdepth - fs.put( - local_join(source, "subdir", glob), - t, - recursive=recursive, - maxdepth=1, - ) - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - assert fs.isfile(fs_join(target, "newdir", "subfile2")) - assert not fs.exists(fs_join(target, "newdir", "nesteddir")) - assert not fs.exists(fs_join(target, "subdir")) - assert not fs.exists(fs_join(target, "newdir", "subdir")) - - fs.rm(fs_join(target, "newdir"), recursive=True) - assert not fs.exists(fs_join(target, "newdir")) - - @pytest.mark.parametrize( - GLOB_EDGE_CASES_TESTS["argnames"], - GLOB_EDGE_CASES_TESTS["argvalues"], - ) - def test_put_glob_edge_cases( - self, - path, - recursive, - maxdepth, - expected, - fs, - fs_join, - fs_target, - local_glob_edge_cases_files, - local_join, - fs_sanitize_path, - ): - # Copy scenario 1g - source = local_glob_edge_cases_files - - target = fs_target - - for new_dir, target_slash in product([True, False], [True, False]): - fs.mkdir(target) - - t = fs_join(target, "newdir") if new_dir else target - t = t + "/" if target_slash else t - - fs.put(local_join(source, path), t, recursive=recursive, maxdepth=maxdepth) - - output = fs.find(target) - if new_dir: - prefixed_expected = [ - fs_sanitize_path(fs_join(target, "newdir", p)) for p in expected - ] - else: - prefixed_expected = [ - fs_sanitize_path(fs_join(target, p)) for p in expected - ] - assert sorted(output) == sorted(prefixed_expected) - - try: - fs.rm(target, recursive=True) - except FileNotFoundError: - pass - - def test_put_list_of_files_to_existing_directory( - self, - fs, - fs_join, - fs_target, - local_join, - local_bulk_operations_scenario_0, - supports_empty_directories, - ): - # Copy scenario 2a - source = local_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - if not supports_empty_directories: - # Force target directory to exist by adding a dummy file - dummy = fs_join(target, "dummy") - fs.touch(dummy) - assert fs.isdir(target) - - source_files = [ - local_join(source, "file1"), - local_join(source, "file2"), - local_join(source, "subdir", "subfile1"), - ] - - for target_slash in [False, True]: - t = target + "/" if target_slash else target - - fs.put(source_files, t) - assert fs.isfile(fs_join(target, "file1")) - assert fs.isfile(fs_join(target, "file2")) - assert fs.isfile(fs_join(target, "subfile1")) - - fs.rm( - [ - fs_join(target, "file1"), - fs_join(target, "file2"), - fs_join(target, "subfile1"), - ], - recursive=True, - ) - assert fs.ls(target, detail=False) == ( - [] if supports_empty_directories else [dummy] - ) - - def test_put_list_of_files_to_new_directory( - self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 - ): - # Copy scenario 2b - source = local_bulk_operations_scenario_0 - - target = fs_target - fs.mkdir(target) - - source_files = [ - local_join(source, "file1"), - local_join(source, "file2"), - local_join(source, "subdir", "subfile1"), - ] - - fs.put(source_files, fs_join(target, "newdir") + "/") # Note trailing slash - assert fs.isdir(fs_join(target, "newdir")) - assert fs.isfile(fs_join(target, "newdir", "file1")) - assert fs.isfile(fs_join(target, "newdir", "file2")) - assert fs.isfile(fs_join(target, "newdir", "subfile1")) - - def test_put_directory_recursive( - self, fs, fs_join, fs_target, local_fs, local_join, local_path - ): - # https://github.com/fsspec/filesystem_spec/issues/1062 - # Recursive cp/get/put of source directory into non-existent target directory. - src = local_join(local_path, "src") - src_file = local_join(src, "file") - local_fs.mkdir(src) - local_fs.touch(src_file) - - target = fs_target - - # put without slash - assert not fs.exists(target) - for loop in range(2): - fs.put(src, target, recursive=True) - assert fs.isdir(target) - - if loop == 0: - assert fs.isfile(fs_join(target, "file")) - assert not fs.exists(fs_join(target, "src")) - else: - assert fs.isfile(fs_join(target, "file")) - assert fs.isdir(fs_join(target, "src")) - assert fs.isfile(fs_join(target, "src", "file")) - - fs.rm(target, recursive=True) - - # put with slash - assert not fs.exists(target) - for loop in range(2): - fs.put(src + "/", target, recursive=True) - assert fs.isdir(target) - assert fs.isfile(fs_join(target, "file")) - assert not fs.exists(fs_join(target, "src")) - - def test_put_directory_without_files_with_same_name_prefix( - self, - fs, - fs_join, - fs_target, - local_join, - local_dir_and_file_with_same_name_prefix, - supports_empty_directories, - ): - # Create the test dirs - source = local_dir_and_file_with_same_name_prefix - target = fs_target - - # Test without glob - fs.put(local_join(source, "subdir"), fs_target, recursive=True) - - assert fs.isfile(fs_join(fs_target, "subfile.txt")) - assert not fs.isfile(fs_join(fs_target, "subdir.txt")) - - fs.rm([fs_join(target, "subfile.txt")]) - if supports_empty_directories: - assert fs.ls(target) == [] - else: - assert not fs.exists(target) - - # Test with glob - fs.put(local_join(source, "subdir*"), fs_target, recursive=True) - - assert fs.isdir(fs_join(fs_target, "subdir")) - assert fs.isfile(fs_join(fs_target, "subdir", "subfile.txt")) - assert fs.isfile(fs_join(fs_target, "subdir.txt")) - - def test_copy_with_source_and_destination_as_list( - self, fs, fs_target, fs_join, local_join, local_10_files_with_hashed_names - ): - # Create the test dir - source = local_10_files_with_hashed_names - target = fs_target - - # Create list of files for source and destination - source_files = [] - destination_files = [] - for i in range(10): - hashed_i = md5(str(i).encode("utf-8")).hexdigest() - source_files.append(local_join(source, f"{hashed_i}.txt")) - destination_files.append(fs_join(target, f"{hashed_i}.txt")) - - # Copy and assert order was kept - fs.put(lpath=source_files, rpath=destination_files) - - for i in range(10): - file_content = fs.cat(destination_files[i]).decode("utf-8") - assert file_content == str(i) diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/transaction.py b/bundle/python-cpu/Lib/site-packages/fsspec/transaction.py deleted file mode 100644 index 77293f63ecc5f611e19d849ef236d53e9c258efc..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/transaction.py +++ /dev/null @@ -1,90 +0,0 @@ -from collections import deque - - -class Transaction: - """Filesystem transaction write context - - Gathers files for deferred commit or discard, so that several write - operations can be finalized semi-atomically. This works by having this - instance as the ``.transaction`` attribute of the given filesystem - """ - - def __init__(self, fs, **kwargs): - """ - Parameters - ---------- - fs: FileSystem instance - """ - self.fs = fs - self.files = deque() - - def __enter__(self): - self.start() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """End transaction and commit, if exit is not due to exception""" - # only commit if there was no exception - self.complete(commit=exc_type is None) - if self.fs: - self.fs._intrans = False - self.fs._transaction = None - self.fs = None - - def start(self): - """Start a transaction on this FileSystem""" - self.files = deque() # clean up after previous failed completions - self.fs._intrans = True - - def complete(self, commit=True): - """Finish transaction: commit or discard all deferred files""" - while self.files: - f = self.files.popleft() - if commit: - f.commit() - else: - f.discard() - self.fs._intrans = False - self.fs._transaction = None - self.fs = None - - -class FileActor: - def __init__(self): - self.files = [] - - def commit(self): - for f in self.files: - f.commit() - self.files.clear() - - def discard(self): - for f in self.files: - f.discard() - self.files.clear() - - def append(self, f): - self.files.append(f) - - -class DaskTransaction(Transaction): - def __init__(self, fs): - """ - Parameters - ---------- - fs: FileSystem instance - """ - import distributed - - super().__init__(fs) - client = distributed.default_client() - self.files = client.submit(FileActor, actor=True).result() - - def complete(self, commit=True): - """Finish transaction: commit or discard all deferred files""" - if commit: - self.files.commit().result() - else: - self.files.discard().result() - self.fs._intrans = False - self.fs = None diff --git a/bundle/python-cpu/Lib/site-packages/fsspec/utils.py b/bundle/python-cpu/Lib/site-packages/fsspec/utils.py deleted file mode 100644 index 6010cc1270f60417be1842702d1df435d460909c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/fsspec/utils.py +++ /dev/null @@ -1,757 +0,0 @@ -from __future__ import annotations - -import contextlib -import logging -import math -import os -import re -import sys -import tempfile -from collections.abc import Callable, Iterable, Iterator, Sequence -from functools import partial -from hashlib import md5 -from importlib.metadata import version -from typing import IO, TYPE_CHECKING, Any, TypeVar -from urllib.parse import urlsplit - -if TYPE_CHECKING: - import pathlib - from typing import TypeGuard - - from fsspec.spec import AbstractFileSystem - - -DEFAULT_BLOCK_SIZE = 5 * 2**20 - -T = TypeVar("T") - - -def infer_storage_options( - urlpath: str, inherit_storage_options: dict[str, Any] | None = None -) -> dict[str, Any]: - """Infer storage options from URL path and merge it with existing storage - options. - - Parameters - ---------- - urlpath: str or unicode - Either local absolute file path or URL (hdfs://namenode:8020/file.csv) - inherit_storage_options: dict (optional) - Its contents will get merged with the inferred information from the - given path - - Returns - ------- - Storage options dict. - - Examples - -------- - >>> infer_storage_options('/mnt/datasets/test.csv') # doctest: +SKIP - {"protocol": "file", "path", "/mnt/datasets/test.csv"} - >>> infer_storage_options( - ... 'hdfs://username:pwd@node:123/mnt/datasets/test.csv?q=1', - ... inherit_storage_options={'extra': 'value'}, - ... ) # doctest: +SKIP - {"protocol": "hdfs", "username": "username", "password": "pwd", - "host": "node", "port": 123, "path": "/mnt/datasets/test.csv", - "url_query": "q=1", "extra": "value"} - """ - - # Discover Windows paths including disk name in this special case. - is_filesystem = re.match(r"^[a-zA-Z]:[\\/]", urlpath) - - # Discover URI according to RFC 3986: Scheme names consist of a - # sequence of characters beginning with a letter and followed by - # any combination of letters, digits, plus ("+"), period ("."), - # or hyphen ("-"). - # https://datatracker.ietf.org/doc/html/rfc3986#section-3.1 - is_uri = re.match(r"^[a-zA-Z0-9+.-]+://", urlpath) - - if is_filesystem or is_uri is None: - return {"protocol": "file", "path": urlpath} - - parsed_path = urlsplit(urlpath) - protocol = parsed_path.scheme or "file" - if parsed_path.fragment: - path = "#".join([parsed_path.path, parsed_path.fragment]) - else: - path = parsed_path.path - if protocol == "file": - # Special case parsing file protocol URL on Windows according to: - # https://msdn.microsoft.com/en-us/library/jj710207.aspx - windows_path = re.match(r"^/([a-zA-Z])[:|]([\\/].*)$", path) - if windows_path: - drive, path = windows_path.groups() - path = f"{drive}:{path}" - - if protocol in ["http", "https"]: - # for HTTP, we don't want to parse, as requests will anyway - return {"protocol": protocol, "path": urlpath} - - options: dict[str, Any] = {"protocol": protocol, "path": path} - - if parsed_path.netloc: - # Parse `hostname` from netloc manually because `parsed_path.hostname` - # lowercases the hostname which is not always desirable (e.g. in S3): - # https://github.com/dask/dask/issues/1417 - options["host"] = parsed_path.netloc.rsplit("@", 1)[-1].rsplit(":", 1)[0] - - if protocol in ("s3", "s3a", "gcs", "gs"): - options["path"] = options["host"] + options["path"] - else: - options["host"] = options["host"] - if parsed_path.port: - options["port"] = parsed_path.port - if parsed_path.username: - options["username"] = parsed_path.username - if parsed_path.password: - options["password"] = parsed_path.password - - if parsed_path.query: - options["url_query"] = parsed_path.query - if parsed_path.fragment: - options["url_fragment"] = parsed_path.fragment - - if inherit_storage_options: - update_storage_options(options, inherit_storage_options) - - return options - - -def update_storage_options( - options: dict[str, Any], inherited: dict[str, Any] | None = None -) -> None: - if not inherited: - inherited = {} - collisions = set(options) & set(inherited) - if collisions: - for collision in collisions: - if options.get(collision) != inherited.get(collision): - raise KeyError( - f"Collision between inferred and specified storage " - f"option:\n{collision}" - ) - options.update(inherited) - - -# Compression extensions registered via fsspec.compression.register_compression -compressions: dict[str, str] = {} - - -def infer_compression(filename: str) -> str | None: - """Infer compression, if available, from filename. - - Infer a named compression type, if registered and available, from filename - extension. This includes builtin (gz, bz2, zip) compressions, as well as - optional compressions. See fsspec.compression.register_compression. - """ - extension = os.path.splitext(filename)[-1].strip(".").lower() - if extension in compressions: - return compressions[extension] - return None - - -def build_name_function(max_int: float) -> Callable[[int], str]: - """Returns a function that receives a single integer - and returns it as a string padded by enough zero characters - to align with maximum possible integer - - >>> name_f = build_name_function(57) - - >>> name_f(7) - '07' - >>> name_f(31) - '31' - >>> build_name_function(1000)(42) - '0042' - >>> build_name_function(999)(42) - '042' - >>> build_name_function(0)(0) - '0' - """ - # handle corner cases max_int is 0 or exact power of 10 - max_int += 1e-8 - - pad_length = int(math.ceil(math.log10(max_int))) - - def name_function(i: int) -> str: - return str(i).zfill(pad_length) - - return name_function - - -def seek_delimiter(file: IO[bytes], delimiter: bytes, blocksize: int) -> bool: - r"""Seek current file to file start, file end, or byte after delimiter seq. - - Seeks file to next chunk delimiter, where chunks are defined on file start, - a delimiting sequence, and file end. Use file.tell() to see location afterwards. - Note that file start is a valid split, so must be at offset > 0 to seek for - delimiter. - - Parameters - ---------- - file: a file - delimiter: bytes - a delimiter like ``b'\n'`` or message sentinel, matching file .read() type - blocksize: int - Number of bytes to read from the file at once. - - - Returns - ------- - Returns True if a delimiter was found, False if at file start or end. - - """ - - if file.tell() == 0: - # beginning-of-file, return without seek - return False - - # Interface is for binary IO, with delimiter as bytes, but initialize last - # with result of file.read to preserve compatibility with text IO. - last: bytes | None = None - while True: - current = file.read(blocksize) - if not current: - # end-of-file without delimiter - return False - full = last + current if last else current - try: - if delimiter in full: - i = full.index(delimiter) - file.seek(file.tell() - (len(full) - i) + len(delimiter)) - return True - elif len(current) < blocksize: - # end-of-file without delimiter - return False - except (OSError, ValueError): - pass - last = full[-len(delimiter) :] - - -def read_block( - f: IO[bytes], - offset: int, - length: int | None, - delimiter: bytes | None = None, - split_before: bool = False, -) -> bytes: - """Read a block of bytes from a file - - Parameters - ---------- - f: File - Open file - offset: int - Byte offset to start read - length: int - Number of bytes to read, read through end of file if None - delimiter: bytes (optional) - Ensure reading starts and stops at delimiter bytestring - split_before: bool (optional) - Start/stop read *before* delimiter bytestring. - - - If using the ``delimiter=`` keyword argument we ensure that the read - starts and stops at delimiter boundaries that follow the locations - ``offset`` and ``offset + length``. If ``offset`` is zero then we - start at zero, regardless of delimiter. The bytestring returned WILL - include the terminating delimiter string. - - Examples - -------- - - >>> from io import BytesIO # doctest: +SKIP - >>> f = BytesIO(b'Alice, 100\\nBob, 200\\nCharlie, 300') # doctest: +SKIP - >>> read_block(f, 0, 13) # doctest: +SKIP - b'Alice, 100\\nBo' - - >>> read_block(f, 0, 13, delimiter=b'\\n') # doctest: +SKIP - b'Alice, 100\\nBob, 200\\n' - - >>> read_block(f, 10, 10, delimiter=b'\\n') # doctest: +SKIP - b'Bob, 200\\nCharlie, 300' - """ - if delimiter: - f.seek(offset) - found_start_delim = seek_delimiter(f, delimiter, 2**16) - if length is None: - return f.read() - start = f.tell() - length -= start - offset - - f.seek(start + length) - found_end_delim = seek_delimiter(f, delimiter, 2**16) - end = f.tell() - - # Adjust split location to before delimiter if seek found the - # delimiter sequence, not start or end of file. - if found_start_delim and split_before: - start -= len(delimiter) - - if found_end_delim and split_before: - end -= len(delimiter) - - offset = start - length = end - start - - f.seek(offset) - - # TODO: allow length to be None and read to the end of the file? - assert length is not None - b = f.read(length) - return b - - -def tokenize(*args: Any, **kwargs: Any) -> str: - """Deterministic token - - (modified from dask.base) - - >>> tokenize([1, 2, '3']) - '9d71491b50023b06fc76928e6eddb952' - - >>> tokenize('Hello') == tokenize('Hello') - True - """ - if kwargs: - args += (kwargs,) - try: - h = md5(str(args).encode()) - except ValueError: - # FIPS systems: https://github.com/fsspec/filesystem_spec/issues/380 - h = md5(str(args).encode(), usedforsecurity=False) - return h.hexdigest() - - -def stringify_path(filepath: str | os.PathLike[str] | pathlib.Path) -> str: - """Attempt to convert a path-like object to a string. - - Parameters - ---------- - filepath: object to be converted - - Returns - ------- - filepath_str: maybe a string version of the object - - Notes - ----- - Objects supporting the fspath protocol are coerced according to its - __fspath__ method. - - For backwards compatibility with older Python version, pathlib.Path - objects are specially coerced. - - Any other object is passed through unchanged, which includes bytes, - strings, buffers, or anything else that's not even path-like. - """ - if isinstance(filepath, str): - return filepath - elif hasattr(filepath, "__fspath__"): - return filepath.__fspath__() - elif hasattr(filepath, "path"): - return filepath.path - else: - return filepath # type: ignore[return-value] - - -def make_instance( - cls: Callable[..., T], args: Sequence[Any], kwargs: dict[str, Any] -) -> T: - inst = cls(*args, **kwargs) - inst._determine_worker() # type: ignore[attr-defined] - return inst - - -def common_prefix(paths: Iterable[str]) -> str: - """For a list of paths, find the shortest prefix common to all""" - parts = [p.split("/") for p in paths] - lmax = min(len(p) for p in parts) - end = 0 - for i in range(lmax): - end = all(p[i] == parts[0][i] for p in parts) - if not end: - break - i += end - return "/".join(parts[0][:i]) - - -def other_paths( - paths: list[str], - path2: str | list[str], - exists: bool = False, - flatten: bool = False, -) -> list[str]: - """In bulk file operations, construct a new file tree from a list of files - - Parameters - ---------- - paths: list of str - The input file tree - path2: str or list of str - Root to construct the new list in. If this is already a list of str, we just - assert it has the right number of elements. - exists: bool (optional) - For a str destination, it is already exists (and is a dir), files should - end up inside. - flatten: bool (optional) - Whether to flatten the input directory tree structure so that the output files - are in the same directory. - - Returns - ------- - list of str - """ - - if isinstance(path2, str): - path2 = path2.rstrip("/") - - if flatten: - path2 = ["/".join((path2, p.split("/")[-1])) for p in paths] - else: - cp = common_prefix(paths) - if exists: - cp = cp.rsplit("/", 1)[0] - if not cp and all(not s.startswith("/") for s in paths): - path2 = ["/".join([path2, p]) for p in paths] - else: - path2 = [p.replace(cp, path2, 1) for p in paths] - else: - assert len(paths) == len(path2) - return path2 - - -def is_exception(obj: Any) -> bool: - return isinstance(obj, BaseException) - - -def isfilelike(f: Any) -> TypeGuard[IO[bytes]]: - return all(hasattr(f, attr) for attr in ["read", "close", "tell"]) - - -def get_protocol(url: str) -> str: - url = stringify_path(url) - parts = re.split(r"(\:\:|\://)", url, maxsplit=1) - if len(parts) > 1: - return parts[0] - return "file" - - -def get_file_extension(url: str) -> str: - url = stringify_path(url) - # Only consider the final path component: a "." in a parent directory name - # (e.g. "/path/to.dir/file") is not the file's extension. - ext_parts = url.rsplit("/", 1)[-1].rsplit(".", 1) - if len(ext_parts) > 1: - return ext_parts[-1] - return "" - - -def can_be_local(path: str) -> bool: - """Can the given URL be used with open_local?""" - from fsspec import get_filesystem_class - - try: - return getattr(get_filesystem_class(get_protocol(path)), "local_file", False) - except (ValueError, ImportError): - # not in registry or import failed - return False - - -def get_package_version_without_import(name: str) -> str | None: - """For given package name, try to find the version without importing it - - Import and package.__version__ is still the backup here, so an import - *might* happen. - - Returns either the version string, or None if the package - or the version was not readily found. - """ - if name in sys.modules: - mod = sys.modules[name] - if hasattr(mod, "__version__"): - return mod.__version__ - try: - return version(name) - except: # noqa: E722 - pass - try: - import importlib - - mod = importlib.import_module(name) - return mod.__version__ - except (ImportError, AttributeError): - return None - - -def setup_logging( - logger: logging.Logger | None = None, - logger_name: str | None = None, - level: str = "DEBUG", - clear: bool = True, -) -> logging.Logger: - if logger is None and logger_name is None: - raise ValueError("Provide either logger object or logger name") - logger = logger or logging.getLogger(logger_name) - handle = logging.StreamHandler() - formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(funcName)s -- %(message)s" - ) - handle.setFormatter(formatter) - if clear: - logger.handlers.clear() - logger.addHandler(handle) - logger.setLevel(level) - return logger - - -def _unstrip_protocol(name: str, fs: AbstractFileSystem) -> str: - return fs.unstrip_protocol(name) - - -def mirror_from( - origin_name: str, methods: Iterable[str] -) -> Callable[[type[T]], type[T]]: - """Mirror attributes and methods from the given - origin_name attribute of the instance to the - decorated class""" - - def origin_getter(method: str, self: Any) -> Any: - origin = getattr(self, origin_name) - return getattr(origin, method) - - def wrapper(cls: type[T]) -> type[T]: - for method in methods: - wrapped_method = partial(origin_getter, method) - setattr(cls, method, property(wrapped_method)) - return cls - - return wrapper - - -@contextlib.contextmanager -def nullcontext(obj: T) -> Iterator[T]: - yield obj - - -def merge_offset_ranges( - paths: list[str], - starts: list[int] | int, - ends: list[int] | int, - max_gap: int = 0, - max_block: int | None = None, - sort: bool = True, -) -> tuple[list[str], list[int], list[int]]: - """Merge adjacent byte-offset ranges when the inter-range - gap is <= `max_gap`, and when the merged byte range does not - exceed `max_block` (if specified). By default, this function - will re-order the input paths and byte ranges to ensure sorted - order. If the user can guarantee that the inputs are already - sorted, passing `sort=False` will skip the re-ordering. - """ - # Check input - if not isinstance(paths, list): - raise TypeError - if not isinstance(starts, list): - starts = [starts] * len(paths) - if not isinstance(ends, list): - ends = [ends] * len(paths) - if len(starts) != len(paths) or len(ends) != len(paths): - raise ValueError - - # Early Return - if len(starts) <= 1: - return paths, starts, ends - - starts = [s or 0 for s in starts] - # Sort by paths and then ranges if `sort=True` - if sort: - paths, starts, ends = ( - list(v) - for v in zip( - *sorted( - zip(paths, starts, ends), - ) - ) - ) - remove = [] - for i, (path, start, end) in enumerate(zip(paths, starts, ends)): - if any( - e is not None and p == path and start >= s and end <= e and i != i2 - for i2, (p, s, e) in enumerate(zip(paths, starts, ends)) - ): - remove.append(i) - paths = [p for i, p in enumerate(paths) if i not in remove] - starts = [s for i, s in enumerate(starts) if i not in remove] - ends = [e for i, e in enumerate(ends) if i not in remove] - - if paths: - # Loop through the coupled `paths`, `starts`, and - # `ends`, and merge adjacent blocks when appropriate - new_paths = paths[:1] - new_starts = starts[:1] - new_ends = ends[:1] - for i in range(1, len(paths)): - if paths[i] == paths[i - 1] and new_ends[-1] is None: - continue - elif ( - paths[i] != paths[i - 1] - or ((starts[i] - new_ends[-1]) > max_gap) - or (max_block is not None and (ends[i] - new_starts[-1]) > max_block) - ): - # Cannot merge with previous block. - # Add new `paths`, `starts`, and `ends` elements - new_paths.append(paths[i]) - new_starts.append(starts[i]) - new_ends.append(ends[i]) - else: - # Merge with the previous block by updating the - # last element of `ends` - new_ends[-1] = ends[i] - return new_paths, new_starts, new_ends - - # `paths` is empty. Just return input lists - return paths, starts, ends - - -def file_size(filelike: IO[bytes]) -> int: - """Find length of any open read-mode file-like""" - pos = filelike.tell() - try: - return filelike.seek(0, 2) - finally: - filelike.seek(pos) - - -@contextlib.contextmanager -def atomic_write(path: str, mode: str = "wb"): - """ - A context manager that opens a temporary file next to `path` and, on exit, - replaces `path` with the temporary file, thereby updating `path` - atomically. - """ - fd, fn = tempfile.mkstemp( - dir=os.path.dirname(path), prefix=os.path.basename(path) + "-" - ) - try: - with open(fd, mode) as fp: - yield fp - except BaseException: - with contextlib.suppress(FileNotFoundError): - os.unlink(fn) - raise - else: - os.replace(fn, path) - - -def _translate(pat, STAR, QUESTION_MARK): - # Copied from: https://github.com/python/cpython/pull/106703. - res: list[str] = [] - add = res.append - i, n = 0, len(pat) - while i < n: - c = pat[i] - i = i + 1 - if c == "*": - # compress consecutive `*` into one - if (not res) or res[-1] is not STAR: - add(STAR) - elif c == "?": - add(QUESTION_MARK) - elif c == "[": - j = i - if j < n and pat[j] == "!": - j = j + 1 - if j < n and pat[j] == "]": - j = j + 1 - while j < n and pat[j] != "]": - j = j + 1 - if j >= n: - add("\\[") - else: - stuff = pat[i:j] - if "-" not in stuff: - stuff = stuff.replace("\\", r"\\") - else: - chunks = [] - k = i + 2 if pat[i] == "!" else i + 1 - while True: - k = pat.find("-", k, j) - if k < 0: - break - chunks.append(pat[i:k]) - i = k + 1 - k = k + 3 - chunk = pat[i:j] - if chunk: - chunks.append(chunk) - else: - chunks[-1] += "-" - # Remove empty ranges -- invalid in RE. - for k in range(len(chunks) - 1, 0, -1): - if chunks[k - 1][-1] > chunks[k][0]: - chunks[k - 1] = chunks[k - 1][:-1] + chunks[k][1:] - del chunks[k] - # Escape backslashes and hyphens for set difference (--). - # Hyphens that create ranges shouldn't be escaped. - stuff = "-".join( - s.replace("\\", r"\\").replace("-", r"\-") for s in chunks - ) - # Escape set operations (&&, ~~ and ||). - stuff = re.sub(r"([&~|])", r"\\\1", stuff) - i = j + 1 - if not stuff: - # Empty range: never match. - add("(?!)") - elif stuff == "!": - # Negated empty range: match any character. - add(".") - else: - if stuff[0] == "!": - stuff = "^" + stuff[1:] - elif stuff[0] in ("^", "["): - stuff = "\\" + stuff - add(f"[{stuff}]") - else: - add(re.escape(c)) - assert i == n - return res - - -def glob_translate(pat): - # Copied from: https://github.com/python/cpython/pull/106703. - # The keyword parameters' values are fixed to: - # recursive=True, include_hidden=True, seps=None - """Translate a pathname with shell wildcards to a regular expression.""" - if os.path.altsep: - seps = os.path.sep + os.path.altsep - else: - seps = os.path.sep - escaped_seps = "".join(map(re.escape, seps)) - any_sep = f"[{escaped_seps}]" if len(seps) > 1 else escaped_seps - not_sep = f"[^{escaped_seps}]" - one_last_segment = f"{not_sep}+" - one_segment = f"{one_last_segment}{any_sep}" - any_segments = f"(?:.+{any_sep})?" - any_last_segments = ".*" - results = [] - parts = re.split(any_sep, pat) - last_part_idx = len(parts) - 1 - for idx, part in enumerate(parts): - if part == "*": - results.append(one_segment if idx < last_part_idx else one_last_segment) - continue - if part == "**": - results.append(any_segments if idx < last_part_idx else any_last_segments) - continue - elif "**" in part: - raise ValueError( - "Invalid pattern: '**' can only be an entire path component" - ) - if part: - results.extend(_translate(part, f"{not_sep}*", not_sep)) - if idx < last_part_idx: - results.append(any_sep) - res = "".join(results) - return rf"(?s:{res})\Z" diff --git a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/METADATA deleted file mode 100644 index 1eaaae955f155091e4516629699af17fcfb8e57a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/METADATA +++ /dev/null @@ -1,151 +0,0 @@ -Metadata-Version: 2.3 -Name: ftfy -Version: 6.3.1 -Summary: Fixes mojibake and other problems with Unicode, after the fact -Project-URL: Homepage, https://ftfy.readthedocs.io/en/latest/ -Project-URL: Documentation, https://ftfy.readthedocs.io/en/latest/ -Project-URL: Repository, https://github.com/rspeer/python-ftfy -Project-URL: Issues, https://github.com/rspeer/python-ftfy/issues/ -Project-URL: Changelog, https://github.com/rspeer/python-ftfy/blob/main/CHANGELOG.md -Project-URL: Blog, https://posts.arborelia.net -Author-email: Robyn Speer -License: Apache-2.0 -License-File: LICENSE.txt -Requires-Python: >=3.9 -Requires-Dist: wcwidth -Description-Content-Type: text/markdown - -# ftfy: fixes text for you - -[![PyPI package](https://badge.fury.io/py/ftfy.svg)](https://badge.fury.io/py/ftfy) -[![Docs](https://readthedocs.org/projects/ftfy/badge/?version=latest)](https://ftfy.readthedocs.org/en/latest/) - -```python - ->>> from ftfy import fix_encoding ->>> print(fix_encoding("(ง'⌣')ง")) -(ง'⌣')ง - -``` - -The full documentation of ftfy is available at [ftfy.readthedocs.org](https://ftfy.readthedocs.org). The documentation covers a lot more than this README, so here are some links into it: - -- [Fixing problems and getting explanations](https://ftfy.readthedocs.io/en/latest/explain.html) -- [Configuring ftfy](https://ftfy.readthedocs.io/en/latest/config.html) -- [Encodings ftfy can handle](https://ftfy.readthedocs.io/en/latest/encodings.html) -- [“Fixer” functions](https://ftfy.readthedocs.io/en/latest/fixes.html) -- [Is ftfy an encoding detector?](https://ftfy.readthedocs.io/en/latest/detect.html) -- [Heuristics for detecting mojibake](https://ftfy.readthedocs.io/en/latest/heuristic.html) -- [Support for “bad” encodings](https://ftfy.readthedocs.io/en/latest/bad_encodings.html) -- [Command-line usage](https://ftfy.readthedocs.io/en/latest/cli.html) -- [Citing ftfy](https://ftfy.readthedocs.io/en/latest/cite.html) - -## Testimonials - -- “My life is livable again!” - — [@planarrowspace](https://twitter.com/planarrowspace) -- “A handy piece of magic” - — [@simonw](https://twitter.com/simonw) -- “Saved me a large amount of frustrating dev work” - — [@iancal](https://twitter.com/iancal) -- “ftfy did the right thing right away, with no faffing about. Excellent work, solving a very tricky real-world (whole-world!) problem.” - — Brennan Young -- “I have no idea when I’m gonna need this, but I’m definitely bookmarking it.” - — [/u/ocrow](https://reddit.com/u/ocrow) - -## What it does - -Here are some examples (found in the real world) of what ftfy can do: - -ftfy can fix mojibake (encoding mix-ups), by detecting patterns of characters that were clearly meant to be UTF-8 but were decoded as something else: - - >>> import ftfy - >>> ftfy.fix_text('✔ No problems') - '✔ No problems' - -Does this sound impossible? It's really not. UTF-8 is a well-designed encoding that makes it obvious when it's being misused, and a string of mojibake usually contains all the information we need to recover the original string. - -ftfy can fix multiple layers of mojibake simultaneously: - - >>> ftfy.fix_text('The Mona Lisa doesn’t have eyebrows.') - "The Mona Lisa doesn't have eyebrows." - -It can fix mojibake that has had "curly quotes" applied on top of it, which cannot be consistently decoded until the quotes are uncurled: - - >>> ftfy.fix_text("l’humanité") - "l'humanité" - -ftfy can fix mojibake that would have included the character U+A0 (non-breaking space), but the U+A0 was turned into an ASCII space and then combined with another following space: - - >>> ftfy.fix_text('Ã\xa0 perturber la réflexion') - 'à perturber la réflexion' - >>> ftfy.fix_text('à perturber la réflexion') - 'à perturber la réflexion' - -ftfy can also decode HTML entities that appear outside of HTML, even in cases where the entity has been incorrectly capitalized: - - >>> # by the HTML 5 standard, only 'PÉREZ' is acceptable - >>> ftfy.fix_text('P&EACUTE;REZ') - 'PÉREZ' - -These fixes are not applied in all cases, because ftfy has a strongly-held goal of avoiding false positives -- it should never change correctly-decoded text to something else. - -The following text could be encoded in Windows-1252 and decoded in UTF-8, and it would decode as 'MARQUɅ'. However, the original text is already sensible, so it is unchanged. - - >>> ftfy.fix_text('IL Y MARQUÉ…') - 'IL Y MARQUÉ…' - -## Installing - -ftfy is a Python 3 package that can be installed using `pip` or `uv pip`: - - pip install ftfy - -(Or use `pip3 install ftfy` on systems where Python 2 and 3 are both globally installed and `pip` refers to Python 2.) - -If you use `poetry`, you can use ftfy as a dependency in the usual way (such as `poetry add ftfy`). - -### Local development - -ftfy is developed using [uv](https://github.com/astral-sh/uv). You can build a virtual environment with its local dependencies by running `uv venv`, and test it with `uv run pytest`. - -## Who maintains ftfy? - -I'm Robyn Speer, also known as Elia Robyn Lake. You can find my projects -[on GitHub](https://github.com/rspeer) and my posts on [my own blog](https://posts.arborelia.net). - -## Citing ftfy - -ftfy has been used as a crucial data processing step in major NLP research. - -It's important to give credit appropriately to everyone whose work you build on in research. This includes software, not just high-status contributions such as mathematical models. All I ask when you use ftfy for research is that you cite it. - -ftfy has a citable record [on Zenodo](https://zenodo.org/record/2591652). A citation of ftfy may look like this: - - Robyn Speer. (2019). ftfy (Version 5.5). Zenodo. - http://doi.org/10.5281/zenodo.2591652 - -In BibTeX format, the citation is:: - - @misc{speer-2019-ftfy, - author = {Robyn Speer}, - title = {ftfy}, - note = {Version 5.5}, - year = 2019, - howpublished = {Zenodo}, - doi = {10.5281/zenodo.2591652}, - url = {https://doi.org/10.5281/zenodo.2591652} - } - -## Important license clarifications - -If you do not follow ftfy's license, you do not have a license to ftfy. - -This sounds obvious and tautological, but there are people who think open source licenses mean that they can just do what they want, especially in the field of generative AI. It's a permissive license but you still have to follow it. The [Apache license](https://www.apache.org/licenses/LICENSE-2.0) is the only thing that gives you permission to use and copy ftfy; otherwise, all rights are reserved. - -If you use or distribute ftfy, you must follow the terms of the [Apache license](https://www.apache.org/licenses/LICENSE-2.0), including that you must attribute the author of ftfy (Robyn Speer) correctly. - -You _may not_ make a derived work of ftfy that obscures its authorship, such as by putting its code in an AI training dataset, including the code in AI training at runtime, or using a generative AI that copies code from such a dataset. - -At my discretion, I may notify you of a license violation, and give you a chance to either remedy it or delete all copies of ftfy in your possession. - diff --git a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/RECORD deleted file mode 100644 index 1b04ed7a9a7674dc828ae5c53d2d94a1192e3c3f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/RECORD +++ /dev/null @@ -1,18 +0,0 @@ -../../Scripts/ftfy.exe,sha256=sKtABXs-6ii-iMJvenMgd9m3XBaDNyMHwkhtjhumOwI,46080 -ftfy-6.3.1.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -ftfy-6.3.1.dist-info/METADATA,sha256=g_Z8YoA3djfFYLXogCoAum7lErCh6h2uXf8edFsZ0d8,7257 -ftfy-6.3.1.dist-info/RECORD,, -ftfy-6.3.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -ftfy-6.3.1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87 -ftfy-6.3.1.dist-info/entry_points.txt,sha256=sy4Ei29IqZhQRgYFC17GVrT1JkE6l1-ooJheKulSqO8,39 -ftfy-6.3.1.dist-info/licenses/LICENSE.txt,sha256=oL_FAgvC4eggVRoGJccMlfqomMFec3EA_0iAqhAZeFc,552 -ftfy/__init__.py,sha256=ekX6ZbJnR3UxmgWwpM2_0Eoizn82ukCYQRv57Bsiyzw,29593 -ftfy/bad_codecs/__init__.py,sha256=sPKWae974vjc3JeOsGhZ8ZFNUoPTISx988zBbxJ7SIQ,3282 -ftfy/bad_codecs/sloppy.py,sha256=bzgoZAQIrzRUxb-qJmDVX6_Eh2l143IqstdFFFayTII,6814 -ftfy/bad_codecs/utf8_variants.py,sha256=WlyMbOtUFftePEj_SpxDTXirPAj8HcuxVZ1aSYApVh0,9964 -ftfy/badness.py,sha256=jOR-ucAfX6LTjmUP4EdHS95VN766WXG3IsprQHnfb9M,15561 -ftfy/chardata.py,sha256=tAXqrX2Y3zBslAZoj6tFBy-Yn-_-UwF1uszvKN7cVck,34911 -ftfy/cli.py,sha256=q9DNi4LuhKGrleoH_Z3aB6J0oST34a2WWatap7Odr7s,4137 -ftfy/fixes.py,sha256=G8XQKzMlcsw4aPhRrQgcwAMcxqatf7EDmNaOrsY-C6U,18233 -ftfy/formatting.py,sha256=pbmBunYydgPs3bxMhCXFtaTM0_3qT_yp6esZvjvOKZk,5882 -ftfy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/WHEEL deleted file mode 100644 index cdd68a497cdfa8d3f2b837225beacef711b85047..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: hatchling 1.25.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/entry_points.txt b/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/entry_points.txt deleted file mode 100644 index 99476a44fd1015a1c391431110d1773503340096..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -ftfy = ftfy.cli:main diff --git a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/licenses/LICENSE.txt b/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/licenses/LICENSE.txt deleted file mode 100644 index 275a4299e4d3204f13d5dec6cea39a852267b78d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy-6.3.1.dist-info/licenses/LICENSE.txt +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2023 Robyn Speer - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/bundle/python-cpu/Lib/site-packages/ftfy/__init__.py b/bundle/python-cpu/Lib/site-packages/ftfy/__init__.py deleted file mode 100644 index cc0a12049eb1a3093fd921b18bb9dbcef5a7584e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy/__init__.py +++ /dev/null @@ -1,802 +0,0 @@ -""" -ftfy: fixes text for you - -This is a module for making text less broken. See the `fix_text` function -for more information. -""" - -from __future__ import annotations - -import unicodedata -import warnings -from collections.abc import Iterator -from typing import ( - Any, - BinaryIO, - Callable, - Literal, - NamedTuple, - TextIO, - cast, -) - -from ftfy import bad_codecs, chardata, fixes -from ftfy.badness import is_bad -from ftfy.formatting import display_ljust - -__version__ = "6.3.1" - - -# Though this function does nothing, it lets linters know that we're using -# ftfy.bad_codecs. See the docstring in `bad_codecs/__init__.py` for more. -bad_codecs.ok() - - -class ExplanationStep(NamedTuple): - """ - A step in an ExplainedText, explaining how to decode text. - - The possible actions are: - - - "encode": take in a string and encode it as bytes, with the given encoding - - "decode": take in bytes and decode them as a string, with the given encoding - - "transcode": convert bytes to bytes with a particular named function - - "apply": convert str to str with a particular named function - - The `parameter` is the name of the encoding or function to use. If it's a - function, it must appear in the FIXERS dictionary. - """ - - action: str - parameter: str - - def __repr__(self) -> str: - """ - Get the string representation of an ExplanationStep. We output the - representation of the equivalent tuple, for simplicity. - """ - return repr(tuple(self)) - - -class ExplainedText(NamedTuple): - """ - The return type from ftfy's functions that provide an "explanation" of which - steps it applied to fix the text, such as :func:`fix_and_explain()`. - - When the 'explain' option is disabled, these functions return the same - type, but the `explanation` will be None. - """ - - text: str - explanation: list[ExplanationStep] | None - - -# Functions that can be applied using `apply_plan`. -FIXERS: dict[str, Callable] = { # type: ignore[type-arg] - "unescape_html": fixes.unescape_html, - "remove_terminal_escapes": fixes.remove_terminal_escapes, - "restore_byte_a0": fixes.restore_byte_a0, - "replace_lossy_sequences": fixes.replace_lossy_sequences, - "decode_inconsistent_utf8": fixes.decode_inconsistent_utf8, - "fix_c1_controls": fixes.fix_c1_controls, - "fix_latin_ligatures": fixes.fix_latin_ligatures, - "fix_character_width": fixes.fix_character_width, - "uncurl_quotes": fixes.uncurl_quotes, - "fix_line_breaks": fixes.fix_line_breaks, - "fix_surrogates": fixes.fix_surrogates, - "remove_control_chars": fixes.remove_control_chars, -} - - -class TextFixerConfig(NamedTuple): - r""" - A TextFixerConfig object stores configuration options for ftfy. - - It's implemented as a namedtuple with defaults, so you can instantiate - it by providing the values to change from their defaults as keyword arguments. - For example, to disable 'unescape_html' and keep the rest of the defaults:: - - TextFixerConfig(unescape_html=False) - - Here are the options and their default values: - - - `unescape_html`: "auto" - - Configures whether to replace HTML entities such as & with the character - they represent. "auto" says to do this by default, but disable it when a - literal < character appears, indicating that the input is actual HTML and - entities should be preserved. The value can be True, to always enable this - fixer, or False, to always disable it. - - - `remove_terminal_escapes`: True - - Removes "ANSI" terminal escapes, such as for changing the color of text in a - terminal window. - - - `fix_encoding`: True - - Detect mojibake and attempt to fix it by decoding the text in a different - encoding standard. - - The following four options affect `fix_encoding` works, and do nothing if - `fix_encoding` is False: - - - `restore_byte_a0`: True - - Allow a literal space (U+20) to be interpreted as a non-breaking space - (U+A0) when that would make it part of a fixable mojibake string. - - Because spaces are very common characters, this could lead to false - positives, but we try to apply it only when there's strong evidence for - mojibake. Disabling `restore_byte_a0` is safer from false positives, - but creates false negatives. - - - `replace_lossy_sequences`: True - - Detect mojibake that has been partially replaced by the characters - '�' or '?'. If the mojibake could be decoded otherwise, replace the - detected sequence with '�'. - - - `decode_inconsistent_utf8`: True - - When we see sequences that distinctly look like UTF-8 mojibake, but - there's no consistent way to reinterpret the string in a new encoding, - replace the mojibake with the appropriate UTF-8 characters anyway. - - This helps to decode strings that are concatenated from different - encodings. - - - `fix_c1_controls`: True - - Replace C1 control characters (the useless characters U+80 - U+9B that - come from Latin-1) with their Windows-1252 equivalents, like HTML5 does, - even if the whole string doesn't decode as Latin-1. - - - `fix_latin_ligatures`: True - - Replace common Latin-alphabet ligatures, such as ``fi``, with the - letters they're made of. - - - `fix_character_width`: True - - Replace fullwidth Latin characters and halfwidth Katakana with - their more standard widths. - - - `uncurl_quotes`: True - - Replace curly quotes with straight quotes. - - - `fix_line_breaks`: True - - Replace various forms of line breaks with the standard Unix line - break, ``\n``. - - - `fix_surrogates`: True - - Replace sequences of UTF-16 surrogate codepoints with the character - they were meant to encode. This fixes text that was decoded with the - obsolete UCS-2 standard, and allows it to support high-numbered - codepoints such as emoji. - - - `remove_control_chars`: True - - Remove certain control characters that have no displayed effect on text. - - - `normalization`: "NFC" - - Choose what kind of Unicode normalization is applied. Usually, we apply - NFC normalization, so that letters followed by combining characters become - single combined characters. - - Changing this to "NFKC" applies more compatibility conversions, such as - replacing the 'micro sign' with a standard Greek lowercase mu, which looks - identical. However, some NFKC normalizations change the meaning of text, - such as converting "10³" to "103". - - `normalization` can be None, to apply no normalization. - - - `max_decode_length`: 1_000_000 - - The maximum size of "segment" that ftfy will try to fix all at once. - - - `explain`: True - - Whether to compute 'explanations', lists describing what ftfy changed. - When this is False, the explanation will be None, and the code that - builds the explanation will be skipped, possibly saving time. - - Functions that accept TextFixerConfig and don't return an explanation - will automatically set `explain` to False. - """ - - unescape_html: str | bool = "auto" - remove_terminal_escapes: bool = True - fix_encoding: bool = True - restore_byte_a0: bool = True - replace_lossy_sequences: bool = True - decode_inconsistent_utf8: bool = True - fix_c1_controls: bool = True - fix_latin_ligatures: bool = True - fix_character_width: bool = True - uncurl_quotes: bool = True - fix_line_breaks: bool = True - fix_surrogates: bool = True - remove_control_chars: bool = True - normalization: Literal["NFC", "NFD", "NFKC", "NFKD"] | None = "NFC" - max_decode_length: int = 1000000 - explain: bool = True - - -def _config_from_kwargs( - config: TextFixerConfig, kwargs: dict[str, Any] -) -> TextFixerConfig: - """ - Handle parameters provided as keyword arguments to ftfy's top-level - functions, converting them into a TextFixerConfig. - """ - if "fix_entities" in kwargs: - warnings.warn( - "`fix_entities` has been renamed to `unescape_html`", - DeprecationWarning, - stacklevel=2, - ) - kwargs = kwargs.copy() - kwargs["unescape_html"] = kwargs["fix_entities"] - del kwargs["fix_entities"] - config = config._replace(**kwargs) - return config - - -BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. - -ftfy is designed to fix problems with text. Treating bytes like they're -interchangeable with Unicode text is usually something that introduces -problems with text. - -You should first decode these bytes from the encoding you think they're in. -If you're not sure what encoding they're in: - -- First, try to find out. 'utf-8' is a good assumption. -- If the encoding is simply unknowable, try running your bytes through - ftfy.guess_bytes. As the name implies, this may not always be accurate. - -For more information on the distinction between bytes and text, read the -Python Unicode HOWTO: - - http://docs.python.org/3/howto/unicode.html -""" - - -def _try_fix( - fixer_name: str, - text: str, - config: TextFixerConfig, - steps: list[ExplanationStep] | None, -) -> str: - """ - A helper function used across several 'fixer' steps, deciding whether to - apply the fix and whether to record the fix in `steps`. - """ - if getattr(config, fixer_name): - fixer = FIXERS[fixer_name] - fixed = fixer(text) - if steps is not None and fixed != text: - steps.append(ExplanationStep("apply", fixer_name)) - return cast(str, fixed) - - return text - - -def fix_text(text: str, config: TextFixerConfig | None = None, **kwargs: Any) -> str: - r""" - Given Unicode text as input, fix inconsistencies and glitches in it, - such as mojibake (text that was decoded in the wrong encoding). - - Let's start with some examples: - - >>> fix_text('✔ No problems') - '✔ No problems' - - >>> print(fix_text("¯\\_(ã\x83\x84)_/¯")) - ¯\_(ツ)_/¯ - - >>> fix_text('Broken text… it’s flubberific!') - "Broken text... it's flubberific!" - - >>> fix_text('LOUD NOISES') - 'LOUD NOISES' - - ftfy applies a number of different fixes to the text, and can accept - configuration to select which fixes to apply. - - The configuration takes the form of a :class:`TextFixerConfig` object, - and you can see a description of the options in that class's docstring - or in the full documentation at ftfy.readthedocs.org. - - For convenience and backward compatibility, the configuration can also - take the form of keyword arguments, which will set the equivalently-named - fields of the TextFixerConfig object. - - For example, here are two ways to fix text but skip the "uncurl_quotes" - step:: - - fix_text(text, TextFixerConfig(uncurl_quotes=False)) - fix_text(text, uncurl_quotes=False) - - This function fixes text in independent segments, which are usually lines - of text, or arbitrarily broken up every 1 million codepoints (configurable - with `config.max_decode_length`) if there aren't enough line breaks. The - bound on segment lengths helps to avoid unbounded slowdowns. - - ftfy can also provide an 'explanation', a list of transformations it applied - to the text that would fix more text like it. This function doesn't provide - explanations (because there may be different fixes for different segments - of text). - - To get an explanation, use the :func:`fix_and_explain()` function, which - fixes the string in one segment and explains what it fixed. - """ - - if config is None: - config = TextFixerConfig(explain=False) - config = _config_from_kwargs(config, kwargs) - if isinstance(text, bytes): - raise UnicodeError(BYTES_ERROR_TEXT) - - out = [] - pos = 0 - while pos < len(text): - textbreak = text.find("\n", pos) + 1 - if textbreak == 0: - textbreak = len(text) - if (textbreak - pos) > config.max_decode_length: - textbreak = pos + config.max_decode_length - - segment = text[pos:textbreak] - if config.unescape_html == "auto" and "<" in segment: - config = config._replace(unescape_html=False) - fixed_segment, _ = fix_and_explain(segment, config) - out.append(fixed_segment) - pos = textbreak - return "".join(out) - - -def fix_and_explain( - text: str, config: TextFixerConfig | None = None, **kwargs: Any -) -> ExplainedText: - """ - Fix text as a single segment, returning the fixed text and an explanation - of what was fixed. - - The explanation is a list of steps that can be applied with - :func:`apply_plan`, or if config.explain is False, it will be None. - """ - if config is None: - config = TextFixerConfig() - if isinstance(text, bytes): - raise UnicodeError(BYTES_ERROR_TEXT) - config = _config_from_kwargs(config, kwargs) - - if config.unescape_html == "auto" and "<" in text: - config = config._replace(unescape_html=False) - - if config.explain: - steps: list[ExplanationStep] | None = [] - else: - # If explanations aren't desired, `steps` will be None - steps = None - - while True: - origtext = text - - text = _try_fix("unescape_html", text, config, steps) - - if config.fix_encoding: - if steps is None: - text = fix_encoding(text) - else: - text, encoding_steps = fix_encoding_and_explain(text, config) - if encoding_steps is not None: - steps.extend(encoding_steps) - - for fixer in [ - "fix_c1_controls", - "fix_latin_ligatures", - "fix_character_width", - "uncurl_quotes", - "fix_line_breaks", - "fix_surrogates", - "remove_terminal_escapes", - "remove_control_chars", - ]: - text = _try_fix(fixer, text, config, steps) - - if config.normalization is not None: - fixed = unicodedata.normalize(config.normalization, text) - if steps is not None and fixed != text: - steps.append(ExplanationStep("normalize", config.normalization)) - text = fixed - - if text == origtext: - return ExplainedText(text, steps) - - -def fix_encoding_and_explain( - text: str, config: TextFixerConfig | None = None, **kwargs: Any -) -> ExplainedText: - """ - Apply the steps of ftfy that detect mojibake and fix it. Returns the fixed - text and a list explaining what was fixed. - - This includes fixing text by encoding and decoding it in different encodings, - as well as the subordinate fixes `restore_byte_a0`, `replace_lossy_sequences`, - `decode_inconsistent_utf8`, and `fix_c1_controls`. - - Examples:: - - >>> fix_encoding_and_explain("só") - ExplainedText(text='só', explanation=[('encode', 'latin-1'), ('decode', 'utf-8')]) - - >>> result = fix_encoding_and_explain("voilà le travail") - >>> result.text - 'voilà le travail' - >>> result.explanation - [('encode', 'latin-1'), ('transcode', 'restore_byte_a0'), ('decode', 'utf-8')] - - """ - if config is None: - config = TextFixerConfig() - if isinstance(text, bytes): - raise UnicodeError(BYTES_ERROR_TEXT) - config = _config_from_kwargs(config, kwargs) - - if not config.fix_encoding: - # A weird trivial case: we're asked to fix the encoding, but skip - # fixing the encoding - return ExplainedText(text, []) - - plan_so_far: list[ExplanationStep] = [] - while True: - prevtext = text - text, plan = _fix_encoding_one_step_and_explain(text, config) - if plan is not None: - plan_so_far.extend(plan) - if text == prevtext: - return ExplainedText(text, plan_so_far) - - -def _fix_encoding_one_step_and_explain( - text: str, config: TextFixerConfig -) -> ExplainedText: - """ - Perform one step of fixing the encoding of text. - """ - if config is None: - config = TextFixerConfig() - - if len(text) == 0: - return ExplainedText(text, []) - - # The first plan is to return ASCII text unchanged, as well as text - # that doesn't look like it contains mojibake - if chardata.possible_encoding(text, "ascii") or not is_bad(text): - return ExplainedText(text, []) - - # As we go through the next step, remember the possible encodings - # that we encounter but don't successfully fix yet. We may need them - # later. - possible_1byte_encodings = [] - - # Suppose the text was supposed to be UTF-8, but it was decoded using - # a single-byte encoding instead. When these cases can be fixed, they - # are usually the correct thing to do, so try them next. - for encoding in chardata.CHARMAP_ENCODINGS: - if chardata.possible_encoding(text, encoding): - possible_1byte_encodings.append(encoding) - encoded_bytes = text.encode(encoding) - encode_step = ExplanationStep("encode", encoding) - transcode_steps = [] - - # Now, find out if it's UTF-8 (or close enough). Otherwise, - # remember the encoding for later. - try: - decoding = "utf-8" - # Check encoded_bytes for sequences that would be UTF-8, - # except they have b' ' where b'\xa0' would belong. - # - # Don't do this in the macroman encoding, where it would match - # an en dash followed by a space, leading to false positives. - if ( - config.restore_byte_a0 - and encoding != "macroman" - and chardata.ALTERED_UTF8_RE.search(encoded_bytes) - ): - replaced_bytes = fixes.restore_byte_a0(encoded_bytes) - if replaced_bytes != encoded_bytes: - transcode_steps.append( - ExplanationStep("transcode", "restore_byte_a0") - ) - encoded_bytes = replaced_bytes - - # Replace sequences where information has been lost - if config.replace_lossy_sequences and encoding.startswith("sloppy"): - replaced_bytes = fixes.replace_lossy_sequences(encoded_bytes) - if replaced_bytes != encoded_bytes: - transcode_steps.append( - ExplanationStep("transcode", "replace_lossy_sequences") - ) - encoded_bytes = replaced_bytes - - if 0xED in encoded_bytes or 0xC0 in encoded_bytes: - decoding = "utf-8-variants" - - decode_step = ExplanationStep("decode", decoding) - steps = [encode_step] + transcode_steps + [decode_step] - fixed = encoded_bytes.decode(decoding) - return ExplainedText(fixed, steps) - - except UnicodeDecodeError: - pass - - # Look for a-hat-euro sequences that remain, and fix them in isolation. - if config.decode_inconsistent_utf8 and chardata.UTF8_DETECTOR_RE.search(text): - steps = [ExplanationStep("apply", "decode_inconsistent_utf8")] - fixed = fixes.decode_inconsistent_utf8(text) - if fixed != text: - return ExplainedText(fixed, steps) - - # The next most likely case is that this is Latin-1 that was intended to - # be read as Windows-1252, because those two encodings in particular are - # easily confused. - if "latin-1" in possible_1byte_encodings: - if "windows-1252" in possible_1byte_encodings: - # This text is in the intersection of Latin-1 and - # Windows-1252, so it's probably legit. - return ExplainedText(text, []) - else: - # Otherwise, it means we have characters that are in Latin-1 but - # not in Windows-1252. Those are C1 control characters. Nobody - # wants those. Assume they were meant to be Windows-1252. - try: - fixed = text.encode("latin-1").decode("windows-1252") - if fixed != text: - steps = [ - ExplanationStep("encode", "latin-1"), - ExplanationStep("decode", "windows-1252"), - ] - return ExplainedText(fixed, steps) - except UnicodeDecodeError: - pass - - # Fix individual characters of Latin-1 with a less satisfying explanation - if config.fix_c1_controls and chardata.C1_CONTROL_RE.search(text): - steps = [ExplanationStep("transcode", "fix_c1_controls")] - fixed = fixes.fix_c1_controls(text) - return ExplainedText(fixed, steps) - - # The cases that remain are mixups between two different single-byte - # encodings, and not the common case of Latin-1 vs. Windows-1252. - # - # With the new heuristic in 6.0, it's possible that we're closer to solving - # these in some cases. It would require a lot of testing and tuning, though. - # For now, we leave the text unchanged in these cases. - return ExplainedText(text, []) - - -def fix_encoding( - text: str, config: TextFixerConfig | None = None, **kwargs: Any -) -> str: - """ - Apply just the encoding-fixing steps of ftfy to this text. Returns the - fixed text, discarding the explanation. - - >>> fix_encoding("ó") - 'ó' - >>> fix_encoding("&ATILDE;&SUP3;") - '&ATILDE;&SUP3;' - """ - if config is None: - config = TextFixerConfig(explain=False) - config = _config_from_kwargs(config, kwargs) - fixed, _explan = fix_encoding_and_explain(text, config) - return fixed - - -# Some alternate names for the main functions -ftfy = fix_text - - -def fix_text_segment( - text: str, config: TextFixerConfig | None = None, **kwargs: Any -) -> str: - """ - Fix text as a single segment, with a consistent sequence of steps that - are applied to fix the text. Discard the explanation. - """ - if config is None: - config = TextFixerConfig(explain=False) - config = _config_from_kwargs(config, kwargs) - fixed, _explan = fix_and_explain(text, config) - return fixed - - -def fix_file( - input_file: TextIO | BinaryIO, - encoding: str | None = None, - config: TextFixerConfig | None = None, - **kwargs: Any, -) -> Iterator[str]: - """ - Fix text that is found in a file. - - If the file is being read as Unicode text, use that. If it's being read as - bytes, then we hope an encoding was supplied. If not, unfortunately, we - have to guess what encoding it is. We'll try a few common encodings, but we - make no promises. See the `guess_bytes` function for how this is done. - - The output is a stream of fixed lines of text. - """ - if config is None: - config = TextFixerConfig() - config = _config_from_kwargs(config, kwargs) - - for line in input_file: - if isinstance(line, bytes): - if encoding is None: - line, encoding = guess_bytes(line) - else: - line = line.decode(encoding) - if config.unescape_html == "auto" and "<" in line: - config = config._replace(unescape_html=False) - - fixed_line, _explan = fix_and_explain(line, config) - yield fixed_line - - -def guess_bytes(bstring: bytes) -> tuple[str, str]: - """ - NOTE: Using `guess_bytes` is not the recommended way of using ftfy. ftfy - is not designed to be an encoding detector. - - In the unfortunate situation that you have some bytes in an unknown - encoding, ftfy can guess a reasonable strategy for decoding them, by trying - a few common encodings that can be distinguished from each other. - - Unlike the rest of ftfy, this may not be accurate, and it may *create* - Unicode problems instead of solving them! - - The encodings we try here are: - - - UTF-16 with a byte order mark, because a UTF-16 byte order mark looks - like nothing else - - UTF-8, because it's the global standard, which has been used by a - majority of the Web since 2008 - - "utf-8-variants", or buggy implementations of UTF-8 - - MacRoman, because Microsoft Office thinks it's still a thing, and it - can be distinguished by its line breaks. (If there are no line breaks in - the string, though, you're out of luck.) - - "sloppy-windows-1252", the Latin-1-like encoding that is the most common - single-byte encoding. - """ - if isinstance(bstring, str): - raise UnicodeError( - "This string was already decoded as Unicode. You should pass " - "bytes to guess_bytes, not Unicode." - ) - - if bstring.startswith(b"\xfe\xff") or bstring.startswith(b"\xff\xfe"): - return bstring.decode("utf-16"), "utf-16" - - byteset = set(bstring) - try: - if 0xED in byteset or 0xC0 in byteset: - # Byte 0xed can be used to encode a range of codepoints that - # are UTF-16 surrogates. UTF-8 does not use UTF-16 surrogates, - # so when we see 0xed, it's very likely we're being asked to - # decode CESU-8, the variant that encodes UTF-16 surrogates - # instead of the original characters themselves. - # - # This will occasionally trigger on standard UTF-8, as there - # are some Korean characters that also use byte 0xed, but that's - # not harmful because standard UTF-8 characters will decode the - # same way in our 'utf-8-variants' codec. - # - # Byte 0xc0 is impossible because, numerically, it would only - # encode characters lower than U+0040. Those already have - # single-byte representations, and UTF-8 requires using the - # shortest possible representation. However, Java hides the null - # codepoint, U+0000, in a non-standard longer representation -- it - # encodes it as 0xc0 0x80 instead of 0x00, guaranteeing that 0x00 - # will never appear in the encoded bytes. - # - # The 'utf-8-variants' decoder can handle both of these cases, as - # well as standard UTF-8, at the cost of a bit of speed. - return bstring.decode("utf-8-variants"), "utf-8-variants" - else: - return bstring.decode("utf-8"), "utf-8" - except UnicodeDecodeError: - pass - - if 0x0D in byteset and 0x0A not in byteset: - # Files that contain CR and not LF are likely to be MacRoman. - return bstring.decode("macroman"), "macroman" - - return bstring.decode("sloppy-windows-1252"), "sloppy-windows-1252" - - -def apply_plan(text: str, plan: list[tuple[str, str]]) -> str: - """ - Apply a plan for fixing the encoding of text. - - The plan is a list of tuples of the form (operation, arg). - - `operation` is one of: - - - `'encode'`: convert a string to bytes, using `arg` as the encoding - - `'decode'`: convert bytes to a string, using `arg` as the encoding - - `'transcode'`: convert bytes to bytes, using the function named `arg` - - `'apply'`: convert a string to a string, using the function named `arg` - - The functions that can be applied by 'transcode' and 'apply' are - specifically those that appear in the dictionary named `FIXERS`. They - can also can be imported from the `ftfy.fixes` module. - - Example:: - - >>> mojibake = "schön" - >>> text, plan = fix_and_explain(mojibake) - >>> apply_plan(mojibake, plan) - 'schön' - """ - obj = text - for operation, encoding in plan: - if operation == "encode": - obj = obj.encode(encoding) # type: ignore - elif operation == "decode": - obj = obj.decode(encoding) # type: ignore - elif operation in ("transcode", "apply"): - if encoding in FIXERS: - obj = FIXERS[encoding](obj) - else: - raise ValueError(f"Unknown function to apply: {encoding}") - else: - raise ValueError(f"Unknown plan step: {operation}") - - return obj - - -def explain_unicode(text: str) -> None: - """ - A utility method that's useful for debugging mysterious Unicode. - - It breaks down a string, showing you for each codepoint its number in - hexadecimal, its glyph, its category in the Unicode standard, and its name - in the Unicode standard. - - >>> explain_unicode('(╯°□°)╯︵ ┻━┻') - U+0028 ( [Ps] LEFT PARENTHESIS - U+256F ╯ [So] BOX DRAWINGS LIGHT ARC UP AND LEFT - U+00B0 ° [So] DEGREE SIGN - U+25A1 □ [So] WHITE SQUARE - U+00B0 ° [So] DEGREE SIGN - U+0029 ) [Pe] RIGHT PARENTHESIS - U+256F ╯ [So] BOX DRAWINGS LIGHT ARC UP AND LEFT - U+FE35 ︵ [Ps] PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS - U+0020 [Zs] SPACE - U+253B ┻ [So] BOX DRAWINGS HEAVY UP AND HORIZONTAL - U+2501 ━ [So] BOX DRAWINGS HEAVY HORIZONTAL - U+253B ┻ [So] BOX DRAWINGS HEAVY UP AND HORIZONTAL - """ - for char in text: - if char.isprintable(): - display = char - else: - display = char.encode("unicode-escape").decode("ascii") - print( - "U+{code:04X} {display} [{category}] {name}".format( - display=display_ljust(display, 7), - code=ord(char), - category=unicodedata.category(char), - name=unicodedata.name(char, ""), - ) - ) diff --git a/bundle/python-cpu/Lib/site-packages/ftfy/bad_codecs/__init__.py b/bundle/python-cpu/Lib/site-packages/ftfy/bad_codecs/__init__.py deleted file mode 100644 index a449a38ed1ca3e787ba3b97cf48ecf8c3f15ec06..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy/bad_codecs/__init__.py +++ /dev/null @@ -1,101 +0,0 @@ -r""" -The `ftfy.bad_codecs` module gives Python the ability to decode some common, -flawed encodings. - -Python does not want you to be sloppy with your text. Its encoders and decoders -("codecs") follow the relevant standards whenever possible, which means that -when you get text that *doesn't* follow those standards, you'll probably fail -to decode it. Or you might succeed at decoding it for implementation-specific -reasons, which is perhaps worse. - -There are some encodings out there that Python wishes didn't exist, which are -widely used outside of Python: - -- "utf-8-variants", a family of not-quite-UTF-8 encodings, including the - ever-popular CESU-8 and "Java modified UTF-8". -- "Sloppy" versions of character map encodings, where bytes that don't map to - anything will instead map to the Unicode character with the same number. - -Simply importing this module, or in fact any part of the `ftfy` package, will -make these new "bad codecs" available to Python through the standard Codecs -API. You never have to actually call any functions inside `ftfy.bad_codecs`. - -However, if you want to call something because your code checker insists on it, -you can call ``ftfy.bad_codecs.ok()``. - -A quick example of decoding text that's encoded in CESU-8: - - >>> import ftfy.bad_codecs - >>> print(b'\xed\xa0\xbd\xed\xb8\x8d'.decode('utf-8-variants')) - 😍 -""" - -import codecs -from encodings import normalize_encoding -from typing import Optional - -_CACHE: dict[str, codecs.CodecInfo] = {} - -# Define some aliases for 'utf-8-variants'. All hyphens get turned into -# underscores, because of `normalize_encoding`. -UTF8_VAR_NAMES = ( - "utf_8_variants", - "utf8_variants", - "utf_8_variant", - "utf8_variant", - "utf_8_var", - "utf8_var", - "cesu_8", - "cesu8", - "java_utf_8", - "java_utf8", -) - - -def search_function(encoding: str) -> Optional[codecs.CodecInfo]: - """ - Register our "bad codecs" with Python's codecs API. This involves adding - a search function that takes in an encoding name, and returns a codec - for that encoding if it knows one, or None if it doesn't. - - The encodings this will match are: - - - Encodings of the form 'sloppy-windows-NNNN' or 'sloppy-iso-8859-N', - where the non-sloppy version is an encoding that leaves some bytes - unmapped to characters. - - The 'utf-8-variants' encoding, which has the several aliases seen - above. - """ - if encoding in _CACHE: - return _CACHE[encoding] - - norm_encoding = normalize_encoding(encoding) - codec = None - if norm_encoding in UTF8_VAR_NAMES: - from ftfy.bad_codecs.utf8_variants import CODEC_INFO - - codec = CODEC_INFO - elif norm_encoding.startswith("sloppy_"): - from ftfy.bad_codecs.sloppy import CODECS - - codec = CODECS.get(norm_encoding) - - if codec is not None: - _CACHE[encoding] = codec - - return codec - - -def ok() -> None: - """ - A feel-good function that gives you something to call after importing - this package. - - Why is this here? Pyflakes. Pyflakes gets upset when you import a module - and appear not to use it. It doesn't know that you're using it when - you use the ``unicode.encode`` and ``bytes.decode`` methods with certain - encodings. - """ - - -codecs.register(search_function) diff --git a/bundle/python-cpu/Lib/site-packages/ftfy/bad_codecs/sloppy.py b/bundle/python-cpu/Lib/site-packages/ftfy/bad_codecs/sloppy.py deleted file mode 100644 index 656f01cf260feda58a15492bce82758c1b594d86..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy/bad_codecs/sloppy.py +++ /dev/null @@ -1,166 +0,0 @@ -r""" -`ftfy.bad_codecs.sloppy` provides character-map encodings that fill their "holes" -in a messy but common way: by outputting the Unicode codepoints with the same -numbers. - -This is incredibly ugly, and it's also in the HTML5 standard. - -A single-byte encoding maps each byte to a Unicode character, except that some -bytes are left unmapped. In the commonly-used Windows-1252 encoding, for -example, bytes 0x81 and 0x8D, among others, have no meaning. - -Python, wanting to preserve some sense of decorum, will handle these bytes -as errors. But Windows knows that 0x81 and 0x8D are possible bytes and they're -different from each other. It just hasn't defined what they are in terms of -Unicode. - -Software that has to interoperate with Windows-1252 and Unicode -- such as all -the common Web browsers -- will pick some Unicode characters for them to map -to, and the characters they pick are the Unicode characters with the same -numbers: U+0081 and U+008D. This is the same as what Latin-1 does, and the -resulting characters tend to fall into a range of Unicode that's set aside for -obsolete Latin-1 control characters anyway. - -These sloppy codecs let Python do the same thing, thus interoperating with -other software that works this way. It defines a sloppy version of many -single-byte encodings with holes. (There is no need for a sloppy version of -an encoding without holes: for example, there is no such thing as -sloppy-iso-8859-2 or sloppy-macroman.) - -The following encodings will become defined: - -- sloppy-windows-1250 (Central European, sort of based on ISO-8859-2) -- sloppy-windows-1251 (Cyrillic) -- sloppy-windows-1252 (Western European, based on Latin-1) -- sloppy-windows-1253 (Greek, sort of based on ISO-8859-7) -- sloppy-windows-1254 (Turkish, based on ISO-8859-9) -- sloppy-windows-1255 (Hebrew, based on ISO-8859-8) -- sloppy-windows-1256 (Arabic) -- sloppy-windows-1257 (Baltic, based on ISO-8859-13) -- sloppy-windows-1258 (Vietnamese) -- sloppy-cp874 (Thai, based on ISO-8859-11) -- sloppy-iso-8859-3 (Maltese and Esperanto, I guess) -- sloppy-iso-8859-6 (different Arabic) -- sloppy-iso-8859-7 (Greek) -- sloppy-iso-8859-8 (Hebrew) -- sloppy-iso-8859-11 (Thai) - -Aliases such as "sloppy-cp1252" for "sloppy-windows-1252" will also be -defined. - -Five of these encodings (`sloppy-windows-1250` through `sloppy-windows-1254`) -are used within ftfy. - -Here are some examples, using :func:`ftfy.explain_unicode` to illustrate how -sloppy-windows-1252 merges Windows-1252 with Latin-1: - - >>> from ftfy import explain_unicode - >>> some_bytes = b'\x80\x81\x82' - >>> explain_unicode(some_bytes.decode('latin-1')) - U+0080 \x80 [Cc] - U+0081 \x81 [Cc] - U+0082 \x82 [Cc] - - >>> explain_unicode(some_bytes.decode('windows-1252', 'replace')) - U+20AC € [Sc] EURO SIGN - U+FFFD � [So] REPLACEMENT CHARACTER - U+201A ‚ [Ps] SINGLE LOW-9 QUOTATION MARK - - >>> explain_unicode(some_bytes.decode('sloppy-windows-1252')) - U+20AC € [Sc] EURO SIGN - U+0081 \x81 [Cc] - U+201A ‚ [Ps] SINGLE LOW-9 QUOTATION MARK -""" - -from __future__ import annotations - -import codecs -from encodings import normalize_encoding - -REPLACEMENT_CHAR = "\ufffd" - - -def make_sloppy_codec(encoding: str) -> codecs.CodecInfo: - """ - Take a codec name, and return a 'sloppy' version of that codec that can - encode and decode the unassigned bytes in that encoding. - - Single-byte encodings in the standard library are defined using some - boilerplate classes surrounding the functions that do the actual work, - `codecs.charmap_decode` and `charmap_encode`. This function, given an - encoding name, *defines* those boilerplate classes. - """ - # Make a bytestring of all 256 possible bytes. - all_bytes = bytes(range(256)) - - # Get a list of what they would decode to in Latin-1. - sloppy_chars = list(all_bytes.decode("latin-1")) - - # Get a list of what they decode to in the given encoding. Use the - # replacement character for unassigned bytes. - decoded_chars = all_bytes.decode(encoding, errors="replace") - - # Update the sloppy_chars list. Each byte that was successfully decoded - # gets its decoded value in the list. The unassigned bytes are left as - # they are, which gives their decoding in Latin-1. - for i, char in enumerate(decoded_chars): - if char != REPLACEMENT_CHAR: - sloppy_chars[i] = char - - # For ftfy's own purposes, we're going to allow byte 1A, the "Substitute" - # control code, to encode the Unicode replacement character U+FFFD. - sloppy_chars[0x1A] = REPLACEMENT_CHAR - - # Create the data structures that tell the charmap methods how to encode - # and decode in this sloppy encoding. - decoding_table = "".join(sloppy_chars) - encoding_table = codecs.charmap_build(decoding_table) - - # Now produce all the class boilerplate. Look at the Python source for - # `encodings.cp1252` for comparison; this is almost exactly the same, - # except I made it follow pep8. - class Codec(codecs.Codec): - def encode(self, input: str, errors: str | None = "strict") -> tuple[bytes, int]: - return codecs.charmap_encode(input, errors, encoding_table) - - def decode(self, input: bytes, errors: str | None = "strict") -> tuple[str, int]: - return codecs.charmap_decode(input, errors, decoding_table) # type: ignore[arg-type] - - class IncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input: str, final: bool = False) -> bytes: - return codecs.charmap_encode(input, self.errors, encoding_table)[0] - - class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input: bytes, final: bool = False) -> str: # type: ignore[override] - return codecs.charmap_decode(input, self.errors, decoding_table)[0] # type: ignore[arg-type] - - class StreamWriter(Codec, codecs.StreamWriter): - pass - - class StreamReader(Codec, codecs.StreamReader): - pass - - return codecs.CodecInfo( - name="sloppy-" + encoding, - encode=Codec().encode, - decode=Codec().decode, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamreader=StreamReader, - streamwriter=StreamWriter, - ) - - -# Define a codec for each incomplete encoding. The resulting CODECS dictionary -# can be used by the main module of ftfy.bad_codecs. -CODECS = {} -INCOMPLETE_ENCODINGS = ( - [f"windows-{num}" for num in range(1250, 1259)] - + [f"iso-8859-{num}" for num in (3, 6, 7, 8, 11)] - + [f"cp{num}" for num in range(1250, 1259)] - + ["cp874"] -) - -for _encoding in INCOMPLETE_ENCODINGS: - _new_name = normalize_encoding("sloppy-" + _encoding) - CODECS[_new_name] = make_sloppy_codec(_encoding) diff --git a/bundle/python-cpu/Lib/site-packages/ftfy/bad_codecs/utf8_variants.py b/bundle/python-cpu/Lib/site-packages/ftfy/bad_codecs/utf8_variants.py deleted file mode 100644 index c15a3cf18431668c3817f9d9ff7a5478b4ccc5f8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy/bad_codecs/utf8_variants.py +++ /dev/null @@ -1,256 +0,0 @@ -r""" -This file defines a codec called "utf-8-variants" (or "utf-8-var"), which can -decode text that's been encoded with a popular non-standard version of UTF-8. -This includes CESU-8, the accidental encoding made by layering UTF-8 on top of -UTF-16, as well as Java's twist on CESU-8 that contains a two-byte encoding for -codepoint 0. - -This is particularly relevant in Python 3, which provides no other way of -decoding CESU-8 [1]_. - -The easiest way to use the codec is to simply import `ftfy.bad_codecs`: - - >>> import ftfy.bad_codecs - >>> result = b'here comes a null! \xc0\x80'.decode('utf-8-var') - >>> print(repr(result).lstrip('u')) - 'here comes a null! \x00' - -The codec does not at all enforce "correct" CESU-8. For example, the Unicode -Consortium's not-quite-standard describing CESU-8 requires that there is only -one possible encoding of any character, so it does not allow mixing of valid -UTF-8 and CESU-8. This codec *does* allow that, just like Python 2's UTF-8 -decoder does. - -Characters in the Basic Multilingual Plane still have only one encoding. This -codec still enforces the rule, within the BMP, that characters must appear in -their shortest form. There is one exception: the sequence of bytes `0xc0 0x80`, -instead of just `0x00`, may be used to encode the null character `U+0000`, like -in Java. - -If you encode with this codec, you get legitimate UTF-8. Decoding with this -codec and then re-encoding is not idempotent, although encoding and then -decoding is. So this module won't produce CESU-8 for you. Look for that -functionality in the sister module, "Breaks Text For You", coming approximately -never. - -.. [1] In a pinch, you can decode CESU-8 in Python 2 using the UTF-8 codec: - first decode the bytes (incorrectly), then encode them, then decode them - again, using UTF-8 as the codec every time. But Python 2 is dead, so use - ftfy instead. -""" - -import codecs -import re -from encodings.utf_8 import ( - IncrementalDecoder as UTF8IncrementalDecoder, -) -from encodings.utf_8 import ( - IncrementalEncoder as UTF8IncrementalEncoder, -) -from typing import Callable, Optional - -NAME = "utf-8-variants" - -# This regular expression matches all possible six-byte CESU-8 sequences, -# plus truncations of them at the end of the string. (If any of the -# subgroups matches $, then all the subgroups after it also have to match $, -# as there are no more characters to match.) -CESU8_EXPR = ( - b"(" - b"\xed" - b"([\xa0-\xaf]|$)" - b"([\x80-\xbf]|$)" - b"(\xed|$)" - b"([\xb0-\xbf]|$)" - b"([\x80-\xbf]|$)" - b")" -) - -CESU8_RE = re.compile(CESU8_EXPR) - -# This expression matches isolated surrogate characters that aren't -# CESU-8, which have to be handled carefully on Python 2. -SURROGATE_EXPR = b"(\xed([\xa0-\xbf]|$)([\x80-\xbf]|$))" - -# This expression matches the Java encoding of U+0, including if it's -# truncated and we need more bytes. -NULL_EXPR = b"(\xc0(\x80|$))" - -# This regex matches cases that we need to decode differently from -# standard UTF-8. -SPECIAL_BYTES_RE = re.compile(b"|".join([NULL_EXPR, CESU8_EXPR, SURROGATE_EXPR])) - - -class IncrementalDecoder(UTF8IncrementalDecoder): - """ - An incremental decoder that extends Python's built-in UTF-8 decoder. - - This encoder needs to take in bytes, possibly arriving in a stream, and - output the correctly decoded text. The general strategy for doing this - is to fall back on the real UTF-8 decoder whenever possible, because - the real UTF-8 decoder is way optimized, but to call specialized methods - we define here for the cases the real encoder isn't expecting. - """ - - @staticmethod - def _buffer_decode( # type: ignore[override] - input: bytes, errors: Optional[str], final: bool - ) -> tuple[str, int]: - """ - Decode bytes that may be arriving in a stream, following the Codecs - API. - - `input` is the incoming sequence of bytes. `errors` tells us how to - handle errors, though we delegate all error-handling cases to the real - UTF-8 decoder to ensure correct behavior. `final` indicates whether - this is the end of the sequence, in which case we should raise an - error given incomplete input. - - Returns as much decoded text as possible, and the number of bytes - consumed. - """ - # decoded_segments are the pieces of text we have decoded so far, - # and position is our current position in the byte string. (Bytes - # before this position have been consumed, and bytes after it have - # yet to be decoded.) - decoded_segments = [] - position = 0 - while True: - # Use _buffer_decode_step to decode a segment of text. - decoded, consumed = IncrementalDecoder._buffer_decode_step( - input[position:], errors, final - ) - if consumed == 0: - # Either there's nothing left to decode, or we need to wait - # for more input. Either way, we're done for now. - break - - # Append the decoded text to the list, and update our position. - decoded_segments.append(decoded) - position += consumed - - if final: - # _buffer_decode_step must consume all the bytes when `final` is - # true. - assert position == len(input) - - return "".join(decoded_segments), position - - @staticmethod - def _buffer_decode_step(input: bytes, errors: Optional[str], final: bool) -> tuple[str, int]: - """ - There are three possibilities for each decoding step: - - - Decode as much real UTF-8 as possible. - - Decode a six-byte CESU-8 sequence at the current position. - - Decode a Java-style null at the current position. - - This method figures out which step is appropriate, and does it. - """ - # Get a reference to the superclass method that we'll be using for - # most of the real work. - sup = UTF8IncrementalDecoder._buffer_decode - - # Find the next byte position that indicates a variant of UTF-8. - match = SPECIAL_BYTES_RE.search(input) - if match is None: - return sup(input, errors, final) - - cutoff = match.start() - if cutoff > 0: - return sup(input[:cutoff], errors, True) - - # Some byte sequence that we intend to handle specially matches - # at the beginning of the input. - if input.startswith(b"\xc0"): - if len(input) > 1: - # Decode the two-byte sequence 0xc0 0x80. - return "\u0000", 2 - else: - if final: - # We hit the end of the stream. Let the superclass method - # handle it. - return sup(input, errors, True) - else: - # Wait to see another byte. - return "", 0 - else: - # Decode a possible six-byte sequence starting with 0xed. - return IncrementalDecoder._buffer_decode_surrogates(sup, input, errors, final) - - @staticmethod - def _buffer_decode_surrogates( - sup: Callable[[bytes, Optional[str], bool], tuple[str, int]], - input: bytes, - errors: Optional[str], - final: bool, - ) -> tuple[str, int]: - """ - When we have improperly encoded surrogates, we can still see the - bits that they were meant to represent. - - The surrogates were meant to encode a 20-bit number, to which we - add 0x10000 to get a codepoint. That 20-bit number now appears in - this form: - - 11101101 1010abcd 10efghij 11101101 1011klmn 10opqrst - - The CESU8_RE above matches byte sequences of this form. Then we need - to extract the bits and assemble a codepoint number from them. - """ - if len(input) < 6: - if final: - # We found 0xed near the end of the stream, and there aren't - # six bytes to decode. Delegate to the superclass method to - # handle it as normal UTF-8. It might be a Hangul character - # or an error. - return sup(input, errors, final) - else: - # We found a surrogate, the stream isn't over yet, and we don't - # know enough of the following bytes to decode anything, so - # consume zero bytes and wait. - return "", 0 - else: - if CESU8_RE.match(input): - # Given this is a CESU-8 sequence, do some math to pull out - # the intended 20-bit value, and consume six bytes. - codepoint = ( - ((input[1] & 0x0F) << 16) - + ((input[2] & 0x3F) << 10) - + ((input[4] & 0x0F) << 6) - + (input[5] & 0x3F) - + 0x10000 - ) - return chr(codepoint), 6 - else: - # This looked like a CESU-8 sequence, but it wasn't one. - # 0xed indicates the start of a three-byte sequence, so give - # three bytes to the superclass to decode as usual. - return sup(input[:3], errors, False) - - -# The encoder is identical to UTF-8. -IncrementalEncoder = UTF8IncrementalEncoder - - -class StreamWriter(codecs.StreamWriter): - @staticmethod - def encode(input: str, errors: str = "strict") -> tuple[bytes, int]: - return IncrementalEncoder(errors).encode(input, final=True), len(input) - - -class StreamReader(codecs.StreamReader): - @staticmethod - def decode(input: bytes, errors: str = "strict") -> tuple[str, int]: - return IncrementalDecoder(errors).decode(input, final=True), len(input) - - -CODEC_INFO = codecs.CodecInfo( - name=NAME, - encode=StreamWriter.encode, - decode=StreamReader.decode, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamreader=StreamReader, - streamwriter=StreamWriter, -) diff --git a/bundle/python-cpu/Lib/site-packages/ftfy/badness.py b/bundle/python-cpu/Lib/site-packages/ftfy/badness.py deleted file mode 100644 index 38ec1f44c44cdd3eba35eaa0aaf823ea37fbe0d8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy/badness.py +++ /dev/null @@ -1,420 +0,0 @@ -""" -`ftfy.badness` contains a heuristic that detects likely mojibake. - -This heuristic signals to ftfy which segments of text need to be fixed, and -also indicates when the text can stop being fixed. - -The design of this heuristic is that we categorize the approximately 400 -Unicode characters that occur in UTF-8 mojibake, specifically the characters -that come from mixing up UTF-8 with the other encodings we support. We -identify sequences and contexts of these characters that are much more likely -to be mojibake than intended strings, such as lowercase accented letters -followed immediately by currency symbols. -""" - -import warnings -import re - - -# There are only a few hundred characters that occur in known UTF-8 mojibake, and we can -# characterize them: - -MOJIBAKE_CATEGORIES = { - # Characters that appear in many different contexts. Sequences that contain - # them are not inherently mojibake - "common": ( - "\N{NO-BREAK SPACE}" - "\N{SOFT HYPHEN}" - "\N{MIDDLE DOT}" - "\N{ACUTE ACCENT}" - "\N{EN DASH}" - "\N{EM DASH}" - "\N{HORIZONTAL BAR}" - "\N{HORIZONTAL ELLIPSIS}" - "\N{RIGHT SINGLE QUOTATION MARK}" - ), - # the C1 control character range, which have no uses outside of mojibake anymore - "c1": "\x80-\x9f", - # Characters that are nearly 100% used in mojibake - "bad": ( - "\N{BROKEN BAR}" - "\N{CURRENCY SIGN}" - "\N{DIAERESIS}" - "\N{NOT SIGN}" - "\N{MACRON}" - "\N{CEDILLA}" - "\N{LATIN SMALL LETTER F WITH HOOK}" - "\N{MODIFIER LETTER CIRCUMFLEX ACCENT}" # it's not a modifier - "\N{CARON}" - "\N{BREVE}" - "\N{OGONEK}" - "\N{SMALL TILDE}" - "\N{DAGGER}" - "\N{DOUBLE DAGGER}" - "\N{PER MILLE SIGN}" - "\N{REVERSED NOT SIGN}" - "\N{LOZENGE}" - "\ufffd" - # Theoretically these would appear in 'numeric' contexts, but when they - # co-occur with other mojibake characters, it's not really ambiguous - "\N{FEMININE ORDINAL INDICATOR}" - "\N{MASCULINE ORDINAL INDICATOR}" - ), - # Characters used in legalese - "law": ( - "\N{PILCROW SIGN}" - "\N{SECTION SIGN}" - ), - "currency": ( - "\N{CENT SIGN}" - "\N{POUND SIGN}" - "\N{YEN SIGN}" - "\N{PESETA SIGN}" - "\N{EURO SIGN}" - ), - "start_punctuation": ( - "\N{INVERTED EXCLAMATION MARK}" - "\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}" - "\N{INVERTED QUESTION MARK}" - "\N{COPYRIGHT SIGN}" - "\N{GREEK TONOS}" - "\N{GREEK DIALYTIKA TONOS}" - "\N{LEFT SINGLE QUOTATION MARK}" - "\N{SINGLE LOW-9 QUOTATION MARK}" - "\N{LEFT DOUBLE QUOTATION MARK}" - "\N{DOUBLE LOW-9 QUOTATION MARK}" - "\N{BULLET}" - "\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}" - "\uf8ff" # OS-specific symbol, usually the Apple logo - ), - "end_punctuation": ( - "\N{REGISTERED SIGN}" - "\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}" - "\N{DOUBLE ACUTE ACCENT}" - "\N{RIGHT DOUBLE QUOTATION MARK}" - "\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}" - "\N{TRADE MARK SIGN}" - ), - "numeric": ( - "\N{SUPERSCRIPT TWO}" - "\N{SUPERSCRIPT THREE}" - "\N{SUPERSCRIPT ONE}" - "\N{PLUS-MINUS SIGN}" - "\N{VULGAR FRACTION ONE QUARTER}" - "\N{VULGAR FRACTION ONE HALF}" - "\N{VULGAR FRACTION THREE QUARTERS}" - "\N{MULTIPLICATION SIGN}" - "\N{MICRO SIGN}" - "\N{DIVISION SIGN}" - "\N{FRACTION SLASH}" - "\N{PARTIAL DIFFERENTIAL}" - "\N{INCREMENT}" - "\N{N-ARY PRODUCT}" - "\N{N-ARY SUMMATION}" - "\N{SQUARE ROOT}" - "\N{INFINITY}" - "\N{INTERSECTION}" - "\N{INTEGRAL}" - "\N{ALMOST EQUAL TO}" - "\N{NOT EQUAL TO}" - "\N{IDENTICAL TO}" - "\N{LESS-THAN OR EQUAL TO}" - "\N{GREATER-THAN OR EQUAL TO}" - "\N{NUMERO SIGN}" - ), - # Letters that might be used to make emoticon faces (kaomoji), and - # therefore might need to appear in more improbable-looking contexts. - # - # These are concatenated character ranges for use in a regex. I know - # they look like faces themselves. I think expressing the ranges like - # this helps to illustrate why we need to be careful with these - # characters. - "kaomoji": ( - "Ò-Ö" - "Ù-Ü" - "ò-ö" - "ø-ü" - "\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}" - "\N{LATIN CAPITAL LETTER O WITH MACRON}" - "\N{LATIN CAPITAL LETTER U WITH MACRON}" - "\N{LATIN CAPITAL LETTER U WITH OGONEK}" - "\N{DEGREE SIGN}" - ), - "upper_accented": ( - # LATIN CAPITAL LETTER A WITH GRAVE - LATIN CAPITAL LETTER N WITH TILDE - "\xc0-\xd1" - # skip capital O's and U's that could be used in kaomoji, but - # include Ø because it's very common in Arabic mojibake: - "\N{LATIN CAPITAL LETTER O WITH STROKE}" - "\N{LATIN CAPITAL LETTER U WITH DIAERESIS}" - "\N{LATIN CAPITAL LETTER Y WITH ACUTE}" - "\N{LATIN CAPITAL LETTER A WITH BREVE}" - "\N{LATIN CAPITAL LETTER A WITH MACRON}" - "\N{LATIN CAPITAL LETTER A WITH OGONEK}" - "\N{LATIN CAPITAL LETTER C WITH ACUTE}" - "\N{LATIN CAPITAL LETTER C WITH CARON}" - "\N{LATIN CAPITAL LETTER D WITH CARON}" - "\N{LATIN CAPITAL LETTER D WITH STROKE}" - "\N{LATIN CAPITAL LETTER E WITH OGONEK}" - "\N{LATIN CAPITAL LETTER E WITH CARON}" - "\N{LATIN CAPITAL LETTER E WITH MACRON}" - "\N{LATIN CAPITAL LETTER E WITH DOT ABOVE}" - "\N{LATIN CAPITAL LETTER G WITH BREVE}" - "\N{LATIN CAPITAL LETTER G WITH CEDILLA}" - "\N{LATIN CAPITAL LETTER I WITH DOT ABOVE}" - "\N{LATIN CAPITAL LETTER I WITH MACRON}" - "\N{LATIN CAPITAL LETTER K WITH CEDILLA}" - "\N{LATIN CAPITAL LETTER L WITH ACUTE}" - "\N{LATIN CAPITAL LETTER L WITH CARON}" - "\N{LATIN CAPITAL LETTER L WITH STROKE}" - "\N{LATIN CAPITAL LETTER L WITH CEDILLA}" - "\N{LATIN CAPITAL LETTER N WITH ACUTE}" - "\N{LATIN CAPITAL LETTER N WITH CARON}" - "\N{LATIN CAPITAL LETTER N WITH CEDILLA}" - "\N{LATIN CAPITAL LIGATURE OE}" - "\N{LATIN CAPITAL LETTER R WITH CARON}" - "\N{LATIN CAPITAL LETTER S WITH ACUTE}" - "\N{LATIN CAPITAL LETTER S WITH CEDILLA}" - "\N{LATIN CAPITAL LETTER S WITH CARON}" - "\N{LATIN CAPITAL LETTER T WITH CEDILLA}" - "\N{LATIN CAPITAL LETTER T WITH CARON}" - "\N{LATIN CAPITAL LETTER U WITH RING ABOVE}" - "\N{LATIN CAPITAL LETTER U WITH DOUBLE ACUTE}" - "\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}" - "\N{LATIN CAPITAL LETTER Z WITH ACUTE}" - "\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}" - "\N{LATIN CAPITAL LETTER Z WITH CARON}" - "\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}" - ), - "lower_accented": ( - "\N{LATIN SMALL LETTER SHARP S}" - # LATIN SMALL LETTER A WITH GRAVE - LATIN SMALL LETTER N WITH TILDE - "\xe0-\xf1" - # skip o's and u's that could be used in kaomoji - "\N{LATIN SMALL LETTER A WITH BREVE}" - "\N{LATIN SMALL LETTER A WITH OGONEK}" - "\N{LATIN SMALL LETTER A WITH MACRON}" - "\N{LATIN SMALL LETTER C WITH ACUTE}" - "\N{LATIN SMALL LETTER C WITH CARON}" - "\N{LATIN SMALL LETTER D WITH CARON}" - "\N{LATIN SMALL LETTER D WITH STROKE}" - "\N{LATIN SMALL LETTER E WITH OGONEK}" - "\N{LATIN SMALL LETTER E WITH CARON}" - "\N{LATIN SMALL LETTER E WITH MACRON}" - "\N{LATIN SMALL LETTER E WITH DOT ABOVE}" - "\N{LATIN SMALL LETTER G WITH BREVE}" - "\N{LATIN SMALL LETTER G WITH CEDILLA}" - "\N{LATIN SMALL LETTER I WITH OGONEK}" - "\N{LATIN SMALL LETTER I WITH MACRON}" - "\N{LATIN SMALL LETTER K WITH CEDILLA}" - "\N{LATIN SMALL LETTER L WITH ACUTE}" - "\N{LATIN SMALL LETTER L WITH CARON}" - "\N{LATIN SMALL LETTER L WITH STROKE}" - "\N{LATIN SMALL LETTER L WITH CEDILLA}" - "\N{LATIN SMALL LIGATURE OE}" - "\N{LATIN SMALL LETTER R WITH ACUTE}" - "\N{LATIN SMALL LETTER S WITH ACUTE}" - "\N{LATIN SMALL LETTER S WITH CEDILLA}" - "\N{LATIN SMALL LETTER S WITH CARON}" - "\N{LATIN SMALL LETTER T WITH CARON}" - "\N{LATIN SMALL LETTER U WITH DIAERESIS}" - "\N{LATIN SMALL LETTER Z WITH ACUTE}" - "\N{LATIN SMALL LETTER Z WITH DOT ABOVE}" - "\N{LATIN SMALL LETTER Z WITH CARON}" - "\N{CYRILLIC SMALL LETTER GHE WITH UPTURN}" - "\N{LATIN SMALL LIGATURE FI}" - "\N{LATIN SMALL LIGATURE FL}" - ), - "upper_common": ( - "\N{LATIN CAPITAL LETTER THORN}" - "\N{GREEK CAPITAL LETTER ALPHA}-\N{GREEK CAPITAL LETTER OMEGA}" - # not included under 'accented' because these can commonly - # occur at ends of words, in positions where they'd be detected - # as mojibake - "\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}" - "\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}" - "\N{GREEK CAPITAL LETTER ETA WITH TONOS}" - "\N{GREEK CAPITAL LETTER IOTA WITH TONOS}" - "\N{GREEK CAPITAL LETTER OMICRON WITH TONOS}" - "\N{GREEK CAPITAL LETTER UPSILON WITH TONOS}" - "\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}" - "\N{GREEK CAPITAL LETTER IOTA WITH DIALYTIKA}" - "\N{GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA}" - "\N{CYRILLIC CAPITAL LETTER IO}-\N{CYRILLIC CAPITAL LETTER YA}" - ), - "lower_common": ( - # lowercase thorn does not appear in mojibake - "\N{GREEK SMALL LETTER ALPHA}-\N{GREEK SMALL LETTER OMEGA}" - "\N{GREEK SMALL LETTER ALPHA WITH TONOS}" - "\N{GREEK SMALL LETTER EPSILON WITH TONOS}" - "\N{GREEK SMALL LETTER ETA WITH TONOS}" - "\N{GREEK SMALL LETTER IOTA WITH TONOS}" - "\N{GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS}" - "\N{CYRILLIC SMALL LETTER A}-\N{CYRILLIC SMALL LETTER DZHE}" - ), - "box": ( - # omit the single horizontal line, might be used in kaomoji - "│┌┐┘├┤┬┼" - "\N{BOX DRAWINGS DOUBLE HORIZONTAL}-\N{BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL}" - "▀▄█▌▐░▒▓" - ), -} - - -# We can now build a regular expression that detects unlikely juxtapositions -# of characters, mostly based on their categories. -# -# Another regular expression, which detects sequences that look more specifically -# like UTF-8 mojibake, appears in chardata.py. -# -# This is a verbose regular expression, with whitespace added for somewhat more -# readability. Remember that the only spaces that count as literal spaces in this -# expression are ones inside character classes (square brackets). - -BADNESS_RE = re.compile( - r""" - [{c1}] - | - [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}{law}] [{bad}] - | - [a-zA-Z] [{lower_common}{upper_common}] [{bad}] - | - [{bad}] [{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}{law}] - | - [{lower_accented}{lower_common}{box}{end_punctuation}{currency}{numeric}] [{upper_accented}] - | - [{box}{end_punctuation}{currency}{numeric}] [{lower_accented}] - | - [{lower_accented}{box}{end_punctuation}] [{currency}] - | - \s [{upper_accented}] [{currency}] - | - [{upper_accented}{box}] [{numeric}{law}] - | - [{lower_accented}{upper_accented}{box}{currency}{end_punctuation}] [{start_punctuation}] [{numeric}] - | - [{lower_accented}{upper_accented}{currency}{numeric}{box}{law}] [{end_punctuation}] [{start_punctuation}] - | - [{currency}{numeric}{box}] [{start_punctuation}] - | - [a-z] [{upper_accented}] [{start_punctuation}{currency}] - | - [{box}] [{kaomoji}] - | - [{lower_accented}{upper_accented}{currency}{numeric}{start_punctuation}{end_punctuation}{law}] [{box}] - | - [{box}] [{end_punctuation}] - | - [{lower_accented}{upper_accented}] [{start_punctuation}{end_punctuation}] \w - | - - # The ligature œ when not followed by an unaccented Latin letter - [Œœ][^A-Za-z] - | - - # Degree signs after capital letters - [{upper_accented}]° - | - - # Common Windows-1252 2-character mojibake that isn't covered by the cases above - [ÂÃÎÐ][€œŠš¢£Ÿž\xa0\xad®©°·»{start_punctuation}{end_punctuation}–—´] - | - × [²³] - | - # Windows-1252 mojibake of Arabic words needs to include the 'common' characters. - # To compensate, we require four characters to be matched. - [ØÙ] [{common}{currency}{bad}{numeric}{start_punctuation}ŸŠ®°µ»] - [ØÙ] [{common}{currency}{bad}{numeric}{start_punctuation}ŸŠ®°µ»] - | - - # Windows-1252 mojibake that starts 3-character sequences for some South Asian - # alphabets - à[²µ¹¼½¾] - | - - # MacRoman mojibake that isn't covered by the cases above - √[±∂†≠®™´≤≥¥µø] - | - ≈[°¢] - | - ‚Ä[ìîïòôúùû†°¢π] - | - ‚[âó][àä°ê] - | - - # Windows-1251 mojibake of characters in the U+2000 range - †- | - - # Windows-1251 mojibake of Latin-1 characters and/or the Cyrillic alphabet. - # Because the 2-character sequences involved here may be common, we require - # seeing a 3-character sequence. - [ВГРС][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°µ][ВГРС] - | - # A distinctive five-character sequence of Cyrillic letters, which can be - # Windows-1251 mojibake on top of Latin-1 mojibake of Windows-1252 characters. - # Require a Latin letter nearby. - ГўВЂВ.[A-Za-z ] - | - - # Windows-1252 encodings of 'à' and 'á', as well as \xa0 itself - Ã[\xa0¡] - | - [a-z]\s?[ÃÂ][ ] - | - ^[ÃÂ][ ] - | - - # Cases where  precedes a character as an encoding of exactly the same - # character, and the character is common enough - [a-z.,?!{end_punctuation}]  [ {start_punctuation}{end_punctuation}] - | - - # Windows-1253 mojibake of characters in the U+2000 range - β€[™\xa0Ά\xad®°] - | - - # Windows-1253 mojibake of Latin-1 characters and/or the Greek alphabet - [ΒΓΞΟ][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°][ΒΓΞΟ] - | - - # Windows-1257 mojibake of characters in the U+2000 range - †- """.format( - **MOJIBAKE_CATEGORIES - ), - re.VERBOSE, -) - - -def sequence_weirdness(text: str) -> int: - """ - This was the name of the heuristic used in ftfy 2.x through 5.x. As an - attempt at compatibility with external code that calls the heuristic - directly, we redirect to our new heuristic, :func:`badness`. - """ - warnings.warn( - "`sequence_weirdness()` is an old heuristic, and the current " - "closest equivalent is `ftfy.badness.badness()`" - ) - return badness(text) - - -def badness(text: str) -> int: - """ - Get the 'badness' of a sequence of text, counting the number of unlikely - character sequences. A badness greater than 0 indicates that some of it - seems to be mojibake. - """ - return len(BADNESS_RE.findall(text)) - - -def is_bad(text: str) -> bool: - """ - Returns true iff the given text looks like it contains mojibake. - - This can be faster than `badness`, because it returns when the first match - is found to a regex instead of counting matches. Note that as strings get - longer, they have a higher chance of returning True for `is_bad(string)`. - """ - return bool(BADNESS_RE.search(text)) diff --git a/bundle/python-cpu/Lib/site-packages/ftfy/chardata.py b/bundle/python-cpu/Lib/site-packages/ftfy/chardata.py deleted file mode 100644 index afcc76715707694d4fdf25ca3d33247731e21c00..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy/chardata.py +++ /dev/null @@ -1,691 +0,0 @@ -""" -This gives other modules access to the gritty details about characters and the -encodings that use them. -""" - -from __future__ import annotations - -import html -import itertools -import re -import unicodedata - -# These are the encodings we will try to fix in ftfy, in the -# order that they should be tried. -CHARMAP_ENCODINGS = [ - "latin-1", - "sloppy-windows-1252", - "sloppy-windows-1251", - "sloppy-windows-1250", - "sloppy-windows-1253", - "sloppy-windows-1254", - "sloppy-windows-1257", - "iso-8859-2", - "macroman", - "cp437", -] - -SINGLE_QUOTE_RE = re.compile("[\u02bc\u2018-\u201b]") -DOUBLE_QUOTE_RE = re.compile("[\u201c-\u201f]") - - -def _build_regexes() -> dict[str, re.Pattern[str]]: - """ - ENCODING_REGEXES contain reasonably fast ways to detect if we - could represent a given string in a given encoding. The simplest one is - the 'ascii' detector, which of course just determines if all characters - are between U+0000 and U+007F. - """ - # Define a regex that matches ASCII text. - encoding_regexes = {"ascii": re.compile("^[\x00-\x7f]*$")} - - for encoding in CHARMAP_ENCODINGS: - # Make a sequence of characters that bytes \x80 to \xFF decode to - # in each encoding, as well as byte \x1A, which is used to represent - # the replacement character � in the sloppy-* encodings. - byte_range = bytes(list(range(0x80, 0x100)) + [0x1A]) - charlist = byte_range.decode(encoding) - - # The rest of the ASCII bytes -- bytes \x00 to \x19 and \x1B - # to \x7F -- will decode as those ASCII characters in any encoding we - # support, so we can just include them as ranges. This also lets us - # not worry about escaping regex special characters, because all of - # them are in the \x1B to \x7F range. - regex = f"^[\x00-\x19\x1b-\x7f{charlist}]*$" - encoding_regexes[encoding] = re.compile(regex) - return encoding_regexes - - -ENCODING_REGEXES = _build_regexes() - - -def _build_html_entities() -> dict[str, str]: - entities = {} - # Create a dictionary based on the built-in HTML5 entity dictionary. - # Add a limited set of HTML entities that we'll also decode if they've - # been case-folded to uppercase, such as decoding &NTILDE; as "Ñ". - for name, char in html.entities.html5.items(): # type: ignore - if name.endswith(";"): - entities["&" + name] = char - - # Restrict the set of characters we can attempt to decode if their - # name has been uppercased. If we tried to handle all entity names, - # the results would be ambiguous. - if name == name.lower(): - name_upper = name.upper() - entity_upper = "&" + name_upper - if html.unescape(entity_upper) == entity_upper: - entities[entity_upper] = char.upper() - return entities - - -HTML_ENTITY_RE = re.compile(r"&#?[0-9A-Za-z]{1,24};") -HTML_ENTITIES = _build_html_entities() - - -def possible_encoding(text: str, encoding: str) -> bool: - """ - Given text and a single-byte encoding, check whether that text could have - been decoded from that single-byte encoding. - - In other words, check whether it can be encoded in that encoding, possibly - sloppily. - """ - return bool(ENCODING_REGEXES[encoding].match(text)) - - -def _build_control_char_mapping() -> dict[int, None]: - """ - Build a translate mapping that strips likely-unintended control characters. - See :func:`ftfy.fixes.remove_control_chars` for a description of these - codepoint ranges and why they should be removed. - """ - control_chars: dict[int, None] = {} - - for i in itertools.chain( - range(0x00, 0x09), - [0x0B], - range(0x0E, 0x20), - [0x7F], - range(0x206A, 0x2070), - [0xFEFF], - range(0xFFF9, 0xFFFD), - ): - control_chars[i] = None - - return control_chars - - -CONTROL_CHARS = _build_control_char_mapping() - - -# Recognize UTF-8 sequences that would be valid if it weren't for a b'\xa0' -# that some Windows-1252 program converted to a plain space. -# -# The smaller values are included on a case-by-case basis, because we don't want -# to decode likely input sequences to unlikely characters. These are the ones -# that *do* form likely characters before 0xa0: -# -# 0xc2 -> U+A0 NO-BREAK SPACE -# 0xc3 -> U+E0 LATIN SMALL LETTER A WITH GRAVE -# 0xc5 -> U+160 LATIN CAPITAL LETTER S WITH CARON -# 0xce -> U+3A0 GREEK CAPITAL LETTER PI -# 0xd0 -> U+420 CYRILLIC CAPITAL LETTER ER -# 0xd9 -> U+660 ARABIC-INDIC DIGIT ZERO -# -# In three-character sequences, we exclude some lead bytes in some cases. -# -# When the lead byte is immediately followed by 0xA0, we shouldn't accept -# a space there, because it leads to some less-likely character ranges: -# -# 0xe0 -> Samaritan script -# 0xe1 -> Mongolian script (corresponds to Latin-1 'á' which is too common) -# -# We accept 0xe2 and 0xe3, which cover many scripts. Bytes 0xe4 and -# higher point mostly to CJK characters, which we generally don't want to -# decode near Latin lowercase letters. -# -# In four-character sequences, the lead byte must be F0, because that accounts -# for almost all of the usage of high-numbered codepoints (tag characters whose -# UTF-8 starts with the byte F3 are only used in some rare new emoji sequences). -# -# This is meant to be applied to encodings of text that tests true for `is_bad`. -# Any of these could represent characters that legitimately appear surrounded by -# spaces, particularly U+C5 (Å), which is a word in multiple languages! -# -# We should consider checking for b'\x85' being converted to ... in the future. -# I've seen it once, but the text still wasn't recoverable. - -ALTERED_UTF8_RE = re.compile( - b"[\xc2\xc3\xc5\xce\xd0\xd9][ ]" - b"|[\xe2\xe3][ ][\x80-\x84\x86-\x9f\xa1-\xbf]" - b"|[\xe0-\xe3][\x80-\x84\x86-\x9f\xa1-\xbf][ ]" - b"|[\xf0][ ][\x80-\xbf][\x80-\xbf]" - b"|[\xf0][\x80-\xbf][ ][\x80-\xbf]" - b"|[\xf0][\x80-\xbf][\x80-\xbf][ ]" -) - - -# This expression matches UTF-8 and CESU-8 sequences where some of the -# continuation bytes have been lost. The byte 0x1a (sometimes written as ^Z) is -# used within ftfy to represent a byte that produced the replacement character -# \ufffd. We don't know which byte it was, but we can at least decode the UTF-8 -# sequence as \ufffd instead of failing to re-decode it at all. -# -# In some cases, we allow the ASCII '?' in place of \ufffd, but at most once per -# sequence. -LOSSY_UTF8_RE = re.compile( - b"[\xc2-\xdf][\x1a]" - b"|[\xc2-\xc3][?]" - b"|\xed[\xa0-\xaf][\x1a?]\xed[\xb0-\xbf][\x1a?\x80-\xbf]" - b"|\xed[\xa0-\xaf][\x1a?\x80-\xbf]\xed[\xb0-\xbf][\x1a?]" - b"|[\xe0-\xef][\x1a?][\x1a\x80-\xbf]" - b"|[\xe0-\xef][\x1a\x80-\xbf][\x1a?]" - b"|[\xf0-\xf4][\x1a?][\x1a\x80-\xbf][\x1a\x80-\xbf]" - b"|[\xf0-\xf4][\x1a\x80-\xbf][\x1a?][\x1a\x80-\xbf]" - b"|[\xf0-\xf4][\x1a\x80-\xbf][\x1a\x80-\xbf][\x1a?]" - b"|\x1a" -) - - -# This regex matches C1 control characters, which occupy some of the positions -# in the Latin-1 character map that Windows assigns to other characters instead. -C1_CONTROL_RE = re.compile(r"[\x80-\x9f]") - - -# A translate mapping that breaks ligatures made of Latin letters. While -# ligatures may be important to the representation of other languages, in Latin -# letters they tend to represent a copy/paste error. It omits ligatures such -# as æ that are frequently used intentionally. -# -# This list additionally includes some Latin digraphs that represent two -# characters for legacy encoding reasons, not for typographical reasons. -# -# Ligatures and digraphs may also be separated by NFKC normalization, but that -# is sometimes more normalization than you want. - -LIGATURES = { - ord("IJ"): "IJ", # Dutch ligatures - ord("ij"): "ij", - ord("ʼn"): "ʼn", # Afrikaans digraph meant to avoid auto-curled quote - ord("DZ"): "DZ", # Serbian/Croatian digraphs for Cyrillic conversion - ord("Dz"): "Dz", - ord("dz"): "dz", - ord("DŽ"): "DŽ", - ord("Dž"): "Dž", - ord("dž"): "dž", - ord("LJ"): "LJ", - ord("Lj"): "Lj", - ord("lj"): "lj", - ord("NJ"): "NJ", - ord("Nj"): "Nj", - ord("nj"): "nj", - ord("ff"): "ff", # Latin typographical ligatures - ord("fi"): "fi", - ord("fl"): "fl", - ord("ffi"): "ffi", - ord("ffl"): "ffl", - ord("ſt"): "ſt", - ord("st"): "st", -} - - -def _build_width_map() -> dict[int, str]: - """ - Build a translate mapping that replaces halfwidth and fullwidth forms - with their standard-width forms. - """ - # Though it's not listed as a fullwidth character, we'll want to convert - # U+3000 IDEOGRAPHIC SPACE to U+20 SPACE on the same principle, so start - # with that in the dictionary. - width_map = {0x3000: " "} - for i in range(0xFF01, 0xFFF0): - char = chr(i) - alternate = unicodedata.normalize("NFKC", char) - if alternate != char: - width_map[i] = alternate - return width_map - - -WIDTH_MAP = _build_width_map() - - -# Character classes that help us pinpoint embedded mojibake. These can -# include common characters, because we'll also check them for 'badness'. -# -# Though they go on for many lines, the members of this dictionary are -# single concatenated strings. -# -# This code is generated using scripts/char_data_table.py. -UTF8_CLUES: dict[str, str] = { - # Letters that decode to 0xC2 - 0xDF in a Latin-1-like encoding - "utf8_first_of_2": ( - "\N{LATIN CAPITAL LETTER A WITH BREVE}" # windows-1250:C3 - "\N{LATIN CAPITAL LETTER A WITH CIRCUMFLEX}" # latin-1:C2 - "\N{LATIN CAPITAL LETTER A WITH DIAERESIS}" # latin-1:C4 - "\N{LATIN CAPITAL LETTER A WITH MACRON}" # windows-1257:C2 - "\N{LATIN CAPITAL LETTER A WITH RING ABOVE}" # latin-1:C5 - "\N{LATIN CAPITAL LETTER A WITH TILDE}" # latin-1:C3 - "\N{LATIN CAPITAL LETTER AE}" # latin-1:C6 - "\N{LATIN CAPITAL LETTER C WITH ACUTE}" # windows-1250:C6 - "\N{LATIN CAPITAL LETTER C WITH CARON}" # windows-1250:C8 - "\N{LATIN CAPITAL LETTER C WITH CEDILLA}" # latin-1:C7 - "\N{LATIN CAPITAL LETTER D WITH CARON}" # windows-1250:CF - "\N{LATIN CAPITAL LETTER D WITH STROKE}" # windows-1250:D0 - "\N{LATIN CAPITAL LETTER E WITH ACUTE}" # latin-1:C9 - "\N{LATIN CAPITAL LETTER E WITH CARON}" # windows-1250:CC - "\N{LATIN CAPITAL LETTER E WITH CIRCUMFLEX}" # latin-1:CA - "\N{LATIN CAPITAL LETTER E WITH DIAERESIS}" # latin-1:CB - "\N{LATIN CAPITAL LETTER E WITH DOT ABOVE}" # windows-1257:CB - "\N{LATIN CAPITAL LETTER E WITH GRAVE}" # latin-1:C8 - "\N{LATIN CAPITAL LETTER E WITH MACRON}" # windows-1257:C7 - "\N{LATIN CAPITAL LETTER E WITH OGONEK}" # windows-1250:CA - "\N{LATIN CAPITAL LETTER ETH}" # latin-1:D0 - "\N{LATIN CAPITAL LETTER G WITH BREVE}" # windows-1254:D0 - "\N{LATIN CAPITAL LETTER G WITH CEDILLA}" # windows-1257:CC - "\N{LATIN CAPITAL LETTER I WITH ACUTE}" # latin-1:CD - "\N{LATIN CAPITAL LETTER I WITH CIRCUMFLEX}" # latin-1:CE - "\N{LATIN CAPITAL LETTER I WITH DIAERESIS}" # latin-1:CF - "\N{LATIN CAPITAL LETTER I WITH DOT ABOVE}" # windows-1254:DD - "\N{LATIN CAPITAL LETTER I WITH GRAVE}" # latin-1:CC - "\N{LATIN CAPITAL LETTER I WITH MACRON}" # windows-1257:CE - "\N{LATIN CAPITAL LETTER K WITH CEDILLA}" # windows-1257:CD - "\N{LATIN CAPITAL LETTER L WITH ACUTE}" # windows-1250:C5 - "\N{LATIN CAPITAL LETTER L WITH CEDILLA}" # windows-1257:CF - "\N{LATIN CAPITAL LETTER L WITH STROKE}" # windows-1257:D9 - "\N{LATIN CAPITAL LETTER N WITH ACUTE}" # windows-1250:D1 - "\N{LATIN CAPITAL LETTER N WITH CARON}" # windows-1250:D2 - "\N{LATIN CAPITAL LETTER N WITH CEDILLA}" # windows-1257:D2 - "\N{LATIN CAPITAL LETTER N WITH TILDE}" # latin-1:D1 - "\N{LATIN CAPITAL LETTER O WITH ACUTE}" # latin-1:D3 - "\N{LATIN CAPITAL LETTER O WITH CIRCUMFLEX}" # latin-1:D4 - "\N{LATIN CAPITAL LETTER O WITH DIAERESIS}" # latin-1:D6 - "\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}" # windows-1250:D5 - "\N{LATIN CAPITAL LETTER O WITH GRAVE}" # latin-1:D2 - "\N{LATIN CAPITAL LETTER O WITH MACRON}" # windows-1257:D4 - "\N{LATIN CAPITAL LETTER O WITH STROKE}" # latin-1:D8 - "\N{LATIN CAPITAL LETTER O WITH TILDE}" # latin-1:D5 - "\N{LATIN CAPITAL LETTER R WITH CARON}" # windows-1250:D8 - "\N{LATIN CAPITAL LETTER S WITH ACUTE}" # windows-1257:DA - "\N{LATIN CAPITAL LETTER S WITH CARON}" # windows-1257:D0 - "\N{LATIN CAPITAL LETTER S WITH CEDILLA}" # windows-1254:DE - "\N{LATIN CAPITAL LETTER T WITH CEDILLA}" # windows-1250:DE - "\N{LATIN CAPITAL LETTER THORN}" # latin-1:DE - "\N{LATIN CAPITAL LETTER U WITH ACUTE}" # latin-1:DA - "\N{LATIN CAPITAL LETTER U WITH CIRCUMFLEX}" # latin-1:DB - "\N{LATIN CAPITAL LETTER U WITH DIAERESIS}" # latin-1:DC - "\N{LATIN CAPITAL LETTER U WITH DOUBLE ACUTE}" # windows-1250:DB - "\N{LATIN CAPITAL LETTER U WITH GRAVE}" # latin-1:D9 - "\N{LATIN CAPITAL LETTER U WITH MACRON}" # windows-1257:DB - "\N{LATIN CAPITAL LETTER U WITH OGONEK}" # windows-1257:D8 - "\N{LATIN CAPITAL LETTER U WITH RING ABOVE}" # windows-1250:D9 - "\N{LATIN CAPITAL LETTER Y WITH ACUTE}" # latin-1:DD - "\N{LATIN CAPITAL LETTER Z WITH ACUTE}" # windows-1257:CA - "\N{LATIN CAPITAL LETTER Z WITH CARON}" # windows-1257:DE - "\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}" # windows-1257:DD - "\N{LATIN SMALL LETTER SHARP S}" # latin-1:DF - "\N{MULTIPLICATION SIGN}" # latin-1:D7 - "\N{GREEK CAPITAL LETTER BETA}" # windows-1253:C2 - "\N{GREEK CAPITAL LETTER GAMMA}" # windows-1253:C3 - "\N{GREEK CAPITAL LETTER DELTA}" # windows-1253:C4 - "\N{GREEK CAPITAL LETTER EPSILON}" # windows-1253:C5 - "\N{GREEK CAPITAL LETTER ZETA}" # windows-1253:C6 - "\N{GREEK CAPITAL LETTER ETA}" # windows-1253:C7 - "\N{GREEK CAPITAL LETTER THETA}" # windows-1253:C8 - "\N{GREEK CAPITAL LETTER IOTA}" # windows-1253:C9 - "\N{GREEK CAPITAL LETTER KAPPA}" # windows-1253:CA - "\N{GREEK CAPITAL LETTER LAMDA}" # windows-1253:CB - "\N{GREEK CAPITAL LETTER MU}" # windows-1253:CC - "\N{GREEK CAPITAL LETTER NU}" # windows-1253:CD - "\N{GREEK CAPITAL LETTER XI}" # windows-1253:CE - "\N{GREEK CAPITAL LETTER OMICRON}" # windows-1253:CF - "\N{GREEK CAPITAL LETTER PI}" # windows-1253:D0 - "\N{GREEK CAPITAL LETTER RHO}" # windows-1253:D1 - "\N{GREEK CAPITAL LETTER SIGMA}" # windows-1253:D3 - "\N{GREEK CAPITAL LETTER TAU}" # windows-1253:D4 - "\N{GREEK CAPITAL LETTER UPSILON}" # windows-1253:D5 - "\N{GREEK CAPITAL LETTER PHI}" # windows-1253:D6 - "\N{GREEK CAPITAL LETTER CHI}" # windows-1253:D7 - "\N{GREEK CAPITAL LETTER PSI}" # windows-1253:D8 - "\N{GREEK CAPITAL LETTER OMEGA}" # windows-1253:D9 - "\N{GREEK CAPITAL LETTER IOTA WITH DIALYTIKA}" # windows-1253:DA - "\N{GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA}" # windows-1253:DB - "\N{GREEK SMALL LETTER ALPHA WITH TONOS}" # windows-1253:DC - "\N{GREEK SMALL LETTER EPSILON WITH TONOS}" # windows-1253:DD - "\N{GREEK SMALL LETTER ETA WITH TONOS}" # windows-1253:DE - "\N{GREEK SMALL LETTER IOTA WITH TONOS}" # windows-1253:DF - "\N{CYRILLIC CAPITAL LETTER VE}" # windows-1251:C2 - "\N{CYRILLIC CAPITAL LETTER GHE}" # windows-1251:C3 - "\N{CYRILLIC CAPITAL LETTER DE}" # windows-1251:C4 - "\N{CYRILLIC CAPITAL LETTER IE}" # windows-1251:C5 - "\N{CYRILLIC CAPITAL LETTER ZHE}" # windows-1251:C6 - "\N{CYRILLIC CAPITAL LETTER ZE}" # windows-1251:C7 - "\N{CYRILLIC CAPITAL LETTER I}" # windows-1251:C8 - "\N{CYRILLIC CAPITAL LETTER SHORT I}" # windows-1251:C9 - "\N{CYRILLIC CAPITAL LETTER KA}" # windows-1251:CA - "\N{CYRILLIC CAPITAL LETTER EL}" # windows-1251:CB - "\N{CYRILLIC CAPITAL LETTER EM}" # windows-1251:CC - "\N{CYRILLIC CAPITAL LETTER EN}" # windows-1251:CD - "\N{CYRILLIC CAPITAL LETTER O}" # windows-1251:CE - "\N{CYRILLIC CAPITAL LETTER PE}" # windows-1251:CF - "\N{CYRILLIC CAPITAL LETTER ER}" # windows-1251:D0 - "\N{CYRILLIC CAPITAL LETTER ES}" # windows-1251:D1 - "\N{CYRILLIC CAPITAL LETTER TE}" # windows-1251:D2 - "\N{CYRILLIC CAPITAL LETTER U}" # windows-1251:D3 - "\N{CYRILLIC CAPITAL LETTER EF}" # windows-1251:D4 - "\N{CYRILLIC CAPITAL LETTER HA}" # windows-1251:D5 - "\N{CYRILLIC CAPITAL LETTER TSE}" # windows-1251:D6 - "\N{CYRILLIC CAPITAL LETTER CHE}" # windows-1251:D7 - "\N{CYRILLIC CAPITAL LETTER SHA}" # windows-1251:D8 - "\N{CYRILLIC CAPITAL LETTER SHCHA}" # windows-1251:D9 - "\N{CYRILLIC CAPITAL LETTER HARD SIGN}" # windows-1251:DA - "\N{CYRILLIC CAPITAL LETTER YERU}" # windows-1251:DB - "\N{CYRILLIC CAPITAL LETTER SOFT SIGN}" # windows-1251:DC - "\N{CYRILLIC CAPITAL LETTER E}" # windows-1251:DD - "\N{CYRILLIC CAPITAL LETTER YU}" # windows-1251:DE - "\N{CYRILLIC CAPITAL LETTER YA}" # windows-1251:DF - ), - # Letters that decode to 0xE0 - 0xEF in a Latin-1-like encoding - "utf8_first_of_3": ( - "\N{LATIN SMALL LETTER A WITH ACUTE}" # latin-1:E1 - "\N{LATIN SMALL LETTER A WITH BREVE}" # windows-1250:E3 - "\N{LATIN SMALL LETTER A WITH CIRCUMFLEX}" # latin-1:E2 - "\N{LATIN SMALL LETTER A WITH DIAERESIS}" # latin-1:E4 - "\N{LATIN SMALL LETTER A WITH GRAVE}" # latin-1:E0 - "\N{LATIN SMALL LETTER A WITH MACRON}" # windows-1257:E2 - "\N{LATIN SMALL LETTER A WITH OGONEK}" # windows-1257:E0 - "\N{LATIN SMALL LETTER A WITH RING ABOVE}" # latin-1:E5 - "\N{LATIN SMALL LETTER A WITH TILDE}" # latin-1:E3 - "\N{LATIN SMALL LETTER AE}" # latin-1:E6 - "\N{LATIN SMALL LETTER C WITH ACUTE}" # windows-1250:E6 - "\N{LATIN SMALL LETTER C WITH CARON}" # windows-1250:E8 - "\N{LATIN SMALL LETTER C WITH CEDILLA}" # latin-1:E7 - "\N{LATIN SMALL LETTER D WITH CARON}" # windows-1250:EF - "\N{LATIN SMALL LETTER E WITH ACUTE}" # latin-1:E9 - "\N{LATIN SMALL LETTER E WITH CARON}" # windows-1250:EC - "\N{LATIN SMALL LETTER E WITH CIRCUMFLEX}" # latin-1:EA - "\N{LATIN SMALL LETTER E WITH DIAERESIS}" # latin-1:EB - "\N{LATIN SMALL LETTER E WITH DOT ABOVE}" # windows-1257:EB - "\N{LATIN SMALL LETTER E WITH GRAVE}" # latin-1:E8 - "\N{LATIN SMALL LETTER E WITH MACRON}" # windows-1257:E7 - "\N{LATIN SMALL LETTER E WITH OGONEK}" # windows-1250:EA - "\N{LATIN SMALL LETTER E WITH OGONEK}" # windows-1250:EA - "\N{LATIN SMALL LETTER G WITH CEDILLA}" # windows-1257:EC - "\N{LATIN SMALL LETTER I WITH ACUTE}" # latin-1:ED - "\N{LATIN SMALL LETTER I WITH CIRCUMFLEX}" # latin-1:EE - "\N{LATIN SMALL LETTER I WITH DIAERESIS}" # latin-1:EF - "\N{LATIN SMALL LETTER I WITH GRAVE}" # latin-1:EC - "\N{LATIN SMALL LETTER I WITH MACRON}" # windows-1257:EE - "\N{LATIN SMALL LETTER I WITH OGONEK}" # windows-1257:E1 - "\N{LATIN SMALL LETTER K WITH CEDILLA}" # windows-1257:ED - "\N{LATIN SMALL LETTER L WITH ACUTE}" # windows-1250:E5 - "\N{LATIN SMALL LETTER L WITH CEDILLA}" # windows-1257:EF - "\N{LATIN SMALL LETTER R WITH ACUTE}" # windows-1250:E0 - "\N{LATIN SMALL LETTER Z WITH ACUTE}" # windows-1257:EA - "\N{GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS}" # windows-1253:E0 - "\N{GREEK SMALL LETTER ALPHA}" # windows-1253:E1 - "\N{GREEK SMALL LETTER BETA}" # windows-1253:E2 - "\N{GREEK SMALL LETTER GAMMA}" # windows-1253:E3 - "\N{GREEK SMALL LETTER DELTA}" # windows-1253:E4 - "\N{GREEK SMALL LETTER EPSILON}" # windows-1253:E5 - "\N{GREEK SMALL LETTER ZETA}" # windows-1253:E6 - "\N{GREEK SMALL LETTER ETA}" # windows-1253:E7 - "\N{GREEK SMALL LETTER THETA}" # windows-1253:E8 - "\N{GREEK SMALL LETTER IOTA}" # windows-1253:E9 - "\N{GREEK SMALL LETTER KAPPA}" # windows-1253:EA - "\N{GREEK SMALL LETTER LAMDA}" # windows-1253:EB - "\N{GREEK SMALL LETTER MU}" # windows-1253:EC - "\N{GREEK SMALL LETTER NU}" # windows-1253:ED - "\N{GREEK SMALL LETTER XI}" # windows-1253:EE - "\N{GREEK SMALL LETTER OMICRON}" # windows-1253:EF - "\N{CYRILLIC SMALL LETTER A}" # windows-1251:E0 - "\N{CYRILLIC SMALL LETTER BE}" # windows-1251:E1 - "\N{CYRILLIC SMALL LETTER VE}" # windows-1251:E2 - "\N{CYRILLIC SMALL LETTER GHE}" # windows-1251:E3 - "\N{CYRILLIC SMALL LETTER DE}" # windows-1251:E4 - "\N{CYRILLIC SMALL LETTER IE}" # windows-1251:E5 - "\N{CYRILLIC SMALL LETTER ZHE}" # windows-1251:E6 - "\N{CYRILLIC SMALL LETTER ZE}" # windows-1251:E7 - "\N{CYRILLIC SMALL LETTER I}" # windows-1251:E8 - "\N{CYRILLIC SMALL LETTER SHORT I}" # windows-1251:E9 - "\N{CYRILLIC SMALL LETTER KA}" # windows-1251:EA - "\N{CYRILLIC SMALL LETTER EL}" # windows-1251:EB - "\N{CYRILLIC SMALL LETTER EM}" # windows-1251:EC - "\N{CYRILLIC SMALL LETTER EN}" # windows-1251:ED - "\N{CYRILLIC SMALL LETTER O}" # windows-1251:EE - "\N{CYRILLIC SMALL LETTER PE}" # windows-1251:EF - ), - # Letters that decode to 0xF0 or 0xF3 in a Latin-1-like encoding. - # (Other leading bytes correspond only to unassigned codepoints) - "utf8_first_of_4": ( - "\N{LATIN SMALL LETTER D WITH STROKE}" # windows-1250:F0 - "\N{LATIN SMALL LETTER ETH}" # latin-1:F0 - "\N{LATIN SMALL LETTER G WITH BREVE}" # windows-1254:F0 - "\N{LATIN SMALL LETTER O WITH ACUTE}" # latin-1:F3 - "\N{LATIN SMALL LETTER S WITH CARON}" # windows-1257:F0 - "\N{GREEK SMALL LETTER PI}" # windows-1253:F0 - "\N{GREEK SMALL LETTER SIGMA}" # windows-1253:F3 - "\N{CYRILLIC SMALL LETTER ER}" # windows-1251:F0 - "\N{CYRILLIC SMALL LETTER U}" # windows-1251:F3 - ), - # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding, - # including a space standing in for 0xA0 - "utf8_continuation": ( - "\x80-\xbf" - "\N{SPACE}" # modification of latin-1:A0, NO-BREAK SPACE - "\N{LATIN CAPITAL LETTER A WITH OGONEK}" # windows-1250:A5 - "\N{LATIN CAPITAL LETTER AE}" # windows-1257:AF - "\N{LATIN CAPITAL LETTER L WITH CARON}" # windows-1250:BC - "\N{LATIN CAPITAL LETTER L WITH STROKE}" # windows-1250:A3 - "\N{LATIN CAPITAL LETTER O WITH STROKE}" # windows-1257:A8 - "\N{LATIN CAPITAL LETTER R WITH CEDILLA}" # windows-1257:AA - "\N{LATIN CAPITAL LETTER S WITH ACUTE}" # windows-1250:8C - "\N{LATIN CAPITAL LETTER S WITH CARON}" # windows-1252:8A - "\N{LATIN CAPITAL LETTER S WITH CEDILLA}" # windows-1250:AA - "\N{LATIN CAPITAL LETTER T WITH CARON}" # windows-1250:8D - "\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}" # windows-1252:9F - "\N{LATIN CAPITAL LETTER Z WITH ACUTE}" # windows-1250:8F - "\N{LATIN CAPITAL LETTER Z WITH CARON}" # windows-1252:8E - "\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}" # windows-1250:AF - "\N{LATIN CAPITAL LIGATURE OE}" # windows-1252:8C - "\N{LATIN SMALL LETTER A WITH OGONEK}" # windows-1250:B9 - "\N{LATIN SMALL LETTER AE}" # windows-1257:BF - "\N{LATIN SMALL LETTER F WITH HOOK}" # windows-1252:83 - "\N{LATIN SMALL LETTER L WITH CARON}" # windows-1250:BE - "\N{LATIN SMALL LETTER L WITH STROKE}" # windows-1250:B3 - "\N{LATIN SMALL LETTER O WITH STROKE}" # windows-1257:B8 - "\N{LATIN SMALL LETTER R WITH CEDILLA}" # windows-1257:BA - "\N{LATIN SMALL LETTER S WITH ACUTE}" # windows-1250:9C - "\N{LATIN SMALL LETTER S WITH CARON}" # windows-1252:9A - "\N{LATIN SMALL LETTER S WITH CEDILLA}" # windows-1250:BA - "\N{LATIN SMALL LETTER T WITH CARON}" # windows-1250:9D - "\N{LATIN SMALL LETTER Z WITH ACUTE}" # windows-1250:9F - "\N{LATIN SMALL LETTER Z WITH CARON}" # windows-1252:9E - "\N{LATIN SMALL LETTER Z WITH DOT ABOVE}" # windows-1250:BF - "\N{LATIN SMALL LIGATURE OE}" # windows-1252:9C - "\N{MODIFIER LETTER CIRCUMFLEX ACCENT}" # windows-1252:88 - "\N{CARON}" # windows-1250:A1 - "\N{BREVE}" # windows-1250:A2 - "\N{OGONEK}" # windows-1250:B2 - "\N{SMALL TILDE}" # windows-1252:98 - "\N{DOUBLE ACUTE ACCENT}" # windows-1250:BD - "\N{GREEK TONOS}" # windows-1253:B4 - "\N{GREEK DIALYTIKA TONOS}" # windows-1253:A1 - "\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}" # windows-1253:A2 - "\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}" # windows-1253:B8 - "\N{GREEK CAPITAL LETTER ETA WITH TONOS}" # windows-1253:B9 - "\N{GREEK CAPITAL LETTER IOTA WITH TONOS}" # windows-1253:BA - "\N{GREEK CAPITAL LETTER OMICRON WITH TONOS}" # windows-1253:BC - "\N{GREEK CAPITAL LETTER UPSILON WITH TONOS}" # windows-1253:BE - "\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}" # windows-1253:BF - "\N{CYRILLIC CAPITAL LETTER IO}" # windows-1251:A8 - "\N{CYRILLIC CAPITAL LETTER DJE}" # windows-1251:80 - "\N{CYRILLIC CAPITAL LETTER GJE}" # windows-1251:81 - "\N{CYRILLIC CAPITAL LETTER UKRAINIAN IE}" # windows-1251:AA - "\N{CYRILLIC CAPITAL LETTER DZE}" # windows-1251:BD - "\N{CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I}" # windows-1251:B2 - "\N{CYRILLIC CAPITAL LETTER YI}" # windows-1251:AF - "\N{CYRILLIC CAPITAL LETTER JE}" # windows-1251:A3 - "\N{CYRILLIC CAPITAL LETTER LJE}" # windows-1251:8A - "\N{CYRILLIC CAPITAL LETTER NJE}" # windows-1251:8C - "\N{CYRILLIC CAPITAL LETTER TSHE}" # windows-1251:8E - "\N{CYRILLIC CAPITAL LETTER KJE}" # windows-1251:8D - "\N{CYRILLIC CAPITAL LETTER SHORT U}" # windows-1251:A1 - "\N{CYRILLIC CAPITAL LETTER DZHE}" # windows-1251:8F - "\N{CYRILLIC SMALL LETTER IO}" # windows-1251:B8 - "\N{CYRILLIC SMALL LETTER DJE}" # windows-1251:90 - "\N{CYRILLIC SMALL LETTER GJE}" # windows-1251:83 - "\N{CYRILLIC SMALL LETTER UKRAINIAN IE}" # windows-1251:BA - "\N{CYRILLIC SMALL LETTER DZE}" # windows-1251:BE - "\N{CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I}" # windows-1251:B3 - "\N{CYRILLIC SMALL LETTER YI}" # windows-1251:BF - "\N{CYRILLIC SMALL LETTER JE}" # windows-1251:BC - "\N{CYRILLIC SMALL LETTER LJE}" # windows-1251:9A - "\N{CYRILLIC SMALL LETTER NJE}" # windows-1251:9C - "\N{CYRILLIC SMALL LETTER TSHE}" # windows-1251:9E - "\N{CYRILLIC SMALL LETTER KJE}" # windows-1251:9D - "\N{CYRILLIC SMALL LETTER SHORT U}" # windows-1251:A2 - "\N{CYRILLIC SMALL LETTER DZHE}" # windows-1251:9F - "\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}" # windows-1251:A5 - "\N{CYRILLIC SMALL LETTER GHE WITH UPTURN}" # windows-1251:B4 - "\N{EN DASH}" # windows-1252:96 - "\N{EM DASH}" # windows-1252:97 - "\N{HORIZONTAL BAR}" # windows-1253:AF - "\N{LEFT SINGLE QUOTATION MARK}" # windows-1252:91 - "\N{RIGHT SINGLE QUOTATION MARK}" # windows-1252:92 - "\N{SINGLE LOW-9 QUOTATION MARK}" # windows-1252:82 - "\N{LEFT DOUBLE QUOTATION MARK}" # windows-1252:93 - "\N{RIGHT DOUBLE QUOTATION MARK}" # windows-1252:94 - "\N{DOUBLE LOW-9 QUOTATION MARK}" # windows-1252:84 - "\N{DAGGER}" # windows-1252:86 - "\N{DOUBLE DAGGER}" # windows-1252:87 - "\N{BULLET}" # windows-1252:95 - "\N{HORIZONTAL ELLIPSIS}" # windows-1252:85 - "\N{PER MILLE SIGN}" # windows-1252:89 - "\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}" # windows-1252:8B - "\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}" # windows-1252:9B - "\N{EURO SIGN}" # windows-1252:80 - "\N{NUMERO SIGN}" # windows-1251:B9 - "\N{TRADE MARK SIGN}" # windows-1252:99 - ), - # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding, - # and don't usually stand for themselves when adjacent to mojibake. - # This excludes spaces, dashes, 'bullet', quotation marks, and ellipses. - "utf8_continuation_strict": ( - "\x80-\xbf" - "\N{LATIN CAPITAL LETTER A WITH OGONEK}" # windows-1250:A5 - "\N{LATIN CAPITAL LETTER AE}" # windows-1257:AF - "\N{LATIN CAPITAL LETTER L WITH CARON}" # windows-1250:BC - "\N{LATIN CAPITAL LETTER L WITH STROKE}" # windows-1250:A3 - "\N{LATIN CAPITAL LETTER O WITH STROKE}" # windows-1257:A8 - "\N{LATIN CAPITAL LETTER R WITH CEDILLA}" # windows-1257:AA - "\N{LATIN CAPITAL LETTER S WITH ACUTE}" # windows-1250:8C - "\N{LATIN CAPITAL LETTER S WITH CARON}" # windows-1252:8A - "\N{LATIN CAPITAL LETTER S WITH CEDILLA}" # windows-1250:AA - "\N{LATIN CAPITAL LETTER T WITH CARON}" # windows-1250:8D - "\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}" # windows-1252:9F - "\N{LATIN CAPITAL LETTER Z WITH ACUTE}" # windows-1250:8F - "\N{LATIN CAPITAL LETTER Z WITH CARON}" # windows-1252:8E - "\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}" # windows-1250:AF - "\N{LATIN CAPITAL LIGATURE OE}" # windows-1252:8C - "\N{LATIN SMALL LETTER A WITH OGONEK}" # windows-1250:B9 - "\N{LATIN SMALL LETTER AE}" # windows-1257:BF - "\N{LATIN SMALL LETTER F WITH HOOK}" # windows-1252:83 - "\N{LATIN SMALL LETTER L WITH CARON}" # windows-1250:BE - "\N{LATIN SMALL LETTER L WITH STROKE}" # windows-1250:B3 - "\N{LATIN SMALL LETTER O WITH STROKE}" # windows-1257:B8 - "\N{LATIN SMALL LETTER R WITH CEDILLA}" # windows-1257:BA - "\N{LATIN SMALL LETTER S WITH ACUTE}" # windows-1250:9C - "\N{LATIN SMALL LETTER S WITH CARON}" # windows-1252:9A - "\N{LATIN SMALL LETTER S WITH CEDILLA}" # windows-1250:BA - "\N{LATIN SMALL LETTER T WITH CARON}" # windows-1250:9D - "\N{LATIN SMALL LETTER Z WITH ACUTE}" # windows-1250:9F - "\N{LATIN SMALL LETTER Z WITH CARON}" # windows-1252:9E - "\N{LATIN SMALL LETTER Z WITH DOT ABOVE}" # windows-1250:BF - "\N{LATIN SMALL LIGATURE OE}" # windows-1252:9C - "\N{MODIFIER LETTER CIRCUMFLEX ACCENT}" # windows-1252:88 - "\N{CARON}" # windows-1250:A1 - "\N{BREVE}" # windows-1250:A2 - "\N{OGONEK}" # windows-1250:B2 - "\N{SMALL TILDE}" # windows-1252:98 - "\N{DOUBLE ACUTE ACCENT}" # windows-1250:BD - "\N{GREEK TONOS}" # windows-1253:B4 - "\N{GREEK DIALYTIKA TONOS}" # windows-1253:A1 - "\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}" # windows-1253:A2 - "\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}" # windows-1253:B8 - "\N{GREEK CAPITAL LETTER ETA WITH TONOS}" # windows-1253:B9 - "\N{GREEK CAPITAL LETTER IOTA WITH TONOS}" # windows-1253:BA - "\N{GREEK CAPITAL LETTER OMICRON WITH TONOS}" # windows-1253:BC - "\N{GREEK CAPITAL LETTER UPSILON WITH TONOS}" # windows-1253:BE - "\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}" # windows-1253:BF - "\N{CYRILLIC CAPITAL LETTER IO}" # windows-1251:A8 - "\N{CYRILLIC CAPITAL LETTER DJE}" # windows-1251:80 - "\N{CYRILLIC CAPITAL LETTER GJE}" # windows-1251:81 - "\N{CYRILLIC CAPITAL LETTER UKRAINIAN IE}" # windows-1251:AA - "\N{CYRILLIC CAPITAL LETTER DZE}" # windows-1251:BD - "\N{CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I}" # windows-1251:B2 - "\N{CYRILLIC CAPITAL LETTER YI}" # windows-1251:AF - "\N{CYRILLIC CAPITAL LETTER JE}" # windows-1251:A3 - "\N{CYRILLIC CAPITAL LETTER LJE}" # windows-1251:8A - "\N{CYRILLIC CAPITAL LETTER NJE}" # windows-1251:8C - "\N{CYRILLIC CAPITAL LETTER TSHE}" # windows-1251:8E - "\N{CYRILLIC CAPITAL LETTER KJE}" # windows-1251:8D - "\N{CYRILLIC CAPITAL LETTER SHORT U}" # windows-1251:A1 - "\N{CYRILLIC CAPITAL LETTER DZHE}" # windows-1251:8F - "\N{CYRILLIC SMALL LETTER IO}" # windows-1251:B8 - "\N{CYRILLIC SMALL LETTER DJE}" # windows-1251:90 - "\N{CYRILLIC SMALL LETTER GJE}" # windows-1251:83 - "\N{CYRILLIC SMALL LETTER UKRAINIAN IE}" # windows-1251:BA - "\N{CYRILLIC SMALL LETTER DZE}" # windows-1251:BE - "\N{CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I}" # windows-1251:B3 - "\N{CYRILLIC SMALL LETTER YI}" # windows-1251:BF - "\N{CYRILLIC SMALL LETTER JE}" # windows-1251:BC - "\N{CYRILLIC SMALL LETTER LJE}" # windows-1251:9A - "\N{CYRILLIC SMALL LETTER NJE}" # windows-1251:9C - "\N{CYRILLIC SMALL LETTER TSHE}" # windows-1251:9E - "\N{CYRILLIC SMALL LETTER KJE}" # windows-1251:9D - "\N{CYRILLIC SMALL LETTER SHORT U}" # windows-1251:A2 - "\N{CYRILLIC SMALL LETTER DZHE}" # windows-1251:9F - "\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}" # windows-1251:A5 - "\N{CYRILLIC SMALL LETTER GHE WITH UPTURN}" # windows-1251:B4 - "\N{DAGGER}" # windows-1252:86 - "\N{DOUBLE DAGGER}" # windows-1252:87 - "\N{PER MILLE SIGN}" # windows-1252:89 - "\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}" # windows-1252:8B - "\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}" # windows-1252:9B - "\N{EURO SIGN}" # windows-1252:80 - "\N{NUMERO SIGN}" # windows-1251:B9 - "\N{TRADE MARK SIGN}" # windows-1252:99 - ), -} - -# This regex uses UTF8_CLUES to find sequences of likely mojibake. -# It matches them with + so that several adjacent UTF-8-looking sequences -# get coalesced into one, allowing them to be fixed more efficiently -# and not requiring every individual subsequence to be detected as 'badness'. -# -# We accept spaces in place of "utf8_continuation", because spaces might have -# been intended to be U+A0 NO-BREAK SPACE. -# -# We do a lookbehind to make sure the previous character isn't a -# "utf8_continuation_strict" character, so that we don't fix just a few -# characters in a huge garble and make the situation worse. -# -# Unfortunately, the matches to this regular expression won't show their -# surrounding context, and including context would make the expression much -# less efficient. The 'badness' rules that require context, such as a preceding -# lowercase letter, will prevent some cases of inconsistent UTF-8 from being -# fixed when they don't see it. -UTF8_DETECTOR_RE = re.compile( - """ - (? None: - """ - Run ftfy as a command-line utility. - """ - import argparse - - parser = argparse.ArgumentParser( - description=f"ftfy (fixes text for you), version {__version__}" - ) - parser.add_argument( - "filename", - default="-", - nargs="?", - help="The file whose Unicode is to be fixed. Defaults to -, meaning standard input.", - ) - parser.add_argument( - "-o", - "--output", - type=str, - default="-", - help="The file to output to. Defaults to -, meaning standard output.", - ) - parser.add_argument( - "-g", - "--guess", - action="store_true", - help="Ask ftfy to guess the encoding of your input. This is risky. Overrides -e.", - ) - parser.add_argument( - "-e", - "--encoding", - type=str, - default="utf-8", - help="The encoding of the input. Defaults to UTF-8.", - ) - parser.add_argument( - "-n", - "--normalization", - type=str, - default="NFC", - help='The normalization of Unicode to apply. Defaults to NFC. Can be "none".', - ) - parser.add_argument( - "--preserve-entities", - action="store_true", - help="Leave HTML entities as they are. The default " - "is to decode them, as long as no HTML tags have appeared in the file.", - ) - - args = parser.parse_args() - - encoding = args.encoding - if args.guess: - encoding = None - - if args.filename == "-": - # Get a standard input stream made of bytes, so we can decode it as - # whatever encoding is necessary. - file = sys.stdin.buffer - else: - file = open(args.filename, "rb") - - if args.output == "-": - outfile = sys.stdout - else: - if os.path.realpath(args.output) == os.path.realpath(args.filename): - sys.stderr.write(SAME_FILE_ERROR_TEXT) - sys.exit(1) - outfile = open(args.output, "w", encoding="utf-8") - - normalization = args.normalization - if normalization.lower() == "none": - normalization = None - - unescape_html: Union[str, bool] - if args.preserve_entities: - unescape_html = False - else: - unescape_html = "auto" - - config = TextFixerConfig(unescape_html=unescape_html, normalization=normalization) - - try: - for line in fix_file(file, encoding=encoding, config=config): - try: - outfile.write(line) - except UnicodeEncodeError: - if sys.platform == "win32": - sys.stderr.write(ENCODE_ERROR_TEXT_WINDOWS) - else: - sys.stderr.write(ENCODE_ERROR_TEXT_UNIX) - sys.exit(1) - except UnicodeDecodeError as err: - sys.stderr.write(DECODE_ERROR_TEXT % (encoding, err)) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/bundle/python-cpu/Lib/site-packages/ftfy/fixes.py b/bundle/python-cpu/Lib/site-packages/ftfy/fixes.py deleted file mode 100644 index 41d3c2f817f0dedfb6887a64a97bb6e5dc0a118f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy/fixes.py +++ /dev/null @@ -1,510 +0,0 @@ -""" -The `ftfy.fixes` module contains the individual fixes that :func:`ftfy.fix_text` -can perform, and provides the functions that are named in "explanations" -such as the output of :func:`ftfy.fix_and_explain`. - -Two of these functions are particularly useful on their own, as more robust -versions of functions in the Python standard library: - -- :func:`ftfy.fixes.decode_escapes` -- :func:`ftfy.fixes.unescape_html` -""" - -import codecs -import html -import re -import warnings -from re import Match -from typing import Any - -import ftfy -from ftfy.badness import is_bad -from ftfy.chardata import ( - ALTERED_UTF8_RE, - C1_CONTROL_RE, - CONTROL_CHARS, - DOUBLE_QUOTE_RE, - HTML_ENTITIES, - HTML_ENTITY_RE, - LIGATURES, - LOSSY_UTF8_RE, - SINGLE_QUOTE_RE, - UTF8_DETECTOR_RE, - WIDTH_MAP, -) - - -def fix_encoding_and_explain(text: str) -> Any: - """ - Deprecated copy of `ftfy.fix_encoding_and_explain()`. - """ - warnings.warn( - "`fix_encoding_and_explain()` has moved to the main module of ftfy.", - DeprecationWarning, - stacklevel=2, - ) - return ftfy.fix_encoding_and_explain(text) - - -def fix_encoding(text: str) -> str: - """ - Deprecated copy of `ftfy.fix_encoding()`. - """ - warnings.warn( - "`fix_encoding()` has moved to the main module of ftfy.", - DeprecationWarning, - stacklevel=2, - ) - return ftfy.fix_encoding(text) - - -def apply_plan(text: str, plan: list[tuple[str, str]]) -> str: - """ - Deprecated copy of `ftfy.apply_plan()`. - """ - warnings.warn( - "`apply_plan()` has moved to the main module of ftfy.", - DeprecationWarning, - stacklevel=2, - ) - return ftfy.apply_plan(text, plan) - - -def _unescape_fixup(match: Match[str]) -> str: - """ - Replace one matched HTML entity with the character it represents, - if possible. - """ - text = match.group(0) - if text in HTML_ENTITIES: - return HTML_ENTITIES[text] - elif text.startswith("&#"): - unescaped: str = html.unescape(text) - - # If html.unescape only decoded part of the string, that's not what - # we want. The semicolon should be consumed. - if ";" in unescaped: - return text - else: - return unescaped - else: - return text - - -def unescape_html(text: str) -> str: - """ - Decode HTML entities and character references, including some nonstandard - ones written in all-caps. - - Python has a built-in called `html.unescape` that can decode HTML escapes, - including a bunch of messy edge cases such as decoding escapes without - semicolons such as "&". - - If you know you've got HTML-escaped text, applying `html.unescape` is the - right way to convert it to plain text. But in ambiguous situations, that - would create false positives. For example, the informally written text - "this¬ that" should not automatically be decoded as "this¬ that". - - In this function, we decode the escape sequences that appear in the - `html.entities.html5` dictionary, as long as they are the unambiguous ones - that end in semicolons. - - We also decode all-caps versions of Latin letters and common symbols. - If a database contains the name 'P&EACUTE;REZ', we can read that and intuit - that it was supposed to say 'PÉREZ'. This is limited to a smaller set of - entities, because there are many instances where entity names are - case-sensitive in complicated ways. - - >>> unescape_html('<tag>') - '' - - >>> unescape_html('𝒥ohn ℋancock') - '𝒥ohn ℋancock' - - >>> unescape_html('✓') - '✓' - - >>> unescape_html('Pérez') - 'Pérez' - - >>> unescape_html('P&EACUTE;REZ') - 'PÉREZ' - - >>> unescape_html('BUNDESSTRA&SZLIG;E') - 'BUNDESSTRASSE' - - >>> unescape_html('ñ Ñ &NTILDE; &nTILDE;') - 'ñ Ñ Ñ &nTILDE;' - """ - return HTML_ENTITY_RE.sub(_unescape_fixup, text) - - -ANSI_RE = re.compile("\033\\[((?:\\d|;)*)([a-zA-Z])") - - -def remove_terminal_escapes(text: str) -> str: - r""" - Strip out "ANSI" terminal escape sequences, such as those that produce - colored text on Unix. - - >>> print(remove_terminal_escapes( - ... "\033[36;44mI'm blue, da ba dee da ba doo...\033[0m" - ... )) - I'm blue, da ba dee da ba doo... - """ - return ANSI_RE.sub("", text) - - -def uncurl_quotes(text: str) -> str: - r""" - Replace curly quotation marks with straight equivalents. - - >>> print(uncurl_quotes('\u201chere\u2019s a test\u201d')) - "here's a test" - """ - return SINGLE_QUOTE_RE.sub("'", DOUBLE_QUOTE_RE.sub('"', text)) - - -def fix_latin_ligatures(text: str) -> str: - """ - Replace single-character ligatures of Latin letters, such as 'fi', with the - characters that they contain, as in 'fi'. Latin ligatures are usually not - intended in text strings (though they're lovely in *rendered* text). If - you have such a ligature in your string, it is probably a result of a - copy-and-paste glitch. - - We leave ligatures in other scripts alone to be safe. They may be intended, - and removing them may lose information. If you want to take apart nearly - all ligatures, use NFKC normalization. - - >>> print(fix_latin_ligatures("fluffiest")) - fluffiest - """ - return text.translate(LIGATURES) - - -def fix_character_width(text: str) -> str: - """ - The ASCII characters, katakana, and Hangul characters have alternate - "halfwidth" or "fullwidth" forms that help text line up in a grid. - - If you don't need these width properties, you probably want to replace - these characters with their standard form, which is what this function - does. - - Note that this replaces the ideographic space, U+3000, with the ASCII - space, U+20. - - >>> print(fix_character_width("LOUD NOISES")) - LOUD NOISES - >>> print(fix_character_width("Uターン")) # this means "U-turn" - Uターン - """ - return text.translate(WIDTH_MAP) - - -def fix_line_breaks(text: str) -> str: - r""" - Convert all line breaks to Unix style. - - This will convert the following sequences into the standard \\n - line break: - - - CRLF (\\r\\n), used on Windows and in some communication protocols - - CR (\\r), once used on Mac OS Classic, and now kept alive by misguided - software such as Microsoft Office for Mac - - LINE SEPARATOR (\\u2028) and PARAGRAPH SEPARATOR (\\u2029), defined by - Unicode and used to sow confusion and discord - - NEXT LINE (\\x85), a C1 control character that is certainly not what you - meant - - The NEXT LINE character is a bit of an odd case, because it - usually won't show up if `fix_encoding` is also being run. - \\x85 is very common mojibake for \\u2026, HORIZONTAL ELLIPSIS. - - >>> print(fix_line_breaks( - ... "This string is made of two things:\u2029" - ... "1. Unicode\u2028" - ... "2. Spite" - ... )) - This string is made of two things: - 1. Unicode - 2. Spite - - For further testing and examples, let's define a function to make sure - we can see the control characters in their escaped form: - - >>> def eprint(text): - ... print(text.encode('unicode-escape').decode('ascii')) - - >>> eprint(fix_line_breaks("Content-type: text/plain\r\n\r\nHi.")) - Content-type: text/plain\n\nHi. - - >>> eprint(fix_line_breaks("This is how Microsoft \r trolls Mac users")) - This is how Microsoft \n trolls Mac users - - >>> eprint(fix_line_breaks("What is this \x85 I don't even")) - What is this \n I don't even - """ - return ( - text.replace("\r\n", "\n") - .replace("\r", "\n") - .replace("\u2028", "\n") - .replace("\u2029", "\n") - .replace("\u0085", "\n") - ) - - -SURROGATE_RE = re.compile("[\ud800-\udfff]") -SURROGATE_PAIR_RE = re.compile("[\ud800-\udbff][\udc00-\udfff]") - - -def convert_surrogate_pair(match: Match[str]) -> str: - """ - Convert a surrogate pair to the single codepoint it represents. - - This implements the formula described at: - http://en.wikipedia.org/wiki/Universal_Character_Set_characters#Surrogates - """ - pair = match.group(0) - codept = 0x10000 + (ord(pair[0]) - 0xD800) * 0x400 + (ord(pair[1]) - 0xDC00) - return chr(codept) - - -def fix_surrogates(text: str) -> str: - """ - Replace 16-bit surrogate codepoints with the characters they represent - (when properly paired), or with \ufffd otherwise. - - >>> high_surrogate = chr(0xd83d) - >>> low_surrogate = chr(0xdca9) - >>> print(fix_surrogates(high_surrogate + low_surrogate)) - 💩 - >>> print(fix_surrogates(low_surrogate + high_surrogate)) - �� - - The above doctest had to be very carefully written, because even putting - the Unicode escapes of the surrogates in the docstring was causing - various tools to fail, which I think just goes to show why this fixer is - necessary. - """ - if SURROGATE_RE.search(text): - text = SURROGATE_PAIR_RE.sub(convert_surrogate_pair, text) - text = SURROGATE_RE.sub("\ufffd", text) - return text - - -def remove_control_chars(text: str) -> str: - """ - Remove various control characters that you probably didn't intend to be in - your text. Many of these characters appear in the table of "Characters not - suitable for use with markup" at - http://www.unicode.org/reports/tr20/tr20-9.html. - - This includes: - - - ASCII control characters, except for the important whitespace characters - (U+00 to U+08, U+0B, U+0E to U+1F, U+7F) - - Deprecated Arabic control characters (U+206A to U+206F) - - Interlinear annotation characters (U+FFF9 to U+FFFB) - - The Object Replacement Character (U+FFFC) - - The byte order mark (U+FEFF) - - However, these similar characters are left alone: - - - Control characters that produce whitespace (U+09, U+0A, U+0C, U+0D, - U+2028, and U+2029) - - C1 control characters (U+80 to U+9F) -- even though they are basically - never used intentionally, they are important clues about what mojibake - has happened - - Control characters that affect glyph rendering, such as joiners and - right-to-left marks (U+200C to U+200F, U+202A to U+202E) - - Musical notation control characters (U+1D173 to U+1D17A) because wow if - you're using those you probably have a good reason - - Tag characters, because they are now used in emoji sequences such as - "Flag of Wales" - """ - return text.translate(CONTROL_CHARS) - - -def remove_bom(text: str) -> str: - r""" - Remove a byte-order mark that was accidentally decoded as if it were part - of the text. - - >>> print(remove_bom(chr(0xfeff) + "Where do you want to go today?")) - Where do you want to go today? - """ - return text.lstrip(chr(0xFEFF)) - - -# Define a regex to match valid escape sequences in Python string literals. -ESCAPE_SEQUENCE_RE = re.compile( - r""" - ( \\U........ # 8-digit hex escapes - | \\u.... # 4-digit hex escapes - | \\x.. # 2-digit hex escapes - | \\[0-7]{1,3} # Octal escapes - | \\N\{[^}]+\} # Unicode characters by name - | \\[\\'"abfnrtv] # Single-character escapes - )""", - re.UNICODE | re.VERBOSE, -) - - -def decode_escapes(text: str) -> str: - r""" - Decode backslashed escape sequences, including \\x, \\u, and \\U character - references, even in the presence of other Unicode. - - This function has to be called specifically. It's not run automatically by - ftfy, because escaped text is not necessarily a mistake, and there is no - way to distinguish when it is. - - This is what Python's "string-escape" and "unicode-escape" codecs were - meant to do, but in contrast, this actually works. It will decode the - string exactly the same way that the Python interpreter decodes its string - literals. - - >>> factoid = '\\u20a1 is the currency symbol for the colón.' - >>> print(factoid[1:]) - u20a1 is the currency symbol for the colón. - >>> print(decode_escapes(factoid)) - ₡ is the currency symbol for the colón. - - Even though Python itself can read string literals with a combination of - escapes and literal Unicode -- you're looking at one right now -- the - "unicode-escape" codec doesn't work on literal Unicode. (See - http://stackoverflow.com/a/24519338/773754 for more details.) - - Instead, this function searches for just the parts of a string that - represent escape sequences, and decodes them, leaving the rest alone. All - valid escape sequences are made of ASCII characters, and this allows - "unicode-escape" to work correctly. - """ - - def decode_match(match: Match[str]) -> str: - "Given a regex match, decode the escape sequence it contains." - return codecs.decode(match.group(0), "unicode-escape") - - return ESCAPE_SEQUENCE_RE.sub(decode_match, text) - - -# This regex implements an exception to restore_byte_a0, so we can decode the -# very common mojibake of (for example) "à la mode" as "à la mode", not "àla -# mode". -# -# If byte C3 appears with a single space after it -- most commonly this shows -# up as " à " appearing as an entire word -- we'll insert \xa0 while keeping -# the space. Without this change, we would decode "à" as the start of the next -# word, such as "àla". It's almost always intended to be a separate word, as in -# "à la", but when mojibake turns this into "Ã\xa0 la", the two kinds of spaces -# get coalesced into "à la". -# -# We make exceptions for the Portuguese words "às", "àquele", "àquela", -# "àquilo" and their plurals -- these are contractions of, for example, "a -# aquele" and are very common. Note that the final letter is important to -# distinguish this case from French "à quel point". -# -# Other instances in Portuguese, such as "àfrica", seem to be typos (intended -# to be "África" with the accent in the other direction). -# -# Unfortunately, "à" is a common letter in Catalan, and mojibake of words that -# contain it will end up with inserted spaces. We can't do the right thing with -# every word. The cost is that the mojibake text "fà cil" will be interpreted as -# "fà cil", not "fàcil". -A_GRAVE_WORD_RE = re.compile(b"\xc3 (?! |quele|quela|quilo|s )") - - -def restore_byte_a0(byts: bytes) -> bytes: - """ - Some mojibake has been additionally altered by a process that said "hmm, - byte A0, that's basically a space!" and replaced it with an ASCII space. - When the A0 is part of a sequence that we intend to decode as UTF-8, - changing byte A0 to 20 would make it fail to decode. - - This process finds sequences that would convincingly decode as UTF-8 if - byte 20 were changed to A0, and puts back the A0. For the purpose of - deciding whether this is a good idea, this step gets a cost of twice - the number of bytes that are changed. - - This is used as a step within `fix_encoding`. - """ - byts = A_GRAVE_WORD_RE.sub(b"\xc3\xa0 ", byts) - - def replacement(match: Match[bytes]) -> bytes: - "The function to apply when this regex matches." - return match.group(0).replace(b"\x20", b"\xa0") - - return ALTERED_UTF8_RE.sub(replacement, byts) - - -def replace_lossy_sequences(byts: bytes) -> bytes: - """ - This function identifies sequences where information has been lost in - a "sloppy" codec, indicated by byte 1A, and if they would otherwise look - like a UTF-8 sequence, it replaces them with the UTF-8 sequence for U+FFFD. - - A further explanation: - - ftfy can now fix text in a few cases that it would previously fix - incompletely, because of the fact that it can't successfully apply the fix - to the entire string. A very common case of this is when characters have - been erroneously decoded as windows-1252, but instead of the "sloppy" - windows-1252 that passes through unassigned bytes, the unassigned bytes get - turned into U+FFFD (�), so we can't tell what they were. - - This most commonly happens with curly quotation marks that appear - ``“ like this â€�``. - - We can do better by building on ftfy's "sloppy codecs" to let them handle - less-sloppy but more-lossy text. When they encounter the character ``�``, - instead of refusing to encode it, they encode it as byte 1A -- an - ASCII control code called SUBSTITUTE that once was meant for about the same - purpose. We can then apply a fixer that looks for UTF-8 sequences where - some continuation bytes have been replaced by byte 1A, and decode the whole - sequence as �; if that doesn't work, it'll just turn the byte back into � - itself. - - As a result, the above text ``“ like this â€�`` will decode as - ``“ like this �``. - - If U+1A was actually in the original string, then the sloppy codecs will - not be used, and this function will not be run, so your weird control - character will be left alone but wacky fixes like this won't be possible. - - This is used as a transcoder within `fix_encoding`. - """ - return LOSSY_UTF8_RE.sub("\ufffd".encode(), byts) - - -def decode_inconsistent_utf8(text: str) -> str: - """ - Sometimes, text from one encoding ends up embedded within text from a - different one. This is common enough that we need to be able to fix it. - - This is used as a transcoder within `fix_encoding`. - """ - - def fix_embedded_mojibake(match: Match[str]) -> str: - substr = match.group(0) - - # Require the match to be shorter, so that this doesn't recurse infinitely - if len(substr) < len(text) and is_bad(substr): - return ftfy.fix_encoding(substr) - else: - return substr - - return UTF8_DETECTOR_RE.sub(fix_embedded_mojibake, text) - - -def _c1_fixer(match: Match[str]) -> str: - return match.group(0).encode("latin-1").decode("sloppy-windows-1252") - - -def fix_c1_controls(text: str) -> str: - """ - If text still contains C1 control characters, treat them as their - Windows-1252 equivalents. This matches what Web browsers do. - """ - return C1_CONTROL_RE.sub(_c1_fixer, text) diff --git a/bundle/python-cpu/Lib/site-packages/ftfy/formatting.py b/bundle/python-cpu/Lib/site-packages/ftfy/formatting.py deleted file mode 100644 index 18df64b082ddfe26f079578de57a6bb6f5d2df03..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/ftfy/formatting.py +++ /dev/null @@ -1,166 +0,0 @@ -""" -This module provides functions for justifying Unicode text in a monospaced -display such as a terminal. - -We used to have our own implementation here, but now we mostly rely on -the 'wcwidth' library. -""" - -from unicodedata import normalize - -from wcwidth import wcswidth, wcwidth - -from ftfy.fixes import remove_terminal_escapes - - -def character_width(char: str) -> int: - r""" - Determine the width that a character is likely to be displayed as in - a monospaced terminal. The width for a printable character will - always be 0, 1, or 2. - - Nonprintable or control characters will return -1, a convention that comes - from wcwidth. - - >>> character_width('車') - 2 - >>> character_width('A') - 1 - >>> character_width('\N{ZERO WIDTH JOINER}') - 0 - >>> character_width('\n') - -1 - """ - return int(wcwidth(char)) - - -def monospaced_width(text: str) -> int: - r""" - Return the number of character cells that this string is likely to occupy - when displayed in a monospaced, modern, Unicode-aware terminal emulator. - We refer to this as the "display width" of the string. - - This can be useful for formatting text that may contain non-spacing - characters, or CJK characters that take up two character cells. - - Returns -1 if the string contains a non-printable or control character. - - >>> monospaced_width('ちゃぶ台返し') - 12 - >>> len('ちゃぶ台返し') - 6 - >>> monospaced_width('owl\N{SOFT HYPHEN}flavored') - 11 - >>> monospaced_width('example\x80') - -1 - - A more complex example: The Korean word 'ibnida' can be written with 3 - pre-composed characters or 7 jamo. Either way, it *looks* the same and - takes up 6 character cells. - - >>> monospaced_width('입니다') - 6 - >>> monospaced_width('\u110b\u1175\u11b8\u1102\u1175\u1103\u1161') - 6 - - The word "blue" with terminal escapes to make it blue still takes up only - 4 characters, when shown as intended. - >>> monospaced_width('\x1b[34mblue\x1b[m') - 4 - """ - # NFC-normalize the text first, so that we don't need special cases for - # Hangul jamo. - # - # Remove terminal escapes before calculating width, because if they are - # displayed as intended, they will have zero width. - return int(wcswidth(remove_terminal_escapes(normalize("NFC", text)))) - - -def display_ljust(text: str, width: int, fillchar: str = " ") -> str: - """ - Return `text` left-justified in a Unicode string whose display width, - in a monospaced terminal, should be at least `width` character cells. - The rest of the string will be padded with `fillchar`, which must be - a width-1 character. - - "Left" here means toward the beginning of the string, which may actually - appear on the right in an RTL context. This is similar to the use of the - word "left" in "left parenthesis". - - >>> lines = ['Table flip', '(╯°□°)╯︵ ┻━┻', 'ちゃぶ台返し'] - >>> for line in lines: - ... print(display_ljust(line, 20, '▒')) - Table flip▒▒▒▒▒▒▒▒▒▒ - (╯°□°)╯︵ ┻━┻▒▒▒▒▒▒▒ - ちゃぶ台返し▒▒▒▒▒▒▒▒ - - This example, and the similar ones that follow, should come out justified - correctly when viewed in a monospaced terminal. It will probably not look - correct if you're viewing this code or documentation in a Web browser. - """ - if character_width(fillchar) != 1: - raise ValueError("The padding character must have display width 1") - - text_width = monospaced_width(text) - if text_width == -1: - # There's a control character here, so just don't add padding - return text - - padding = max(0, width - text_width) - return text + fillchar * padding - - -def display_rjust(text: str, width: int, fillchar: str = " ") -> str: - """ - Return `text` right-justified in a Unicode string whose display width, - in a monospaced terminal, should be at least `width` character cells. - The rest of the string will be padded with `fillchar`, which must be - a width-1 character. - - "Right" here means toward the end of the string, which may actually be on - the left in an RTL context. This is similar to the use of the word "right" - in "right parenthesis". - - >>> lines = ['Table flip', '(╯°□°)╯︵ ┻━┻', 'ちゃぶ台返し'] - >>> for line in lines: - ... print(display_rjust(line, 20, '▒')) - ▒▒▒▒▒▒▒▒▒▒Table flip - ▒▒▒▒▒▒▒(╯°□°)╯︵ ┻━┻ - ▒▒▒▒▒▒▒▒ちゃぶ台返し - """ - if character_width(fillchar) != 1: - raise ValueError("The padding character must have display width 1") - - text_width = monospaced_width(text) - if text_width == -1: - return text - - padding = max(0, width - text_width) - return fillchar * padding + text - - -def display_center(text: str, width: int, fillchar: str = " ") -> str: - """ - Return `text` centered in a Unicode string whose display width, in a - monospaced terminal, should be at least `width` character cells. The rest - of the string will be padded with `fillchar`, which must be a width-1 - character. - - >>> lines = ['Table flip', '(╯°□°)╯︵ ┻━┻', 'ちゃぶ台返し'] - >>> for line in lines: - ... print(display_center(line, 20, '▒')) - ▒▒▒▒▒Table flip▒▒▒▒▒ - ▒▒▒(╯°□°)╯︵ ┻━┻▒▒▒▒ - ▒▒▒▒ちゃぶ台返し▒▒▒▒ - """ - if character_width(fillchar) != 1: - raise ValueError("The padding character must have display width 1") - - text_width = monospaced_width(text) - if text_width == -1: - return text - - padding = max(0, width - text_width) - left_padding = padding // 2 - right_padding = padding - left_padding - return fillchar * left_padding + text + fillchar * right_padding diff --git a/bundle/python-cpu/Lib/site-packages/ftfy/py.typed b/bundle/python-cpu/Lib/site-packages/ftfy/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/functorch/_C.cp310-win_amd64.pyd b/bundle/python-cpu/Lib/site-packages/functorch/_C.cp310-win_amd64.pyd deleted file mode 100644 index 9b6ce625c33ae8ae29f830e24084336e04b15587..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/_C.cp310-win_amd64.pyd +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a62dd7fecafc54ebca29fdaa55a2456dbff3afbab9ec52f3c05e7b222215494d -size 321536 diff --git a/bundle/python-cpu/Lib/site-packages/functorch/__init__.py b/bundle/python-cpu/Lib/site-packages/functorch/__init__.py deleted file mode 100644 index 9d2790da74333710de2bb6e2b99528fe1b85e2e2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -import torch -from torch._functorch.deprecated import ( - combine_state_for_ensemble, - functionalize, - grad, - grad_and_value, - hessian, - jacfwd, - jacrev, - jvp, - make_functional, - make_functional_with_buffers, - vjp, - vmap, -) - -# utilities. Maybe these should go in their own namespace in the future? -from torch._functorch.make_functional import ( - FunctionalModule, - FunctionalModuleWithBuffers, -) - -# Was never documented -from torch._functorch.python_key import make_fx - - -# Top-level APIs. Please think carefully before adding something to the -# top-level namespace: -# - private helper functions should go into torch._functorch -# - very experimental things should go into functorch.experimental -# - compilation related things should go into functorch.compile - - -__version__ = torch.__version__ diff --git a/bundle/python-cpu/Lib/site-packages/functorch/_src/__init__.py b/bundle/python-cpu/Lib/site-packages/functorch/_src/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/functorch/_src/aot_autograd/__init__.py b/bundle/python-cpu/Lib/site-packages/functorch/_src/aot_autograd/__init__.py deleted file mode 100644 index bef6245f7a0f900c71b5350dcec65c19c260cd78..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/_src/aot_autograd/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# This file has moved to under torch/_functorch. It is not public API. -# If you are not a PyTorch developer and you are relying on the following -# imports, please file an issue. -from torch._functorch.aot_autograd import ( - aot_autograd_decompositions, - KNOWN_TYPES, - PytreeThunk, -) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/_src/eager_transforms/__init__.py b/bundle/python-cpu/Lib/site-packages/functorch/_src/eager_transforms/__init__.py deleted file mode 100644 index 37df7424c4056760388c6b0d17288e4571fde359..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/_src/eager_transforms/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# This file has moved to under torch/_functorch. It is not public API. -# If you are not a PyTorch developer and you are relying on the following -# imports, please file an issue. -from torch._functorch.eager_transforms import ( - _assert_wrapped_functional, - _unwrap_functional_tensor, -) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/_src/make_functional/__init__.py b/bundle/python-cpu/Lib/site-packages/functorch/_src/make_functional/__init__.py deleted file mode 100644 index c96ce28510b5172edbc3941b232c2221e75397de..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/_src/make_functional/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# This file has moved to under torch/_functorch. It is not public API. -# If you are not a PyTorch developer and you are relying on the following -# imports, please file an issue. -from torch._functorch.make_functional import _swap_state diff --git a/bundle/python-cpu/Lib/site-packages/functorch/_src/vmap/__init__.py b/bundle/python-cpu/Lib/site-packages/functorch/_src/vmap/__init__.py deleted file mode 100644 index ec274ba24ad6b3f28798c42bfe92c8b14670a51e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/_src/vmap/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# This file has moved to under torch/_functorch. It is not public API. -# If you are not a PyTorch developer and you are relying on the following -# imports, please file an issue. -from torch._functorch.vmap import ( - _add_batch_dim, - _broadcast_to_and_flatten, - _create_batched_inputs, - _get_name, - _process_batched_inputs, - _remove_batch_dim, - _unwrap_batched, - _validate_and_get_batch_size, - Tensor, - tree_flatten, - tree_unflatten, -) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/compile/__init__.py b/bundle/python-cpu/Lib/site-packages/functorch/compile/__init__.py deleted file mode 100644 index cff66e3aeef73f96273b6c4c77c74e3f1153f4b5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/compile/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -from torch._functorch import config -from torch._functorch.aot_autograd import ( - aot_function, - aot_module, - aot_module_simplified, - compiled_function, - compiled_module, - get_aot_compilation_context, - get_aot_graph_name, - get_graph_being_compiled, - make_boxed_compiler, - make_boxed_func, -) -from torch._functorch.compilers import ( - debug_compile, - default_decompositions, - draw_graph_compile, - memory_efficient_fusion, - nnc_jit, - nop, - print_compile, - ts_compile, -) -from torch._functorch.fx_minifier import minifier -from torch._functorch.partitioners import ( - default_partition, - draw_graph, - min_cut_rematerialization_partition, -) -from torch._functorch.python_key import pythonkey_decompose diff --git a/bundle/python-cpu/Lib/site-packages/functorch/dim/__init__.py b/bundle/python-cpu/Lib/site-packages/functorch/dim/__init__.py deleted file mode 100644 index b47289b1bd6ae3e1d315706632b944fd7c4dcd16..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/dim/__init__.py +++ /dev/null @@ -1,181 +0,0 @@ -import dis -import inspect -from typing import Sequence, Union - -import functorch._C -import torch -from functorch._C import dim as _C - -from .tree_map import tree_flatten, tree_map -from .wrap_type import wrap_type - - -_C._patch_tensor_class() -dims, DimList, dimlists = _C.dims, _C.DimList, _C.dimlists - - -class DimensionMismatchError(Exception): - pass - - -class DimensionBindError(Exception): - pass - - -from . import op_properties - - -# use dict to avoid writing C++ bindings for set -pointwise = dict.fromkeys(op_properties.pointwise, True) - -use_c = True -if not use_c: - from . import reference - - -class _Tensor: - # fast path around slow wrapping/unwrapping logic for simply queries used - # by the implementation... - - @property - def dims(self): - return tuple(d for d in self._levels if isinstance(d, Dim)) - - def dim(self): - return self.ndim - - if use_c: - __torch_function__ = classmethod(_C.__torch_function__) - expand = _C._instancemethod(_C.expand) - else: - __torch_function__ = reference.__torch_function__ - expand = reference.expand - - index = _C._instancemethod(_C.index) - - def __repr__(self): - tensor, levels, ndim = self._tensor, self._levels, self.ndim - return f"{tensor}\nwith dims={tuple(l + ndim if isinstance(l, int) else l for l in levels)} sizes={tuple(tensor.size())}" - - -TensorLike = (_Tensor, torch.Tensor) - - -class Dim(_C.Dim, _Tensor): - # note that _C.Dim comes before tensor because we want the Dim API for things like size to take precendence. - # Tensor defines format, but we want to print Dims with special formatting - __format__ = object.__format__ - - -class Tensor(_Tensor, _C.Tensor): - if not use_c: - from_batched = staticmethod(_C.Tensor_from_batched) - from_positional = staticmethod(_C.Tensor_from_positional) - sum = _C._instancemethod(_C.Tensor_sum) - - -def cat(tensors, dim, new_dim): - n = dims() - return stack(tensors, n, dim).index([n, dim], new_dim) - - -if use_c: - _wrap = _C._wrap - - def _def(name, *args, **kwargs): - orig = getattr(torch.Tensor, name) - setattr(_Tensor, name, _C._instancemethod(_wrap(orig, *args, **kwargs))) - - t__getitem__ = _C._instancemethod(_C.__getitem__) - stack = _C.stack - split = _C._instancemethod(_C.split) -else: - _wrap, _def = reference._wrap, reference._def - t__getitem__ = reference.t__getitem__ - stack = reference.stack - split = reference.split - -# note: there is no python reference -t__setitem__ = _C._instancemethod(_C.__setitem__) -# this is patched in the C API because otherwise torch.Tensor will -# no longer be considered a sequence and things will break -# torch.Tensor.__getitem__ = t__getitem__ - -_Tensor.__getitem__ = t__getitem__ -# torch.Tensor.__setitem__ = t__setitem__ -_Tensor.__setitem__ = t__setitem__ - -torch.Tensor.split = split -_Tensor.split = split -torch.Tensor.expand = _C._instancemethod(_C.expand) -torch.Tensor.index = _C._instancemethod(_C.index) -wrap_type(use_c, _Tensor, torch.Tensor, _Tensor.__torch_function__) -del _Tensor.ndim - -if use_c: - _Tensor.order = _C._instancemethod(_C.order) -else: - _Tensor.order = reference.positional - -_def("mean") -_def("sum") -_def("all") -_def("amax") -_def("amin") -_def("aminmax") -_def("any") -_def("count_nonzero") -_def("logsumexp") -_def("nanmean") -_def("nansum") -_def("prod") -_def("std", keepdim_offset=2) -_def("var", keepdim_offset=2) -_def("max", single_dim=True) -_def("min", single_dim=True) -_def("argmax", single_dim=True) -_def("argmin", single_dim=True) -_def("kthvalue", single_dim=True) -_def("median", single_dim=True) -_def("nanmedian", single_dim=True) -_def("mode", single_dim=True) -_def("sort", reduce=False) -_def("argsort", reduce=False) -_def("unbind", single_dim=True) -_def("chunk", dim_offset=1, reduce=False) -_def("cummax", single_dim=True, reduce=False) -_def("cummin", single_dim=True, reduce=False) -_def("cumprod", single_dim=True, reduce=False) -_def("cumprod_", single_dim=True, reduce=False) -_def("cumsum", single_dim=True, reduce=False) -_def("cumsum_", single_dim=True, reduce=False) -_def("logcumsumexp", single_dim=True, reduce=False) -_def("renorm", dim_offset=1, single_dim=True, reduce=False) -_def("softmax", single_dim=True, reduce=False) -softmax = _wrap(torch.nn.functional.softmax, single_dim=True, reduce=False) - -# stuff to handle in the future, because they require special -# binding logic for dims -# cross -# diag_embed -# diagonal -# diagonal_scatter -# diff -# nanquantile -# quantile -# roll -# rot90 -# topk (new dimes on output) -# should these all be subsumed by inplace indexing? -# index_add_ -# index_add -# index_copy -# index_copy_ -# index_fill -# index_fill_ -# index_select -# scatter -# scatter_ -# scatter_add -# scatter_add_ -# scatter_reduce diff --git a/bundle/python-cpu/Lib/site-packages/functorch/dim/batch_tensor.py b/bundle/python-cpu/Lib/site-packages/functorch/dim/batch_tensor.py deleted file mode 100644 index 234e731e825bd0a55a80a1a55d90ea43b0586080..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/dim/batch_tensor.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -from contextlib import contextmanager - -from torch._C._functorch import _vmap_add_layers, _vmap_remove_layers - - -_enabled = False - - -@contextmanager -def _enable_layers(dims): - global _enabled - assert not _enabled - input = sorted((d._level, d.size) for d in dims if not isinstance(d, int)) - n = len(input) - try: - _vmap_add_layers(input) - _enabled = True - yield - finally: - _enabled = False - _vmap_remove_layers(n) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/dim/delayed_mul_tensor.py b/bundle/python-cpu/Lib/site-packages/functorch/dim/delayed_mul_tensor.py deleted file mode 100644 index 397b1b56796f7be73987cfa4f7eb58fb58ea0a40..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/dim/delayed_mul_tensor.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -import torch - -from . import _Tensor, Tensor -from .reference import _dims, _enable_layers, llist, ltuple - - -class DelayedMulTensor(_Tensor): - def __init__(self, lhs, rhs): - self._lhs, self._rhs = lhs, rhs - self._data = None - self._levels_data = None - self._has_device = lhs._has_device or rhs._has_device - self._batchtensor_data = None - self._tensor_data = None - - @property - def _levels(self): - if self._levels_data is None: - levels = llist(self._lhs._levels) - for l in self._rhs._levels: - if l not in levels: - levels.append(l) - self._levels_data = ltuple(levels) - return self._levels_data - - @property - def _batchtensor(self): - if self._batchtensor_data is None: - with _enable_layers(self._levels): - print("bt multiply fallback") - self._batchtensor_data = self._lhs._batchtensor * self._rhs._batchtensor - return self._batchtensor_data - - @property - def _tensor(self): - if self._tensor_data is None: - self._tensor_data = Tensor.from_batched( - self._batchtensor, self._has_device - )._tensor - return self._tensor_data - - @property - def ndim(self): - return self._batchtensor.ndim - - @property - def dims(self): - return ltuple(super().dims) - - def sum(self, dim): - dims = _dims(dim, 0, False, False) - n = ord("a") - all_levels = self._levels - - def to_char(d): - return chr(n + all_levels.index(d)) - - plhs, levelslhs = self._lhs._tensor, self._lhs._levels - prhs, levelsrhs = self._rhs._tensor, self._rhs._levels - new_dims = tuple(d for d in self.dims if d not in dims) - new_levels = [l for l in self._levels if l not in dims] - fmt = "".join( - [ - *(to_char(d) for d in levelslhs), - ",", - *(to_char(d) for d in levelsrhs), - "->", - *(to_char(d) for d in new_levels), - ] - ) - result_data = torch.einsum(fmt, (plhs, prhs)) - return Tensor.from_positional(result_data, new_levels, True) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/dim/dim.py b/bundle/python-cpu/Lib/site-packages/functorch/dim/dim.py deleted file mode 100644 index a18c8600c0575f13f8c24d80c653105b7a98173f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/dim/dim.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -import dis -import inspect -from dataclasses import dataclass -from typing import Union - -from . import DimList - - -_vmap_levels = [] - - -@dataclass -class LevelInfo: - level: int - alive: bool = True - - -class Dim: - def __init__(self, name: str, size: Union[None, int] = None): - self.name = name - self._size = None - self._vmap_level = None - if size is not None: - self.size = size - - def __del__(self): - if self._vmap_level is not None: - _vmap_active_levels[self._vmap_stack].alive = False # noqa: F821 - while ( - not _vmap_levels[-1].alive - and current_level() == _vmap_levels[-1].level # noqa: F821 - ): - _vmap_decrement_nesting() # noqa: F821 - _vmap_levels.pop() - - @property - def size(self): - assert self.is_bound - return self._size - - @size.setter - def size(self, size: int): - from . import DimensionBindError - - if self._size is None: - self._size = size - self._vmap_level = _vmap_increment_nesting(size, "same") # noqa: F821 - self._vmap_stack = len(_vmap_levels) - _vmap_levels.append(LevelInfo(self._vmap_level)) - - elif self._size != size: - raise DimensionBindError( - f"Dim '{self}' previously bound to a dimension of size {self._size} cannot bind to a dimension of size {size}" - ) - - @property - def is_bound(self): - return self._size is not None - - def __repr__(self): - return self.name - - -def extract_name(inst): - assert inst.opname == "STORE_FAST" or inst.opname == "STORE_NAME" - return inst.argval - - -_cache = {} - - -def dims(lists=0): - frame = inspect.currentframe() - assert frame is not None - calling_frame = frame.f_back - assert calling_frame is not None - code, lasti = calling_frame.f_code, calling_frame.f_lasti - key = (code, lasti) - if key not in _cache: - first = lasti // 2 + 1 - instructions = list(dis.get_instructions(calling_frame.f_code)) - unpack = instructions[first] - - if unpack.opname == "STORE_FAST" or unpack.opname == "STORE_NAME": - # just a single dim, not a list - name = unpack.argval - ctor = Dim if lists == 0 else DimList - _cache[key] = lambda: ctor(name=name) - else: - assert unpack.opname == "UNPACK_SEQUENCE" - ndims = unpack.argval - names = tuple( - extract_name(instructions[first + 1 + i]) for i in range(ndims) - ) - first_list = len(names) - lists - _cache[key] = lambda: tuple( - Dim(n) if i < first_list else DimList(name=n) - for i, n in enumerate(names) - ) - return _cache[key]() - - -def _dim_set(positional, arg): - def convert(a): - if isinstance(a, Dim): - return a - else: - assert isinstance(a, int) - return positional[a] - - if arg is None: - return positional - elif not isinstance(arg, (Dim, int)): - return tuple(convert(a) for a in arg) - else: - return (convert(arg),) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/dim/magic_trace.py b/bundle/python-cpu/Lib/site-packages/functorch/dim/magic_trace.py deleted file mode 100644 index 8ad92632a2ef08b300493d9e63c8a7d3ab8e9749..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/dim/magic_trace.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -import os -import signal -import subprocess -from contextlib import contextmanager - - -@contextmanager -def magic_trace(output="trace.fxt", magic_trace_cache="/tmp/magic-trace"): - pid = os.getpid() - if not os.path.exists(magic_trace_cache): - print(f"Downloading magic_trace to: {magic_trace_cache}") - subprocess.run( - [ - "wget", - "-O", - magic_trace_cache, - "-q", - "https://github.com/janestreet/magic-trace/releases/download/v1.0.2/magic-trace", - ] - ) - subprocess.run(["chmod", "+x", magic_trace_cache]) - args = [magic_trace_cache, "attach", "-pid", str(pid), "-o", output] - p = subprocess.Popen(args, stderr=subprocess.PIPE, encoding="utf-8") - while True: - x = p.stderr.readline() - print(x) - if "Attached" in x: - break - try: - yield - finally: - p.send_signal(signal.SIGINT) - r = p.wait() - print(p.stderr.read()) - p.stderr.close() - if r != 0: - raise ValueError(f"magic_trace exited abnormally: {r}") diff --git a/bundle/python-cpu/Lib/site-packages/functorch/dim/op_properties.py b/bundle/python-cpu/Lib/site-packages/functorch/dim/op_properties.py deleted file mode 100644 index 3fda08afb60f75d635b5590735e7320700ed8594..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/dim/op_properties.py +++ /dev/null @@ -1,312 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -import torch - - -# pointwise operators can go through a faster pathway - -tensor_magic_methods = ["add", ""] -pointwise_magic_methods_with_reverse = ( - "add", - "sub", - "mul", - "floordiv", - "div", - "truediv", - "mod", - "pow", - "lshift", - "rshift", - "and", - "or", - "xor", -) -pointwise_magic_methods = ( - *(x for m in pointwise_magic_methods_with_reverse for x in (m, "r" + m)), - "eq", - "gt", - "le", - "lt", - "ge", - "gt", - "ne", - "neg", - "pos", - "abs", - "invert", - "iadd", - "isub", - "imul", - "ifloordiv", - "idiv", - "itruediv", - "imod", - "ipow", - "ilshift", - "irshift", - "iand", - "ior", - "ixor", - "int", - "long", - "float", - "complex", -) - -pointwise_methods = (*(f"__{m}__" for m in pointwise_magic_methods),) - -pointwise = ( - *(getattr(torch.Tensor, m) for m in pointwise_methods), - torch.nn.functional.dropout, - torch.where, - torch.Tensor.abs, - torch.abs, - torch.Tensor.acos, - torch.acos, - torch.Tensor.acosh, - torch.acosh, - torch.Tensor.add, - torch.add, - torch.Tensor.addcdiv, - torch.addcdiv, - torch.Tensor.addcmul, - torch.addcmul, - torch.Tensor.addr, - torch.addr, - torch.Tensor.angle, - torch.angle, - torch.Tensor.asin, - torch.asin, - torch.Tensor.asinh, - torch.asinh, - torch.Tensor.atan, - torch.atan, - torch.Tensor.atan2, - torch.atan2, - torch.Tensor.atanh, - torch.atanh, - torch.Tensor.bitwise_and, - torch.bitwise_and, - torch.Tensor.bitwise_left_shift, - torch.bitwise_left_shift, - torch.Tensor.bitwise_not, - torch.bitwise_not, - torch.Tensor.bitwise_or, - torch.bitwise_or, - torch.Tensor.bitwise_right_shift, - torch.bitwise_right_shift, - torch.Tensor.bitwise_xor, - torch.bitwise_xor, - torch.Tensor.ceil, - torch.ceil, - torch.celu, - torch.nn.functional.celu, - torch.Tensor.clamp, - torch.clamp, - torch.Tensor.clamp_max, - torch.clamp_max, - torch.Tensor.clamp_min, - torch.clamp_min, - torch.Tensor.copysign, - torch.copysign, - torch.Tensor.cos, - torch.cos, - torch.Tensor.cosh, - torch.cosh, - torch.Tensor.deg2rad, - torch.deg2rad, - torch.Tensor.digamma, - torch.digamma, - torch.Tensor.div, - torch.div, - torch.dropout, - torch.nn.functional.dropout, - torch.nn.functional.elu, - torch.Tensor.eq, - torch.eq, - torch.Tensor.erf, - torch.erf, - torch.Tensor.erfc, - torch.erfc, - torch.Tensor.erfinv, - torch.erfinv, - torch.Tensor.exp, - torch.exp, - torch.Tensor.exp2, - torch.exp2, - torch.Tensor.expm1, - torch.expm1, - torch.feature_dropout, - torch.Tensor.float_power, - torch.float_power, - torch.Tensor.floor, - torch.floor, - torch.Tensor.floor_divide, - torch.floor_divide, - torch.Tensor.fmod, - torch.fmod, - torch.Tensor.frac, - torch.frac, - torch.Tensor.frexp, - torch.frexp, - torch.Tensor.gcd, - torch.gcd, - torch.Tensor.ge, - torch.ge, - torch.nn.functional.gelu, - torch.nn.functional.glu, - torch.Tensor.gt, - torch.gt, - torch.Tensor.hardshrink, - torch.hardshrink, - torch.nn.functional.hardshrink, - torch.nn.functional.hardsigmoid, - torch.nn.functional.hardswish, - torch.nn.functional.hardtanh, - torch.Tensor.heaviside, - torch.heaviside, - torch.Tensor.hypot, - torch.hypot, - torch.Tensor.i0, - torch.i0, - torch.Tensor.igamma, - torch.igamma, - torch.Tensor.igammac, - torch.igammac, - torch.Tensor.isclose, - torch.isclose, - torch.Tensor.isfinite, - torch.isfinite, - torch.Tensor.isinf, - torch.isinf, - torch.Tensor.isnan, - torch.isnan, - torch.Tensor.isneginf, - torch.isneginf, - torch.Tensor.isposinf, - torch.isposinf, - torch.Tensor.isreal, - torch.isreal, - torch.Tensor.kron, - torch.kron, - torch.Tensor.lcm, - torch.lcm, - torch.Tensor.ldexp, - torch.ldexp, - torch.Tensor.le, - torch.le, - torch.nn.functional.leaky_relu, - torch.Tensor.lerp, - torch.lerp, - torch.Tensor.lgamma, - torch.lgamma, - torch.Tensor.log, - torch.log, - torch.Tensor.log10, - torch.log10, - torch.Tensor.log1p, - torch.log1p, - torch.Tensor.log2, - torch.log2, - torch.nn.functional.logsigmoid, - torch.Tensor.logical_and, - torch.logical_and, - torch.Tensor.logical_not, - torch.logical_not, - torch.Tensor.logical_or, - torch.logical_or, - torch.Tensor.logical_xor, - torch.logical_xor, - torch.Tensor.logit, - torch.logit, - torch.Tensor.lt, - torch.lt, - torch.Tensor.maximum, - torch.maximum, - torch.Tensor.minimum, - torch.minimum, - torch.nn.functional.mish, - torch.Tensor.mvlgamma, - torch.mvlgamma, - torch.Tensor.nan_to_num, - torch.nan_to_num, - torch.Tensor.ne, - torch.ne, - torch.Tensor.neg, - torch.neg, - torch.Tensor.nextafter, - torch.nextafter, - torch.Tensor.outer, - torch.outer, - torch.polar, - torch.Tensor.polygamma, - torch.polygamma, - torch.Tensor.positive, - torch.positive, - torch.Tensor.pow, - torch.pow, - torch.Tensor.prelu, - torch.prelu, - torch.nn.functional.prelu, - torch.Tensor.rad2deg, - torch.rad2deg, - torch.Tensor.reciprocal, - torch.reciprocal, - torch.Tensor.relu, - torch.relu, - torch.nn.functional.relu, - torch.nn.functional.relu6, - torch.Tensor.remainder, - torch.remainder, - torch.Tensor.round, - torch.round, - torch.rrelu, - torch.nn.functional.rrelu, - torch.Tensor.rsqrt, - torch.rsqrt, - torch.rsub, - torch.selu, - torch.nn.functional.selu, - torch.Tensor.sgn, - torch.sgn, - torch.Tensor.sigmoid, - torch.sigmoid, - torch.nn.functional.sigmoid, - torch.Tensor.sign, - torch.sign, - torch.Tensor.signbit, - torch.signbit, - torch.nn.functional.silu, - torch.Tensor.sin, - torch.sin, - torch.Tensor.sinc, - torch.sinc, - torch.Tensor.sinh, - torch.sinh, - torch.nn.functional.softplus, - torch.nn.functional.softshrink, - torch.Tensor.sqrt, - torch.sqrt, - torch.Tensor.square, - torch.square, - torch.Tensor.sub, - torch.sub, - torch.Tensor.tan, - torch.tan, - torch.Tensor.tanh, - torch.tanh, - torch.nn.functional.tanh, - torch.threshold, - torch.nn.functional.threshold, - torch.trapz, - torch.Tensor.true_divide, - torch.true_divide, - torch.Tensor.trunc, - torch.trunc, - torch.Tensor.xlogy, - torch.xlogy, - torch.rand_like, -) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/dim/reference.py b/bundle/python-cpu/Lib/site-packages/functorch/dim/reference.py deleted file mode 100644 index b000f908e3bfe4c42c9ccf8696aeaee540e4abb5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/dim/reference.py +++ /dev/null @@ -1,645 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -# reference python implementations for C ops -import torch -from functorch._C import dim as _C - -from . import op_properties -from .batch_tensor import _enable_layers -from .tree_map import tree_flatten, tree_map - - -DimList = _C.DimList -import operator -from functools import reduce - - -# use dict to avoid writing C++ bindings for set -pointwise = set(op_properties.pointwise) - - -def prod(x): - return reduce(operator.mul, x, 1) - - -def _wrap_dim(d, N, keepdim): - from . import Dim - - if isinstance(d, Dim): - assert not keepdim, "cannot preserve first-class dimensions with keepdim=True" - return d - elif d >= 0: - return d - N - else: - return d - - -def _dims(d, N, keepdim, single_dim): - from . import Dim - - if isinstance(d, (Dim, int)): - return ltuple((_wrap_dim(d, N, keepdim),)) - assert not single_dim, f"expected a single dimension or int but found: {d}" - return ltuple(_wrap_dim(x, N, keepdim) for x in d) - - -def _bind_dims_to_size(lhs_size, rhs, lhs_debug): - from . import DimensionMismatchError - - not_bound = tuple((i, r) for i, r in enumerate(rhs) if not r.is_bound) - if len(not_bound) == 1: - idx, d = not_bound[0] - rhs_so_far = prod(r.size for r in rhs if r.is_bound) - if lhs_size % rhs_so_far != 0: - rhs_s = tuple("?" if not r.is_bound else str(r.size) for r in rhs) - raise DimensionMismatchError( - f"inferred dimension does not evenly fit into larger dimension: {lhs_size} vs {rhs_s}" - ) - new_size = lhs_size // rhs_so_far - d.size = new_size - elif len(not_bound) > 1: - rhs_s = tuple("?" if not r.is_bound else str(r.size) for r in rhs) - raise DimensionMismatchError( - f"cannot infer the size of two dimensions at once: {rhs} with sizes {rhs_s}" - ) - else: - rhs_size = prod(r.size for r in rhs) - if lhs_size != rhs_size: - raise DimensionMismatchError( - f"Dimension sizes to do not match ({lhs_size} != {rhs_size}) when matching {lhs_debug} to {rhs}" - ) - - -def _tensor_levels(inp): - from . import _Tensor - - if isinstance(inp, _Tensor): - return inp._tensor, llist(inp._levels), inp._has_device - else: - return inp, llist(range(-inp.ndim, 0)), True - - -def _match_levels(v, from_levels, to_levels): - view = [] - permute = [] - requires_view = False - size = v.size() - for t in to_levels: - try: - idx = from_levels.index(t) - permute.append(idx) - view.append(size[idx]) - except ValueError: - view.append(1) - requires_view = True - if permute != list(range(len(permute))): - v = v.permute(*permute) - if requires_view: - v = v.view(*view) - return v - - -# make a single dimension positional but do not permute it, -# used to do multi-tensor operators where the dim being acted on -# should not physically move if possible -def _positional_no_permute(self, dim, expand_dim=False): - from . import Tensor - - ptensor, levels = self._tensor, llist(self._levels) - try: - idx = levels.index(dim) - except ValueError: - if not expand_dim: - raise - idx = 0 - ptensor = ptensor.expand(dim.size, *ptensor.size()) - levels.insert(0, 0) - idx_batched = 0 - for i in range(idx): - if isinstance(levels[i], int): - levels[i] -= 1 - idx_batched += 1 - levels[idx] = -idx_batched - 1 - return Tensor.from_positional(ptensor, levels, self._has_device), idx_batched - - -def seq(a, b): - from . import Dim - - if isinstance(a, Dim) != isinstance(b, Dim): - return False - if isinstance(a, Dim): - return a is b - else: - return a == b - - -class isin: - def __contains__(self, item): - for x in self: - if seq(item, x): - return True - return False - - def index(self, item): - for i, x in enumerate(self): - if seq(item, x): - return i - raise ValueError - - -class llist(isin, list): - pass - - -class ltuple(isin, tuple): - pass - - -empty_dict = {} - - -@classmethod -def __torch_function__(self, orig, cls, args, kwargs=empty_dict): - from . import _Tensor, Tensor, TensorLike - from .delayed_mul_tensor import DelayedMulTensor - - if orig is torch.Tensor.__mul__: - lhs, rhs = args - if ( - isinstance(lhs, _Tensor) - and isinstance(rhs, _Tensor) - and lhs.ndim == 0 - and rhs.ndim == 0 - ): - return DelayedMulTensor(lhs, rhs) - all_dims = llist() - flat_args, unflatten = tree_flatten((args, kwargs)) - device_holding_tensor = None - for f in flat_args: - if isinstance(f, _Tensor): - if f._has_device: - device_holding_tensor = f._batchtensor - for d in f.dims: - if d not in all_dims: - all_dims.append(d) - - def unwrap(t): - if isinstance(t, _Tensor): - r = t._batchtensor - if device_holding_tensor is not None and not t._has_device: - r = r.to(device=device_holding_tensor.device) - return r - return t - - if orig in pointwise: - result_levels = llist() - arg_levels = llist() - to_expand = [] - for i, f in enumerate(flat_args): - if isinstance(f, TensorLike): - ptensor, levels, _ = _tensor_levels(f) - if ( - isinstance(f, _Tensor) - and not f._has_device - and device_holding_tensor is not None - ): - ptensor = ptensor.to(device=device_holding_tensor.device) - flat_args[i] = ptensor - for l in levels: - if l not in result_levels: - result_levels.append(l) - to_expand.append((i, levels)) - - for i, levels in to_expand: - flat_args[i] = _match_levels(flat_args[i], levels, result_levels) - args, kwargs = unflatten(flat_args) - result = orig(*args, **kwargs) - - def wrap(t): - if isinstance(t, TensorLike): - return Tensor.from_positional( - t, result_levels, device_holding_tensor is not None - ) - return t - - return tree_map(wrap, result) - else: - - def wrap(t): - if isinstance(t, TensorLike): - return Tensor.from_batched(t, device_holding_tensor is not None) - return t - - with _enable_layers(all_dims): - print(f"batch_tensor for {orig}") - args, kwargs = unflatten(unwrap(f) for f in flat_args) - result = orig(*args, **kwargs) - # print("END", orig) - return tree_map(wrap, result) - - -def positional(self, *dims): - from . import Dim, DimensionBindError, Tensor - - ptensor, levels = self._tensor, llist(self._levels) - flat_dims = llist() - view = [] - needs_view = False - ndim = self.ndim - for d in dims: - if isinstance(d, DimList): - flat_dims.extend(d) - view.extend(e.size for e in d) - elif isinstance(d, Dim): - flat_dims.append(d) - view.append(d.size) - elif isinstance(d, int): - d = _wrap_dim(d, ndim, False) - flat_dims.append(d) - view.append(ptensor.size(d)) - else: - flat_dims.extend(d) - view.append(prod(e.size for e in d)) - needs_view = True - - permute = list(range(len(levels))) - nflat = len(flat_dims) - for i, d in enumerate(flat_dims): - try: - idx = levels.index(d) - except ValueError as e: - raise DimensionBindError( - f"tensor of dimensions {self.dims} does not contain dim {d}" - ) from e - p = permute[idx] - del levels[idx] - del permute[idx] - levels.insert(i, 0) - permute.insert(i, p) - ptensor = ptensor.permute(*permute) - seen = 0 - for i in range(len(levels) - 1, -1, -1): - if isinstance(levels[i], int): - seen += 1 - levels[i] = -seen - result = Tensor.from_positional(ptensor, levels, self._has_device) - if needs_view: - result = result.reshape(*view, *result.size()[len(flat_dims) :]) - return result - - -def _contains_dim(input): - from . import Dim - - for i in input: - if isinstance(i, Dim): - return True - - -def expand(self, *sizes): - if not _contains_dim(sizes): - return self.__torch_function__(torch.Tensor.expand, None, (self, *sizes)) - dims = sizes - sizes = [d.size for d in dims] + [-1] * self.ndim - self = self.expand(*sizes) - return self[dims] - - -_not_present = object() - - -def _getarg(name, offset, args, kwargs, default): - if len(args) > offset: - return args[offset] - return kwargs.get(name, default) - - -def _patcharg(name, offset, args, kwargs, value): - if len(args) > offset: - args[offset] = value - else: - kwargs[name] = value - - -def _wrap( - orig, dim_offset=0, keepdim_offset=1, dim_name="dim", single_dim=False, reduce=True -): - from . import Dim, Tensor, TensorLike - - def fn(self, *args, **kwargs): - dim = _getarg(dim_name, dim_offset, args, kwargs, _not_present) - if dim is _not_present or (single_dim and not isinstance(dim, Dim)): - with _enable_layers(self.dims): - print(f"dim fallback batch_tensor for {orig}") - return Tensor.from_batched( - orig(self._batchtensor, *args, **kwargs), self._has_device - ) - keepdim = ( - _getarg("keepdim", keepdim_offset, args, kwargs, False) if reduce else False - ) - t, levels = self._tensor, llist(self._levels) - dims = _dims(dim, self._batchtensor.ndim, keepdim, single_dim) - dim_indices = tuple(levels.index(d) for d in dims) - if reduce and not keepdim: - new_levels = [l for i, l in enumerate(levels) if i not in dim_indices] - else: - new_levels = levels - - if len(dim_indices) == 1: - dim_indices = dim_indices[ - 0 - ] # so that dims that really only take a single argument work... - args = list(args) - _patcharg(dim_name, dim_offset, args, kwargs, dim_indices) - - def wrap(t): - if isinstance(t, TensorLike): - return Tensor.from_positional(t, new_levels, self._has_device) - return t - - with _enable_layers(new_levels): - print(f"dim used batch_tensor for {orig}") - r = orig(t, *args, **kwargs) - return tree_map(wrap, r) - - return fn - - -def _def(name, *args, **kwargs): - from . import _Tensor - - orig = getattr(torch.Tensor, name) - setattr(_Tensor, name, _wrap(orig, *args, **kwargs)) - - -no_slice = slice(None) - -_orig_getitem = torch.Tensor.__getitem__ - - -class dim_tracker: - def __init__(self) -> None: - self.dims = llist() - self.count = [] - - def record(self, d): - if d not in self.dims: - self.dims.append(d) - self.count.append(1) - - def __getitem__(self, d): - return self.count[self.dims.index(d)] - - -def t__getitem__(self, input): - from . import _Tensor, Dim, DimensionBindError, DimList, Tensor, TensorLike - - # * bail to original example if we have a single non-Dim tensor, or a non-tensor - # * locate ... or an unbound tensor list, and determine its size, bind dim list - # (remember that None does not count to the total dim count) - # * bind simple dims and dim-packs to their sizes, count the number of uses of each dim, - # produce the re-view if needed - # * for each single-use dim index, replace with no_slice and mark that it will be added - # (keep track of whether we have to call super) - # * call super if needed - # * if we have dims to bind, bind them (it will help if we eliminated ... and None before) - # this handles bool indexing handling, as well as some other simple cases. - - is_simple = ( - not isinstance(input, Dim) - and not isinstance(input, (tuple, list)) - and - # WAR for functorch bug where zero time tensors in getitem are not handled correctly. - not (isinstance(input, TensorLike) and input.ndim == 0) - ) - - if is_simple: - if isinstance(self, _Tensor): - return _Tensor.__torch_function__(_orig_getitem, None, (self, input)) - else: - return _orig_getitem(self, input) - - # can further optimize this case - if not isinstance(input, tuple): - input = [input] - else: - input = list(input) - - dims_indexed = 0 - expanding_object = None - dimlists = [] - for i, s in enumerate(input): - if s is ... or isinstance(s, DimList) and not s.is_bound: - if expanding_object is not None: - msg = ( - "at most one ... or unbound dimension list can exist in indexing list but" - f" found 2 at offsets {i} and {expanding_object}" - ) - raise DimensionBindError(msg) - expanding_object = i - - if isinstance(s, DimList): - dims_indexed += len(s) if s.is_bound else 0 - dimlists.append(i) - elif s is not None and s is not ...: - dims_indexed += 1 - - ndim = self.ndim - if dims_indexed > ndim: - raise IndexError( - f"at least {dims_indexed} indices were supplied but the tensor only has {ndim} dimensions." - ) - if expanding_object is not None: - expanding_ndims = ndim - dims_indexed - obj = input[expanding_object] - if obj is ...: - input[expanding_object : expanding_object + 1] = [ - no_slice - ] * expanding_ndims - else: - obj.bind_len(expanding_ndims) - # flatten the dimslists into the indexing - for i in reversed(dimlists): - input[i : i + 1] = input[i] - dims_indexed = 0 - requires_view = False - size = self.size() - view_sizes = [] - dims_seen = dim_tracker() - - def add_dims(t): - if not isinstance(t, _Tensor): - return - for d in t.dims: - dims_seen.record(d) - - add_dims(self) - dim_packs = [] - for i, idx in enumerate(input): - if idx is None: - input[i] = no_slice - view_sizes.append(1) - requires_view = True - else: - sz = size[dims_indexed] - if isinstance(idx, Dim): - idx.size = sz - dims_seen.record(idx) - view_sizes.append(sz) - elif isinstance(idx, (tuple, list)) and idx and isinstance(idx[0], Dim): - for d in idx: - dims_seen.record(idx) - _bind_dims_to_size(sz, idx, f"offset {i}") - view_sizes.extend(d.size for d in idx) - requires_view = True - dim_packs.append(i) - else: - add_dims(idx) - view_sizes.append(sz) - dims_indexed += 1 - if requires_view: - self = self.view(*view_sizes) - for i in reversed(dim_packs): - input[i : i + 1] = input[i] - - # currenty: - # input is flat, containing either Dim, or Tensor, or something valid for standard indexing - # self may have first-class dims as well. - - # to index: - # drop the first class dims from self, they just become direct indices of their positions - - # figure out the dimensions of the indexing tensors: union of all the dims in the tensors in the index. - # these dimensions will appear and need to be bound at the first place tensor occures - - if isinstance(self, _Tensor): - ptensor_self, levels = self._tensor, list(self._levels) - # indices to ptensor rather than self which has first-class dimensions - input_it = iter(input) - flat_inputs = [next(input_it) if isinstance(l, int) else l for l in levels] - has_device = self._has_device - to_pad = 0 - else: - ptensor_self, flat_inputs = self, input - to_pad = ptensor_self.ndim - len(flat_inputs) - has_device = True - - result_levels = [] - index_levels = [] - tensor_insert_point = None - to_expand = {} - requires_getindex = False - for i, inp in enumerate(flat_inputs): - if isinstance(inp, Dim) and dims_seen[inp] == 1: - flat_inputs[i] = no_slice - result_levels.append(inp) - elif isinstance(inp, TensorLike): - requires_getindex = True - if tensor_insert_point is None: - tensor_insert_point = len(result_levels) - ptensor, levels, _ = _tensor_levels(inp) - to_expand[i] = levels - flat_inputs[i] = ptensor - for l in levels: - if l not in index_levels: - index_levels.append(l) - else: - requires_getindex = True - result_levels.append(0) - - if tensor_insert_point is not None: - result_levels[tensor_insert_point:tensor_insert_point] = index_levels - - for i, levels in to_expand.items(): - flat_inputs[i] = _match_levels(flat_inputs[i], levels, index_levels) - - if requires_getindex: - result = _orig_getitem(ptensor_self, flat_inputs) - else: - result = ptensor_self - - next_positional = -1 - if to_pad > 0: - result_levels.extend([0] * to_pad) - for i, r in enumerate(reversed(result_levels)): - if isinstance(r, int): - result_levels[-1 - i] = next_positional - next_positional -= 1 - - return Tensor.from_positional(result, result_levels, has_device) - - -# XXX - dim is optional and can be the outer-most dimension... -def stack(tensors, new_dim, dim=0, out=None): - if isinstance(dim, int): - return torch.stack(tensors, dim, out).index(dim, new_dim) - index = None - if out is not None: - out, index = _positional_no_permute(out, dim, expand_dim=True) - ptensors = [] - for t in tensors: - pt, pi = _positional_no_permute(t, dim, expand_dim=True) - if index is not None and pi != index: - pt = pt.move_dim(pi, index) - else: - index = pi - ptensors.append(pt) - pr = torch.stack(ptensors, index, out=out) - return pr.index((index, index + 1), (new_dim, dim)) - - -_orig_split = torch.Tensor.split - - -def split(self, split_size_or_sections, dim=0): - from . import _Tensor, Dim - - if isinstance(split_size_or_sections, int) or any( - isinstance(t, int) for t in split_size_or_sections - ): - if isinstance(dim, Dim): - raise ValueError( - "when dim is specified as a Dim object, split sizes must also be dimensions." - ) - return _orig_split(self, split_size_or_sections, dim=dim) - - if isinstance(dim, Dim): - assert isinstance(self, _Tensor), f"Tensor does not have dimension {dim}" - self, dim = _positional_no_permute(self, dim) - - size = self.size(dim) - total_bound_size = 0 - unbound = [] - sizes = [] - for i, d in enumerate(split_size_or_sections): - if d.is_bound: - sizes.append(d.size) - total_bound_size += d.size - else: - sizes.append(0) - unbound.append(i) - - if unbound: - assert ( - total_bound_size <= size - ), f"result dimensions are larger than original: {total_bound_size} vs {size} ({split_size_or_sections})" - remaining_size = size - total_bound_size - chunk_size = -(-remaining_size // len(unbound)) - for u in unbound: - sz = min(chunk_size, remaining_size) - split_size_or_sections[u].size = sz - sizes[u] = sz - remaining_size -= sz - else: - assert ( - total_bound_size == size - ), f"result dimensions do not match original: {total_bound_size} vs {size} ({split_size_or_sections})" - return tuple( - t.index(dim, d) - for d, t in zip(split_size_or_sections, _orig_split(self, sizes, dim=dim)) - ) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/dim/tree_map.py b/bundle/python-cpu/Lib/site-packages/functorch/dim/tree_map.py deleted file mode 100644 index 014a44b95d1890bc8456bc64be6f093e2b433cb6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/dim/tree_map.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -from functorch._C import dim - - -tree_flatten = dim.tree_flatten - - -def tree_map(fn, tree): - vs, unflatten = tree_flatten(tree) - return unflatten(fn(v) for v in vs) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/dim/wrap_type.py b/bundle/python-cpu/Lib/site-packages/functorch/dim/wrap_type.py deleted file mode 100644 index 797ce6c34c5cbbe519e07b3c68bceb9350b98b94..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/dim/wrap_type.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -from types import ( - BuiltinMethodType, - FunctionType, - GetSetDescriptorType, - MethodDescriptorType, - WrapperDescriptorType, -) - -from functorch._C import dim as _C - - -_wrap_method = _C._wrap_method - -FUNC_TYPES = ( - FunctionType, - MethodDescriptorType, - BuiltinMethodType, - WrapperDescriptorType, -) -PROPERTY_TYPES = (GetSetDescriptorType, property) - - -def _py_wrap_method(orig, __torch_function__): - def impl(*args, **kwargs): - return __torch_function__(orig, None, args, kwargs) - - return impl - - -def wrap_type(use_c, to_patch, pattern, __torch_function__): - if use_c: - wrap_method = _wrap_method - else: - wrap_method = _py_wrap_method - - all = {} - for t in reversed(pattern.mro()[:-1]): # skip object - all.update(t.__dict__) - - def wrap_attr(orig): - return property(wrap_method(orig.__get__, __torch_function__)) - - for name, obj in all.items(): - if name in ( - "__dict__", - "__new__", - "__init__", - "__repr__", - "__weakref__", - "__doc__", - "__module__", - "__dir__", - ): - continue - - # skip things that have been overloaded - # things that come from object like `__eq__` still need to be patched, however. - if hasattr(to_patch, name) and getattr(to_patch, name) is not getattr( - object, name, None - ): - continue - - if isinstance(obj, FUNC_TYPES): - setattr(to_patch, name, wrap_method(obj, __torch_function__)) - elif isinstance(obj, PROPERTY_TYPES): - setattr(to_patch, name, wrap_attr(obj)) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/einops/__init__.py b/bundle/python-cpu/Lib/site-packages/functorch/einops/__init__.py deleted file mode 100644 index ef17a03c09e65c20bff38b7ed2d2b1e05f18f780..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/einops/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .rearrange import rearrange - - -__all__ = ["rearrange"] diff --git a/bundle/python-cpu/Lib/site-packages/functorch/einops/_parsing.py b/bundle/python-cpu/Lib/site-packages/functorch/einops/_parsing.py deleted file mode 100644 index 88268cde8f867de4c1ec40e378cde8e3909258ef..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/einops/_parsing.py +++ /dev/null @@ -1,303 +0,0 @@ -"""Adapted from https://github.com/arogozhnikov/einops/blob/36c7bb16e57d6e57f8f3050f9e07abdf3f00469f/einops/parsing.py. - -MIT License - -Copyright (c) 2018 Alex Rogozhnikov - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" -from __future__ import annotations - -import keyword -import warnings -from typing import Collection, List, Mapping, Optional, Set, Tuple, Union - - -_ellipsis: str = "\u2026" # NB, this is a single unicode symbol. String is used as it is not a list, but can be iterated - - -class AnonymousAxis: - """Used by `ParsedExpression` to represent an axis with a size (> 1), but no associated identifier. - - Note: Different instances of this class are not equal to each other, even if they have the same value. - """ - - def __init__(self, value: str) -> None: - self.value = int(value) - if self.value < 1: - raise ValueError( - f"Anonymous axis should have positive length, not {self.value}" - ) - - def __repr__(self) -> str: - return f"{self.value}-axis" - - -class ParsedExpression: - """Structure containing information about one side of an `einops`-style pattern (e.g. 'b c (h w)').""" - - def __init__( - self, - expression: str, - *, - allow_underscore: bool = False, - allow_duplicates: bool = False, - ) -> None: - """Parse the expression and store relevant metadata. - - Args: - expression (str): the `einops`-pattern to parse - allow_underscore (bool): whether to allow axis identifier names to begin with an underscore - allow_duplicates (bool): whether to allow an identifier to appear more than once in the expression - """ - self.has_ellipsis: bool = False - self.has_ellipsis_parenthesized: Optional[bool] = None - self.identifiers: Set[Union[str, AnonymousAxis]] = set() - # that's axes like 2, 3, 4 or 5. Axes with size 1 are exceptional and replaced with empty composition - self.has_non_unitary_anonymous_axes: bool = False - # composition keeps structure of composite axes, see how different corner cases are handled in tests - self.composition: List[Union[List[Union[str, AnonymousAxis]], str]] = [] - if "." in expression: - if "..." not in expression: - raise ValueError( - "Expression may contain dots only inside ellipsis (...)" - ) - if str.count(expression, "...") != 1 or str.count(expression, ".") != 3: - raise ValueError( - "Expression may contain dots only inside ellipsis (...); only one ellipsis for tensor " - ) - expression = expression.replace("...", _ellipsis) - self.has_ellipsis = True - - bracket_group: Optional[List[Union[str, AnonymousAxis]]] = None - - def add_axis_name(x: str) -> None: - if x in self.identifiers: - if not (allow_underscore and x == "_") and not allow_duplicates: - raise ValueError( - f"Indexing expression contains duplicate dimension '{x}'" - ) - if x == _ellipsis: - self.identifiers.add(_ellipsis) - if bracket_group is None: - self.composition.append(_ellipsis) - self.has_ellipsis_parenthesized = False - else: - bracket_group.append(_ellipsis) - self.has_ellipsis_parenthesized = True - else: - is_number = str.isdecimal(x) - if is_number and int(x) == 1: - # handling the case of anonymous axis of length 1 - if bracket_group is None: - self.composition.append([]) - else: - pass # no need to think about 1s inside parenthesis - return - is_axis_name, reason = self.check_axis_name_return_reason( - x, allow_underscore=allow_underscore - ) - if not (is_number or is_axis_name): - raise ValueError(f"Invalid axis identifier: {x}\n{reason}") - axis_name: Union[str, AnonymousAxis] = ( - AnonymousAxis(x) if is_number else x - ) - self.identifiers.add(axis_name) - if is_number: - self.has_non_unitary_anonymous_axes = True - if bracket_group is None: - self.composition.append([axis_name]) - else: - bracket_group.append(axis_name) - - current_identifier = None - for char in expression: - if char in "() ": - if current_identifier is not None: - add_axis_name(current_identifier) - current_identifier = None - if char == "(": - if bracket_group is not None: - raise ValueError( - "Axis composition is one-level (brackets inside brackets not allowed)" - ) - bracket_group = [] - elif char == ")": - if bracket_group is None: - raise ValueError("Brackets are not balanced") - self.composition.append(bracket_group) - bracket_group = None - elif str.isalnum(char) or char in ["_", _ellipsis]: - if current_identifier is None: - current_identifier = char - else: - current_identifier += char - else: - raise ValueError(f"Unknown character '{char}'") - - if bracket_group is not None: - raise ValueError(f"Imbalanced parentheses in expression: '{expression}'") - if current_identifier is not None: - add_axis_name(current_identifier) - - @staticmethod - def check_axis_name_return_reason( - name: str, allow_underscore: bool = False - ) -> Tuple[bool, str]: - """Check if the given axis name is valid, and a message explaining why if not. - - Valid axes names are python identifiers except keywords, and should not start or end with an underscore. - - Args: - name (str): the axis name to check - allow_underscore (bool): whether axis names are allowed to start with an underscore - - Returns: - Tuple[bool, str]: whether the axis name is valid, a message explaining why if not - """ - if not str.isidentifier(name): - return False, "not a valid python identifier" - elif name[0] == "_" or name[-1] == "_": - if name == "_" and allow_underscore: - return True, "" - return False, "axis name should should not start or end with underscore" - else: - if keyword.iskeyword(name): - warnings.warn( - f"It is discouraged to use axes names that are keywords: {name}", - RuntimeWarning, - ) - if name in ["axis"]: - warnings.warn( - "It is discouraged to use 'axis' as an axis name and will raise an error in future", - FutureWarning, - ) - return True, "" - - @staticmethod - def check_axis_name(name: str) -> bool: - """Check if the name is a valid axis name. - - Args: - name (str): the axis name to check - - Returns: - bool: whether the axis name is valid - """ - is_valid, _ = ParsedExpression.check_axis_name_return_reason(name) - return is_valid - - -def parse_pattern( - pattern: str, axes_lengths: Mapping[str, int] -) -> Tuple[ParsedExpression, ParsedExpression]: - """Parse an `einops`-style pattern into a left-hand side and right-hand side `ParsedExpression` object. - - Args: - pattern (str): the `einops`-style rearrangement pattern - axes_lengths (Mapping[str, int]): any additional length specifications for dimensions - - Returns: - Tuple[ParsedExpression, ParsedExpression]: a tuple containing the left-hand side and right-hand side expressions - """ - # adapted from einops.einops._prepare_transformation_recipe - # https://github.com/arogozhnikov/einops/blob/230ac1526c1f42c9e1f7373912c7f8047496df11/einops/einops.py - try: - left_str, right_str = pattern.split("->") - except ValueError: - raise ValueError("Pattern must contain a single '->' separator") from None - - if _ellipsis in axes_lengths: - raise ValueError(f"'{_ellipsis}' is not an allowed axis identifier") - - left = ParsedExpression(left_str) - right = ParsedExpression(right_str) - - if not left.has_ellipsis and right.has_ellipsis: - raise ValueError( - f"Ellipsis found in right side, but not left side of a pattern {pattern}" - ) - if left.has_ellipsis and left.has_ellipsis_parenthesized: - raise ValueError( - f"Ellipsis is parenthesis in the left side is not allowed: {pattern}" - ) - - return left, right - - -def validate_rearrange_expressions( - left: ParsedExpression, right: ParsedExpression, axes_lengths: Mapping[str, int] -) -> None: - """Perform expression validations that are specific to the `rearrange` operation. - - Args: - left (ParsedExpression): left-hand side expression - right (ParsedExpression): right-hand side expression - axes_lengths (Mapping[str, int]): any additional length specifications for dimensions - """ - for length in axes_lengths.values(): - if (length_type := type(length)) is not int: - raise TypeError( - f"rearrange axis lengths must be integers, got: {length_type}" - ) - - if left.has_non_unitary_anonymous_axes or right.has_non_unitary_anonymous_axes: - raise ValueError("rearrange only supports unnamed axes of size 1") - - difference = set.symmetric_difference(left.identifiers, right.identifiers) - if len(difference) > 0: - raise ValueError( - f"Identifiers only on one side of rearrange expression (should be on both): {difference}" - ) - - unmatched_axes = axes_lengths.keys() - left.identifiers - if len(unmatched_axes) > 0: - raise ValueError( - f"Identifiers not found in rearrange expression: {unmatched_axes}" - ) - - -def comma_separate(collection: Collection[Union[str, Collection[str]]]) -> str: - """Convert a collection of strings representing first class dims into a comma-separated string. - - Args: - collection (Collection[Union[str, Collection[str]]]): the collection of strings to convert - - Returns: - str: the comma-separated string - - Examples: - >>> comma_separate(('d0',)) - 'd0' - - >>> comma_separate(('d0', 'd1', 'd2', 'd3')) - 'd0, d1, d2, d3' - - >>> comma_separate([('d1', 'd4')]) - '(d1, d4)' - - >>> comma_separate([('d0',), (), ('d1',), ('d2',), ('d3', 'd4')]) - '(d0,), (), (d1,), (d2,), (d3, d4)' - """ - return ", ".join( - item - if isinstance(item, str) - else f"({comma_separate(item)}{',' if len(item) == 1 else ''})" - for item in collection - ) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/einops/rearrange.py b/bundle/python-cpu/Lib/site-packages/functorch/einops/rearrange.py deleted file mode 100644 index 5e9214ac7c87351e1086e2d3addd2640a1552cb7..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/einops/rearrange.py +++ /dev/null @@ -1,208 +0,0 @@ -from __future__ import annotations - -import functools -from typing import Callable, Dict, List, Sequence, Tuple, Union - -import torch -from functorch._C import dim as _C - -from ._parsing import ( - _ellipsis, - AnonymousAxis, - comma_separate, - parse_pattern, - validate_rearrange_expressions, -) - - -__all__ = ["rearrange"] - -dims = _C.dims - - -@functools.lru_cache(256) -def _create_rearrange_callable( - tensor_ndim: int, pattern: str, **axes_lengths: int -) -> Callable[[torch.Tensor], torch.Tensor]: - r"""Translate an `einops`-style pattern into a callable that performs the rearrange using first-class dimensions. - - Since the an equivalent result is computed for tensors with the same number of dimensions, with the same pattern and - specified axes lengths, this function can be memoized. - - Args: - tensor_ndim (int): the number of dimensions in the tensor to rearrange - pattern (str): the `einops`-style rearrangement pattern - axes_lengths (int): any additional length specifications for dimensions - - Returns: - Callable[[torch.Tensor], torch.Tensor]: a callable that performs the rearrangement - """ - left, right = parse_pattern(pattern, axes_lengths) - validate_rearrange_expressions(left, right, axes_lengths) - - n_anon_dims = sum(not dim for dim in left.composition) - if left.has_ellipsis: - n_ellipsis_dims = tensor_ndim - (len(left.composition) - 1) - n_named_dims = len(left.identifiers) - 1 - - if (pattern_ndim := n_anon_dims + n_named_dims) > tensor_ndim: - raise ValueError( - f"Number of dimensions in pattern ({pattern_ndim}) must be less than or equal to the number of " - f"dimensions in the tensor ({tensor_ndim})" - ) - else: - n_ellipsis_dims = 0 - n_named_dims = len(left.identifiers) - - if (pattern_ndim := len(left.composition)) != tensor_ndim: - raise ValueError( - f"Number of dimensions in pattern ({pattern_ndim}) must be equal to the number of dimensions in " - f"the tensor ({tensor_ndim})" - ) - n_dims = n_named_dims + n_ellipsis_dims + n_anon_dims - - if n_dims == 0: - # an identity rearrangement on a 0-dimension tensor - return lambda tensor: tensor - - first_class_dims: Tuple[str, ...] = tuple(f"d{i}" for i in range(n_dims)) - identifier_dim_map: Dict[Union[str, AnonymousAxis], Tuple[str, ...]] = {} - anon_axes: List[AnonymousAxis] = [] - - # map the left-hand side identifiers to strings representing first class dims - dims_i = 0 - for dimension in left.composition: - if isinstance(dimension, list): - for identifier in dimension: - # non-unitary anon axes are not allowed in rearrange & unitary anon axes are represented as empty lists - assert isinstance(identifier, str) - identifier_dim_map[identifier] = (first_class_dims[dims_i],) - dims_i += 1 - if not dimension: - # unitary anonymous axis - anon_axis = AnonymousAxis("1") - identifier_dim_map[anon_axis] = (first_class_dims[dims_i],) - anon_axes.append(anon_axis) - dimension.append(anon_axis) - dims_i += 1 - elif dimension == _ellipsis: - identifier = _ellipsis - identifier_dim_map[identifier] = tuple( - first_class_dims[dims_i + j] for j in range(n_ellipsis_dims) - ) - dims_i += n_ellipsis_dims - else: - raise ValueError(f"Unexpected dimension: {dimension}") - - def composition_to_dims( - composition: Sequence[Union[List[Union[str, AnonymousAxis]], str]] - ) -> List[Union[str, Tuple[str, ...]]]: - """Convert a `ParsedExpression.composition` into a `Tensor.__getitem__` index of strings representing first - class dims.""" - dim_composition: List[Union[str, Tuple[str, ...]]] = [] - for dimension in composition: - if isinstance(dimension, list): - dim_composition.append( - tuple( - dim - for identifier in dimension - for dim in identifier_dim_map[identifier] - ) - ) - elif dimension == _ellipsis: - dim_composition.extend(identifier_dim_map[_ellipsis]) - else: - raise ValueError(f"Unexpected dimension: {dimension}") - return dim_composition - - left_dims = composition_to_dims(left.composition) - right_dims = composition_to_dims(right.composition) - anon_dims = tuple(identifier_dim_map[axis][0] for axis in anon_axes) - specified_lengths = tuple( - (identifier_dim_map[axis][0], length) for axis, length in axes_lengths.items() - ) - - custom_rearrange_callable_name = "do_rearrange" - custom_rearrange_callable_code = ( - ( - f"def {custom_rearrange_callable_name}(tensor):\n" - f" {comma_separate(first_class_dims)} = dims({n_dims})\n" - ) - + ( - "".join( - f" {dim}.size = {length}\n" for (dim, length) in specified_lengths - ) - if specified_lengths - else "" - ) - + f" tensor = tensor[{comma_separate(left_dims)}].order({comma_separate(right_dims)})\n" - + ( - f" return tensor.sum({comma_separate([anon_dims])}, keepdim=False)\n" - if anon_dims - else " return tensor\n" - ) - ) - - exec(custom_rearrange_callable_code) - return locals()[custom_rearrange_callable_name] - - -def rearrange( - tensor: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor, ...]], - pattern: str, - **axes_lengths: int, -) -> torch.Tensor: - r"""A native implementation of `einops.rearrange`, a reader-friendly smart element reordering for multidimensional - tensors. This operation includes functionality of transpose (axes permutation), reshape (view), squeeze, unsqueeze, - stack, concatenate and other operations. - - See: https://einops.rocks/api/rearrange/ - - Args: - tensor (Tensor or sequence of Tensor): the tensor(s) to rearrange - pattern (str): the rearrangement pattern - axes_lengths (int): any additional length specifications for dimensions - - Returns: - Tensor: the rearranged tensor - - Examples: - >>> # suppose we have a set of 32 images in "h w c" format (height-width-channel) - >>> images = torch.randn((32, 30, 40, 3)) - - >>> # stack along first (batch) axis, output is a single array - >>> rearrange(images, 'b h w c -> b h w c').shape - torch.Size([32, 30, 40, 3]) - - >>> # concatenate images along height (vertical axis), 960 = 32 * 30 - >>> rearrange(images, 'b h w c -> (b h) w c').shape - torch.Size([960, 40, 3]) - - >>> # concatenated images along horizontal axis, 1280 = 32 * 40 - >>> rearrange(images, 'b h w c -> h (b w) c').shape - torch.Size([30, 1280, 3]) - - >>> # reordered axes to "b c h w" format for deep learning - >>> rearrange(images, 'b h w c -> b c h w').shape - torch.Size([32, 3, 30, 40]) - - >>> # flattened each image into a vector, 3600 = 30 * 40 * 3 - >>> rearrange(images, 'b h w c -> b (c h w)').shape - torch.Size([32, 3600]) - - >>> # split each image into 4 smaller (top-left, top-right, bottom-left, bottom-right), 128 = 32 * 2 * 2 - >>> rearrange(images, 'b (h1 h) (w1 w) c -> (b h1 w1) h w c', h1=2, w1=2).shape - torch.Size([128, 15, 20, 3]) - - >>> # space-to-depth operation - >>> rearrange(images, 'b (h h1) (w w1) c -> b h w (c h1 w1)', h1=2, w1=2).shape - torch.Size([32, 15, 20, 12]) - """ - if not isinstance(tensor, torch.Tensor): - tensor = torch.stack(tensor) - - rearrange_callable = _create_rearrange_callable( - tensor.ndim, pattern, **axes_lengths - ) - - return rearrange_callable(tensor) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/experimental/__init__.py b/bundle/python-cpu/Lib/site-packages/functorch/experimental/__init__.py deleted file mode 100644 index 00af1abcc55fd733e473cfa413da38ef5898bbcb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/experimental/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# PyTorch forward-mode is not mature yet -from functorch import functionalize -from torch._functorch.apis import chunk_vmap -from torch._functorch.batch_norm_replacement import replace_all_batch_norm_modules_ -from torch._functorch.eager_transforms import hessian, jacfwd, jvp diff --git a/bundle/python-cpu/Lib/site-packages/functorch/experimental/control_flow.py b/bundle/python-cpu/Lib/site-packages/functorch/experimental/control_flow.py deleted file mode 100644 index 7f048e20c84b409c1d3d75dc9831b2f6fa4d4bad..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/experimental/control_flow.py +++ /dev/null @@ -1,7 +0,0 @@ -from torch import cond # noqa: F401 -from torch._higher_order_ops.cond import UnsupportedAliasMutationException # noqa: F401 -from torch._higher_order_ops.map import ( # noqa: F401 - _stack_pytree, - _unstack_pytree, - map, -) diff --git a/bundle/python-cpu/Lib/site-packages/functorch/experimental/ops.py b/bundle/python-cpu/Lib/site-packages/functorch/experimental/ops.py deleted file mode 100644 index 144515478f0c537ee28ba46f72e3a70f08858f38..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/functorch/experimental/ops.py +++ /dev/null @@ -1 +0,0 @@ -from torch._ops import HigherOrderOperator # noqa: F401 diff --git a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/METADATA deleted file mode 100644 index 8a2f639061cc4a203f7109d8335d28076442c61d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/METADATA +++ /dev/null @@ -1,202 +0,0 @@ -Metadata-Version: 2.4 -Name: h11 -Version: 0.16.0 -Summary: A pure-Python, bring-your-own-I/O implementation of HTTP/1.1 -Home-page: https://github.com/python-hyper/h11 -Author: Nathaniel J. Smith -Author-email: njs@pobox.com -License: MIT -Classifier: Development Status :: 3 - Alpha -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Topic :: Internet :: WWW/HTTP -Classifier: Topic :: System :: Networking -Requires-Python: >=3.8 -License-File: LICENSE.txt -Dynamic: author -Dynamic: author-email -Dynamic: classifier -Dynamic: description -Dynamic: home-page -Dynamic: license -Dynamic: license-file -Dynamic: requires-python -Dynamic: summary - -h11 -=== - -.. image:: https://travis-ci.org/python-hyper/h11.svg?branch=master - :target: https://travis-ci.org/python-hyper/h11 - :alt: Automated test status - -.. image:: https://codecov.io/gh/python-hyper/h11/branch/master/graph/badge.svg - :target: https://codecov.io/gh/python-hyper/h11 - :alt: Test coverage - -.. image:: https://readthedocs.org/projects/h11/badge/?version=latest - :target: http://h11.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - -This is a little HTTP/1.1 library written from scratch in Python, -heavily inspired by `hyper-h2 `_. - -It's a "bring-your-own-I/O" library; h11 contains no IO code -whatsoever. This means you can hook h11 up to your favorite network -API, and that could be anything you want: synchronous, threaded, -asynchronous, or your own implementation of `RFC 6214 -`_ -- h11 won't judge you. -(Compare this to the current state of the art, where every time a `new -network API `_ comes along then someone -gets to start over reimplementing the entire HTTP protocol from -scratch.) Cory Benfield made an `excellent blog post describing the -benefits of this approach -`_, or if you like video -then here's his `PyCon 2016 talk on the same theme -`_. - -This also means that h11 is not immediately useful out of the box: -it's a toolkit for building programs that speak HTTP, not something -that could directly replace ``requests`` or ``twisted.web`` or -whatever. But h11 makes it much easier to implement something like -``requests`` or ``twisted.web``. - -At a high level, working with h11 goes like this: - -1) First, create an ``h11.Connection`` object to track the state of a - single HTTP/1.1 connection. - -2) When you read data off the network, pass it to - ``conn.receive_data(...)``; you'll get back a list of objects - representing high-level HTTP "events". - -3) When you want to send a high-level HTTP event, create the - corresponding "event" object and pass it to ``conn.send(...)``; - this will give you back some bytes that you can then push out - through the network. - -For example, a client might instantiate and then send a -``h11.Request`` object, then zero or more ``h11.Data`` objects for the -request body (e.g., if this is a POST), and then a -``h11.EndOfMessage`` to indicate the end of the message. Then the -server would then send back a ``h11.Response``, some ``h11.Data``, and -its own ``h11.EndOfMessage``. If either side violates the protocol, -you'll get a ``h11.ProtocolError`` exception. - -h11 is suitable for implementing both servers and clients, and has a -pleasantly symmetric API: the events you send as a client are exactly -the ones that you receive as a server and vice-versa. - -`Here's an example of a tiny HTTP client -`_ - -It also has `a fine manual `_. - -FAQ ---- - -*Whyyyyy?* - -I wanted to play with HTTP in `Curio -`__ and `Trio -`__, which at the time didn't have any -HTTP libraries. So I thought, no big deal, Python has, like, a dozen -different implementations of HTTP, surely I can find one that's -reusable. I didn't find one, but I did find Cory's call-to-arms -blog-post. So I figured, well, fine, if I have to implement HTTP from -scratch, at least I can make sure no-one *else* has to ever again. - -*Should I use it?* - -Maybe. You should be aware that it's a very young project. But, it's -feature complete and has an exhaustive test-suite and complete docs, -so the next step is for people to try using it and see how it goes -:-). If you do then please let us know -- if nothing else we'll want -to talk to you before making any incompatible changes! - -*What are the features/limitations?* - -Roughly speaking, it's trying to be a robust, complete, and non-hacky -implementation of the first "chapter" of the HTTP/1.1 spec: `RFC 7230: -HTTP/1.1 Message Syntax and Routing -`_. That is, it mostly focuses on -implementing HTTP at the level of taking bytes on and off the wire, -and the headers related to that, and tries to be anal about spec -conformance. It doesn't know about higher-level concerns like URL -routing, conditional GETs, cross-origin cookie policies, or content -negotiation. But it does know how to take care of framing, -cross-version differences in keep-alive handling, and the "obsolete -line folding" rule, so you can focus your energies on the hard / -interesting parts for your application, and it tries to support the -full specification in the sense that any useful HTTP/1.1 conformant -application should be able to use h11. - -It's pure Python, and has no dependencies outside of the standard -library. - -It has a test suite with 100.0% coverage for both statements and -branches. - -Currently it supports Python 3 (testing on 3.8-3.12) and PyPy 3. -The last Python 2-compatible version was h11 0.11.x. -(Originally it had a Cython wrapper for `http-parser -`_ and a beautiful nested state -machine implemented with ``yield from`` to postprocess the output. But -I had to take these out -- the new *parser* needs fewer lines-of-code -than the old *parser wrapper*, is written in pure Python, uses no -exotic language syntax, and has more features. It's sad, really; that -old state machine was really slick. I just need a few sentences here -to mourn that.) - -I don't know how fast it is. I haven't benchmarked or profiled it yet, -so it's probably got a few pointless hot spots, and I've been trying -to err on the side of simplicity and robustness instead of -micro-optimization. But at the architectural level I tried hard to -avoid fundamentally bad decisions, e.g., I believe that all the -parsing algorithms remain linear-time even in the face of pathological -input like slowloris, and there are no byte-by-byte loops. (I also -believe that it maintains bounded memory usage in the face of -arbitrary/pathological input.) - -The whole library is ~800 lines-of-code. You can read and understand -the whole thing in less than an hour. Most of the energy invested in -this so far has been spent on trying to keep things simple by -minimizing special-cases and ad hoc state manipulation; even though it -is now quite small and simple, I'm still annoyed that I haven't -figured out how to make it even smaller and simpler. (Unfortunately, -HTTP does not lend itself to simplicity.) - -The API is ~feature complete and I don't expect the general outlines -to change much, but you can't judge an API's ergonomics until you -actually document and use it, so I'd expect some changes in the -details. - -*How do I try it?* - -.. code-block:: sh - - $ pip install h11 - $ git clone git@github.com:python-hyper/h11 - $ cd h11/examples - $ python basic-client.py - -and go from there. - -*License?* - -MIT - -*Code of conduct?* - -Contributors are requested to follow our `code of conduct -`_ in -all project spaces. diff --git a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/RECORD deleted file mode 100644 index 8e8d23bccaf5c72a3ac77d1a889dff9ce1caea57..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/RECORD +++ /dev/null @@ -1,19 +0,0 @@ -h11-0.16.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -h11-0.16.0.dist-info/METADATA,sha256=KPMmCYrAn8unm48YD5YIfIQf4kViFct7hyqcfVzRnWQ,8348 -h11-0.16.0.dist-info/RECORD,, -h11-0.16.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -h11-0.16.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91 -h11-0.16.0.dist-info/licenses/LICENSE.txt,sha256=N9tbuFkm2yikJ6JYZ_ELEjIAOuob5pzLhRE4rbjm82E,1124 -h11-0.16.0.dist-info/top_level.txt,sha256=F7dC4jl3zeh8TGHEPaWJrMbeuoWbS379Gwdi-Yvdcis,4 -h11/__init__.py,sha256=iO1KzkSO42yZ6ffg-VMgbx_ZVTWGUY00nRYEWn-s3kY,1507 -h11/_abnf.py,sha256=ybixr0xsupnkA6GFAyMubuXF6Tc1lb_hF890NgCsfNc,4815 -h11/_connection.py,sha256=k9YRVf6koZqbttBW36xSWaJpWdZwa-xQVU9AHEo9DuI,26863 -h11/_events.py,sha256=I97aXoal1Wu7dkL548BANBUCkOIbe-x5CioYA9IBY14,11792 -h11/_headers.py,sha256=P7D-lBNxHwdLZPLimmYwrPG-9ZkjElvvJZJdZAgSP-4,10412 -h11/_readers.py,sha256=a4RypORUCC3d0q_kxPuBIM7jTD8iLt5X91TH0FsduN4,8590 -h11/_receivebuffer.py,sha256=xrspsdsNgWFxRfQcTXxR8RrdjRXXTK0Io5cQYWpJ1Ws,5252 -h11/_state.py,sha256=_5LG_BGR8FCcFQeBPH-TMHgm_-B-EUcWCnQof_9XjFE,13231 -h11/_util.py,sha256=LWkkjXyJaFlAy6Lt39w73UStklFT5ovcvo0TkY7RYuk,4888 -h11/_version.py,sha256=GVSsbPSPDcOuF6ptfIiXnVJoaEm3ygXbMnqlr_Giahw,686 -h11/_writers.py,sha256=oFKm6PtjeHfbj4RLX7VB7KDc1gIY53gXG3_HR9ltmTA,5081 -h11/py.typed,sha256=sow9soTwP9T_gEAQSVh7Gb8855h04Nwmhs2We-JRgZM,7 diff --git a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/WHEEL deleted file mode 100644 index 1eb3c49d99559863120cfb8433fc8738fba43ba9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/WHEEL +++ /dev/null @@ -1,5 +0,0 @@ -Wheel-Version: 1.0 -Generator: setuptools (78.1.0) -Root-Is-Purelib: true -Tag: py3-none-any - diff --git a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/licenses/LICENSE.txt b/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/licenses/LICENSE.txt deleted file mode 100644 index 8f080eae848f759c9173bfc0c79506357ebe5090..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/licenses/LICENSE.txt +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Nathaniel J. Smith and other contributors - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/top_level.txt b/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/top_level.txt deleted file mode 100644 index 0d24def711344ec6f4da2108f7d5c9261eb35f8b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11-0.16.0.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -h11 diff --git a/bundle/python-cpu/Lib/site-packages/h11/__init__.py b/bundle/python-cpu/Lib/site-packages/h11/__init__.py deleted file mode 100644 index 989e92c3458681a6f0be72ae4105ea742750d328..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/__init__.py +++ /dev/null @@ -1,62 +0,0 @@ -# A highish-level implementation of the HTTP/1.1 wire protocol (RFC 7230), -# containing no networking code at all, loosely modelled on hyper-h2's generic -# implementation of HTTP/2 (and in particular the h2.connection.H2Connection -# class). There's still a bunch of subtle details you need to get right if you -# want to make this actually useful, because it doesn't implement all the -# semantics to check that what you're asking to write to the wire is sensible, -# but at least it gets you out of dealing with the wire itself. - -from h11._connection import Connection, NEED_DATA, PAUSED -from h11._events import ( - ConnectionClosed, - Data, - EndOfMessage, - Event, - InformationalResponse, - Request, - Response, -) -from h11._state import ( - CLIENT, - CLOSED, - DONE, - ERROR, - IDLE, - MIGHT_SWITCH_PROTOCOL, - MUST_CLOSE, - SEND_BODY, - SEND_RESPONSE, - SERVER, - SWITCHED_PROTOCOL, -) -from h11._util import LocalProtocolError, ProtocolError, RemoteProtocolError -from h11._version import __version__ - -PRODUCT_ID = "python-h11/" + __version__ - - -__all__ = ( - "Connection", - "NEED_DATA", - "PAUSED", - "ConnectionClosed", - "Data", - "EndOfMessage", - "Event", - "InformationalResponse", - "Request", - "Response", - "CLIENT", - "CLOSED", - "DONE", - "ERROR", - "IDLE", - "MUST_CLOSE", - "SEND_BODY", - "SEND_RESPONSE", - "SERVER", - "SWITCHED_PROTOCOL", - "ProtocolError", - "LocalProtocolError", - "RemoteProtocolError", -) diff --git a/bundle/python-cpu/Lib/site-packages/h11/_abnf.py b/bundle/python-cpu/Lib/site-packages/h11/_abnf.py deleted file mode 100644 index 933587fba22290d7eb7df4c88e12f1e61702b8ce..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/_abnf.py +++ /dev/null @@ -1,132 +0,0 @@ -# We use native strings for all the re patterns, to take advantage of string -# formatting, and then convert to bytestrings when compiling the final re -# objects. - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#whitespace -# OWS = *( SP / HTAB ) -# ; optional whitespace -OWS = r"[ \t]*" - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#rule.token.separators -# token = 1*tchar -# -# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" -# / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" -# / DIGIT / ALPHA -# ; any VCHAR, except delimiters -token = r"[-!#$%&'*+.^_`|~0-9a-zA-Z]+" - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#header.fields -# field-name = token -field_name = token - -# The standard says: -# -# field-value = *( field-content / obs-fold ) -# field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] -# field-vchar = VCHAR / obs-text -# obs-fold = CRLF 1*( SP / HTAB ) -# ; obsolete line folding -# ; see Section 3.2.4 -# -# https://tools.ietf.org/html/rfc5234#appendix-B.1 -# -# VCHAR = %x21-7E -# ; visible (printing) characters -# -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#rule.quoted-string -# obs-text = %x80-FF -# -# However, the standard definition of field-content is WRONG! It disallows -# fields containing a single visible character surrounded by whitespace, -# e.g. "foo a bar". -# -# See: https://www.rfc-editor.org/errata_search.php?rfc=7230&eid=4189 -# -# So our definition of field_content attempts to fix it up... -# -# Also, we allow lots of control characters, because apparently people assume -# that they're legal in practice (e.g., google analytics makes cookies with -# \x01 in them!): -# https://github.com/python-hyper/h11/issues/57 -# We still don't allow NUL or whitespace, because those are often treated as -# meta-characters and letting them through can lead to nasty issues like SSRF. -vchar = r"[\x21-\x7e]" -vchar_or_obs_text = r"[^\x00\s]" -field_vchar = vchar_or_obs_text -field_content = r"{field_vchar}+(?:[ \t]+{field_vchar}+)*".format(**globals()) - -# We handle obs-fold at a different level, and our fixed-up field_content -# already grows to swallow the whole value, so ? instead of * -field_value = r"({field_content})?".format(**globals()) - -# header-field = field-name ":" OWS field-value OWS -header_field = ( - r"(?P{field_name})" - r":" - r"{OWS}" - r"(?P{field_value})" - r"{OWS}".format(**globals()) -) - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#request.line -# -# request-line = method SP request-target SP HTTP-version CRLF -# method = token -# HTTP-version = HTTP-name "/" DIGIT "." DIGIT -# HTTP-name = %x48.54.54.50 ; "HTTP", case-sensitive -# -# request-target is complicated (see RFC 7230 sec 5.3) -- could be path, full -# URL, host+port (for connect), or even "*", but in any case we are guaranteed -# that it contists of the visible printing characters. -method = token -request_target = r"{vchar}+".format(**globals()) -http_version = r"HTTP/(?P[0-9]\.[0-9])" -request_line = ( - r"(?P{method})" - r" " - r"(?P{request_target})" - r" " - r"{http_version}".format(**globals()) -) - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#status.line -# -# status-line = HTTP-version SP status-code SP reason-phrase CRLF -# status-code = 3DIGIT -# reason-phrase = *( HTAB / SP / VCHAR / obs-text ) -status_code = r"[0-9]{3}" -reason_phrase = r"([ \t]|{vchar_or_obs_text})*".format(**globals()) -status_line = ( - r"{http_version}" - r" " - r"(?P{status_code})" - # However, there are apparently a few too many servers out there that just - # leave out the reason phrase: - # https://github.com/scrapy/scrapy/issues/345#issuecomment-281756036 - # https://github.com/seanmonstar/httparse/issues/29 - # so make it optional. ?: is a non-capturing group. - r"(?: (?P{reason_phrase}))?".format(**globals()) -) - -HEXDIG = r"[0-9A-Fa-f]" -# Actually -# -# chunk-size = 1*HEXDIG -# -# but we impose an upper-limit to avoid ridiculosity. len(str(2**64)) == 20 -chunk_size = r"({HEXDIG}){{1,20}}".format(**globals()) -# Actually -# -# chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] ) -# -# but we aren't parsing the things so we don't really care. -chunk_ext = r";.*" -chunk_header = ( - r"(?P{chunk_size})" - r"(?P{chunk_ext})?" - r"{OWS}\r\n".format( - **globals() - ) # Even though the specification does not allow for extra whitespaces, - # we are lenient with trailing whitespaces because some servers on the wild use it. -) diff --git a/bundle/python-cpu/Lib/site-packages/h11/_connection.py b/bundle/python-cpu/Lib/site-packages/h11/_connection.py deleted file mode 100644 index e37d82a82a882c072cb938a90eb4486b51cdad99..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/_connection.py +++ /dev/null @@ -1,659 +0,0 @@ -# This contains the main Connection class. Everything in h11 revolves around -# this. -from typing import ( - Any, - Callable, - cast, - Dict, - List, - Optional, - overload, - Tuple, - Type, - Union, -) - -from ._events import ( - ConnectionClosed, - Data, - EndOfMessage, - Event, - InformationalResponse, - Request, - Response, -) -from ._headers import get_comma_header, has_expect_100_continue, set_comma_header -from ._readers import READERS, ReadersType -from ._receivebuffer import ReceiveBuffer -from ._state import ( - _SWITCH_CONNECT, - _SWITCH_UPGRADE, - CLIENT, - ConnectionState, - DONE, - ERROR, - MIGHT_SWITCH_PROTOCOL, - SEND_BODY, - SERVER, - SWITCHED_PROTOCOL, -) -from ._util import ( # Import the internal things we need - LocalProtocolError, - RemoteProtocolError, - Sentinel, -) -from ._writers import WRITERS, WritersType - -# Everything in __all__ gets re-exported as part of the h11 public API. -__all__ = ["Connection", "NEED_DATA", "PAUSED"] - - -class NEED_DATA(Sentinel, metaclass=Sentinel): - pass - - -class PAUSED(Sentinel, metaclass=Sentinel): - pass - - -# If we ever have this much buffered without it making a complete parseable -# event, we error out. The only time we really buffer is when reading the -# request/response line + headers together, so this is effectively the limit on -# the size of that. -# -# Some precedents for defaults: -# - node.js: 80 * 1024 -# - tomcat: 8 * 1024 -# - IIS: 16 * 1024 -# - Apache: <8 KiB per line> -DEFAULT_MAX_INCOMPLETE_EVENT_SIZE = 16 * 1024 - - -# RFC 7230's rules for connection lifecycles: -# - If either side says they want to close the connection, then the connection -# must close. -# - HTTP/1.1 defaults to keep-alive unless someone says Connection: close -# - HTTP/1.0 defaults to close unless both sides say Connection: keep-alive -# (and even this is a mess -- e.g. if you're implementing a proxy then -# sending Connection: keep-alive is forbidden). -# -# We simplify life by simply not supporting keep-alive with HTTP/1.0 peers. So -# our rule is: -# - If someone says Connection: close, we will close -# - If someone uses HTTP/1.0, we will close. -def _keep_alive(event: Union[Request, Response]) -> bool: - connection = get_comma_header(event.headers, b"connection") - if b"close" in connection: - return False - if getattr(event, "http_version", b"1.1") < b"1.1": - return False - return True - - -def _body_framing( - request_method: bytes, event: Union[Request, Response] -) -> Tuple[str, Union[Tuple[()], Tuple[int]]]: - # Called when we enter SEND_BODY to figure out framing information for - # this body. - # - # These are the only two events that can trigger a SEND_BODY state: - assert type(event) in (Request, Response) - # Returns one of: - # - # ("content-length", count) - # ("chunked", ()) - # ("http/1.0", ()) - # - # which are (lookup key, *args) for constructing body reader/writer - # objects. - # - # Reference: https://tools.ietf.org/html/rfc7230#section-3.3.3 - # - # Step 1: some responses always have an empty body, regardless of what the - # headers say. - if type(event) is Response: - if ( - event.status_code in (204, 304) - or request_method == b"HEAD" - or (request_method == b"CONNECT" and 200 <= event.status_code < 300) - ): - return ("content-length", (0,)) - # Section 3.3.3 also lists another case -- responses with status_code - # < 200. For us these are InformationalResponses, not Responses, so - # they can't get into this function in the first place. - assert event.status_code >= 200 - - # Step 2: check for Transfer-Encoding (T-E beats C-L): - transfer_encodings = get_comma_header(event.headers, b"transfer-encoding") - if transfer_encodings: - assert transfer_encodings == [b"chunked"] - return ("chunked", ()) - - # Step 3: check for Content-Length - content_lengths = get_comma_header(event.headers, b"content-length") - if content_lengths: - return ("content-length", (int(content_lengths[0]),)) - - # Step 4: no applicable headers; fallback/default depends on type - if type(event) is Request: - return ("content-length", (0,)) - else: - return ("http/1.0", ()) - - -################################################################ -# -# The main Connection class -# -################################################################ - - -class Connection: - """An object encapsulating the state of an HTTP connection. - - Args: - our_role: If you're implementing a client, pass :data:`h11.CLIENT`. If - you're implementing a server, pass :data:`h11.SERVER`. - - max_incomplete_event_size (int): - The maximum number of bytes we're willing to buffer of an - incomplete event. In practice this mostly sets a limit on the - maximum size of the request/response line + headers. If this is - exceeded, then :meth:`next_event` will raise - :exc:`RemoteProtocolError`. - - """ - - def __init__( - self, - our_role: Type[Sentinel], - max_incomplete_event_size: int = DEFAULT_MAX_INCOMPLETE_EVENT_SIZE, - ) -> None: - self._max_incomplete_event_size = max_incomplete_event_size - # State and role tracking - if our_role not in (CLIENT, SERVER): - raise ValueError(f"expected CLIENT or SERVER, not {our_role!r}") - self.our_role = our_role - self.their_role: Type[Sentinel] - if our_role is CLIENT: - self.their_role = SERVER - else: - self.their_role = CLIENT - self._cstate = ConnectionState() - - # Callables for converting data->events or vice-versa given the - # current state - self._writer = self._get_io_object(self.our_role, None, WRITERS) - self._reader = self._get_io_object(self.their_role, None, READERS) - - # Holds any unprocessed received data - self._receive_buffer = ReceiveBuffer() - # If this is true, then it indicates that the incoming connection was - # closed *after* the end of whatever's in self._receive_buffer: - self._receive_buffer_closed = False - - # Extra bits of state that don't fit into the state machine. - # - # These two are only used to interpret framing headers for figuring - # out how to read/write response bodies. their_http_version is also - # made available as a convenient public API. - self.their_http_version: Optional[bytes] = None - self._request_method: Optional[bytes] = None - # This is pure flow-control and doesn't at all affect the set of legal - # transitions, so no need to bother ConnectionState with it: - self.client_is_waiting_for_100_continue = False - - @property - def states(self) -> Dict[Type[Sentinel], Type[Sentinel]]: - """A dictionary like:: - - {CLIENT: , SERVER: } - - See :ref:`state-machine` for details. - - """ - return dict(self._cstate.states) - - @property - def our_state(self) -> Type[Sentinel]: - """The current state of whichever role we are playing. See - :ref:`state-machine` for details. - """ - return self._cstate.states[self.our_role] - - @property - def their_state(self) -> Type[Sentinel]: - """The current state of whichever role we are NOT playing. See - :ref:`state-machine` for details. - """ - return self._cstate.states[self.their_role] - - @property - def they_are_waiting_for_100_continue(self) -> bool: - return self.their_role is CLIENT and self.client_is_waiting_for_100_continue - - def start_next_cycle(self) -> None: - """Attempt to reset our connection state for a new request/response - cycle. - - If both client and server are in :data:`DONE` state, then resets them - both to :data:`IDLE` state in preparation for a new request/response - cycle on this same connection. Otherwise, raises a - :exc:`LocalProtocolError`. - - See :ref:`keepalive-and-pipelining`. - - """ - old_states = dict(self._cstate.states) - self._cstate.start_next_cycle() - self._request_method = None - # self.their_http_version gets left alone, since it presumably lasts - # beyond a single request/response cycle - assert not self.client_is_waiting_for_100_continue - self._respond_to_state_changes(old_states) - - def _process_error(self, role: Type[Sentinel]) -> None: - old_states = dict(self._cstate.states) - self._cstate.process_error(role) - self._respond_to_state_changes(old_states) - - def _server_switch_event(self, event: Event) -> Optional[Type[Sentinel]]: - if type(event) is InformationalResponse and event.status_code == 101: - return _SWITCH_UPGRADE - if type(event) is Response: - if ( - _SWITCH_CONNECT in self._cstate.pending_switch_proposals - and 200 <= event.status_code < 300 - ): - return _SWITCH_CONNECT - return None - - # All events go through here - def _process_event(self, role: Type[Sentinel], event: Event) -> None: - # First, pass the event through the state machine to make sure it - # succeeds. - old_states = dict(self._cstate.states) - if role is CLIENT and type(event) is Request: - if event.method == b"CONNECT": - self._cstate.process_client_switch_proposal(_SWITCH_CONNECT) - if get_comma_header(event.headers, b"upgrade"): - self._cstate.process_client_switch_proposal(_SWITCH_UPGRADE) - server_switch_event = None - if role is SERVER: - server_switch_event = self._server_switch_event(event) - self._cstate.process_event(role, type(event), server_switch_event) - - # Then perform the updates triggered by it. - - if type(event) is Request: - self._request_method = event.method - - if role is self.their_role and type(event) in ( - Request, - Response, - InformationalResponse, - ): - event = cast(Union[Request, Response, InformationalResponse], event) - self.their_http_version = event.http_version - - # Keep alive handling - # - # RFC 7230 doesn't really say what one should do if Connection: close - # shows up on a 1xx InformationalResponse. I think the idea is that - # this is not supposed to happen. In any case, if it does happen, we - # ignore it. - if type(event) in (Request, Response) and not _keep_alive( - cast(Union[Request, Response], event) - ): - self._cstate.process_keep_alive_disabled() - - # 100-continue - if type(event) is Request and has_expect_100_continue(event): - self.client_is_waiting_for_100_continue = True - if type(event) in (InformationalResponse, Response): - self.client_is_waiting_for_100_continue = False - if role is CLIENT and type(event) in (Data, EndOfMessage): - self.client_is_waiting_for_100_continue = False - - self._respond_to_state_changes(old_states, event) - - def _get_io_object( - self, - role: Type[Sentinel], - event: Optional[Event], - io_dict: Union[ReadersType, WritersType], - ) -> Optional[Callable[..., Any]]: - # event may be None; it's only used when entering SEND_BODY - state = self._cstate.states[role] - if state is SEND_BODY: - # Special case: the io_dict has a dict of reader/writer factories - # that depend on the request/response framing. - framing_type, args = _body_framing( - cast(bytes, self._request_method), cast(Union[Request, Response], event) - ) - return io_dict[SEND_BODY][framing_type](*args) # type: ignore[index] - else: - # General case: the io_dict just has the appropriate reader/writer - # for this state - return io_dict.get((role, state)) # type: ignore[return-value] - - # This must be called after any action that might have caused - # self._cstate.states to change. - def _respond_to_state_changes( - self, - old_states: Dict[Type[Sentinel], Type[Sentinel]], - event: Optional[Event] = None, - ) -> None: - # Update reader/writer - if self.our_state != old_states[self.our_role]: - self._writer = self._get_io_object(self.our_role, event, WRITERS) - if self.their_state != old_states[self.their_role]: - self._reader = self._get_io_object(self.their_role, event, READERS) - - @property - def trailing_data(self) -> Tuple[bytes, bool]: - """Data that has been received, but not yet processed, represented as - a tuple with two elements, where the first is a byte-string containing - the unprocessed data itself, and the second is a bool that is True if - the receive connection was closed. - - See :ref:`switching-protocols` for discussion of why you'd want this. - """ - return (bytes(self._receive_buffer), self._receive_buffer_closed) - - def receive_data(self, data: bytes) -> None: - """Add data to our internal receive buffer. - - This does not actually do any processing on the data, just stores - it. To trigger processing, you have to call :meth:`next_event`. - - Args: - data (:term:`bytes-like object`): - The new data that was just received. - - Special case: If *data* is an empty byte-string like ``b""``, - then this indicates that the remote side has closed the - connection (end of file). Normally this is convenient, because - standard Python APIs like :meth:`file.read` or - :meth:`socket.recv` use ``b""`` to indicate end-of-file, while - other failures to read are indicated using other mechanisms - like raising :exc:`TimeoutError`. When using such an API you - can just blindly pass through whatever you get from ``read`` - to :meth:`receive_data`, and everything will work. - - But, if you have an API where reading an empty string is a - valid non-EOF condition, then you need to be aware of this and - make sure to check for such strings and avoid passing them to - :meth:`receive_data`. - - Returns: - Nothing, but after calling this you should call :meth:`next_event` - to parse the newly received data. - - Raises: - RuntimeError: - Raised if you pass an empty *data*, indicating EOF, and then - pass a non-empty *data*, indicating more data that somehow - arrived after the EOF. - - (Calling ``receive_data(b"")`` multiple times is fine, - and equivalent to calling it once.) - - """ - if data: - if self._receive_buffer_closed: - raise RuntimeError("received close, then received more data?") - self._receive_buffer += data - else: - self._receive_buffer_closed = True - - def _extract_next_receive_event( - self, - ) -> Union[Event, Type[NEED_DATA], Type[PAUSED]]: - state = self.their_state - # We don't pause immediately when they enter DONE, because even in - # DONE state we can still process a ConnectionClosed() event. But - # if we have data in our buffer, then we definitely aren't getting - # a ConnectionClosed() immediately and we need to pause. - if state is DONE and self._receive_buffer: - return PAUSED - if state is MIGHT_SWITCH_PROTOCOL or state is SWITCHED_PROTOCOL: - return PAUSED - assert self._reader is not None - event = self._reader(self._receive_buffer) - if event is None: - if not self._receive_buffer and self._receive_buffer_closed: - # In some unusual cases (basically just HTTP/1.0 bodies), EOF - # triggers an actual protocol event; in that case, we want to - # return that event, and then the state will change and we'll - # get called again to generate the actual ConnectionClosed(). - if hasattr(self._reader, "read_eof"): - event = self._reader.read_eof() - else: - event = ConnectionClosed() - if event is None: - event = NEED_DATA - return event # type: ignore[no-any-return] - - def next_event(self) -> Union[Event, Type[NEED_DATA], Type[PAUSED]]: - """Parse the next event out of our receive buffer, update our internal - state, and return it. - - This is a mutating operation -- think of it like calling :func:`next` - on an iterator. - - Returns: - : One of three things: - - 1) An event object -- see :ref:`events`. - - 2) The special constant :data:`NEED_DATA`, which indicates that - you need to read more data from your socket and pass it to - :meth:`receive_data` before this method will be able to return - any more events. - - 3) The special constant :data:`PAUSED`, which indicates that we - are not in a state where we can process incoming data (usually - because the peer has finished their part of the current - request/response cycle, and you have not yet called - :meth:`start_next_cycle`). See :ref:`flow-control` for details. - - Raises: - RemoteProtocolError: - The peer has misbehaved. You should close the connection - (possibly after sending some kind of 4xx response). - - Once this method returns :class:`ConnectionClosed` once, then all - subsequent calls will also return :class:`ConnectionClosed`. - - If this method raises any exception besides :exc:`RemoteProtocolError` - then that's a bug -- if it happens please file a bug report! - - If this method raises any exception then it also sets - :attr:`Connection.their_state` to :data:`ERROR` -- see - :ref:`error-handling` for discussion. - - """ - - if self.their_state is ERROR: - raise RemoteProtocolError("Can't receive data when peer state is ERROR") - try: - event = self._extract_next_receive_event() - if event not in [NEED_DATA, PAUSED]: - self._process_event(self.their_role, cast(Event, event)) - if event is NEED_DATA: - if len(self._receive_buffer) > self._max_incomplete_event_size: - # 431 is "Request header fields too large" which is pretty - # much the only situation where we can get here - raise RemoteProtocolError( - "Receive buffer too long", error_status_hint=431 - ) - if self._receive_buffer_closed: - # We're still trying to complete some event, but that's - # never going to happen because no more data is coming - raise RemoteProtocolError("peer unexpectedly closed connection") - return event - except BaseException as exc: - self._process_error(self.their_role) - if isinstance(exc, LocalProtocolError): - exc._reraise_as_remote_protocol_error() - else: - raise - - @overload - def send(self, event: ConnectionClosed) -> None: - ... - - @overload - def send( - self, event: Union[Request, InformationalResponse, Response, Data, EndOfMessage] - ) -> bytes: - ... - - @overload - def send(self, event: Event) -> Optional[bytes]: - ... - - def send(self, event: Event) -> Optional[bytes]: - """Convert a high-level event into bytes that can be sent to the peer, - while updating our internal state machine. - - Args: - event: The :ref:`event ` to send. - - Returns: - If ``type(event) is ConnectionClosed``, then returns - ``None``. Otherwise, returns a :term:`bytes-like object`. - - Raises: - LocalProtocolError: - Sending this event at this time would violate our - understanding of the HTTP/1.1 protocol. - - If this method raises any exception then it also sets - :attr:`Connection.our_state` to :data:`ERROR` -- see - :ref:`error-handling` for discussion. - - """ - data_list = self.send_with_data_passthrough(event) - if data_list is None: - return None - else: - return b"".join(data_list) - - def send_with_data_passthrough(self, event: Event) -> Optional[List[bytes]]: - """Identical to :meth:`send`, except that in situations where - :meth:`send` returns a single :term:`bytes-like object`, this instead - returns a list of them -- and when sending a :class:`Data` event, this - list is guaranteed to contain the exact object you passed in as - :attr:`Data.data`. See :ref:`sendfile` for discussion. - - """ - if self.our_state is ERROR: - raise LocalProtocolError("Can't send data when our state is ERROR") - try: - if type(event) is Response: - event = self._clean_up_response_headers_for_sending(event) - # We want to call _process_event before calling the writer, - # because if someone tries to do something invalid then this will - # give a sensible error message, while our writers all just assume - # they will only receive valid events. But, _process_event might - # change self._writer. So we have to do a little dance: - writer = self._writer - self._process_event(self.our_role, event) - if type(event) is ConnectionClosed: - return None - else: - # In any situation where writer is None, process_event should - # have raised ProtocolError - assert writer is not None - data_list: List[bytes] = [] - writer(event, data_list.append) - return data_list - except: - self._process_error(self.our_role) - raise - - def send_failed(self) -> None: - """Notify the state machine that we failed to send the data it gave - us. - - This causes :attr:`Connection.our_state` to immediately become - :data:`ERROR` -- see :ref:`error-handling` for discussion. - - """ - self._process_error(self.our_role) - - # When sending a Response, we take responsibility for a few things: - # - # - Sometimes you MUST set Connection: close. We take care of those - # times. (You can also set it yourself if you want, and if you do then - # we'll respect that and close the connection at the right time. But you - # don't have to worry about that unless you want to.) - # - # - The user has to set Content-Length if they want it. Otherwise, for - # responses that have bodies (e.g. not HEAD), then we will automatically - # select the right mechanism for streaming a body of unknown length, - # which depends on depending on the peer's HTTP version. - # - # This function's *only* responsibility is making sure headers are set up - # right -- everything downstream just looks at the headers. There are no - # side channels. - def _clean_up_response_headers_for_sending(self, response: Response) -> Response: - assert type(response) is Response - - headers = response.headers - need_close = False - - # HEAD requests need some special handling: they always act like they - # have Content-Length: 0, and that's how _body_framing treats - # them. But their headers are supposed to match what we would send if - # the request was a GET. (Technically there is one deviation allowed: - # we're allowed to leave out the framing headers -- see - # https://tools.ietf.org/html/rfc7231#section-4.3.2 . But it's just as - # easy to get them right.) - method_for_choosing_headers = cast(bytes, self._request_method) - if method_for_choosing_headers == b"HEAD": - method_for_choosing_headers = b"GET" - framing_type, _ = _body_framing(method_for_choosing_headers, response) - if framing_type in ("chunked", "http/1.0"): - # This response has a body of unknown length. - # If our peer is HTTP/1.1, we use Transfer-Encoding: chunked - # If our peer is HTTP/1.0, we use no framing headers, and close the - # connection afterwards. - # - # Make sure to clear Content-Length (in principle user could have - # set both and then we ignored Content-Length b/c - # Transfer-Encoding overwrote it -- this would be naughty of them, - # but the HTTP spec says that if our peer does this then we have - # to fix it instead of erroring out, so we'll accord the user the - # same respect). - headers = set_comma_header(headers, b"content-length", []) - if self.their_http_version is None or self.their_http_version < b"1.1": - # Either we never got a valid request and are sending back an - # error (their_http_version is None), so we assume the worst; - # or else we did get a valid HTTP/1.0 request, so we know that - # they don't understand chunked encoding. - headers = set_comma_header(headers, b"transfer-encoding", []) - # This is actually redundant ATM, since currently we - # unconditionally disable keep-alive when talking to HTTP/1.0 - # peers. But let's be defensive just in case we add - # Connection: keep-alive support later: - if self._request_method != b"HEAD": - need_close = True - else: - headers = set_comma_header(headers, b"transfer-encoding", [b"chunked"]) - - if not self._cstate.keep_alive or need_close: - # Make sure Connection: close is set - connection = set(get_comma_header(headers, b"connection")) - connection.discard(b"keep-alive") - connection.add(b"close") - headers = set_comma_header(headers, b"connection", sorted(connection)) - - return Response( - headers=headers, - status_code=response.status_code, - http_version=response.http_version, - reason=response.reason, - ) diff --git a/bundle/python-cpu/Lib/site-packages/h11/_events.py b/bundle/python-cpu/Lib/site-packages/h11/_events.py deleted file mode 100644 index ca1c3adbde2c4e7710482a18e3471f91f1da610e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/_events.py +++ /dev/null @@ -1,369 +0,0 @@ -# High level events that make up HTTP/1.1 conversations. Loosely inspired by -# the corresponding events in hyper-h2: -# -# http://python-hyper.org/h2/en/stable/api.html#events -# -# Don't subclass these. Stuff will break. - -import re -from abc import ABC -from dataclasses import dataclass -from typing import List, Tuple, Union - -from ._abnf import method, request_target -from ._headers import Headers, normalize_and_validate -from ._util import bytesify, LocalProtocolError, validate - -# Everything in __all__ gets re-exported as part of the h11 public API. -__all__ = [ - "Event", - "Request", - "InformationalResponse", - "Response", - "Data", - "EndOfMessage", - "ConnectionClosed", -] - -method_re = re.compile(method.encode("ascii")) -request_target_re = re.compile(request_target.encode("ascii")) - - -class Event(ABC): - """ - Base class for h11 events. - """ - - __slots__ = () - - -@dataclass(init=False, frozen=True) -class Request(Event): - """The beginning of an HTTP request. - - Fields: - - .. attribute:: method - - An HTTP method, e.g. ``b"GET"`` or ``b"POST"``. Always a byte - string. :term:`Bytes-like objects ` and native - strings containing only ascii characters will be automatically - converted to byte strings. - - .. attribute:: target - - The target of an HTTP request, e.g. ``b"/index.html"``, or one of the - more exotic formats described in `RFC 7320, section 5.3 - `_. Always a byte - string. :term:`Bytes-like objects ` and native - strings containing only ascii characters will be automatically - converted to byte strings. - - .. attribute:: headers - - Request headers, represented as a list of (name, value) pairs. See - :ref:`the header normalization rules ` for details. - - .. attribute:: http_version - - The HTTP protocol version, represented as a byte string like - ``b"1.1"``. See :ref:`the HTTP version normalization rules - ` for details. - - """ - - __slots__ = ("method", "headers", "target", "http_version") - - method: bytes - headers: Headers - target: bytes - http_version: bytes - - def __init__( - self, - *, - method: Union[bytes, str], - headers: Union[Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]]], - target: Union[bytes, str], - http_version: Union[bytes, str] = b"1.1", - _parsed: bool = False, - ) -> None: - super().__init__() - if isinstance(headers, Headers): - object.__setattr__(self, "headers", headers) - else: - object.__setattr__( - self, "headers", normalize_and_validate(headers, _parsed=_parsed) - ) - if not _parsed: - object.__setattr__(self, "method", bytesify(method)) - object.__setattr__(self, "target", bytesify(target)) - object.__setattr__(self, "http_version", bytesify(http_version)) - else: - object.__setattr__(self, "method", method) - object.__setattr__(self, "target", target) - object.__setattr__(self, "http_version", http_version) - - # "A server MUST respond with a 400 (Bad Request) status code to any - # HTTP/1.1 request message that lacks a Host header field and to any - # request message that contains more than one Host header field or a - # Host header field with an invalid field-value." - # -- https://tools.ietf.org/html/rfc7230#section-5.4 - host_count = 0 - for name, value in self.headers: - if name == b"host": - host_count += 1 - if self.http_version == b"1.1" and host_count == 0: - raise LocalProtocolError("Missing mandatory Host: header") - if host_count > 1: - raise LocalProtocolError("Found multiple Host: headers") - - validate(method_re, self.method, "Illegal method characters") - validate(request_target_re, self.target, "Illegal target characters") - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class _ResponseBase(Event): - __slots__ = ("headers", "http_version", "reason", "status_code") - - headers: Headers - http_version: bytes - reason: bytes - status_code: int - - def __init__( - self, - *, - headers: Union[Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]]], - status_code: int, - http_version: Union[bytes, str] = b"1.1", - reason: Union[bytes, str] = b"", - _parsed: bool = False, - ) -> None: - super().__init__() - if isinstance(headers, Headers): - object.__setattr__(self, "headers", headers) - else: - object.__setattr__( - self, "headers", normalize_and_validate(headers, _parsed=_parsed) - ) - if not _parsed: - object.__setattr__(self, "reason", bytesify(reason)) - object.__setattr__(self, "http_version", bytesify(http_version)) - if not isinstance(status_code, int): - raise LocalProtocolError("status code must be integer") - # Because IntEnum objects are instances of int, but aren't - # duck-compatible (sigh), see gh-72. - object.__setattr__(self, "status_code", int(status_code)) - else: - object.__setattr__(self, "reason", reason) - object.__setattr__(self, "http_version", http_version) - object.__setattr__(self, "status_code", status_code) - - self.__post_init__() - - def __post_init__(self) -> None: - pass - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class InformationalResponse(_ResponseBase): - """An HTTP informational response. - - Fields: - - .. attribute:: status_code - - The status code of this response, as an integer. For an - :class:`InformationalResponse`, this is always in the range [100, - 200). - - .. attribute:: headers - - Request headers, represented as a list of (name, value) pairs. See - :ref:`the header normalization rules ` for - details. - - .. attribute:: http_version - - The HTTP protocol version, represented as a byte string like - ``b"1.1"``. See :ref:`the HTTP version normalization rules - ` for details. - - .. attribute:: reason - - The reason phrase of this response, as a byte string. For example: - ``b"OK"``, or ``b"Not Found"``. - - """ - - def __post_init__(self) -> None: - if not (100 <= self.status_code < 200): - raise LocalProtocolError( - "InformationalResponse status_code should be in range " - "[100, 200), not {}".format(self.status_code) - ) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class Response(_ResponseBase): - """The beginning of an HTTP response. - - Fields: - - .. attribute:: status_code - - The status code of this response, as an integer. For an - :class:`Response`, this is always in the range [200, - 1000). - - .. attribute:: headers - - Request headers, represented as a list of (name, value) pairs. See - :ref:`the header normalization rules ` for details. - - .. attribute:: http_version - - The HTTP protocol version, represented as a byte string like - ``b"1.1"``. See :ref:`the HTTP version normalization rules - ` for details. - - .. attribute:: reason - - The reason phrase of this response, as a byte string. For example: - ``b"OK"``, or ``b"Not Found"``. - - """ - - def __post_init__(self) -> None: - if not (200 <= self.status_code < 1000): - raise LocalProtocolError( - "Response status_code should be in range [200, 1000), not {}".format( - self.status_code - ) - ) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class Data(Event): - """Part of an HTTP message body. - - Fields: - - .. attribute:: data - - A :term:`bytes-like object` containing part of a message body. Or, if - using the ``combine=False`` argument to :meth:`Connection.send`, then - any object that your socket writing code knows what to do with, and for - which calling :func:`len` returns the number of bytes that will be - written -- see :ref:`sendfile` for details. - - .. attribute:: chunk_start - - A marker that indicates whether this data object is from the start of a - chunked transfer encoding chunk. This field is ignored when when a Data - event is provided to :meth:`Connection.send`: it is only valid on - events emitted from :meth:`Connection.next_event`. You probably - shouldn't use this attribute at all; see - :ref:`chunk-delimiters-are-bad` for details. - - .. attribute:: chunk_end - - A marker that indicates whether this data object is the last for a - given chunked transfer encoding chunk. This field is ignored when when - a Data event is provided to :meth:`Connection.send`: it is only valid - on events emitted from :meth:`Connection.next_event`. You probably - shouldn't use this attribute at all; see - :ref:`chunk-delimiters-are-bad` for details. - - """ - - __slots__ = ("data", "chunk_start", "chunk_end") - - data: bytes - chunk_start: bool - chunk_end: bool - - def __init__( - self, data: bytes, chunk_start: bool = False, chunk_end: bool = False - ) -> None: - object.__setattr__(self, "data", data) - object.__setattr__(self, "chunk_start", chunk_start) - object.__setattr__(self, "chunk_end", chunk_end) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -# XX FIXME: "A recipient MUST ignore (or consider as an error) any fields that -# are forbidden to be sent in a trailer, since processing them as if they were -# present in the header section might bypass external security filters." -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#chunked.trailer.part -# Unfortunately, the list of forbidden fields is long and vague :-/ -@dataclass(init=False, frozen=True) -class EndOfMessage(Event): - """The end of an HTTP message. - - Fields: - - .. attribute:: headers - - Default value: ``[]`` - - Any trailing headers attached to this message, represented as a list of - (name, value) pairs. See :ref:`the header normalization rules - ` for details. - - Must be empty unless ``Transfer-Encoding: chunked`` is in use. - - """ - - __slots__ = ("headers",) - - headers: Headers - - def __init__( - self, - *, - headers: Union[ - Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]], None - ] = None, - _parsed: bool = False, - ) -> None: - super().__init__() - if headers is None: - headers = Headers([]) - elif not isinstance(headers, Headers): - headers = normalize_and_validate(headers, _parsed=_parsed) - - object.__setattr__(self, "headers", headers) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(frozen=True) -class ConnectionClosed(Event): - """This event indicates that the sender has closed their outgoing - connection. - - Note that this does not necessarily mean that they can't *receive* further - data, because TCP connections are composed to two one-way channels which - can be closed independently. See :ref:`closing` for details. - - No fields. - """ - - pass diff --git a/bundle/python-cpu/Lib/site-packages/h11/_headers.py b/bundle/python-cpu/Lib/site-packages/h11/_headers.py deleted file mode 100644 index 31da3e2b23b55a624b36f105e62a6902e63286aa..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/_headers.py +++ /dev/null @@ -1,282 +0,0 @@ -import re -from typing import AnyStr, cast, List, overload, Sequence, Tuple, TYPE_CHECKING, Union - -from ._abnf import field_name, field_value -from ._util import bytesify, LocalProtocolError, validate - -if TYPE_CHECKING: - from ._events import Request - -try: - from typing import Literal -except ImportError: - from typing_extensions import Literal # type: ignore - -CONTENT_LENGTH_MAX_DIGITS = 20 # allow up to 1 billion TB - 1 - - -# Facts -# ----- -# -# Headers are: -# keys: case-insensitive ascii -# values: mixture of ascii and raw bytes -# -# "Historically, HTTP has allowed field content with text in the ISO-8859-1 -# charset [ISO-8859-1], supporting other charsets only through use of -# [RFC2047] encoding. In practice, most HTTP header field values use only a -# subset of the US-ASCII charset [USASCII]. Newly defined header fields SHOULD -# limit their field values to US-ASCII octets. A recipient SHOULD treat other -# octets in field content (obs-text) as opaque data." -# And it deprecates all non-ascii values -# -# Leading/trailing whitespace in header names is forbidden -# -# Values get leading/trailing whitespace stripped -# -# Content-Disposition actually needs to contain unicode semantically; to -# accomplish this it has a terrifically weird way of encoding the filename -# itself as ascii (and even this still has lots of cross-browser -# incompatibilities) -# -# Order is important: -# "a proxy MUST NOT change the order of these field values when forwarding a -# message" -# (and there are several headers where the order indicates a preference) -# -# Multiple occurences of the same header: -# "A sender MUST NOT generate multiple header fields with the same field name -# in a message unless either the entire field value for that header field is -# defined as a comma-separated list [or the header is Set-Cookie which gets a -# special exception]" - RFC 7230. (cookies are in RFC 6265) -# -# So every header aside from Set-Cookie can be merged by b", ".join if it -# occurs repeatedly. But, of course, they can't necessarily be split by -# .split(b","), because quoting. -# -# Given all this mess (case insensitive, duplicates allowed, order is -# important, ...), there doesn't appear to be any standard way to handle -# headers in Python -- they're almost like dicts, but... actually just -# aren't. For now we punt and just use a super simple representation: headers -# are a list of pairs -# -# [(name1, value1), (name2, value2), ...] -# -# where all entries are bytestrings, names are lowercase and have no -# leading/trailing whitespace, and values are bytestrings with no -# leading/trailing whitespace. Searching and updating are done via naive O(n) -# methods. -# -# Maybe a dict-of-lists would be better? - -_content_length_re = re.compile(rb"[0-9]+") -_field_name_re = re.compile(field_name.encode("ascii")) -_field_value_re = re.compile(field_value.encode("ascii")) - - -class Headers(Sequence[Tuple[bytes, bytes]]): - """ - A list-like interface that allows iterating over headers as byte-pairs - of (lowercased-name, value). - - Internally we actually store the representation as three-tuples, - including both the raw original casing, in order to preserve casing - over-the-wire, and the lowercased name, for case-insensitive comparisions. - - r = Request( - method="GET", - target="/", - headers=[("Host", "example.org"), ("Connection", "keep-alive")], - http_version="1.1", - ) - assert r.headers == [ - (b"host", b"example.org"), - (b"connection", b"keep-alive") - ] - assert r.headers.raw_items() == [ - (b"Host", b"example.org"), - (b"Connection", b"keep-alive") - ] - """ - - __slots__ = "_full_items" - - def __init__(self, full_items: List[Tuple[bytes, bytes, bytes]]) -> None: - self._full_items = full_items - - def __bool__(self) -> bool: - return bool(self._full_items) - - def __eq__(self, other: object) -> bool: - return list(self) == list(other) # type: ignore - - def __len__(self) -> int: - return len(self._full_items) - - def __repr__(self) -> str: - return "" % repr(list(self)) - - def __getitem__(self, idx: int) -> Tuple[bytes, bytes]: # type: ignore[override] - _, name, value = self._full_items[idx] - return (name, value) - - def raw_items(self) -> List[Tuple[bytes, bytes]]: - return [(raw_name, value) for raw_name, _, value in self._full_items] - - -HeaderTypes = Union[ - List[Tuple[bytes, bytes]], - List[Tuple[bytes, str]], - List[Tuple[str, bytes]], - List[Tuple[str, str]], -] - - -@overload -def normalize_and_validate(headers: Headers, _parsed: Literal[True]) -> Headers: - ... - - -@overload -def normalize_and_validate(headers: HeaderTypes, _parsed: Literal[False]) -> Headers: - ... - - -@overload -def normalize_and_validate( - headers: Union[Headers, HeaderTypes], _parsed: bool = False -) -> Headers: - ... - - -def normalize_and_validate( - headers: Union[Headers, HeaderTypes], _parsed: bool = False -) -> Headers: - new_headers = [] - seen_content_length = None - saw_transfer_encoding = False - for name, value in headers: - # For headers coming out of the parser, we can safely skip some steps, - # because it always returns bytes and has already run these regexes - # over the data: - if not _parsed: - name = bytesify(name) - value = bytesify(value) - validate(_field_name_re, name, "Illegal header name {!r}", name) - validate(_field_value_re, value, "Illegal header value {!r}", value) - assert isinstance(name, bytes) - assert isinstance(value, bytes) - - raw_name = name - name = name.lower() - if name == b"content-length": - lengths = {length.strip() for length in value.split(b",")} - if len(lengths) != 1: - raise LocalProtocolError("conflicting Content-Length headers") - value = lengths.pop() - validate(_content_length_re, value, "bad Content-Length") - if len(value) > CONTENT_LENGTH_MAX_DIGITS: - raise LocalProtocolError("bad Content-Length") - if seen_content_length is None: - seen_content_length = value - new_headers.append((raw_name, name, value)) - elif seen_content_length != value: - raise LocalProtocolError("conflicting Content-Length headers") - elif name == b"transfer-encoding": - # "A server that receives a request message with a transfer coding - # it does not understand SHOULD respond with 501 (Not - # Implemented)." - # https://tools.ietf.org/html/rfc7230#section-3.3.1 - if saw_transfer_encoding: - raise LocalProtocolError( - "multiple Transfer-Encoding headers", error_status_hint=501 - ) - # "All transfer-coding names are case-insensitive" - # -- https://tools.ietf.org/html/rfc7230#section-4 - value = value.lower() - if value != b"chunked": - raise LocalProtocolError( - "Only Transfer-Encoding: chunked is supported", - error_status_hint=501, - ) - saw_transfer_encoding = True - new_headers.append((raw_name, name, value)) - else: - new_headers.append((raw_name, name, value)) - return Headers(new_headers) - - -def get_comma_header(headers: Headers, name: bytes) -> List[bytes]: - # Should only be used for headers whose value is a list of - # comma-separated, case-insensitive values. - # - # The header name `name` is expected to be lower-case bytes. - # - # Connection: meets these criteria (including cast insensitivity). - # - # Content-Length: technically is just a single value (1*DIGIT), but the - # standard makes reference to implementations that do multiple values, and - # using this doesn't hurt. Ditto, case insensitivity doesn't things either - # way. - # - # Transfer-Encoding: is more complex (allows for quoted strings), so - # splitting on , is actually wrong. For example, this is legal: - # - # Transfer-Encoding: foo; options="1,2", chunked - # - # and should be parsed as - # - # foo; options="1,2" - # chunked - # - # but this naive function will parse it as - # - # foo; options="1 - # 2" - # chunked - # - # However, this is okay because the only thing we are going to do with - # any Transfer-Encoding is reject ones that aren't just "chunked", so - # both of these will be treated the same anyway. - # - # Expect: the only legal value is the literal string - # "100-continue". Splitting on commas is harmless. Case insensitive. - # - out: List[bytes] = [] - for _, found_name, found_raw_value in headers._full_items: - if found_name == name: - found_raw_value = found_raw_value.lower() - for found_split_value in found_raw_value.split(b","): - found_split_value = found_split_value.strip() - if found_split_value: - out.append(found_split_value) - return out - - -def set_comma_header(headers: Headers, name: bytes, new_values: List[bytes]) -> Headers: - # The header name `name` is expected to be lower-case bytes. - # - # Note that when we store the header we use title casing for the header - # names, in order to match the conventional HTTP header style. - # - # Simply calling `.title()` is a blunt approach, but it's correct - # here given the cases where we're using `set_comma_header`... - # - # Connection, Content-Length, Transfer-Encoding. - new_headers: List[Tuple[bytes, bytes]] = [] - for found_raw_name, found_name, found_raw_value in headers._full_items: - if found_name != name: - new_headers.append((found_raw_name, found_raw_value)) - for new_value in new_values: - new_headers.append((name.title(), new_value)) - return normalize_and_validate(new_headers) - - -def has_expect_100_continue(request: "Request") -> bool: - # https://tools.ietf.org/html/rfc7231#section-5.1.1 - # "A server that receives a 100-continue expectation in an HTTP/1.0 request - # MUST ignore that expectation." - if request.http_version < b"1.1": - return False - expect = get_comma_header(request.headers, b"expect") - return b"100-continue" in expect diff --git a/bundle/python-cpu/Lib/site-packages/h11/_readers.py b/bundle/python-cpu/Lib/site-packages/h11/_readers.py deleted file mode 100644 index 576804cc282032526e0a932c9853d586a094bad0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/_readers.py +++ /dev/null @@ -1,250 +0,0 @@ -# Code to read HTTP data -# -# Strategy: each reader is a callable which takes a ReceiveBuffer object, and -# either: -# 1) consumes some of it and returns an Event -# 2) raises a LocalProtocolError (for consistency -- e.g. we call validate() -# and it might raise a LocalProtocolError, so simpler just to always use -# this) -# 3) returns None, meaning "I need more data" -# -# If they have a .read_eof attribute, then this will be called if an EOF is -# received -- but this is optional. Either way, the actual ConnectionClosed -# event will be generated afterwards. -# -# READERS is a dict describing how to pick a reader. It maps states to either: -# - a reader -# - or, for body readers, a dict of per-framing reader factories - -import re -from typing import Any, Callable, Dict, Iterable, NoReturn, Optional, Tuple, Type, Union - -from ._abnf import chunk_header, header_field, request_line, status_line -from ._events import Data, EndOfMessage, InformationalResponse, Request, Response -from ._receivebuffer import ReceiveBuffer -from ._state import ( - CLIENT, - CLOSED, - DONE, - IDLE, - MUST_CLOSE, - SEND_BODY, - SEND_RESPONSE, - SERVER, -) -from ._util import LocalProtocolError, RemoteProtocolError, Sentinel, validate - -__all__ = ["READERS"] - -header_field_re = re.compile(header_field.encode("ascii")) -obs_fold_re = re.compile(rb"[ \t]+") - - -def _obsolete_line_fold(lines: Iterable[bytes]) -> Iterable[bytes]: - it = iter(lines) - last: Optional[bytes] = None - for line in it: - match = obs_fold_re.match(line) - if match: - if last is None: - raise LocalProtocolError("continuation line at start of headers") - if not isinstance(last, bytearray): - # Cast to a mutable type, avoiding copy on append to ensure O(n) time - last = bytearray(last) - last += b" " - last += line[match.end() :] - else: - if last is not None: - yield last - last = line - if last is not None: - yield last - - -def _decode_header_lines( - lines: Iterable[bytes], -) -> Iterable[Tuple[bytes, bytes]]: - for line in _obsolete_line_fold(lines): - matches = validate(header_field_re, line, "illegal header line: {!r}", line) - yield (matches["field_name"], matches["field_value"]) - - -request_line_re = re.compile(request_line.encode("ascii")) - - -def maybe_read_from_IDLE_client(buf: ReceiveBuffer) -> Optional[Request]: - lines = buf.maybe_extract_lines() - if lines is None: - if buf.is_next_line_obviously_invalid_request_line(): - raise LocalProtocolError("illegal request line") - return None - if not lines: - raise LocalProtocolError("no request line received") - matches = validate( - request_line_re, lines[0], "illegal request line: {!r}", lines[0] - ) - return Request( - headers=list(_decode_header_lines(lines[1:])), _parsed=True, **matches - ) - - -status_line_re = re.compile(status_line.encode("ascii")) - - -def maybe_read_from_SEND_RESPONSE_server( - buf: ReceiveBuffer, -) -> Union[InformationalResponse, Response, None]: - lines = buf.maybe_extract_lines() - if lines is None: - if buf.is_next_line_obviously_invalid_request_line(): - raise LocalProtocolError("illegal request line") - return None - if not lines: - raise LocalProtocolError("no response line received") - matches = validate(status_line_re, lines[0], "illegal status line: {!r}", lines[0]) - http_version = ( - b"1.1" if matches["http_version"] is None else matches["http_version"] - ) - reason = b"" if matches["reason"] is None else matches["reason"] - status_code = int(matches["status_code"]) - class_: Union[Type[InformationalResponse], Type[Response]] = ( - InformationalResponse if status_code < 200 else Response - ) - return class_( - headers=list(_decode_header_lines(lines[1:])), - _parsed=True, - status_code=status_code, - reason=reason, - http_version=http_version, - ) - - -class ContentLengthReader: - def __init__(self, length: int) -> None: - self._length = length - self._remaining = length - - def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: - if self._remaining == 0: - return EndOfMessage() - data = buf.maybe_extract_at_most(self._remaining) - if data is None: - return None - self._remaining -= len(data) - return Data(data=data) - - def read_eof(self) -> NoReturn: - raise RemoteProtocolError( - "peer closed connection without sending complete message body " - "(received {} bytes, expected {})".format( - self._length - self._remaining, self._length - ) - ) - - -chunk_header_re = re.compile(chunk_header.encode("ascii")) - - -class ChunkedReader: - def __init__(self) -> None: - self._bytes_in_chunk = 0 - # After reading a chunk, we have to throw away the trailing \r\n. - # This tracks the bytes that we need to match and throw away. - self._bytes_to_discard = b"" - self._reading_trailer = False - - def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: - if self._reading_trailer: - lines = buf.maybe_extract_lines() - if lines is None: - return None - return EndOfMessage(headers=list(_decode_header_lines(lines))) - if self._bytes_to_discard: - data = buf.maybe_extract_at_most(len(self._bytes_to_discard)) - if data is None: - return None - if data != self._bytes_to_discard[: len(data)]: - raise LocalProtocolError( - f"malformed chunk footer: {data!r} (expected {self._bytes_to_discard!r})" - ) - self._bytes_to_discard = self._bytes_to_discard[len(data) :] - if self._bytes_to_discard: - return None - # else, fall through and read some more - assert self._bytes_to_discard == b"" - if self._bytes_in_chunk == 0: - # We need to refill our chunk count - chunk_header = buf.maybe_extract_next_line() - if chunk_header is None: - return None - matches = validate( - chunk_header_re, - chunk_header, - "illegal chunk header: {!r}", - chunk_header, - ) - # XX FIXME: we discard chunk extensions. Does anyone care? - self._bytes_in_chunk = int(matches["chunk_size"], base=16) - if self._bytes_in_chunk == 0: - self._reading_trailer = True - return self(buf) - chunk_start = True - else: - chunk_start = False - assert self._bytes_in_chunk > 0 - data = buf.maybe_extract_at_most(self._bytes_in_chunk) - if data is None: - return None - self._bytes_in_chunk -= len(data) - if self._bytes_in_chunk == 0: - self._bytes_to_discard = b"\r\n" - chunk_end = True - else: - chunk_end = False - return Data(data=data, chunk_start=chunk_start, chunk_end=chunk_end) - - def read_eof(self) -> NoReturn: - raise RemoteProtocolError( - "peer closed connection without sending complete message body " - "(incomplete chunked read)" - ) - - -class Http10Reader: - def __call__(self, buf: ReceiveBuffer) -> Optional[Data]: - data = buf.maybe_extract_at_most(999999999) - if data is None: - return None - return Data(data=data) - - def read_eof(self) -> EndOfMessage: - return EndOfMessage() - - -def expect_nothing(buf: ReceiveBuffer) -> None: - if buf: - raise LocalProtocolError("Got data when expecting EOF") - return None - - -ReadersType = Dict[ - Union[Type[Sentinel], Tuple[Type[Sentinel], Type[Sentinel]]], - Union[Callable[..., Any], Dict[str, Callable[..., Any]]], -] - -READERS: ReadersType = { - (CLIENT, IDLE): maybe_read_from_IDLE_client, - (SERVER, IDLE): maybe_read_from_SEND_RESPONSE_server, - (SERVER, SEND_RESPONSE): maybe_read_from_SEND_RESPONSE_server, - (CLIENT, DONE): expect_nothing, - (CLIENT, MUST_CLOSE): expect_nothing, - (CLIENT, CLOSED): expect_nothing, - (SERVER, DONE): expect_nothing, - (SERVER, MUST_CLOSE): expect_nothing, - (SERVER, CLOSED): expect_nothing, - SEND_BODY: { - "chunked": ChunkedReader, - "content-length": ContentLengthReader, - "http/1.0": Http10Reader, - }, -} diff --git a/bundle/python-cpu/Lib/site-packages/h11/_receivebuffer.py b/bundle/python-cpu/Lib/site-packages/h11/_receivebuffer.py deleted file mode 100644 index e5c4e08a56f5081e87103f38b4add6ce1b730204..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/_receivebuffer.py +++ /dev/null @@ -1,153 +0,0 @@ -import re -import sys -from typing import List, Optional, Union - -__all__ = ["ReceiveBuffer"] - - -# Operations we want to support: -# - find next \r\n or \r\n\r\n (\n or \n\n are also acceptable), -# or wait until there is one -# - read at-most-N bytes -# Goals: -# - on average, do this fast -# - worst case, do this in O(n) where n is the number of bytes processed -# Plan: -# - store bytearray, offset, how far we've searched for a separator token -# - use the how-far-we've-searched data to avoid rescanning -# - while doing a stream of uninterrupted processing, advance offset instead -# of constantly copying -# WARNING: -# - I haven't benchmarked or profiled any of this yet. -# -# Note that starting in Python 3.4, deleting the initial n bytes from a -# bytearray is amortized O(n), thanks to some excellent work by Antoine -# Martin: -# -# https://bugs.python.org/issue19087 -# -# This means that if we only supported 3.4+, we could get rid of the code here -# involving self._start and self.compress, because it's doing exactly the same -# thing that bytearray now does internally. -# -# BUT unfortunately, we still support 2.7, and reading short segments out of a -# long buffer MUST be O(bytes read) to avoid DoS issues, so we can't actually -# delete this code. Yet: -# -# https://pythonclock.org/ -# -# (Two things to double-check first though: make sure PyPy also has the -# optimization, and benchmark to make sure it's a win, since we do have a -# slightly clever thing where we delay calling compress() until we've -# processed a whole event, which could in theory be slightly more efficient -# than the internal bytearray support.) -blank_line_regex = re.compile(b"\n\r?\n", re.MULTILINE) - - -class ReceiveBuffer: - def __init__(self) -> None: - self._data = bytearray() - self._next_line_search = 0 - self._multiple_lines_search = 0 - - def __iadd__(self, byteslike: Union[bytes, bytearray]) -> "ReceiveBuffer": - self._data += byteslike - return self - - def __bool__(self) -> bool: - return bool(len(self)) - - def __len__(self) -> int: - return len(self._data) - - # for @property unprocessed_data - def __bytes__(self) -> bytes: - return bytes(self._data) - - def _extract(self, count: int) -> bytearray: - # extracting an initial slice of the data buffer and return it - out = self._data[:count] - del self._data[:count] - - self._next_line_search = 0 - self._multiple_lines_search = 0 - - return out - - def maybe_extract_at_most(self, count: int) -> Optional[bytearray]: - """ - Extract a fixed number of bytes from the buffer. - """ - out = self._data[:count] - if not out: - return None - - return self._extract(count) - - def maybe_extract_next_line(self) -> Optional[bytearray]: - """ - Extract the first line, if it is completed in the buffer. - """ - # Only search in buffer space that we've not already looked at. - search_start_index = max(0, self._next_line_search - 1) - partial_idx = self._data.find(b"\r\n", search_start_index) - - if partial_idx == -1: - self._next_line_search = len(self._data) - return None - - # + 2 is to compensate len(b"\r\n") - idx = partial_idx + 2 - - return self._extract(idx) - - def maybe_extract_lines(self) -> Optional[List[bytearray]]: - """ - Extract everything up to the first blank line, and return a list of lines. - """ - # Handle the case where we have an immediate empty line. - if self._data[:1] == b"\n": - self._extract(1) - return [] - - if self._data[:2] == b"\r\n": - self._extract(2) - return [] - - # Only search in buffer space that we've not already looked at. - match = blank_line_regex.search(self._data, self._multiple_lines_search) - if match is None: - self._multiple_lines_search = max(0, len(self._data) - 2) - return None - - # Truncate the buffer and return it. - idx = match.span(0)[-1] - out = self._extract(idx) - lines = out.split(b"\n") - - for line in lines: - if line.endswith(b"\r"): - del line[-1] - - assert lines[-2] == lines[-1] == b"" - - del lines[-2:] - - return lines - - # In theory we should wait until `\r\n` before starting to validate - # incoming data. However it's interesting to detect (very) invalid data - # early given they might not even contain `\r\n` at all (hence only - # timeout will get rid of them). - # This is not a 100% effective detection but more of a cheap sanity check - # allowing for early abort in some useful cases. - # This is especially interesting when peer is messing up with HTTPS and - # sent us a TLS stream where we were expecting plain HTTP given all - # versions of TLS so far start handshake with a 0x16 message type code. - def is_next_line_obviously_invalid_request_line(self) -> bool: - try: - # HTTP header line must not contain non-printable characters - # and should not start with a space - return self._data[0] < 0x21 - except IndexError: - return False diff --git a/bundle/python-cpu/Lib/site-packages/h11/_state.py b/bundle/python-cpu/Lib/site-packages/h11/_state.py deleted file mode 100644 index 3ad444b043e3f3d6c05c2d9d84d5119312bfaa34..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/_state.py +++ /dev/null @@ -1,365 +0,0 @@ -################################################################ -# The core state machine -################################################################ -# -# Rule 1: everything that affects the state machine and state transitions must -# live here in this file. As much as possible goes into the table-based -# representation, but for the bits that don't quite fit, the actual code and -# state must nonetheless live here. -# -# Rule 2: this file does not know about what role we're playing; it only knows -# about HTTP request/response cycles in the abstract. This ensures that we -# don't cheat and apply different rules to local and remote parties. -# -# -# Theory of operation -# =================== -# -# Possibly the simplest way to think about this is that we actually have 5 -# different state machines here. Yes, 5. These are: -# -# 1) The client state, with its complicated automaton (see the docs) -# 2) The server state, with its complicated automaton (see the docs) -# 3) The keep-alive state, with possible states {True, False} -# 4) The SWITCH_CONNECT state, with possible states {False, True} -# 5) The SWITCH_UPGRADE state, with possible states {False, True} -# -# For (3)-(5), the first state listed is the initial state. -# -# (1)-(3) are stored explicitly in member variables. The last -# two are stored implicitly in the pending_switch_proposals set as: -# (state of 4) == (_SWITCH_CONNECT in pending_switch_proposals) -# (state of 5) == (_SWITCH_UPGRADE in pending_switch_proposals) -# -# And each of these machines has two different kinds of transitions: -# -# a) Event-triggered -# b) State-triggered -# -# Event triggered is the obvious thing that you'd think it is: some event -# happens, and if it's the right event at the right time then a transition -# happens. But there are somewhat complicated rules for which machines can -# "see" which events. (As a rule of thumb, if a machine "sees" an event, this -# means two things: the event can affect the machine, and if the machine is -# not in a state where it expects that event then it's an error.) These rules -# are: -# -# 1) The client machine sees all h11.events objects emitted by the client. -# -# 2) The server machine sees all h11.events objects emitted by the server. -# -# It also sees the client's Request event. -# -# And sometimes, server events are annotated with a _SWITCH_* event. For -# example, we can have a (Response, _SWITCH_CONNECT) event, which is -# different from a regular Response event. -# -# 3) The keep-alive machine sees the process_keep_alive_disabled() event -# (which is derived from Request/Response events), and this event -# transitions it from True -> False, or from False -> False. There's no way -# to transition back. -# -# 4&5) The _SWITCH_* machines transition from False->True when we get a -# Request that proposes the relevant type of switch (via -# process_client_switch_proposals), and they go from True->False when we -# get a Response that has no _SWITCH_* annotation. -# -# So that's event-triggered transitions. -# -# State-triggered transitions are less standard. What they do here is couple -# the machines together. The way this works is, when certain *joint* -# configurations of states are achieved, then we automatically transition to a -# new *joint* state. So, for example, if we're ever in a joint state with -# -# client: DONE -# keep-alive: False -# -# then the client state immediately transitions to: -# -# client: MUST_CLOSE -# -# This is fundamentally different from an event-based transition, because it -# doesn't matter how we arrived at the {client: DONE, keep-alive: False} state -# -- maybe the client transitioned SEND_BODY -> DONE, or keep-alive -# transitioned True -> False. Either way, once this precondition is satisfied, -# this transition is immediately triggered. -# -# What if two conflicting state-based transitions get enabled at the same -# time? In practice there's only one case where this arises (client DONE -> -# MIGHT_SWITCH_PROTOCOL versus DONE -> MUST_CLOSE), and we resolve it by -# explicitly prioritizing the DONE -> MIGHT_SWITCH_PROTOCOL transition. -# -# Implementation -# -------------- -# -# The event-triggered transitions for the server and client machines are all -# stored explicitly in a table. Ditto for the state-triggered transitions that -# involve just the server and client state. -# -# The transitions for the other machines, and the state-triggered transitions -# that involve the other machines, are written out as explicit Python code. -# -# It'd be nice if there were some cleaner way to do all this. This isn't -# *too* terrible, but I feel like it could probably be better. -# -# WARNING -# ------- -# -# The script that generates the state machine diagrams for the docs knows how -# to read out the EVENT_TRIGGERED_TRANSITIONS and STATE_TRIGGERED_TRANSITIONS -# tables. But it can't automatically read the transitions that are written -# directly in Python code. So if you touch those, you need to also update the -# script to keep it in sync! -from typing import cast, Dict, Optional, Set, Tuple, Type, Union - -from ._events import * -from ._util import LocalProtocolError, Sentinel - -# Everything in __all__ gets re-exported as part of the h11 public API. -__all__ = [ - "CLIENT", - "SERVER", - "IDLE", - "SEND_RESPONSE", - "SEND_BODY", - "DONE", - "MUST_CLOSE", - "CLOSED", - "MIGHT_SWITCH_PROTOCOL", - "SWITCHED_PROTOCOL", - "ERROR", -] - - -class CLIENT(Sentinel, metaclass=Sentinel): - pass - - -class SERVER(Sentinel, metaclass=Sentinel): - pass - - -# States -class IDLE(Sentinel, metaclass=Sentinel): - pass - - -class SEND_RESPONSE(Sentinel, metaclass=Sentinel): - pass - - -class SEND_BODY(Sentinel, metaclass=Sentinel): - pass - - -class DONE(Sentinel, metaclass=Sentinel): - pass - - -class MUST_CLOSE(Sentinel, metaclass=Sentinel): - pass - - -class CLOSED(Sentinel, metaclass=Sentinel): - pass - - -class ERROR(Sentinel, metaclass=Sentinel): - pass - - -# Switch types -class MIGHT_SWITCH_PROTOCOL(Sentinel, metaclass=Sentinel): - pass - - -class SWITCHED_PROTOCOL(Sentinel, metaclass=Sentinel): - pass - - -class _SWITCH_UPGRADE(Sentinel, metaclass=Sentinel): - pass - - -class _SWITCH_CONNECT(Sentinel, metaclass=Sentinel): - pass - - -EventTransitionType = Dict[ - Type[Sentinel], - Dict[ - Type[Sentinel], - Dict[Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]], Type[Sentinel]], - ], -] - -EVENT_TRIGGERED_TRANSITIONS: EventTransitionType = { - CLIENT: { - IDLE: {Request: SEND_BODY, ConnectionClosed: CLOSED}, - SEND_BODY: {Data: SEND_BODY, EndOfMessage: DONE}, - DONE: {ConnectionClosed: CLOSED}, - MUST_CLOSE: {ConnectionClosed: CLOSED}, - CLOSED: {ConnectionClosed: CLOSED}, - MIGHT_SWITCH_PROTOCOL: {}, - SWITCHED_PROTOCOL: {}, - ERROR: {}, - }, - SERVER: { - IDLE: { - ConnectionClosed: CLOSED, - Response: SEND_BODY, - # Special case: server sees client Request events, in this form - (Request, CLIENT): SEND_RESPONSE, - }, - SEND_RESPONSE: { - InformationalResponse: SEND_RESPONSE, - Response: SEND_BODY, - (InformationalResponse, _SWITCH_UPGRADE): SWITCHED_PROTOCOL, - (Response, _SWITCH_CONNECT): SWITCHED_PROTOCOL, - }, - SEND_BODY: {Data: SEND_BODY, EndOfMessage: DONE}, - DONE: {ConnectionClosed: CLOSED}, - MUST_CLOSE: {ConnectionClosed: CLOSED}, - CLOSED: {ConnectionClosed: CLOSED}, - SWITCHED_PROTOCOL: {}, - ERROR: {}, - }, -} - -StateTransitionType = Dict[ - Tuple[Type[Sentinel], Type[Sentinel]], Dict[Type[Sentinel], Type[Sentinel]] -] - -# NB: there are also some special-case state-triggered transitions hard-coded -# into _fire_state_triggered_transitions below. -STATE_TRIGGERED_TRANSITIONS: StateTransitionType = { - # (Client state, Server state) -> new states - # Protocol negotiation - (MIGHT_SWITCH_PROTOCOL, SWITCHED_PROTOCOL): {CLIENT: SWITCHED_PROTOCOL}, - # Socket shutdown - (CLOSED, DONE): {SERVER: MUST_CLOSE}, - (CLOSED, IDLE): {SERVER: MUST_CLOSE}, - (ERROR, DONE): {SERVER: MUST_CLOSE}, - (DONE, CLOSED): {CLIENT: MUST_CLOSE}, - (IDLE, CLOSED): {CLIENT: MUST_CLOSE}, - (DONE, ERROR): {CLIENT: MUST_CLOSE}, -} - - -class ConnectionState: - def __init__(self) -> None: - # Extra bits of state that don't quite fit into the state model. - - # If this is False then it enables the automatic DONE -> MUST_CLOSE - # transition. Don't set this directly; call .keep_alive_disabled() - self.keep_alive = True - - # This is a subset of {UPGRADE, CONNECT}, containing the proposals - # made by the client for switching protocols. - self.pending_switch_proposals: Set[Type[Sentinel]] = set() - - self.states: Dict[Type[Sentinel], Type[Sentinel]] = {CLIENT: IDLE, SERVER: IDLE} - - def process_error(self, role: Type[Sentinel]) -> None: - self.states[role] = ERROR - self._fire_state_triggered_transitions() - - def process_keep_alive_disabled(self) -> None: - self.keep_alive = False - self._fire_state_triggered_transitions() - - def process_client_switch_proposal(self, switch_event: Type[Sentinel]) -> None: - self.pending_switch_proposals.add(switch_event) - self._fire_state_triggered_transitions() - - def process_event( - self, - role: Type[Sentinel], - event_type: Type[Event], - server_switch_event: Optional[Type[Sentinel]] = None, - ) -> None: - _event_type: Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]] = event_type - if server_switch_event is not None: - assert role is SERVER - if server_switch_event not in self.pending_switch_proposals: - raise LocalProtocolError( - "Received server _SWITCH_UPGRADE event without a pending proposal" - ) - _event_type = (event_type, server_switch_event) - if server_switch_event is None and _event_type is Response: - self.pending_switch_proposals = set() - self._fire_event_triggered_transitions(role, _event_type) - # Special case: the server state does get to see Request - # events. - if _event_type is Request: - assert role is CLIENT - self._fire_event_triggered_transitions(SERVER, (Request, CLIENT)) - self._fire_state_triggered_transitions() - - def _fire_event_triggered_transitions( - self, - role: Type[Sentinel], - event_type: Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]], - ) -> None: - state = self.states[role] - try: - new_state = EVENT_TRIGGERED_TRANSITIONS[role][state][event_type] - except KeyError: - event_type = cast(Type[Event], event_type) - raise LocalProtocolError( - "can't handle event type {} when role={} and state={}".format( - event_type.__name__, role, self.states[role] - ) - ) from None - self.states[role] = new_state - - def _fire_state_triggered_transitions(self) -> None: - # We apply these rules repeatedly until converging on a fixed point - while True: - start_states = dict(self.states) - - # It could happen that both these special-case transitions are - # enabled at the same time: - # - # DONE -> MIGHT_SWITCH_PROTOCOL - # DONE -> MUST_CLOSE - # - # For example, this will always be true of a HTTP/1.0 client - # requesting CONNECT. If this happens, the protocol switch takes - # priority. From there the client will either go to - # SWITCHED_PROTOCOL, in which case it's none of our business when - # they close the connection, or else the server will deny the - # request, in which case the client will go back to DONE and then - # from there to MUST_CLOSE. - if self.pending_switch_proposals: - if self.states[CLIENT] is DONE: - self.states[CLIENT] = MIGHT_SWITCH_PROTOCOL - - if not self.pending_switch_proposals: - if self.states[CLIENT] is MIGHT_SWITCH_PROTOCOL: - self.states[CLIENT] = DONE - - if not self.keep_alive: - for role in (CLIENT, SERVER): - if self.states[role] is DONE: - self.states[role] = MUST_CLOSE - - # Tabular state-triggered transitions - joint_state = (self.states[CLIENT], self.states[SERVER]) - changes = STATE_TRIGGERED_TRANSITIONS.get(joint_state, {}) - self.states.update(changes) - - if self.states == start_states: - # Fixed point reached - return - - def start_next_cycle(self) -> None: - if self.states != {CLIENT: DONE, SERVER: DONE}: - raise LocalProtocolError( - f"not in a reusable state. self.states={self.states}" - ) - # Can't reach DONE/DONE with any of these active, but still, let's be - # sure. - assert self.keep_alive - assert not self.pending_switch_proposals - self.states = {CLIENT: IDLE, SERVER: IDLE} diff --git a/bundle/python-cpu/Lib/site-packages/h11/_util.py b/bundle/python-cpu/Lib/site-packages/h11/_util.py deleted file mode 100644 index 6718445290770e028ea2f1f662026c9a0b0991db..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/_util.py +++ /dev/null @@ -1,135 +0,0 @@ -from typing import Any, Dict, NoReturn, Pattern, Tuple, Type, TypeVar, Union - -__all__ = [ - "ProtocolError", - "LocalProtocolError", - "RemoteProtocolError", - "validate", - "bytesify", -] - - -class ProtocolError(Exception): - """Exception indicating a violation of the HTTP/1.1 protocol. - - This as an abstract base class, with two concrete base classes: - :exc:`LocalProtocolError`, which indicates that you tried to do something - that HTTP/1.1 says is illegal, and :exc:`RemoteProtocolError`, which - indicates that the remote peer tried to do something that HTTP/1.1 says is - illegal. See :ref:`error-handling` for details. - - In addition to the normal :exc:`Exception` features, it has one attribute: - - .. attribute:: error_status_hint - - This gives a suggestion as to what status code a server might use if - this error occurred as part of a request. - - For a :exc:`RemoteProtocolError`, this is useful as a suggestion for - how you might want to respond to a misbehaving peer, if you're - implementing a server. - - For a :exc:`LocalProtocolError`, this can be taken as a suggestion for - how your peer might have responded to *you* if h11 had allowed you to - continue. - - The default is 400 Bad Request, a generic catch-all for protocol - violations. - - """ - - def __init__(self, msg: str, error_status_hint: int = 400) -> None: - if type(self) is ProtocolError: - raise TypeError("tried to directly instantiate ProtocolError") - Exception.__init__(self, msg) - self.error_status_hint = error_status_hint - - -# Strategy: there are a number of public APIs where a LocalProtocolError can -# be raised (send(), all the different event constructors, ...), and only one -# public API where RemoteProtocolError can be raised -# (receive_data()). Therefore we always raise LocalProtocolError internally, -# and then receive_data will translate this into a RemoteProtocolError. -# -# Internally: -# LocalProtocolError is the generic "ProtocolError". -# Externally: -# LocalProtocolError is for local errors and RemoteProtocolError is for -# remote errors. -class LocalProtocolError(ProtocolError): - def _reraise_as_remote_protocol_error(self) -> NoReturn: - # After catching a LocalProtocolError, use this method to re-raise it - # as a RemoteProtocolError. This method must be called from inside an - # except: block. - # - # An easy way to get an equivalent RemoteProtocolError is just to - # modify 'self' in place. - self.__class__ = RemoteProtocolError # type: ignore - # But the re-raising is somewhat non-trivial -- you might think that - # now that we've modified the in-flight exception object, that just - # doing 'raise' to re-raise it would be enough. But it turns out that - # this doesn't work, because Python tracks the exception type - # (exc_info[0]) separately from the exception object (exc_info[1]), - # and we only modified the latter. So we really do need to re-raise - # the new type explicitly. - # On py3, the traceback is part of the exception object, so our - # in-place modification preserved it and we can just re-raise: - raise self - - -class RemoteProtocolError(ProtocolError): - pass - - -def validate( - regex: Pattern[bytes], data: bytes, msg: str = "malformed data", *format_args: Any -) -> Dict[str, bytes]: - match = regex.fullmatch(data) - if not match: - if format_args: - msg = msg.format(*format_args) - raise LocalProtocolError(msg) - return match.groupdict() - - -# Sentinel values -# -# - Inherit identity-based comparison and hashing from object -# - Have a nice repr -# - Have a *bonus property*: type(sentinel) is sentinel -# -# The bonus property is useful if you want to take the return value from -# next_event() and do some sort of dispatch based on type(event). - -_T_Sentinel = TypeVar("_T_Sentinel", bound="Sentinel") - - -class Sentinel(type): - def __new__( - cls: Type[_T_Sentinel], - name: str, - bases: Tuple[type, ...], - namespace: Dict[str, Any], - **kwds: Any - ) -> _T_Sentinel: - assert bases == (Sentinel,) - v = super().__new__(cls, name, bases, namespace, **kwds) - v.__class__ = v # type: ignore - return v - - def __repr__(self) -> str: - return self.__name__ - - -# Used for methods, request targets, HTTP versions, header names, and header -# values. Accepts ascii-strings, or bytes/bytearray/memoryview/..., and always -# returns bytes. -def bytesify(s: Union[bytes, bytearray, memoryview, int, str]) -> bytes: - # Fast-path: - if type(s) is bytes: - return s - if isinstance(s, str): - s = s.encode("ascii") - if isinstance(s, int): - raise TypeError("expected bytes-like object, not int") - return bytes(s) diff --git a/bundle/python-cpu/Lib/site-packages/h11/_version.py b/bundle/python-cpu/Lib/site-packages/h11/_version.py deleted file mode 100644 index 76e7327b8617c9d12236f511414d5eb58e98a44b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/_version.py +++ /dev/null @@ -1,16 +0,0 @@ -# This file must be kept very simple, because it is consumed from several -# places -- it is imported by h11/__init__.py, execfile'd by setup.py, etc. - -# We use a simple scheme: -# 1.0.0 -> 1.0.0+dev -> 1.1.0 -> 1.1.0+dev -# where the +dev versions are never released into the wild, they're just what -# we stick into the VCS in between releases. -# -# This is compatible with PEP 440: -# http://legacy.python.org/dev/peps/pep-0440/ -# via the use of the "local suffix" "+dev", which is disallowed on index -# servers and causes 1.0.0+dev to sort after plain 1.0.0, which is what we -# want. (Contrast with the special suffix 1.0.0.dev, which sorts *before* -# 1.0.0.) - -__version__ = "0.16.0" diff --git a/bundle/python-cpu/Lib/site-packages/h11/_writers.py b/bundle/python-cpu/Lib/site-packages/h11/_writers.py deleted file mode 100644 index 939cdb912a9debaea07fbf3a9ac04549c44d077c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/_writers.py +++ /dev/null @@ -1,145 +0,0 @@ -# Code to read HTTP data -# -# Strategy: each writer takes an event + a write-some-bytes function, which is -# calls. -# -# WRITERS is a dict describing how to pick a reader. It maps states to either: -# - a writer -# - or, for body writers, a dict of framin-dependent writer factories - -from typing import Any, Callable, Dict, List, Tuple, Type, Union - -from ._events import Data, EndOfMessage, Event, InformationalResponse, Request, Response -from ._headers import Headers -from ._state import CLIENT, IDLE, SEND_BODY, SEND_RESPONSE, SERVER -from ._util import LocalProtocolError, Sentinel - -__all__ = ["WRITERS"] - -Writer = Callable[[bytes], Any] - - -def write_headers(headers: Headers, write: Writer) -> None: - # "Since the Host field-value is critical information for handling a - # request, a user agent SHOULD generate Host as the first header field - # following the request-line." - RFC 7230 - raw_items = headers._full_items - for raw_name, name, value in raw_items: - if name == b"host": - write(b"%s: %s\r\n" % (raw_name, value)) - for raw_name, name, value in raw_items: - if name != b"host": - write(b"%s: %s\r\n" % (raw_name, value)) - write(b"\r\n") - - -def write_request(request: Request, write: Writer) -> None: - if request.http_version != b"1.1": - raise LocalProtocolError("I only send HTTP/1.1") - write(b"%s %s HTTP/1.1\r\n" % (request.method, request.target)) - write_headers(request.headers, write) - - -# Shared between InformationalResponse and Response -def write_any_response( - response: Union[InformationalResponse, Response], write: Writer -) -> None: - if response.http_version != b"1.1": - raise LocalProtocolError("I only send HTTP/1.1") - status_bytes = str(response.status_code).encode("ascii") - # We don't bother sending ascii status messages like "OK"; they're - # optional and ignored by the protocol. (But the space after the numeric - # status code is mandatory.) - # - # XX FIXME: could at least make an effort to pull out the status message - # from stdlib's http.HTTPStatus table. Or maybe just steal their enums - # (either by import or copy/paste). We already accept them as status codes - # since they're of type IntEnum < int. - write(b"HTTP/1.1 %s %s\r\n" % (status_bytes, response.reason)) - write_headers(response.headers, write) - - -class BodyWriter: - def __call__(self, event: Event, write: Writer) -> None: - if type(event) is Data: - self.send_data(event.data, write) - elif type(event) is EndOfMessage: - self.send_eom(event.headers, write) - else: # pragma: no cover - assert False - - def send_data(self, data: bytes, write: Writer) -> None: - pass - - def send_eom(self, headers: Headers, write: Writer) -> None: - pass - - -# -# These are all careful not to do anything to 'data' except call len(data) and -# write(data). This allows us to transparently pass-through funny objects, -# like placeholder objects referring to files on disk that will be sent via -# sendfile(2). -# -class ContentLengthWriter(BodyWriter): - def __init__(self, length: int) -> None: - self._length = length - - def send_data(self, data: bytes, write: Writer) -> None: - self._length -= len(data) - if self._length < 0: - raise LocalProtocolError("Too much data for declared Content-Length") - write(data) - - def send_eom(self, headers: Headers, write: Writer) -> None: - if self._length != 0: - raise LocalProtocolError("Too little data for declared Content-Length") - if headers: - raise LocalProtocolError("Content-Length and trailers don't mix") - - -class ChunkedWriter(BodyWriter): - def send_data(self, data: bytes, write: Writer) -> None: - # if we encoded 0-length data in the naive way, it would look like an - # end-of-message. - if not data: - return - write(b"%x\r\n" % len(data)) - write(data) - write(b"\r\n") - - def send_eom(self, headers: Headers, write: Writer) -> None: - write(b"0\r\n") - write_headers(headers, write) - - -class Http10Writer(BodyWriter): - def send_data(self, data: bytes, write: Writer) -> None: - write(data) - - def send_eom(self, headers: Headers, write: Writer) -> None: - if headers: - raise LocalProtocolError("can't send trailers to HTTP/1.0 client") - # no need to close the socket ourselves, that will be taken care of by - # Connection: close machinery - - -WritersType = Dict[ - Union[Tuple[Type[Sentinel], Type[Sentinel]], Type[Sentinel]], - Union[ - Dict[str, Type[BodyWriter]], - Callable[[Union[InformationalResponse, Response], Writer], None], - Callable[[Request, Writer], None], - ], -] - -WRITERS: WritersType = { - (CLIENT, IDLE): write_request, - (SERVER, IDLE): write_any_response, - (SERVER, SEND_RESPONSE): write_any_response, - SEND_BODY: { - "chunked": ChunkedWriter, - "content-length": ContentLengthWriter, - "http/1.0": Http10Writer, - }, -} diff --git a/bundle/python-cpu/Lib/site-packages/h11/py.typed b/bundle/python-cpu/Lib/site-packages/h11/py.typed deleted file mode 100644 index f5642f79f21d872f010979dcf6f0c4a415acc19d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/h11/py.typed +++ /dev/null @@ -1 +0,0 @@ -Marker diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/METADATA deleted file mode 100644 index 8b6d9e472f616203e1327bb916bef888179fc1bc..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/METADATA +++ /dev/null @@ -1,347 +0,0 @@ -Metadata-Version: 2.4 -Name: huggingface_hub -Version: 0.36.2 -Summary: Client library to download and publish models, datasets and other repos on the huggingface.co hub -Home-page: https://github.com/huggingface/huggingface_hub -Author: Hugging Face, Inc. -Author-email: julien@huggingface.co -License: Apache -Keywords: model-hub machine-learning models natural-language-processing deep-learning pytorch pretrained-models -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Education -Classifier: Intended Audience :: Science/Research -Classifier: License :: OSI Approved :: Apache Software License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence -Requires-Python: >=3.8.0 -Description-Content-Type: text/markdown -License-File: LICENSE -Requires-Dist: filelock -Requires-Dist: fsspec>=2023.5.0 -Requires-Dist: hf-xet<2.0.0,>=1.1.3; platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "arm64" or platform_machine == "aarch64" -Requires-Dist: packaging>=20.9 -Requires-Dist: pyyaml>=5.1 -Requires-Dist: requests -Requires-Dist: tqdm>=4.42.1 -Requires-Dist: typing-extensions>=3.7.4.3 -Provides-Extra: cli -Requires-Dist: InquirerPy==0.3.4; extra == "cli" -Provides-Extra: inference -Requires-Dist: aiohttp; extra == "inference" -Provides-Extra: oauth -Requires-Dist: authlib>=1.3.2; extra == "oauth" -Requires-Dist: fastapi; extra == "oauth" -Requires-Dist: httpx; extra == "oauth" -Requires-Dist: itsdangerous; extra == "oauth" -Provides-Extra: torch -Requires-Dist: torch; extra == "torch" -Requires-Dist: safetensors[torch]; extra == "torch" -Provides-Extra: hf-transfer -Requires-Dist: hf_transfer>=0.1.4; extra == "hf-transfer" -Provides-Extra: fastai -Requires-Dist: toml; extra == "fastai" -Requires-Dist: fastai>=2.4; extra == "fastai" -Requires-Dist: fastcore>=1.3.27; extra == "fastai" -Provides-Extra: tensorflow -Requires-Dist: tensorflow; extra == "tensorflow" -Requires-Dist: pydot; extra == "tensorflow" -Requires-Dist: graphviz; extra == "tensorflow" -Provides-Extra: tensorflow-testing -Requires-Dist: tensorflow; extra == "tensorflow-testing" -Requires-Dist: keras<3.0; extra == "tensorflow-testing" -Provides-Extra: hf-xet -Requires-Dist: hf-xet<2.0.0,>=1.1.2; extra == "hf-xet" -Provides-Extra: mcp -Requires-Dist: mcp>=1.8.0; extra == "mcp" -Requires-Dist: typer; extra == "mcp" -Requires-Dist: aiohttp; extra == "mcp" -Provides-Extra: testing -Requires-Dist: InquirerPy==0.3.4; extra == "testing" -Requires-Dist: aiohttp; extra == "testing" -Requires-Dist: authlib>=1.3.2; extra == "testing" -Requires-Dist: fastapi; extra == "testing" -Requires-Dist: httpx; extra == "testing" -Requires-Dist: itsdangerous; extra == "testing" -Requires-Dist: jedi; extra == "testing" -Requires-Dist: Jinja2; extra == "testing" -Requires-Dist: pytest<8.2.2,>=8.1.1; extra == "testing" -Requires-Dist: pytest-cov; extra == "testing" -Requires-Dist: pytest-env; extra == "testing" -Requires-Dist: pytest-xdist; extra == "testing" -Requires-Dist: pytest-vcr; extra == "testing" -Requires-Dist: pytest-asyncio; extra == "testing" -Requires-Dist: pytest-rerunfailures<16.0; extra == "testing" -Requires-Dist: pytest-mock; extra == "testing" -Requires-Dist: urllib3<2.0; extra == "testing" -Requires-Dist: soundfile; extra == "testing" -Requires-Dist: Pillow; extra == "testing" -Requires-Dist: gradio>=4.0.0; extra == "testing" -Requires-Dist: numpy; extra == "testing" -Requires-Dist: fastapi; extra == "testing" -Provides-Extra: typing -Requires-Dist: typing-extensions>=4.8.0; extra == "typing" -Requires-Dist: types-PyYAML; extra == "typing" -Requires-Dist: types-requests; extra == "typing" -Requires-Dist: types-simplejson; extra == "typing" -Requires-Dist: types-toml; extra == "typing" -Requires-Dist: types-tqdm; extra == "typing" -Requires-Dist: types-urllib3; extra == "typing" -Provides-Extra: quality -Requires-Dist: ruff>=0.9.0; extra == "quality" -Requires-Dist: mypy<1.15.0,>=1.14.1; python_version == "3.8" and extra == "quality" -Requires-Dist: mypy==1.15.0; python_version >= "3.9" and extra == "quality" -Requires-Dist: libcst>=1.4.0; extra == "quality" -Requires-Dist: ty; extra == "quality" -Provides-Extra: all -Requires-Dist: InquirerPy==0.3.4; extra == "all" -Requires-Dist: aiohttp; extra == "all" -Requires-Dist: authlib>=1.3.2; extra == "all" -Requires-Dist: fastapi; extra == "all" -Requires-Dist: httpx; extra == "all" -Requires-Dist: itsdangerous; extra == "all" -Requires-Dist: jedi; extra == "all" -Requires-Dist: Jinja2; extra == "all" -Requires-Dist: pytest<8.2.2,>=8.1.1; extra == "all" -Requires-Dist: pytest-cov; extra == "all" -Requires-Dist: pytest-env; extra == "all" -Requires-Dist: pytest-xdist; extra == "all" -Requires-Dist: pytest-vcr; extra == "all" -Requires-Dist: pytest-asyncio; extra == "all" -Requires-Dist: pytest-rerunfailures<16.0; extra == "all" -Requires-Dist: pytest-mock; extra == "all" -Requires-Dist: urllib3<2.0; extra == "all" -Requires-Dist: soundfile; extra == "all" -Requires-Dist: Pillow; extra == "all" -Requires-Dist: gradio>=4.0.0; extra == "all" -Requires-Dist: numpy; extra == "all" -Requires-Dist: fastapi; extra == "all" -Requires-Dist: ruff>=0.9.0; extra == "all" -Requires-Dist: mypy<1.15.0,>=1.14.1; python_version == "3.8" and extra == "all" -Requires-Dist: mypy==1.15.0; python_version >= "3.9" and extra == "all" -Requires-Dist: libcst>=1.4.0; extra == "all" -Requires-Dist: ty; extra == "all" -Requires-Dist: typing-extensions>=4.8.0; extra == "all" -Requires-Dist: types-PyYAML; extra == "all" -Requires-Dist: types-requests; extra == "all" -Requires-Dist: types-simplejson; extra == "all" -Requires-Dist: types-toml; extra == "all" -Requires-Dist: types-tqdm; extra == "all" -Requires-Dist: types-urllib3; extra == "all" -Provides-Extra: dev -Requires-Dist: InquirerPy==0.3.4; extra == "dev" -Requires-Dist: aiohttp; extra == "dev" -Requires-Dist: authlib>=1.3.2; extra == "dev" -Requires-Dist: fastapi; extra == "dev" -Requires-Dist: httpx; extra == "dev" -Requires-Dist: itsdangerous; extra == "dev" -Requires-Dist: jedi; extra == "dev" -Requires-Dist: Jinja2; extra == "dev" -Requires-Dist: pytest<8.2.2,>=8.1.1; extra == "dev" -Requires-Dist: pytest-cov; extra == "dev" -Requires-Dist: pytest-env; extra == "dev" -Requires-Dist: pytest-xdist; extra == "dev" -Requires-Dist: pytest-vcr; extra == "dev" -Requires-Dist: pytest-asyncio; extra == "dev" -Requires-Dist: pytest-rerunfailures<16.0; extra == "dev" -Requires-Dist: pytest-mock; extra == "dev" -Requires-Dist: urllib3<2.0; extra == "dev" -Requires-Dist: soundfile; extra == "dev" -Requires-Dist: Pillow; extra == "dev" -Requires-Dist: gradio>=4.0.0; extra == "dev" -Requires-Dist: numpy; extra == "dev" -Requires-Dist: fastapi; extra == "dev" -Requires-Dist: ruff>=0.9.0; extra == "dev" -Requires-Dist: mypy<1.15.0,>=1.14.1; python_version == "3.8" and extra == "dev" -Requires-Dist: mypy==1.15.0; python_version >= "3.9" and extra == "dev" -Requires-Dist: libcst>=1.4.0; extra == "dev" -Requires-Dist: ty; extra == "dev" -Requires-Dist: typing-extensions>=4.8.0; extra == "dev" -Requires-Dist: types-PyYAML; extra == "dev" -Requires-Dist: types-requests; extra == "dev" -Requires-Dist: types-simplejson; extra == "dev" -Requires-Dist: types-toml; extra == "dev" -Requires-Dist: types-tqdm; extra == "dev" -Requires-Dist: types-urllib3; extra == "dev" -Dynamic: author -Dynamic: author-email -Dynamic: classifier -Dynamic: description -Dynamic: description-content-type -Dynamic: home-page -Dynamic: keywords -Dynamic: license -Dynamic: license-file -Dynamic: provides-extra -Dynamic: requires-dist -Dynamic: requires-python -Dynamic: summary - -

- - - - huggingface_hub library logo - -
-
-

- -

- The official Python client for the Huggingface Hub. -

- -

- Documentation - GitHub release - PyPi version - PyPI - Downloads - Code coverage -

- -

-

- English | - Deutsch | - हिंदी | - 한국어 | - 中文(简体) -

-

- ---- - -**Documentation**: https://hf.co/docs/huggingface_hub - -**Source Code**: https://github.com/huggingface/huggingface_hub - ---- - -## Welcome to the huggingface_hub library - -The `huggingface_hub` library allows you to interact with the [Hugging Face Hub](https://huggingface.co/), a platform democratizing open-source Machine Learning for creators and collaborators. Discover pre-trained models and datasets for your projects or play with the thousands of machine learning apps hosted on the Hub. You can also create and share your own models, datasets and demos with the community. The `huggingface_hub` library provides a simple way to do all these things with Python. - -## Key features - -- [Download files](https://huggingface.co/docs/huggingface_hub/en/guides/download) from the Hub. -- [Upload files](https://huggingface.co/docs/huggingface_hub/en/guides/upload) to the Hub. -- [Manage your repositories](https://huggingface.co/docs/huggingface_hub/en/guides/repository). -- [Run Inference](https://huggingface.co/docs/huggingface_hub/en/guides/inference) on deployed models. -- [Search](https://huggingface.co/docs/huggingface_hub/en/guides/search) for models, datasets and Spaces. -- [Share Model Cards](https://huggingface.co/docs/huggingface_hub/en/guides/model-cards) to document your models. -- [Engage with the community](https://huggingface.co/docs/huggingface_hub/en/guides/community) through PRs and comments. - -## Installation - -Install the `huggingface_hub` package with [pip](https://pypi.org/project/huggingface-hub/): - -```bash -pip install huggingface_hub -``` - -If you prefer, you can also install it with [conda](https://huggingface.co/docs/huggingface_hub/en/installation#install-with-conda). - -In order to keep the package minimal by default, `huggingface_hub` comes with optional dependencies useful for some use cases. For example, if you want have a complete experience for Inference, run: - -```bash -pip install "huggingface_hub[inference]" -``` - -To learn more installation and optional dependencies, check out the [installation guide](https://huggingface.co/docs/huggingface_hub/en/installation). - -## Quick start - -### Download files - -Download a single file - -```py -from huggingface_hub import hf_hub_download - -hf_hub_download(repo_id="tiiuae/falcon-7b-instruct", filename="config.json") -``` - -Or an entire repository - -```py -from huggingface_hub import snapshot_download - -snapshot_download("stabilityai/stable-diffusion-2-1") -``` - -Files will be downloaded in a local cache folder. More details in [this guide](https://huggingface.co/docs/huggingface_hub/en/guides/manage-cache). - -### Login - -The Hugging Face Hub uses tokens to authenticate applications (see [docs](https://huggingface.co/docs/hub/security-tokens)). To log in your machine, run the following CLI: - -```bash -hf auth login -# or using an environment variable -hf auth login --token $HUGGINGFACE_TOKEN -``` - -### Create a repository - -```py -from huggingface_hub import create_repo - -create_repo(repo_id="super-cool-model") -``` - -### Upload files - -Upload a single file - -```py -from huggingface_hub import upload_file - -upload_file( - path_or_fileobj="/home/lysandre/dummy-test/README.md", - path_in_repo="README.md", - repo_id="lysandre/test-model", -) -``` - -Or an entire folder - -```py -from huggingface_hub import upload_folder - -upload_folder( - folder_path="/path/to/local/space", - repo_id="username/my-cool-space", - repo_type="space", -) -``` - -For details in the [upload guide](https://huggingface.co/docs/huggingface_hub/en/guides/upload). - -## Integrating to the Hub. - -We're partnering with cool open source ML libraries to provide free model hosting and versioning. You can find the existing integrations [here](https://huggingface.co/docs/hub/libraries). - -The advantages are: - -- Free model or dataset hosting for libraries and their users. -- Built-in file versioning, even with very large files, thanks to a git-based approach. -- In-browser widgets to play with the uploaded models. -- Anyone can upload a new model for your library, they just need to add the corresponding tag for the model to be discoverable. -- Fast downloads! We use Cloudfront (a CDN) to geo-replicate downloads so they're blazing fast from anywhere on the globe. -- Usage stats and more features to come. - -If you would like to integrate your library, feel free to open an issue to begin the discussion. We wrote a [step-by-step guide](https://huggingface.co/docs/hub/adding-a-library) with ❤️ showing how to do this integration. - -## Contributions (feature requests, bugs, etc.) are super welcome 💙💚💛💜🧡❤️ - -Everyone is welcome to contribute, and we value everybody's contribution. Code is not the only way to help the community. -Answering questions, helping others, reaching out and improving the documentations are immensely valuable to the community. -We wrote a [contribution guide](https://github.com/huggingface/huggingface_hub/blob/main/CONTRIBUTING.md) to summarize -how to get started to contribute to this repository. diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/RECORD deleted file mode 100644 index fc4b36120533c92ee4b489e15154eade881c6fb4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/RECORD +++ /dev/null @@ -1,175 +0,0 @@ -../../Scripts/hf.exe,sha256=6b3F9YsO-TTSmnLZxyi2IC9GTO1vTUJnrXwgAXHsB-s,46080 -../../Scripts/huggingface-cli.exe,sha256=NaeK-hT3nvNkpqFkkde82ORSwpGUJIENPz36fNXAipY,46080 -../../Scripts/tiny-agents.exe,sha256=L1BYEbf2F2tRbsBknty05KaiyOBowt1oPa_ejWfGe1g,46080 -huggingface_hub-0.36.2.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -huggingface_hub-0.36.2.dist-info/METADATA,sha256=TcKwo_snvLqLCR-YvM_0uUNp0mnJdorq7NC5nZcGZdk,15201 -huggingface_hub-0.36.2.dist-info/RECORD,, -huggingface_hub-0.36.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -huggingface_hub-0.36.2.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91 -huggingface_hub-0.36.2.dist-info/entry_points.txt,sha256=FGUdvu8z-x7lvoJ4udumhcg3AtzigPraCn_ZbjEhIto,218 -huggingface_hub-0.36.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357 -huggingface_hub-0.36.2.dist-info/top_level.txt,sha256=8KzlQJAY4miUvjAssOAJodqKOw3harNzuiwGQ9qLSSk,16 -huggingface_hub/__init__.py,sha256=5Ya2RhJIISj9JBGTZLxNxtypOEBxNORUpeh2NBs2TjU,52675 -huggingface_hub/_commit_api.py,sha256=pGESDsicpWMeZnct-71635KgTfvUoyok_hPl9ZgIIWI,41010 -huggingface_hub/_commit_scheduler.py,sha256=P64poLZoTJnSyR39SN6w5s9bLyngKstWee03fpoVETQ,14660 -huggingface_hub/_inference_endpoints.py,sha256=ahmbPcEXsJ_JcMb9TDgdkD8Z2z9uytkFG3_1o6dTm8g,17598 -huggingface_hub/_jobs_api.py,sha256=OFcbChcXsLvaX4oGumsHscZKAzsueYIhh0Z6Y4ycpio,10883 -huggingface_hub/_local_folder.py,sha256=2iHXNgIT3UdSt2PvCovd0NzgVxTRypKb-rvAFLK-gZU,17305 -huggingface_hub/_login.py,sha256=TWNkZpMPkDuttQ36uoi-ozLQ1IcXVsZ42tbcQ-b-h0Q,20248 -huggingface_hub/_oauth.py,sha256=75ya9toHxC0WRKsLOAI212CrssRjTSxs16mHWWNMb3w,18714 -huggingface_hub/_snapshot_download.py,sha256=b-NzYQcvktsAirIfGQKgzQwu8w0S6lhBTvnJ5S6saw8,16166 -huggingface_hub/_space_api.py,sha256=jb6rF8qLtjaNU12D-8ygAPM26xDiHCu8CHXHowhGTmg,5470 -huggingface_hub/_tensorboard_logger.py,sha256=tUdQzx-wXF4yjoGJG2izqZrn-IPMflMBWMkl1sKYzo0,8420 -huggingface_hub/_upload_large_folder.py,sha256=l2YWLZttOw69EGdihT3y_Nhr5mweLGooZG9L8smNoHY,30066 -huggingface_hub/_webhooks_payload.py,sha256=Xm3KaK7tCOGBlXkuZvbym6zjHXrT1XCrbUFWuXiBmNY,3617 -huggingface_hub/_webhooks_server.py,sha256=RLrQuCHlDH_qUQJQOm11fKFDEhIUR2IxwazuKy-T9Uo,15672 -huggingface_hub/cli/__init__.py,sha256=xzX1qgAvrtAX4gP59WrPlvOZFLuzuTgcjvanQvcpgHc,928 -huggingface_hub/cli/_cli_utils.py,sha256=Nt6CjbkYqQQRuh70bUXVA6rZpbZt_Sa1WqBUxjQLu6g,2095 -huggingface_hub/cli/auth.py,sha256=XSsbU7-_TS5IXdASkgUCdQeoXVG82VUyGYvOS4oLLRs,7317 -huggingface_hub/cli/cache.py,sha256=fQjYfbRUapeHsK10Y6w_Ixu9JKyuZyM7pJzExJGd_2c,15855 -huggingface_hub/cli/download.py,sha256=8b5wqhMYg3X9tar9EEeWdPZk9um1kZTI_WgBqyiatqs,7141 -huggingface_hub/cli/hf.py,sha256=SQ73_SXEQnWVJkhKT_6bwNQBHQXGOdI5qqlTTtI0XH0,2328 -huggingface_hub/cli/jobs.py,sha256=eA6Q7iy_-7vjU4SjYPvn71b2aVo2qt3q-pVxLyXCWqg,44317 -huggingface_hub/cli/lfs.py,sha256=J9MkKOGUW6GjBrKs2zZUCOaAGxpatxsEoSbBjuhDJV8,7230 -huggingface_hub/cli/repo.py,sha256=CuOqQZ7WELLk9Raf3tnyXILt9e93OrlS8Dyxx3BqdQA,10618 -huggingface_hub/cli/repo_files.py,sha256=9oeeQJx8Z0ygbTElw1o5T6dGtRbeolcXENt_ouEBvjk,4844 -huggingface_hub/cli/system.py,sha256=eLSYME7ywt5Ae3tYQnS43Tai2pR2JLtA1KGImzPt5pM,1707 -huggingface_hub/cli/upload.py,sha256=lOHR_JzfM2XL_pYK3Z1HlGnaAI-fw7xGY46Lccvbsy4,14362 -huggingface_hub/cli/upload_large_folder.py,sha256=w4RIW0yZKTnNnhDOB6yISnIo_h_Hy13KwWVzrFzczpY,6164 -huggingface_hub/commands/__init__.py,sha256=AkbM2a-iGh0Vq_xAWhK3mu3uZ44km8-X5uWjKcvcrUQ,928 -huggingface_hub/commands/_cli_utils.py,sha256=ePYTIEWnU677nPvdNC5AdYcEB1400L6qYEUxMkVUzME,2329 -huggingface_hub/commands/delete_cache.py,sha256=035yACUtVUIG8tEtc5vexDoFFphzdk5IXkFTlD4WMiw,17738 -huggingface_hub/commands/download.py,sha256=0QY9ho7eiAPvFndBPttGtH6vXNk3r9AioltNwc8h1Z4,8310 -huggingface_hub/commands/env.py,sha256=qv4SmjuzUz9exo4RDMY2HqabLCKE1oRb55cBA6LN9R4,1342 -huggingface_hub/commands/huggingface_cli.py,sha256=gDi7JueyiLD0bGclTEYfHPQWpAY_WBdPfHT7vkqa5v0,2654 -huggingface_hub/commands/lfs.py,sha256=xdbnNRO04UuQemEhUGT809jFgQn9Rj-SnyT_0Ph-VYg,7342 -huggingface_hub/commands/repo.py,sha256=WcRDFqUYKB0Kz0zFopegiG614ot6VOYTAf6jht0BMss,6042 -huggingface_hub/commands/repo_files.py,sha256=ftjLCC3XCY-AMmiYiZPIdRMmIqZbqVZw-BSjBLcZup4,5054 -huggingface_hub/commands/scan_cache.py,sha256=gQlhBZgWkUzH4wrIYnvgV7CA4C7rvV2SuY0x2JCB7g0,8675 -huggingface_hub/commands/tag.py,sha256=4fgQuXJHG59lTVyOjIUZjxdJDL4JZW4q10XDPSo-gss,6382 -huggingface_hub/commands/upload.py,sha256=eAJIig4ljtO9FRyGjiz6HbHS-Q4MOQziRgzjQrl5Koo,14576 -huggingface_hub/commands/upload_large_folder.py,sha256=_1id84BFtbL8HgFRKZ-el_uPrijamz1qWlzO16KbUAc,6254 -huggingface_hub/commands/user.py,sha256=dDpi0mLYvTeYf0fhPVQyEJsn7Wrk6gWvR5YHC6RgebU,7516 -huggingface_hub/commands/version.py,sha256=rGpCbvxImY9eQqXrshYt609Iws27R75WARmKQrIo6Ok,1390 -huggingface_hub/community.py,sha256=exJxrySnXURAijkVOcreuwM5JAuuz2L1xTSDkd223wk,12365 -huggingface_hub/constants.py,sha256=nILseAp4rqLu_KQTZDpPGOhepVAPanD7azbomAvovj0,10313 -huggingface_hub/dataclasses.py,sha256=rjQfuX9MeTXZQrCQC8JvkjpARDehOiSluE7Kz1L7Ueg,17337 -huggingface_hub/errors.py,sha256=HVqmnJODe1wy1cYsx7AfjrwE4DD-gdKVvMTYTBfLjpA,11265 -huggingface_hub/fastai_utils.py,sha256=m7wwWk-TdhIB1CJMigAzzUBP4eLQALutEzwjWf9Ej-o,16755 -huggingface_hub/file_download.py,sha256=C76FMg1Rg7401K9UpwOAnFd1UG2ko0bL9AES2mM7Ntg,79254 -huggingface_hub/hf_api.py,sha256=REMm9AFgUtyizI6tkEy6glX2Aa7-TH7-uWhlhl0q0fE,487935 -huggingface_hub/hf_file_system.py,sha256=uLeublBZhWd4309fE3eFHIN8G7RCrX2_6_gr0BYjuzQ,48338 -huggingface_hub/hub_mixin.py,sha256=Ii3w9o7XgGbj6UNPnieW5IDfaCd8OEKpIH1hRkncRDQ,38208 -huggingface_hub/inference/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -huggingface_hub/inference/_client.py,sha256=9cAIkBFuzFC5f6jVp62MJNDSUcPqxsFluhQLi6FqXdc,157536 -huggingface_hub/inference/_common.py,sha256=dI3OPg0320OOB0FRy_kqftW9F3ghEnBVA5Gi4VaSctg,15778 -huggingface_hub/inference/_generated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -huggingface_hub/inference/_generated/_async_client.py,sha256=DSOAXJ_TxRubPisWnVKzepXalDA7PcE-NG3oczo8iMw,163445 -huggingface_hub/inference/_generated/types/__init__.py,sha256=9WvrGQ8aThtKSNzZF06j-CIE2ZuItne8FFnea1p1u38,6557 -huggingface_hub/inference/_generated/types/audio_classification.py,sha256=Jg3mzfGhCSH6CfvVvgJSiFpkz6v4nNA0G4LJXacEgNc,1573 -huggingface_hub/inference/_generated/types/audio_to_audio.py,sha256=2Ep4WkePL7oJwcp5nRJqApwviumGHbft9HhXE9XLHj4,891 -huggingface_hub/inference/_generated/types/automatic_speech_recognition.py,sha256=8CEphr6rvRHgq1L5Md3tq14V0tEAmzJkemh1_7gSswo,5515 -huggingface_hub/inference/_generated/types/base.py,sha256=4XG49q0-2SOftYQ8HXQnWLxiJktou-a7IoG3kdOv-kg,6751 -huggingface_hub/inference/_generated/types/chat_completion.py,sha256=j1Y8G4g5yGs4g7N4sXWbipF8TwkQG0J-ftL9OxejkBw,11254 -huggingface_hub/inference/_generated/types/depth_estimation.py,sha256=rcpe9MhYMeLjflOwBs3KMZPr6WjOH3FYEThStG-FJ3M,929 -huggingface_hub/inference/_generated/types/document_question_answering.py,sha256=6BEYGwJcqGlah4RBJDAvWFTEXkO0mosBiMy82432nAM,3202 -huggingface_hub/inference/_generated/types/feature_extraction.py,sha256=NMWVL_TLSG5SS5bdt1-fflkZ75UMlMKeTMtmdnUTADc,1537 -huggingface_hub/inference/_generated/types/fill_mask.py,sha256=OrTgQ7Ndn0_dWK5thQhZwTOHbQni8j0iJcx9llyhRds,1708 -huggingface_hub/inference/_generated/types/image_classification.py,sha256=A-Y024o8723_n8mGVos4TwdAkVL62McGeL1iIo4VzNs,1585 -huggingface_hub/inference/_generated/types/image_segmentation.py,sha256=vrkI4SuP1Iq_iLXc-2pQhYY3SHN4gzvFBoZqbUHxU7o,1950 -huggingface_hub/inference/_generated/types/image_to_image.py,sha256=snvGbmCdqchxGef25MceD7LSKAmVkIgnoX5t71rdlAQ,2290 -huggingface_hub/inference/_generated/types/image_to_text.py,sha256=OaFEBAfgT-fOVzJ7xVermGf7VODhrc9-Jg38WrM7-2o,4810 -huggingface_hub/inference/_generated/types/image_to_video.py,sha256=bC-L_cNsDhk4s_IdSiprJ9d1NeMGePLcUp7UPpco21w,2240 -huggingface_hub/inference/_generated/types/object_detection.py,sha256=VuFlb1281qTXoSgJDmquGz-VNfEZLo2H0Rh_F6MF6ts,2000 -huggingface_hub/inference/_generated/types/question_answering.py,sha256=zw38a9_9l2k1ifYZefjkioqZ4asfSRM9M4nU3gSCmAQ,2898 -huggingface_hub/inference/_generated/types/sentence_similarity.py,sha256=w5Nj1g18eBzopZwxuDLI-fEsyaCK2KrHA5yf_XfSjgo,1052 -huggingface_hub/inference/_generated/types/summarization.py,sha256=WGGr8uDLrZg8JQgF9ZMUP9euw6uZo6zwkVZ-IfvCFI0,1487 -huggingface_hub/inference/_generated/types/table_question_answering.py,sha256=cJnIPA2fIbQP2Ejn7X_esY48qGWoXg30fnNOqCXiOVQ,2293 -huggingface_hub/inference/_generated/types/text2text_generation.py,sha256=v-418w1JNNSZ2tuW9DUl6a36TQQCADa438A3ufvcbOw,1609 -huggingface_hub/inference/_generated/types/text_classification.py,sha256=FarAjygLEfPofLfKeabzJ7PKEBItlHGoUNUOzyLRpL4,1445 -huggingface_hub/inference/_generated/types/text_generation.py,sha256=28u-1zU7elk2teP3y4u1VAtDDHzY0JZ2KEEJe5d5uvg,5922 -huggingface_hub/inference/_generated/types/text_to_audio.py,sha256=1HR9Q6s9MXqtKGTvHPLGVMum5-eg7O-Pgv6Nd0v8_HU,4741 -huggingface_hub/inference/_generated/types/text_to_image.py,sha256=sGGi1Fa0n5Pmd6G3I-F2SBJcJ1M7Gmqnng6sfi0AVzs,1903 -huggingface_hub/inference/_generated/types/text_to_speech.py,sha256=ROFuR32ijROCeqbv81Jos0lmaA8SRWyIUsWrdD4yWow,4760 -huggingface_hub/inference/_generated/types/text_to_video.py,sha256=yHXVNs3t6aYO7visrBlB5cH7kjoysxF9510aofcf_18,1790 -huggingface_hub/inference/_generated/types/token_classification.py,sha256=iblAcgfxXeaLYJ14NdiiCMIQuBlarUknLkXUklhvcLI,1915 -huggingface_hub/inference/_generated/types/translation.py,sha256=xww4X5cfCYv_F0oINWLwqJRPCT6SV3VBAJuPjTs_j7o,1763 -huggingface_hub/inference/_generated/types/video_classification.py,sha256=TyydjQw2NRLK9sDGzJUVnkDeo848ebmCx588Ur8I9q0,1680 -huggingface_hub/inference/_generated/types/visual_question_answering.py,sha256=AWrQ6qo4gZa3PGedaNpzDFqx5yOYyjhnUB6iuZEj_uo,1673 -huggingface_hub/inference/_generated/types/zero_shot_classification.py,sha256=BAiebPjsqoNa8EU35Dx0pfIv8W2c4GSl-TJckV1MaxQ,1738 -huggingface_hub/inference/_generated/types/zero_shot_image_classification.py,sha256=8J9n6VqFARkWvPfAZNWEG70AlrMGldU95EGQQwn06zI,1487 -huggingface_hub/inference/_generated/types/zero_shot_object_detection.py,sha256=GUd81LIV7oEbRWayDlAVgyLmY596r1M3AW0jXDp1yTA,1630 -huggingface_hub/inference/_mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -huggingface_hub/inference/_mcp/_cli_hacks.py,sha256=KX9HZJPa1p8ngY3mtYGGlVUXfg4vYbbBRs-8HLToP04,3284 -huggingface_hub/inference/_mcp/agent.py,sha256=jqvQwOajY41RIhCtD-XgVfuWbTouSYCQkIWJ1gHRrJQ,4262 -huggingface_hub/inference/_mcp/cli.py,sha256=AmSUT6wXlE6EWmI0SfQgTWYnL07322zGwwk2yMZZlBc,9640 -huggingface_hub/inference/_mcp/constants.py,sha256=kldRfaidXMdyMl_jLosaQomgWDv4shvnFe3dnQNwXSU,2511 -huggingface_hub/inference/_mcp/mcp_client.py,sha256=9rcwOO7L2Ih0oGLkeY9o5gbkwEBmsDkHKf4XAmp4Mvc,16784 -huggingface_hub/inference/_mcp/types.py,sha256=3gq-P_mrmvPI6KWBqjCxavtMPiGz10YXog7wg4oJYAo,941 -huggingface_hub/inference/_mcp/utils.py,sha256=KFsGOC8dytS3VgaugBzibdteWasZ9CAnp83U2SyIlMw,4188 -huggingface_hub/inference/_providers/__init__.py,sha256=UxPnzOdVcJgroPEatuahb4fsHaObUYPrwUCzv5ADCa4,9019 -huggingface_hub/inference/_providers/_common.py,sha256=brZJ1CUxDKooPdmVlm4cuKjvaW_refVY0Y7CbGQe7e4,12373 -huggingface_hub/inference/_providers/black_forest_labs.py,sha256=FIukZoIFt_FDrTTDfpF-Vko5sXnmH0QvVIsMtV2Jzm8,2852 -huggingface_hub/inference/_providers/cerebras.py,sha256=QOJ-1U-os7uE7p6eUnn_P_APq-yQhx28be7c3Tq2EuA,210 -huggingface_hub/inference/_providers/clarifai.py,sha256=1cEXQwhGk4DRKiPCQUa5y-L6okTo4781EImQC8yJVOw,380 -huggingface_hub/inference/_providers/cohere.py,sha256=O3tC-qIUL91mx_mE8bOHCtDWcQuKOUauhUoXSUBUCZ8,1253 -huggingface_hub/inference/_providers/fal_ai.py,sha256=pCr5qP6R1W1CrEw-_nKdNuP3UqsUi58yL18w4r7mXRo,9989 -huggingface_hub/inference/_providers/featherless_ai.py,sha256=QxBz-32O4PztxixrIjrfKuTOzvfqyUi-cVsw0Hf_zlY,1382 -huggingface_hub/inference/_providers/fireworks_ai.py,sha256=Id226ITfPkOcFMFzly3MW9l-dZl9l4qizL4JEHWkBFk,1215 -huggingface_hub/inference/_providers/groq.py,sha256=JTk2JV4ZOlaohho7zLAFQtk92kGVsPmLJ1hmzcwsqvQ,315 -huggingface_hub/inference/_providers/hf_inference.py,sha256=0yi3cR-EJ4HYx3mSzOsMOTVmvVBkaajTzTfKB8JXQpk,9540 -huggingface_hub/inference/_providers/hyperbolic.py,sha256=OQIBi2j3aNvuaSQ8BUK1K1PVeRXdrxc80G-6YmBa-ns,1985 -huggingface_hub/inference/_providers/nebius.py,sha256=VJpTF2JZ58rznc9wxdk-57vwF8sV2vESw_WkXjXqCho,3580 -huggingface_hub/inference/_providers/novita.py,sha256=HGVC8wPraRQUuI5uBoye1Y4Wqe4X116B71GhhbWy5yM,2514 -huggingface_hub/inference/_providers/nscale.py,sha256=qWUsWinQmUbNUqehyKn34tVoWehu8gd-OZ2F4uj2SWM,1802 -huggingface_hub/inference/_providers/openai.py,sha256=GCVYeNdjWIgpQQ7E_Xv8IebmdhTi0S6WfFosz3nLtps,1089 -huggingface_hub/inference/_providers/publicai.py,sha256=1I2W6rORloB5QHSvky4njZO2XKLTwA-kPdNoauoT5rg,210 -huggingface_hub/inference/_providers/replicate.py,sha256=otVfPkfBtlWrpjQub4V__t7g_w8Ewc7ZU3efiOauW-I,3820 -huggingface_hub/inference/_providers/sambanova.py,sha256=Unt3H3jr_kgI9vzRjmmW1DFyoEuPkKCcgIIloiOj3j8,2037 -huggingface_hub/inference/_providers/scaleway.py,sha256=Jy81kXWbXCHBpx6xmyzdEfXGSyhUfjKOLHuDSvhHWGo,1209 -huggingface_hub/inference/_providers/together.py,sha256=KHF19CS3qXS7G1-CwcMiD8Z5wzPKEKi4F2DzqAthbBE,3439 -huggingface_hub/inference/_providers/zai_org.py,sha256=plGzMZuLrChZvgpS3CCPqI6ImotZZxNLgfxnR7v6tw8,646 -huggingface_hub/inference_api.py,sha256=b4-NhPSn9b44nYKV8tDKXodmE4JVdEymMWL4CVGkzlE,8323 -huggingface_hub/keras_mixin.py,sha256=gDm8PBcTqYhfrEvhu1_ptxzxbVOF3h0wAArn90UyzRA,19547 -huggingface_hub/lfs.py,sha256=v0mTThnULTmFv8MVWfrkQEwkiFXzWWx7xyp2VLf-EPo,17020 -huggingface_hub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -huggingface_hub/repocard.py,sha256=8tmR7SYVQZ4iBFYCmOj0yl6Ohc9Vv136s-KQKkxBq7U,34865 -huggingface_hub/repocard_data.py,sha256=hr4ReFpEQMNdh_9Dx-L-IJoI1ElHyk-h-8ZRqwVYYOE,34082 -huggingface_hub/repository.py,sha256=axZcbAh4ufXEaMgPbrS1WWgvshd-mFvYnRZAZ_yYljQ,54541 -huggingface_hub/serialization/__init__.py,sha256=kn-Fa-m4FzMnN8lNsF-SwFcfzug4CucexybGKyvZ8S0,1041 -huggingface_hub/serialization/_base.py,sha256=VGQ4Z9Abg2gsL_1rTGSS9p-3tkkG9eaERjlzBTLGkdU,8109 -huggingface_hub/serialization/_dduf.py,sha256=s42239rLiHwaJE36QDEmS5GH7DSmQ__BffiHJO5RjIg,15424 -huggingface_hub/serialization/_tensorflow.py,sha256=Ea3wN1bKgyb_9opj-FtH-WpIp0ptkovKimroZOudX5c,3608 -huggingface_hub/serialization/_torch.py,sha256=dw3RMkr0CYAr_TwPG_rma-ueHBRTXpfEJtrVKAvvtN4,45143 -huggingface_hub/templates/datasetcard_template.md,sha256=W-EMqR6wndbrnZorkVv56URWPG49l7MATGeI015kTvs,5503 -huggingface_hub/templates/modelcard_template.md,sha256=4AqArS3cqdtbit5Bo-DhjcnDFR-pza5hErLLTPM4Yuc,6870 -huggingface_hub/utils/__init__.py,sha256=ORfVkn5D0wuLIq12jjhTzn5_c4F8fRPxB7TG-iednuQ,3722 -huggingface_hub/utils/_auth.py,sha256=Ixve2vxdftHXXk2R2vfyLzlVoDT39Tkq-Hrou9KCUvw,8286 -huggingface_hub/utils/_cache_assets.py,sha256=kai77HPQMfYpROouMBQCr_gdBCaeTm996Sqj0dExbNg,5728 -huggingface_hub/utils/_cache_manager.py,sha256=XbeYoZMj8_JCl6eqRviHO6DxGSS29r5Pj38xLlao96Y,34364 -huggingface_hub/utils/_chunk_utils.py,sha256=MH7-6FwCDZ8noV6dGRytCOJGSfcZmDBvsvVotdI8TvQ,2109 -huggingface_hub/utils/_datetime.py,sha256=kCS5jaKV25kOncX1xujbXsz5iDLcjLcLw85semGNzxQ,2770 -huggingface_hub/utils/_deprecation.py,sha256=HZhRGGUX_QMKBBBwHHlffLtmCSK01TOpeXHefZbPfwI,4872 -huggingface_hub/utils/_dotenv.py,sha256=RzHqC8HgzVxE-N4DFBcnemvX0NHmXcV0My2ASK0U1OQ,2017 -huggingface_hub/utils/_experimental.py,sha256=3-c8irbn9sJr2CwWbzhGkIrdXKg8_x7BifhHFy32ei8,2470 -huggingface_hub/utils/_fixes.py,sha256=xQV1QkUn2WpLqLjtXNiyn9gh-454K6AF-Q3kwkYAQD8,4437 -huggingface_hub/utils/_git_credential.py,sha256=ao9rq-rVHn8lghSVZEjDAX4kIkNi7bayY361TDSgSpg,4619 -huggingface_hub/utils/_headers.py,sha256=w4ayq4hLGaZ3B7nwdEi5Zu23SmmDuOwv58It78wkakk,8868 -huggingface_hub/utils/_hf_folder.py,sha256=WNjTnu0Q7tqcSS9EsP4ssCJrrJMcCvAt8P_-LEtmOU8,2487 -huggingface_hub/utils/_http.py,sha256=Cx8MxnXVvlOfg1w30RR03KcFSoIE0WjV1ZX2svwWmx4,25671 -huggingface_hub/utils/_lfs.py,sha256=EC0Oz6Wiwl8foRNkUOzrETXzAWlbgpnpxo5a410ovFY,3957 -huggingface_hub/utils/_pagination.py,sha256=EX5tRasSuQDaKbXuGYbInBK2odnSWNHgzw2tSgqeBRI,1906 -huggingface_hub/utils/_paths.py,sha256=w1ZhFmmD5ykWjp_hAvhjtOoa2ZUcOXJrF4a6O3QpAWo,5042 -huggingface_hub/utils/_runtime.py,sha256=L7SOYezdxKcwd4DovAY0UGY3qt27toXO-QjceIDwExk,11634 -huggingface_hub/utils/_safetensors.py,sha256=GW3nyv7xQcuwObKYeYoT9VhURVzG1DZTbKBKho8Bbos,4458 -huggingface_hub/utils/_subprocess.py,sha256=u9FFUDE7TrzQTiuEzlUnHx7S2P57GbYRV8u16GJwrFw,4625 -huggingface_hub/utils/_telemetry.py,sha256=54LXeIJU5pEGghPAh06gqNAR-UoxOjVLvKqAQscwqZs,4890 -huggingface_hub/utils/_typing.py,sha256=z-134-HG_qJc0cjdSXkmDm3vIRyF5aEfbZgJCB_Qp2Y,3628 -huggingface_hub/utils/_validators.py,sha256=u8AacmA9xCCyer8efmzl1EpQUWTe3zVzsWSJSv3uxTU,9190 -huggingface_hub/utils/_xet.py,sha256=f8qfk8YKePAeGUL6lQiQ1w_3bcs78oWwbeACYdUeg5k,7312 -huggingface_hub/utils/_xet_progress_reporting.py,sha256=JK64hv8orABfNnk1_Wd0YyD_5FfeyVeBvelKpjaNIvs,6169 -huggingface_hub/utils/endpoint_helpers.py,sha256=9VtIAlxQ5H_4y30sjCAgbu7XCqAtNLC7aRYxaNn0hLI,2366 -huggingface_hub/utils/insecure_hashlib.py,sha256=iAaepavFZ5Dhfa5n8KozRfQprKmvcjSnt3X58OUl9fQ,1142 -huggingface_hub/utils/logging.py,sha256=N6NXaCcbPbZSF-Oe-TY3ZnmkpmdFVyTOV8ASo-yVXLE,4916 -huggingface_hub/utils/sha.py,sha256=OFnNGCba0sNcT2gUwaVCJnldxlltrHHe0DS_PCpV3C4,2134 -huggingface_hub/utils/tqdm.py,sha256=xAKcyfnNHsZ7L09WuEM5Ew5-MDhiahLACbbN2zMmcLs,10671 diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/WHEEL deleted file mode 100644 index 8acb95590701b87bf84eec079cf4e3989f63b098..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/WHEEL +++ /dev/null @@ -1,5 +0,0 @@ -Wheel-Version: 1.0 -Generator: setuptools (79.0.1) -Root-Is-Purelib: true -Tag: py3-none-any - diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/entry_points.txt b/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/entry_points.txt deleted file mode 100644 index cec0bd9209b86c3bf3de87d735efcf332f6f80eb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/entry_points.txt +++ /dev/null @@ -1,7 +0,0 @@ -[console_scripts] -hf = huggingface_hub.cli.hf:main -huggingface-cli = huggingface_hub.commands.huggingface_cli:main -tiny-agents = huggingface_hub.inference._mcp.cli:app - -[fsspec.specs] -hf = huggingface_hub.HfFileSystem diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/licenses/LICENSE b/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/licenses/LICENSE deleted file mode 100644 index 261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/licenses/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/top_level.txt b/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/top_level.txt deleted file mode 100644 index 6b964ccca3c1b6766042b3fe3b2707ba25372924..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub-0.36.2.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -huggingface_hub diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/__init__.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/__init__.py deleted file mode 100644 index 643ef9ac20f22f22c4d2d383de4840983849ba30..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/__init__.py +++ /dev/null @@ -1,1554 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# *********** -# `huggingface_hub` init has 2 modes: -# - Normal usage: -# If imported to use it, all modules and functions are lazy-loaded. This means -# they exist at top level in module but are imported only the first time they are -# used. This way, `from huggingface_hub import something` will import `something` -# quickly without the hassle of importing all the features from `huggingface_hub`. -# - Static check: -# If statically analyzed, all modules and functions are loaded normally. This way -# static typing check works properly as well as autocomplete in text editors and -# IDEs. -# -# The static model imports are done inside the `if TYPE_CHECKING:` statement at -# the bottom of this file. Since module/functions imports are duplicated, it is -# mandatory to make sure to add them twice when adding one. This is checked in the -# `make quality` command. -# -# To update the static imports, please run the following command and commit the changes. -# ``` -# # Use script -# python utils/check_static_imports.py --update-file -# -# # Or run style on codebase -# make style -# ``` -# -# *********** -# Lazy loader vendored from https://github.com/scientific-python/lazy_loader -import importlib -import os -import sys -from typing import TYPE_CHECKING - - -__version__ = "0.36.2" - -# Alphabetical order of definitions is ensured in tests -# WARNING: any comment added in this dictionary definition will be lost when -# re-generating the file ! -_SUBMOD_ATTRS = { - "_commit_scheduler": [ - "CommitScheduler", - ], - "_inference_endpoints": [ - "InferenceEndpoint", - "InferenceEndpointError", - "InferenceEndpointStatus", - "InferenceEndpointTimeoutError", - "InferenceEndpointType", - ], - "_jobs_api": [ - "JobInfo", - "JobOwner", - "JobStage", - "JobStatus", - ], - "_login": [ - "auth_list", - "auth_switch", - "interpreter_login", - "login", - "logout", - "notebook_login", - ], - "_oauth": [ - "OAuthInfo", - "OAuthOrgInfo", - "OAuthUserInfo", - "attach_huggingface_oauth", - "parse_huggingface_oauth", - ], - "_snapshot_download": [ - "snapshot_download", - ], - "_space_api": [ - "SpaceHardware", - "SpaceRuntime", - "SpaceStage", - "SpaceStorage", - "SpaceVariable", - ], - "_tensorboard_logger": [ - "HFSummaryWriter", - ], - "_webhooks_payload": [ - "WebhookPayload", - "WebhookPayloadComment", - "WebhookPayloadDiscussion", - "WebhookPayloadDiscussionChanges", - "WebhookPayloadEvent", - "WebhookPayloadMovedTo", - "WebhookPayloadRepo", - "WebhookPayloadUrl", - "WebhookPayloadWebhook", - ], - "_webhooks_server": [ - "WebhooksServer", - "webhook_endpoint", - ], - "community": [ - "Discussion", - "DiscussionComment", - "DiscussionCommit", - "DiscussionEvent", - "DiscussionStatusChange", - "DiscussionTitleChange", - "DiscussionWithDetails", - ], - "constants": [ - "CONFIG_NAME", - "FLAX_WEIGHTS_NAME", - "HUGGINGFACE_CO_URL_HOME", - "HUGGINGFACE_CO_URL_TEMPLATE", - "PYTORCH_WEIGHTS_NAME", - "REPO_TYPE_DATASET", - "REPO_TYPE_MODEL", - "REPO_TYPE_SPACE", - "TF2_WEIGHTS_NAME", - "TF_WEIGHTS_NAME", - ], - "fastai_utils": [ - "_save_pretrained_fastai", - "from_pretrained_fastai", - "push_to_hub_fastai", - ], - "file_download": [ - "HfFileMetadata", - "_CACHED_NO_EXIST", - "get_hf_file_metadata", - "hf_hub_download", - "hf_hub_url", - "try_to_load_from_cache", - ], - "hf_api": [ - "Collection", - "CollectionItem", - "CommitInfo", - "CommitOperation", - "CommitOperationAdd", - "CommitOperationCopy", - "CommitOperationDelete", - "DatasetInfo", - "GitCommitInfo", - "GitRefInfo", - "GitRefs", - "HfApi", - "ModelInfo", - "Organization", - "RepoUrl", - "SpaceInfo", - "User", - "UserLikes", - "WebhookInfo", - "WebhookWatchedItem", - "accept_access_request", - "add_collection_item", - "add_space_secret", - "add_space_variable", - "auth_check", - "cancel_access_request", - "cancel_job", - "change_discussion_status", - "comment_discussion", - "create_branch", - "create_collection", - "create_commit", - "create_discussion", - "create_inference_endpoint", - "create_inference_endpoint_from_catalog", - "create_pull_request", - "create_repo", - "create_scheduled_job", - "create_scheduled_uv_job", - "create_tag", - "create_webhook", - "dataset_info", - "delete_branch", - "delete_collection", - "delete_collection_item", - "delete_file", - "delete_folder", - "delete_inference_endpoint", - "delete_repo", - "delete_scheduled_job", - "delete_space_secret", - "delete_space_storage", - "delete_space_variable", - "delete_tag", - "delete_webhook", - "disable_webhook", - "duplicate_space", - "edit_discussion_comment", - "enable_webhook", - "fetch_job_logs", - "file_exists", - "get_collection", - "get_dataset_tags", - "get_discussion_details", - "get_full_repo_name", - "get_inference_endpoint", - "get_model_tags", - "get_organization_overview", - "get_paths_info", - "get_repo_discussions", - "get_safetensors_metadata", - "get_space_runtime", - "get_space_variables", - "get_token_permission", - "get_user_overview", - "get_webhook", - "grant_access", - "inspect_job", - "inspect_scheduled_job", - "list_accepted_access_requests", - "list_collections", - "list_datasets", - "list_inference_catalog", - "list_inference_endpoints", - "list_jobs", - "list_lfs_files", - "list_liked_repos", - "list_models", - "list_organization_members", - "list_papers", - "list_pending_access_requests", - "list_rejected_access_requests", - "list_repo_commits", - "list_repo_files", - "list_repo_likers", - "list_repo_refs", - "list_repo_tree", - "list_spaces", - "list_user_followers", - "list_user_following", - "list_webhooks", - "merge_pull_request", - "model_info", - "move_repo", - "paper_info", - "parse_safetensors_file_metadata", - "pause_inference_endpoint", - "pause_space", - "permanently_delete_lfs_files", - "preupload_lfs_files", - "reject_access_request", - "rename_discussion", - "repo_exists", - "repo_info", - "repo_type_and_id_from_hf_id", - "request_space_hardware", - "request_space_storage", - "restart_space", - "resume_inference_endpoint", - "resume_scheduled_job", - "revision_exists", - "run_as_future", - "run_job", - "run_uv_job", - "scale_to_zero_inference_endpoint", - "set_space_sleep_time", - "space_info", - "super_squash_history", - "suspend_scheduled_job", - "unlike", - "update_collection_item", - "update_collection_metadata", - "update_inference_endpoint", - "update_repo_settings", - "update_repo_visibility", - "update_webhook", - "upload_file", - "upload_folder", - "upload_large_folder", - "whoami", - ], - "hf_file_system": [ - "HfFileSystem", - "HfFileSystemFile", - "HfFileSystemResolvedPath", - "HfFileSystemStreamFile", - ], - "hub_mixin": [ - "ModelHubMixin", - "PyTorchModelHubMixin", - ], - "inference._client": [ - "InferenceClient", - "InferenceTimeoutError", - ], - "inference._generated._async_client": [ - "AsyncInferenceClient", - ], - "inference._generated.types": [ - "AudioClassificationInput", - "AudioClassificationOutputElement", - "AudioClassificationOutputTransform", - "AudioClassificationParameters", - "AudioToAudioInput", - "AudioToAudioOutputElement", - "AutomaticSpeechRecognitionEarlyStoppingEnum", - "AutomaticSpeechRecognitionGenerationParameters", - "AutomaticSpeechRecognitionInput", - "AutomaticSpeechRecognitionOutput", - "AutomaticSpeechRecognitionOutputChunk", - "AutomaticSpeechRecognitionParameters", - "ChatCompletionInput", - "ChatCompletionInputFunctionDefinition", - "ChatCompletionInputFunctionName", - "ChatCompletionInputGrammarType", - "ChatCompletionInputJSONSchema", - "ChatCompletionInputMessage", - "ChatCompletionInputMessageChunk", - "ChatCompletionInputMessageChunkType", - "ChatCompletionInputResponseFormatJSONObject", - "ChatCompletionInputResponseFormatJSONSchema", - "ChatCompletionInputResponseFormatText", - "ChatCompletionInputStreamOptions", - "ChatCompletionInputTool", - "ChatCompletionInputToolCall", - "ChatCompletionInputToolChoiceClass", - "ChatCompletionInputToolChoiceEnum", - "ChatCompletionInputURL", - "ChatCompletionOutput", - "ChatCompletionOutputComplete", - "ChatCompletionOutputFunctionDefinition", - "ChatCompletionOutputLogprob", - "ChatCompletionOutputLogprobs", - "ChatCompletionOutputMessage", - "ChatCompletionOutputToolCall", - "ChatCompletionOutputTopLogprob", - "ChatCompletionOutputUsage", - "ChatCompletionStreamOutput", - "ChatCompletionStreamOutputChoice", - "ChatCompletionStreamOutputDelta", - "ChatCompletionStreamOutputDeltaToolCall", - "ChatCompletionStreamOutputFunction", - "ChatCompletionStreamOutputLogprob", - "ChatCompletionStreamOutputLogprobs", - "ChatCompletionStreamOutputTopLogprob", - "ChatCompletionStreamOutputUsage", - "DepthEstimationInput", - "DepthEstimationOutput", - "DocumentQuestionAnsweringInput", - "DocumentQuestionAnsweringInputData", - "DocumentQuestionAnsweringOutputElement", - "DocumentQuestionAnsweringParameters", - "FeatureExtractionInput", - "FeatureExtractionInputTruncationDirection", - "FillMaskInput", - "FillMaskOutputElement", - "FillMaskParameters", - "ImageClassificationInput", - "ImageClassificationOutputElement", - "ImageClassificationOutputTransform", - "ImageClassificationParameters", - "ImageSegmentationInput", - "ImageSegmentationOutputElement", - "ImageSegmentationParameters", - "ImageSegmentationSubtask", - "ImageToImageInput", - "ImageToImageOutput", - "ImageToImageParameters", - "ImageToImageTargetSize", - "ImageToTextEarlyStoppingEnum", - "ImageToTextGenerationParameters", - "ImageToTextInput", - "ImageToTextOutput", - "ImageToTextParameters", - "ImageToVideoInput", - "ImageToVideoOutput", - "ImageToVideoParameters", - "ImageToVideoTargetSize", - "ObjectDetectionBoundingBox", - "ObjectDetectionInput", - "ObjectDetectionOutputElement", - "ObjectDetectionParameters", - "Padding", - "QuestionAnsweringInput", - "QuestionAnsweringInputData", - "QuestionAnsweringOutputElement", - "QuestionAnsweringParameters", - "SentenceSimilarityInput", - "SentenceSimilarityInputData", - "SummarizationInput", - "SummarizationOutput", - "SummarizationParameters", - "SummarizationTruncationStrategy", - "TableQuestionAnsweringInput", - "TableQuestionAnsweringInputData", - "TableQuestionAnsweringOutputElement", - "TableQuestionAnsweringParameters", - "Text2TextGenerationInput", - "Text2TextGenerationOutput", - "Text2TextGenerationParameters", - "Text2TextGenerationTruncationStrategy", - "TextClassificationInput", - "TextClassificationOutputElement", - "TextClassificationOutputTransform", - "TextClassificationParameters", - "TextGenerationInput", - "TextGenerationInputGenerateParameters", - "TextGenerationInputGrammarType", - "TextGenerationOutput", - "TextGenerationOutputBestOfSequence", - "TextGenerationOutputDetails", - "TextGenerationOutputFinishReason", - "TextGenerationOutputPrefillToken", - "TextGenerationOutputToken", - "TextGenerationStreamOutput", - "TextGenerationStreamOutputStreamDetails", - "TextGenerationStreamOutputToken", - "TextToAudioEarlyStoppingEnum", - "TextToAudioGenerationParameters", - "TextToAudioInput", - "TextToAudioOutput", - "TextToAudioParameters", - "TextToImageInput", - "TextToImageOutput", - "TextToImageParameters", - "TextToSpeechEarlyStoppingEnum", - "TextToSpeechGenerationParameters", - "TextToSpeechInput", - "TextToSpeechOutput", - "TextToSpeechParameters", - "TextToVideoInput", - "TextToVideoOutput", - "TextToVideoParameters", - "TokenClassificationAggregationStrategy", - "TokenClassificationInput", - "TokenClassificationOutputElement", - "TokenClassificationParameters", - "TranslationInput", - "TranslationOutput", - "TranslationParameters", - "TranslationTruncationStrategy", - "TypeEnum", - "VideoClassificationInput", - "VideoClassificationOutputElement", - "VideoClassificationOutputTransform", - "VideoClassificationParameters", - "VisualQuestionAnsweringInput", - "VisualQuestionAnsweringInputData", - "VisualQuestionAnsweringOutputElement", - "VisualQuestionAnsweringParameters", - "ZeroShotClassificationInput", - "ZeroShotClassificationOutputElement", - "ZeroShotClassificationParameters", - "ZeroShotImageClassificationInput", - "ZeroShotImageClassificationOutputElement", - "ZeroShotImageClassificationParameters", - "ZeroShotObjectDetectionBoundingBox", - "ZeroShotObjectDetectionInput", - "ZeroShotObjectDetectionOutputElement", - "ZeroShotObjectDetectionParameters", - ], - "inference._mcp.agent": [ - "Agent", - ], - "inference._mcp.mcp_client": [ - "MCPClient", - ], - "inference_api": [ - "InferenceApi", - ], - "keras_mixin": [ - "KerasModelHubMixin", - "from_pretrained_keras", - "push_to_hub_keras", - "save_pretrained_keras", - ], - "repocard": [ - "DatasetCard", - "ModelCard", - "RepoCard", - "SpaceCard", - "metadata_eval_result", - "metadata_load", - "metadata_save", - "metadata_update", - ], - "repocard_data": [ - "CardData", - "DatasetCardData", - "EvalResult", - "ModelCardData", - "SpaceCardData", - ], - "repository": [ - "Repository", - ], - "serialization": [ - "StateDictSplit", - "get_tf_storage_size", - "get_torch_storage_id", - "get_torch_storage_size", - "load_state_dict_from_file", - "load_torch_model", - "save_torch_model", - "save_torch_state_dict", - "split_state_dict_into_shards_factory", - "split_tf_state_dict_into_shards", - "split_torch_state_dict_into_shards", - ], - "serialization._dduf": [ - "DDUFEntry", - "export_entries_as_dduf", - "export_folder_as_dduf", - "read_dduf_file", - ], - "utils": [ - "CacheNotFound", - "CachedFileInfo", - "CachedRepoInfo", - "CachedRevisionInfo", - "CorruptedCacheException", - "DeleteCacheStrategy", - "HFCacheInfo", - "HfFolder", - "cached_assets_path", - "configure_http_backend", - "dump_environment_info", - "get_session", - "get_token", - "logging", - "scan_cache_dir", - ], -} - -# WARNING: __all__ is generated automatically, Any manual edit will be lost when re-generating this file ! -# -# To update the static imports, please run the following command and commit the changes. -# ``` -# # Use script -# python utils/check_all_variable.py --update -# -# # Or run style on codebase -# make style -# ``` - -__all__ = [ - "Agent", - "AsyncInferenceClient", - "AudioClassificationInput", - "AudioClassificationOutputElement", - "AudioClassificationOutputTransform", - "AudioClassificationParameters", - "AudioToAudioInput", - "AudioToAudioOutputElement", - "AutomaticSpeechRecognitionEarlyStoppingEnum", - "AutomaticSpeechRecognitionGenerationParameters", - "AutomaticSpeechRecognitionInput", - "AutomaticSpeechRecognitionOutput", - "AutomaticSpeechRecognitionOutputChunk", - "AutomaticSpeechRecognitionParameters", - "CONFIG_NAME", - "CacheNotFound", - "CachedFileInfo", - "CachedRepoInfo", - "CachedRevisionInfo", - "CardData", - "ChatCompletionInput", - "ChatCompletionInputFunctionDefinition", - "ChatCompletionInputFunctionName", - "ChatCompletionInputGrammarType", - "ChatCompletionInputJSONSchema", - "ChatCompletionInputMessage", - "ChatCompletionInputMessageChunk", - "ChatCompletionInputMessageChunkType", - "ChatCompletionInputResponseFormatJSONObject", - "ChatCompletionInputResponseFormatJSONSchema", - "ChatCompletionInputResponseFormatText", - "ChatCompletionInputStreamOptions", - "ChatCompletionInputTool", - "ChatCompletionInputToolCall", - "ChatCompletionInputToolChoiceClass", - "ChatCompletionInputToolChoiceEnum", - "ChatCompletionInputURL", - "ChatCompletionOutput", - "ChatCompletionOutputComplete", - "ChatCompletionOutputFunctionDefinition", - "ChatCompletionOutputLogprob", - "ChatCompletionOutputLogprobs", - "ChatCompletionOutputMessage", - "ChatCompletionOutputToolCall", - "ChatCompletionOutputTopLogprob", - "ChatCompletionOutputUsage", - "ChatCompletionStreamOutput", - "ChatCompletionStreamOutputChoice", - "ChatCompletionStreamOutputDelta", - "ChatCompletionStreamOutputDeltaToolCall", - "ChatCompletionStreamOutputFunction", - "ChatCompletionStreamOutputLogprob", - "ChatCompletionStreamOutputLogprobs", - "ChatCompletionStreamOutputTopLogprob", - "ChatCompletionStreamOutputUsage", - "Collection", - "CollectionItem", - "CommitInfo", - "CommitOperation", - "CommitOperationAdd", - "CommitOperationCopy", - "CommitOperationDelete", - "CommitScheduler", - "CorruptedCacheException", - "DDUFEntry", - "DatasetCard", - "DatasetCardData", - "DatasetInfo", - "DeleteCacheStrategy", - "DepthEstimationInput", - "DepthEstimationOutput", - "Discussion", - "DiscussionComment", - "DiscussionCommit", - "DiscussionEvent", - "DiscussionStatusChange", - "DiscussionTitleChange", - "DiscussionWithDetails", - "DocumentQuestionAnsweringInput", - "DocumentQuestionAnsweringInputData", - "DocumentQuestionAnsweringOutputElement", - "DocumentQuestionAnsweringParameters", - "EvalResult", - "FLAX_WEIGHTS_NAME", - "FeatureExtractionInput", - "FeatureExtractionInputTruncationDirection", - "FillMaskInput", - "FillMaskOutputElement", - "FillMaskParameters", - "GitCommitInfo", - "GitRefInfo", - "GitRefs", - "HFCacheInfo", - "HFSummaryWriter", - "HUGGINGFACE_CO_URL_HOME", - "HUGGINGFACE_CO_URL_TEMPLATE", - "HfApi", - "HfFileMetadata", - "HfFileSystem", - "HfFileSystemFile", - "HfFileSystemResolvedPath", - "HfFileSystemStreamFile", - "HfFolder", - "ImageClassificationInput", - "ImageClassificationOutputElement", - "ImageClassificationOutputTransform", - "ImageClassificationParameters", - "ImageSegmentationInput", - "ImageSegmentationOutputElement", - "ImageSegmentationParameters", - "ImageSegmentationSubtask", - "ImageToImageInput", - "ImageToImageOutput", - "ImageToImageParameters", - "ImageToImageTargetSize", - "ImageToTextEarlyStoppingEnum", - "ImageToTextGenerationParameters", - "ImageToTextInput", - "ImageToTextOutput", - "ImageToTextParameters", - "ImageToVideoInput", - "ImageToVideoOutput", - "ImageToVideoParameters", - "ImageToVideoTargetSize", - "InferenceApi", - "InferenceClient", - "InferenceEndpoint", - "InferenceEndpointError", - "InferenceEndpointStatus", - "InferenceEndpointTimeoutError", - "InferenceEndpointType", - "InferenceTimeoutError", - "JobInfo", - "JobOwner", - "JobStage", - "JobStatus", - "KerasModelHubMixin", - "MCPClient", - "ModelCard", - "ModelCardData", - "ModelHubMixin", - "ModelInfo", - "OAuthInfo", - "OAuthOrgInfo", - "OAuthUserInfo", - "ObjectDetectionBoundingBox", - "ObjectDetectionInput", - "ObjectDetectionOutputElement", - "ObjectDetectionParameters", - "Organization", - "PYTORCH_WEIGHTS_NAME", - "Padding", - "PyTorchModelHubMixin", - "QuestionAnsweringInput", - "QuestionAnsweringInputData", - "QuestionAnsweringOutputElement", - "QuestionAnsweringParameters", - "REPO_TYPE_DATASET", - "REPO_TYPE_MODEL", - "REPO_TYPE_SPACE", - "RepoCard", - "RepoUrl", - "Repository", - "SentenceSimilarityInput", - "SentenceSimilarityInputData", - "SpaceCard", - "SpaceCardData", - "SpaceHardware", - "SpaceInfo", - "SpaceRuntime", - "SpaceStage", - "SpaceStorage", - "SpaceVariable", - "StateDictSplit", - "SummarizationInput", - "SummarizationOutput", - "SummarizationParameters", - "SummarizationTruncationStrategy", - "TF2_WEIGHTS_NAME", - "TF_WEIGHTS_NAME", - "TableQuestionAnsweringInput", - "TableQuestionAnsweringInputData", - "TableQuestionAnsweringOutputElement", - "TableQuestionAnsweringParameters", - "Text2TextGenerationInput", - "Text2TextGenerationOutput", - "Text2TextGenerationParameters", - "Text2TextGenerationTruncationStrategy", - "TextClassificationInput", - "TextClassificationOutputElement", - "TextClassificationOutputTransform", - "TextClassificationParameters", - "TextGenerationInput", - "TextGenerationInputGenerateParameters", - "TextGenerationInputGrammarType", - "TextGenerationOutput", - "TextGenerationOutputBestOfSequence", - "TextGenerationOutputDetails", - "TextGenerationOutputFinishReason", - "TextGenerationOutputPrefillToken", - "TextGenerationOutputToken", - "TextGenerationStreamOutput", - "TextGenerationStreamOutputStreamDetails", - "TextGenerationStreamOutputToken", - "TextToAudioEarlyStoppingEnum", - "TextToAudioGenerationParameters", - "TextToAudioInput", - "TextToAudioOutput", - "TextToAudioParameters", - "TextToImageInput", - "TextToImageOutput", - "TextToImageParameters", - "TextToSpeechEarlyStoppingEnum", - "TextToSpeechGenerationParameters", - "TextToSpeechInput", - "TextToSpeechOutput", - "TextToSpeechParameters", - "TextToVideoInput", - "TextToVideoOutput", - "TextToVideoParameters", - "TokenClassificationAggregationStrategy", - "TokenClassificationInput", - "TokenClassificationOutputElement", - "TokenClassificationParameters", - "TranslationInput", - "TranslationOutput", - "TranslationParameters", - "TranslationTruncationStrategy", - "TypeEnum", - "User", - "UserLikes", - "VideoClassificationInput", - "VideoClassificationOutputElement", - "VideoClassificationOutputTransform", - "VideoClassificationParameters", - "VisualQuestionAnsweringInput", - "VisualQuestionAnsweringInputData", - "VisualQuestionAnsweringOutputElement", - "VisualQuestionAnsweringParameters", - "WebhookInfo", - "WebhookPayload", - "WebhookPayloadComment", - "WebhookPayloadDiscussion", - "WebhookPayloadDiscussionChanges", - "WebhookPayloadEvent", - "WebhookPayloadMovedTo", - "WebhookPayloadRepo", - "WebhookPayloadUrl", - "WebhookPayloadWebhook", - "WebhookWatchedItem", - "WebhooksServer", - "ZeroShotClassificationInput", - "ZeroShotClassificationOutputElement", - "ZeroShotClassificationParameters", - "ZeroShotImageClassificationInput", - "ZeroShotImageClassificationOutputElement", - "ZeroShotImageClassificationParameters", - "ZeroShotObjectDetectionBoundingBox", - "ZeroShotObjectDetectionInput", - "ZeroShotObjectDetectionOutputElement", - "ZeroShotObjectDetectionParameters", - "_CACHED_NO_EXIST", - "_save_pretrained_fastai", - "accept_access_request", - "add_collection_item", - "add_space_secret", - "add_space_variable", - "attach_huggingface_oauth", - "auth_check", - "auth_list", - "auth_switch", - "cached_assets_path", - "cancel_access_request", - "cancel_job", - "change_discussion_status", - "comment_discussion", - "configure_http_backend", - "create_branch", - "create_collection", - "create_commit", - "create_discussion", - "create_inference_endpoint", - "create_inference_endpoint_from_catalog", - "create_pull_request", - "create_repo", - "create_scheduled_job", - "create_scheduled_uv_job", - "create_tag", - "create_webhook", - "dataset_info", - "delete_branch", - "delete_collection", - "delete_collection_item", - "delete_file", - "delete_folder", - "delete_inference_endpoint", - "delete_repo", - "delete_scheduled_job", - "delete_space_secret", - "delete_space_storage", - "delete_space_variable", - "delete_tag", - "delete_webhook", - "disable_webhook", - "dump_environment_info", - "duplicate_space", - "edit_discussion_comment", - "enable_webhook", - "export_entries_as_dduf", - "export_folder_as_dduf", - "fetch_job_logs", - "file_exists", - "from_pretrained_fastai", - "from_pretrained_keras", - "get_collection", - "get_dataset_tags", - "get_discussion_details", - "get_full_repo_name", - "get_hf_file_metadata", - "get_inference_endpoint", - "get_model_tags", - "get_organization_overview", - "get_paths_info", - "get_repo_discussions", - "get_safetensors_metadata", - "get_session", - "get_space_runtime", - "get_space_variables", - "get_tf_storage_size", - "get_token", - "get_token_permission", - "get_torch_storage_id", - "get_torch_storage_size", - "get_user_overview", - "get_webhook", - "grant_access", - "hf_hub_download", - "hf_hub_url", - "inspect_job", - "inspect_scheduled_job", - "interpreter_login", - "list_accepted_access_requests", - "list_collections", - "list_datasets", - "list_inference_catalog", - "list_inference_endpoints", - "list_jobs", - "list_lfs_files", - "list_liked_repos", - "list_models", - "list_organization_members", - "list_papers", - "list_pending_access_requests", - "list_rejected_access_requests", - "list_repo_commits", - "list_repo_files", - "list_repo_likers", - "list_repo_refs", - "list_repo_tree", - "list_spaces", - "list_user_followers", - "list_user_following", - "list_webhooks", - "load_state_dict_from_file", - "load_torch_model", - "logging", - "login", - "logout", - "merge_pull_request", - "metadata_eval_result", - "metadata_load", - "metadata_save", - "metadata_update", - "model_info", - "move_repo", - "notebook_login", - "paper_info", - "parse_huggingface_oauth", - "parse_safetensors_file_metadata", - "pause_inference_endpoint", - "pause_space", - "permanently_delete_lfs_files", - "preupload_lfs_files", - "push_to_hub_fastai", - "push_to_hub_keras", - "read_dduf_file", - "reject_access_request", - "rename_discussion", - "repo_exists", - "repo_info", - "repo_type_and_id_from_hf_id", - "request_space_hardware", - "request_space_storage", - "restart_space", - "resume_inference_endpoint", - "resume_scheduled_job", - "revision_exists", - "run_as_future", - "run_job", - "run_uv_job", - "save_pretrained_keras", - "save_torch_model", - "save_torch_state_dict", - "scale_to_zero_inference_endpoint", - "scan_cache_dir", - "set_space_sleep_time", - "snapshot_download", - "space_info", - "split_state_dict_into_shards_factory", - "split_tf_state_dict_into_shards", - "split_torch_state_dict_into_shards", - "super_squash_history", - "suspend_scheduled_job", - "try_to_load_from_cache", - "unlike", - "update_collection_item", - "update_collection_metadata", - "update_inference_endpoint", - "update_repo_settings", - "update_repo_visibility", - "update_webhook", - "upload_file", - "upload_folder", - "upload_large_folder", - "webhook_endpoint", - "whoami", -] - - -def _attach(package_name, submodules=None, submod_attrs=None): - """Attach lazily loaded submodules, functions, or other attributes. - - Typically, modules import submodules and attributes as follows: - - ```py - import mysubmodule - import anothersubmodule - - from .foo import someattr - ``` - - The idea is to replace a package's `__getattr__`, `__dir__`, such that all imports - work exactly the way they would with normal imports, except that the import occurs - upon first use. - - The typical way to call this function, replacing the above imports, is: - - ```python - __getattr__, __dir__ = lazy.attach( - __name__, - ['mysubmodule', 'anothersubmodule'], - {'foo': ['someattr']} - ) - ``` - This functionality requires Python 3.7 or higher. - - Args: - package_name (`str`): - Typically use `__name__`. - submodules (`set`): - List of submodules to attach. - submod_attrs (`dict`): - Dictionary of submodule -> list of attributes / functions. - These attributes are imported as they are used. - - Returns: - __getattr__, __dir__, __all__ - - """ - if submod_attrs is None: - submod_attrs = {} - - if submodules is None: - submodules = set() - else: - submodules = set(submodules) - - attr_to_modules = {attr: mod for mod, attrs in submod_attrs.items() for attr in attrs} - - def __getattr__(name): - if name in submodules: - try: - return importlib.import_module(f"{package_name}.{name}") - except Exception as e: - print(f"Error importing {package_name}.{name}: {e}") - raise - elif name in attr_to_modules: - submod_path = f"{package_name}.{attr_to_modules[name]}" - try: - submod = importlib.import_module(submod_path) - except Exception as e: - print(f"Error importing {submod_path}: {e}") - raise - attr = getattr(submod, name) - - # If the attribute lives in a file (module) with the same - # name as the attribute, ensure that the attribute and *not* - # the module is accessible on the package. - if name == attr_to_modules[name]: - pkg = sys.modules[package_name] - pkg.__dict__[name] = attr - - return attr - else: - raise AttributeError(f"No {package_name} attribute {name}") - - def __dir__(): - return __all__ - - return __getattr__, __dir__ - - -__getattr__, __dir__ = _attach(__name__, submodules=[], submod_attrs=_SUBMOD_ATTRS) - -if os.environ.get("EAGER_IMPORT", ""): - for attr in __all__: - __getattr__(attr) - -# WARNING: any content below this statement is generated automatically. Any manual edit -# will be lost when re-generating this file ! -# -# To update the static imports, please run the following command and commit the changes. -# ``` -# # Use script -# python utils/check_static_imports.py --update -# -# # Or run style on codebase -# make style -# ``` -if TYPE_CHECKING: # pragma: no cover - from ._commit_scheduler import CommitScheduler # noqa: F401 - from ._inference_endpoints import ( - InferenceEndpoint, # noqa: F401 - InferenceEndpointError, # noqa: F401 - InferenceEndpointStatus, # noqa: F401 - InferenceEndpointTimeoutError, # noqa: F401 - InferenceEndpointType, # noqa: F401 - ) - from ._jobs_api import ( - JobInfo, # noqa: F401 - JobOwner, # noqa: F401 - JobStage, # noqa: F401 - JobStatus, # noqa: F401 - ) - from ._login import ( - auth_list, # noqa: F401 - auth_switch, # noqa: F401 - interpreter_login, # noqa: F401 - login, # noqa: F401 - logout, # noqa: F401 - notebook_login, # noqa: F401 - ) - from ._oauth import ( - OAuthInfo, # noqa: F401 - OAuthOrgInfo, # noqa: F401 - OAuthUserInfo, # noqa: F401 - attach_huggingface_oauth, # noqa: F401 - parse_huggingface_oauth, # noqa: F401 - ) - from ._snapshot_download import snapshot_download # noqa: F401 - from ._space_api import ( - SpaceHardware, # noqa: F401 - SpaceRuntime, # noqa: F401 - SpaceStage, # noqa: F401 - SpaceStorage, # noqa: F401 - SpaceVariable, # noqa: F401 - ) - from ._tensorboard_logger import HFSummaryWriter # noqa: F401 - from ._webhooks_payload import ( - WebhookPayload, # noqa: F401 - WebhookPayloadComment, # noqa: F401 - WebhookPayloadDiscussion, # noqa: F401 - WebhookPayloadDiscussionChanges, # noqa: F401 - WebhookPayloadEvent, # noqa: F401 - WebhookPayloadMovedTo, # noqa: F401 - WebhookPayloadRepo, # noqa: F401 - WebhookPayloadUrl, # noqa: F401 - WebhookPayloadWebhook, # noqa: F401 - ) - from ._webhooks_server import ( - WebhooksServer, # noqa: F401 - webhook_endpoint, # noqa: F401 - ) - from .community import ( - Discussion, # noqa: F401 - DiscussionComment, # noqa: F401 - DiscussionCommit, # noqa: F401 - DiscussionEvent, # noqa: F401 - DiscussionStatusChange, # noqa: F401 - DiscussionTitleChange, # noqa: F401 - DiscussionWithDetails, # noqa: F401 - ) - from .constants import ( - CONFIG_NAME, # noqa: F401 - FLAX_WEIGHTS_NAME, # noqa: F401 - HUGGINGFACE_CO_URL_HOME, # noqa: F401 - HUGGINGFACE_CO_URL_TEMPLATE, # noqa: F401 - PYTORCH_WEIGHTS_NAME, # noqa: F401 - REPO_TYPE_DATASET, # noqa: F401 - REPO_TYPE_MODEL, # noqa: F401 - REPO_TYPE_SPACE, # noqa: F401 - TF2_WEIGHTS_NAME, # noqa: F401 - TF_WEIGHTS_NAME, # noqa: F401 - ) - from .fastai_utils import ( - _save_pretrained_fastai, # noqa: F401 - from_pretrained_fastai, # noqa: F401 - push_to_hub_fastai, # noqa: F401 - ) - from .file_download import ( - _CACHED_NO_EXIST, # noqa: F401 - HfFileMetadata, # noqa: F401 - get_hf_file_metadata, # noqa: F401 - hf_hub_download, # noqa: F401 - hf_hub_url, # noqa: F401 - try_to_load_from_cache, # noqa: F401 - ) - from .hf_api import ( - Collection, # noqa: F401 - CollectionItem, # noqa: F401 - CommitInfo, # noqa: F401 - CommitOperation, # noqa: F401 - CommitOperationAdd, # noqa: F401 - CommitOperationCopy, # noqa: F401 - CommitOperationDelete, # noqa: F401 - DatasetInfo, # noqa: F401 - GitCommitInfo, # noqa: F401 - GitRefInfo, # noqa: F401 - GitRefs, # noqa: F401 - HfApi, # noqa: F401 - ModelInfo, # noqa: F401 - Organization, # noqa: F401 - RepoUrl, # noqa: F401 - SpaceInfo, # noqa: F401 - User, # noqa: F401 - UserLikes, # noqa: F401 - WebhookInfo, # noqa: F401 - WebhookWatchedItem, # noqa: F401 - accept_access_request, # noqa: F401 - add_collection_item, # noqa: F401 - add_space_secret, # noqa: F401 - add_space_variable, # noqa: F401 - auth_check, # noqa: F401 - cancel_access_request, # noqa: F401 - cancel_job, # noqa: F401 - change_discussion_status, # noqa: F401 - comment_discussion, # noqa: F401 - create_branch, # noqa: F401 - create_collection, # noqa: F401 - create_commit, # noqa: F401 - create_discussion, # noqa: F401 - create_inference_endpoint, # noqa: F401 - create_inference_endpoint_from_catalog, # noqa: F401 - create_pull_request, # noqa: F401 - create_repo, # noqa: F401 - create_scheduled_job, # noqa: F401 - create_scheduled_uv_job, # noqa: F401 - create_tag, # noqa: F401 - create_webhook, # noqa: F401 - dataset_info, # noqa: F401 - delete_branch, # noqa: F401 - delete_collection, # noqa: F401 - delete_collection_item, # noqa: F401 - delete_file, # noqa: F401 - delete_folder, # noqa: F401 - delete_inference_endpoint, # noqa: F401 - delete_repo, # noqa: F401 - delete_scheduled_job, # noqa: F401 - delete_space_secret, # noqa: F401 - delete_space_storage, # noqa: F401 - delete_space_variable, # noqa: F401 - delete_tag, # noqa: F401 - delete_webhook, # noqa: F401 - disable_webhook, # noqa: F401 - duplicate_space, # noqa: F401 - edit_discussion_comment, # noqa: F401 - enable_webhook, # noqa: F401 - fetch_job_logs, # noqa: F401 - file_exists, # noqa: F401 - get_collection, # noqa: F401 - get_dataset_tags, # noqa: F401 - get_discussion_details, # noqa: F401 - get_full_repo_name, # noqa: F401 - get_inference_endpoint, # noqa: F401 - get_model_tags, # noqa: F401 - get_organization_overview, # noqa: F401 - get_paths_info, # noqa: F401 - get_repo_discussions, # noqa: F401 - get_safetensors_metadata, # noqa: F401 - get_space_runtime, # noqa: F401 - get_space_variables, # noqa: F401 - get_token_permission, # noqa: F401 - get_user_overview, # noqa: F401 - get_webhook, # noqa: F401 - grant_access, # noqa: F401 - inspect_job, # noqa: F401 - inspect_scheduled_job, # noqa: F401 - list_accepted_access_requests, # noqa: F401 - list_collections, # noqa: F401 - list_datasets, # noqa: F401 - list_inference_catalog, # noqa: F401 - list_inference_endpoints, # noqa: F401 - list_jobs, # noqa: F401 - list_lfs_files, # noqa: F401 - list_liked_repos, # noqa: F401 - list_models, # noqa: F401 - list_organization_members, # noqa: F401 - list_papers, # noqa: F401 - list_pending_access_requests, # noqa: F401 - list_rejected_access_requests, # noqa: F401 - list_repo_commits, # noqa: F401 - list_repo_files, # noqa: F401 - list_repo_likers, # noqa: F401 - list_repo_refs, # noqa: F401 - list_repo_tree, # noqa: F401 - list_spaces, # noqa: F401 - list_user_followers, # noqa: F401 - list_user_following, # noqa: F401 - list_webhooks, # noqa: F401 - merge_pull_request, # noqa: F401 - model_info, # noqa: F401 - move_repo, # noqa: F401 - paper_info, # noqa: F401 - parse_safetensors_file_metadata, # noqa: F401 - pause_inference_endpoint, # noqa: F401 - pause_space, # noqa: F401 - permanently_delete_lfs_files, # noqa: F401 - preupload_lfs_files, # noqa: F401 - reject_access_request, # noqa: F401 - rename_discussion, # noqa: F401 - repo_exists, # noqa: F401 - repo_info, # noqa: F401 - repo_type_and_id_from_hf_id, # noqa: F401 - request_space_hardware, # noqa: F401 - request_space_storage, # noqa: F401 - restart_space, # noqa: F401 - resume_inference_endpoint, # noqa: F401 - resume_scheduled_job, # noqa: F401 - revision_exists, # noqa: F401 - run_as_future, # noqa: F401 - run_job, # noqa: F401 - run_uv_job, # noqa: F401 - scale_to_zero_inference_endpoint, # noqa: F401 - set_space_sleep_time, # noqa: F401 - space_info, # noqa: F401 - super_squash_history, # noqa: F401 - suspend_scheduled_job, # noqa: F401 - unlike, # noqa: F401 - update_collection_item, # noqa: F401 - update_collection_metadata, # noqa: F401 - update_inference_endpoint, # noqa: F401 - update_repo_settings, # noqa: F401 - update_repo_visibility, # noqa: F401 - update_webhook, # noqa: F401 - upload_file, # noqa: F401 - upload_folder, # noqa: F401 - upload_large_folder, # noqa: F401 - whoami, # noqa: F401 - ) - from .hf_file_system import ( - HfFileSystem, # noqa: F401 - HfFileSystemFile, # noqa: F401 - HfFileSystemResolvedPath, # noqa: F401 - HfFileSystemStreamFile, # noqa: F401 - ) - from .hub_mixin import ( - ModelHubMixin, # noqa: F401 - PyTorchModelHubMixin, # noqa: F401 - ) - from .inference._client import ( - InferenceClient, # noqa: F401 - InferenceTimeoutError, # noqa: F401 - ) - from .inference._generated._async_client import AsyncInferenceClient # noqa: F401 - from .inference._generated.types import ( - AudioClassificationInput, # noqa: F401 - AudioClassificationOutputElement, # noqa: F401 - AudioClassificationOutputTransform, # noqa: F401 - AudioClassificationParameters, # noqa: F401 - AudioToAudioInput, # noqa: F401 - AudioToAudioOutputElement, # noqa: F401 - AutomaticSpeechRecognitionEarlyStoppingEnum, # noqa: F401 - AutomaticSpeechRecognitionGenerationParameters, # noqa: F401 - AutomaticSpeechRecognitionInput, # noqa: F401 - AutomaticSpeechRecognitionOutput, # noqa: F401 - AutomaticSpeechRecognitionOutputChunk, # noqa: F401 - AutomaticSpeechRecognitionParameters, # noqa: F401 - ChatCompletionInput, # noqa: F401 - ChatCompletionInputFunctionDefinition, # noqa: F401 - ChatCompletionInputFunctionName, # noqa: F401 - ChatCompletionInputGrammarType, # noqa: F401 - ChatCompletionInputJSONSchema, # noqa: F401 - ChatCompletionInputMessage, # noqa: F401 - ChatCompletionInputMessageChunk, # noqa: F401 - ChatCompletionInputMessageChunkType, # noqa: F401 - ChatCompletionInputResponseFormatJSONObject, # noqa: F401 - ChatCompletionInputResponseFormatJSONSchema, # noqa: F401 - ChatCompletionInputResponseFormatText, # noqa: F401 - ChatCompletionInputStreamOptions, # noqa: F401 - ChatCompletionInputTool, # noqa: F401 - ChatCompletionInputToolCall, # noqa: F401 - ChatCompletionInputToolChoiceClass, # noqa: F401 - ChatCompletionInputToolChoiceEnum, # noqa: F401 - ChatCompletionInputURL, # noqa: F401 - ChatCompletionOutput, # noqa: F401 - ChatCompletionOutputComplete, # noqa: F401 - ChatCompletionOutputFunctionDefinition, # noqa: F401 - ChatCompletionOutputLogprob, # noqa: F401 - ChatCompletionOutputLogprobs, # noqa: F401 - ChatCompletionOutputMessage, # noqa: F401 - ChatCompletionOutputToolCall, # noqa: F401 - ChatCompletionOutputTopLogprob, # noqa: F401 - ChatCompletionOutputUsage, # noqa: F401 - ChatCompletionStreamOutput, # noqa: F401 - ChatCompletionStreamOutputChoice, # noqa: F401 - ChatCompletionStreamOutputDelta, # noqa: F401 - ChatCompletionStreamOutputDeltaToolCall, # noqa: F401 - ChatCompletionStreamOutputFunction, # noqa: F401 - ChatCompletionStreamOutputLogprob, # noqa: F401 - ChatCompletionStreamOutputLogprobs, # noqa: F401 - ChatCompletionStreamOutputTopLogprob, # noqa: F401 - ChatCompletionStreamOutputUsage, # noqa: F401 - DepthEstimationInput, # noqa: F401 - DepthEstimationOutput, # noqa: F401 - DocumentQuestionAnsweringInput, # noqa: F401 - DocumentQuestionAnsweringInputData, # noqa: F401 - DocumentQuestionAnsweringOutputElement, # noqa: F401 - DocumentQuestionAnsweringParameters, # noqa: F401 - FeatureExtractionInput, # noqa: F401 - FeatureExtractionInputTruncationDirection, # noqa: F401 - FillMaskInput, # noqa: F401 - FillMaskOutputElement, # noqa: F401 - FillMaskParameters, # noqa: F401 - ImageClassificationInput, # noqa: F401 - ImageClassificationOutputElement, # noqa: F401 - ImageClassificationOutputTransform, # noqa: F401 - ImageClassificationParameters, # noqa: F401 - ImageSegmentationInput, # noqa: F401 - ImageSegmentationOutputElement, # noqa: F401 - ImageSegmentationParameters, # noqa: F401 - ImageSegmentationSubtask, # noqa: F401 - ImageToImageInput, # noqa: F401 - ImageToImageOutput, # noqa: F401 - ImageToImageParameters, # noqa: F401 - ImageToImageTargetSize, # noqa: F401 - ImageToTextEarlyStoppingEnum, # noqa: F401 - ImageToTextGenerationParameters, # noqa: F401 - ImageToTextInput, # noqa: F401 - ImageToTextOutput, # noqa: F401 - ImageToTextParameters, # noqa: F401 - ImageToVideoInput, # noqa: F401 - ImageToVideoOutput, # noqa: F401 - ImageToVideoParameters, # noqa: F401 - ImageToVideoTargetSize, # noqa: F401 - ObjectDetectionBoundingBox, # noqa: F401 - ObjectDetectionInput, # noqa: F401 - ObjectDetectionOutputElement, # noqa: F401 - ObjectDetectionParameters, # noqa: F401 - Padding, # noqa: F401 - QuestionAnsweringInput, # noqa: F401 - QuestionAnsweringInputData, # noqa: F401 - QuestionAnsweringOutputElement, # noqa: F401 - QuestionAnsweringParameters, # noqa: F401 - SentenceSimilarityInput, # noqa: F401 - SentenceSimilarityInputData, # noqa: F401 - SummarizationInput, # noqa: F401 - SummarizationOutput, # noqa: F401 - SummarizationParameters, # noqa: F401 - SummarizationTruncationStrategy, # noqa: F401 - TableQuestionAnsweringInput, # noqa: F401 - TableQuestionAnsweringInputData, # noqa: F401 - TableQuestionAnsweringOutputElement, # noqa: F401 - TableQuestionAnsweringParameters, # noqa: F401 - Text2TextGenerationInput, # noqa: F401 - Text2TextGenerationOutput, # noqa: F401 - Text2TextGenerationParameters, # noqa: F401 - Text2TextGenerationTruncationStrategy, # noqa: F401 - TextClassificationInput, # noqa: F401 - TextClassificationOutputElement, # noqa: F401 - TextClassificationOutputTransform, # noqa: F401 - TextClassificationParameters, # noqa: F401 - TextGenerationInput, # noqa: F401 - TextGenerationInputGenerateParameters, # noqa: F401 - TextGenerationInputGrammarType, # noqa: F401 - TextGenerationOutput, # noqa: F401 - TextGenerationOutputBestOfSequence, # noqa: F401 - TextGenerationOutputDetails, # noqa: F401 - TextGenerationOutputFinishReason, # noqa: F401 - TextGenerationOutputPrefillToken, # noqa: F401 - TextGenerationOutputToken, # noqa: F401 - TextGenerationStreamOutput, # noqa: F401 - TextGenerationStreamOutputStreamDetails, # noqa: F401 - TextGenerationStreamOutputToken, # noqa: F401 - TextToAudioEarlyStoppingEnum, # noqa: F401 - TextToAudioGenerationParameters, # noqa: F401 - TextToAudioInput, # noqa: F401 - TextToAudioOutput, # noqa: F401 - TextToAudioParameters, # noqa: F401 - TextToImageInput, # noqa: F401 - TextToImageOutput, # noqa: F401 - TextToImageParameters, # noqa: F401 - TextToSpeechEarlyStoppingEnum, # noqa: F401 - TextToSpeechGenerationParameters, # noqa: F401 - TextToSpeechInput, # noqa: F401 - TextToSpeechOutput, # noqa: F401 - TextToSpeechParameters, # noqa: F401 - TextToVideoInput, # noqa: F401 - TextToVideoOutput, # noqa: F401 - TextToVideoParameters, # noqa: F401 - TokenClassificationAggregationStrategy, # noqa: F401 - TokenClassificationInput, # noqa: F401 - TokenClassificationOutputElement, # noqa: F401 - TokenClassificationParameters, # noqa: F401 - TranslationInput, # noqa: F401 - TranslationOutput, # noqa: F401 - TranslationParameters, # noqa: F401 - TranslationTruncationStrategy, # noqa: F401 - TypeEnum, # noqa: F401 - VideoClassificationInput, # noqa: F401 - VideoClassificationOutputElement, # noqa: F401 - VideoClassificationOutputTransform, # noqa: F401 - VideoClassificationParameters, # noqa: F401 - VisualQuestionAnsweringInput, # noqa: F401 - VisualQuestionAnsweringInputData, # noqa: F401 - VisualQuestionAnsweringOutputElement, # noqa: F401 - VisualQuestionAnsweringParameters, # noqa: F401 - ZeroShotClassificationInput, # noqa: F401 - ZeroShotClassificationOutputElement, # noqa: F401 - ZeroShotClassificationParameters, # noqa: F401 - ZeroShotImageClassificationInput, # noqa: F401 - ZeroShotImageClassificationOutputElement, # noqa: F401 - ZeroShotImageClassificationParameters, # noqa: F401 - ZeroShotObjectDetectionBoundingBox, # noqa: F401 - ZeroShotObjectDetectionInput, # noqa: F401 - ZeroShotObjectDetectionOutputElement, # noqa: F401 - ZeroShotObjectDetectionParameters, # noqa: F401 - ) - from .inference._mcp.agent import Agent # noqa: F401 - from .inference._mcp.mcp_client import MCPClient # noqa: F401 - from .inference_api import InferenceApi # noqa: F401 - from .keras_mixin import ( - KerasModelHubMixin, # noqa: F401 - from_pretrained_keras, # noqa: F401 - push_to_hub_keras, # noqa: F401 - save_pretrained_keras, # noqa: F401 - ) - from .repocard import ( - DatasetCard, # noqa: F401 - ModelCard, # noqa: F401 - RepoCard, # noqa: F401 - SpaceCard, # noqa: F401 - metadata_eval_result, # noqa: F401 - metadata_load, # noqa: F401 - metadata_save, # noqa: F401 - metadata_update, # noqa: F401 - ) - from .repocard_data import ( - CardData, # noqa: F401 - DatasetCardData, # noqa: F401 - EvalResult, # noqa: F401 - ModelCardData, # noqa: F401 - SpaceCardData, # noqa: F401 - ) - from .repository import Repository # noqa: F401 - from .serialization import ( - StateDictSplit, # noqa: F401 - get_tf_storage_size, # noqa: F401 - get_torch_storage_id, # noqa: F401 - get_torch_storage_size, # noqa: F401 - load_state_dict_from_file, # noqa: F401 - load_torch_model, # noqa: F401 - save_torch_model, # noqa: F401 - save_torch_state_dict, # noqa: F401 - split_state_dict_into_shards_factory, # noqa: F401 - split_tf_state_dict_into_shards, # noqa: F401 - split_torch_state_dict_into_shards, # noqa: F401 - ) - from .serialization._dduf import ( - DDUFEntry, # noqa: F401 - export_entries_as_dduf, # noqa: F401 - export_folder_as_dduf, # noqa: F401 - read_dduf_file, # noqa: F401 - ) - from .utils import ( - CachedFileInfo, # noqa: F401 - CachedRepoInfo, # noqa: F401 - CachedRevisionInfo, # noqa: F401 - CacheNotFound, # noqa: F401 - CorruptedCacheException, # noqa: F401 - DeleteCacheStrategy, # noqa: F401 - HFCacheInfo, # noqa: F401 - HfFolder, # noqa: F401 - cached_assets_path, # noqa: F401 - configure_http_backend, # noqa: F401 - dump_environment_info, # noqa: F401 - get_session, # noqa: F401 - get_token, # noqa: F401 - logging, # noqa: F401 - scan_cache_dir, # noqa: F401 - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_commit_api.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_commit_api.py deleted file mode 100644 index 7ed64b0e5ed550c392f193239a2e00669cc3144a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_commit_api.py +++ /dev/null @@ -1,968 +0,0 @@ -""" -Type definitions and utilities for the `create_commit` API -""" - -import base64 -import io -import os -import warnings -from collections import defaultdict -from contextlib import contextmanager -from dataclasses import dataclass, field -from itertools import groupby -from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Iterable, Iterator, List, Literal, Optional, Tuple, Union - -from tqdm.contrib.concurrent import thread_map - -from . import constants -from .errors import EntryNotFoundError, HfHubHTTPError, XetAuthorizationError, XetRefreshTokenError -from .file_download import hf_hub_url -from .lfs import UploadInfo, lfs_upload, post_lfs_batch_info -from .utils import ( - FORBIDDEN_FOLDERS, - XetTokenType, - are_progress_bars_disabled, - chunk_iterable, - fetch_xet_connection_info_from_repo_info, - get_session, - hf_raise_for_status, - logging, - sha, - tqdm_stream_file, - validate_hf_hub_args, -) -from .utils import tqdm as hf_tqdm -from .utils._runtime import is_xet_available - - -if TYPE_CHECKING: - from .hf_api import RepoFile - - -logger = logging.get_logger(__name__) - - -UploadMode = Literal["lfs", "regular"] - -# Max is 1,000 per request on the Hub for HfApi.get_paths_info -# Otherwise we get: -# HfHubHTTPError: 413 Client Error: Payload Too Large for url: https://huggingface.co/api/datasets/xxx (Request ID: xxx)\n\ntoo many parameters -# See https://github.com/huggingface/huggingface_hub/issues/1503 -FETCH_LFS_BATCH_SIZE = 500 - -UPLOAD_BATCH_MAX_NUM_FILES = 256 - - -@dataclass -class CommitOperationDelete: - """ - Data structure holding necessary info to delete a file or a folder from a repository - on the Hub. - - Args: - path_in_repo (`str`): - Relative filepath in the repo, for example: `"checkpoints/1fec34a/weights.bin"` - for a file or `"checkpoints/1fec34a/"` for a folder. - is_folder (`bool` or `Literal["auto"]`, *optional*) - Whether the Delete Operation applies to a folder or not. If "auto", the path - type (file or folder) is guessed automatically by looking if path ends with - a "/" (folder) or not (file). To explicitly set the path type, you can set - `is_folder=True` or `is_folder=False`. - """ - - path_in_repo: str - is_folder: Union[bool, Literal["auto"]] = "auto" - - def __post_init__(self): - self.path_in_repo = _validate_path_in_repo(self.path_in_repo) - - if self.is_folder == "auto": - self.is_folder = self.path_in_repo.endswith("/") - if not isinstance(self.is_folder, bool): - raise ValueError( - f"Wrong value for `is_folder`. Must be one of [`True`, `False`, `'auto'`]. Got '{self.is_folder}'." - ) - - -@dataclass -class CommitOperationCopy: - """ - Data structure holding necessary info to copy a file in a repository on the Hub. - - Limitations: - - Only LFS files can be copied. To copy a regular file, you need to download it locally and re-upload it - - Cross-repository copies are not supported. - - Note: you can combine a [`CommitOperationCopy`] and a [`CommitOperationDelete`] to rename an LFS file on the Hub. - - Args: - src_path_in_repo (`str`): - Relative filepath in the repo of the file to be copied, e.g. `"checkpoints/1fec34a/weights.bin"`. - path_in_repo (`str`): - Relative filepath in the repo where to copy the file, e.g. `"checkpoints/1fec34a/weights_copy.bin"`. - src_revision (`str`, *optional*): - The git revision of the file to be copied. Can be any valid git revision. - Default to the target commit revision. - """ - - src_path_in_repo: str - path_in_repo: str - src_revision: Optional[str] = None - # set to the OID of the file to be copied if it has already been uploaded - # useful to determine if a commit will be empty or not. - _src_oid: Optional[str] = None - # set to the OID of the file to copy to if it has already been uploaded - # useful to determine if a commit will be empty or not. - _dest_oid: Optional[str] = None - - def __post_init__(self): - self.src_path_in_repo = _validate_path_in_repo(self.src_path_in_repo) - self.path_in_repo = _validate_path_in_repo(self.path_in_repo) - - -@dataclass -class CommitOperationAdd: - """ - Data structure holding necessary info to upload a file to a repository on the Hub. - - Args: - path_in_repo (`str`): - Relative filepath in the repo, for example: `"checkpoints/1fec34a/weights.bin"` - path_or_fileobj (`str`, `Path`, `bytes`, or `BinaryIO`): - Either: - - a path to a local file (as `str` or `pathlib.Path`) to upload - - a buffer of bytes (`bytes`) holding the content of the file to upload - - a "file object" (subclass of `io.BufferedIOBase`), typically obtained - with `open(path, "rb")`. It must support `seek()` and `tell()` methods. - - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If `path_or_fileobj` is not one of `str`, `Path`, `bytes` or `io.BufferedIOBase`. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If `path_or_fileobj` is a `str` or `Path` but not a path to an existing file. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If `path_or_fileobj` is a `io.BufferedIOBase` but it doesn't support both - `seek()` and `tell()`. - """ - - path_in_repo: str - path_or_fileobj: Union[str, Path, bytes, BinaryIO] - upload_info: UploadInfo = field(init=False, repr=False) - - # Internal attributes - - # set to "lfs" or "regular" once known - _upload_mode: Optional[UploadMode] = field(init=False, repr=False, default=None) - - # set to True if .gitignore rules prevent the file from being uploaded as LFS - # (server-side check) - _should_ignore: Optional[bool] = field(init=False, repr=False, default=None) - - # set to the remote OID of the file if it has already been uploaded - # useful to determine if a commit will be empty or not - _remote_oid: Optional[str] = field(init=False, repr=False, default=None) - - # set to True once the file has been uploaded as LFS - _is_uploaded: bool = field(init=False, repr=False, default=False) - - # set to True once the file has been committed - _is_committed: bool = field(init=False, repr=False, default=False) - - def __post_init__(self) -> None: - """Validates `path_or_fileobj` and compute `upload_info`.""" - self.path_in_repo = _validate_path_in_repo(self.path_in_repo) - - # Validate `path_or_fileobj` value - if isinstance(self.path_or_fileobj, Path): - self.path_or_fileobj = str(self.path_or_fileobj) - if isinstance(self.path_or_fileobj, str): - path_or_fileobj = os.path.normpath(os.path.expanduser(self.path_or_fileobj)) - if not os.path.isfile(path_or_fileobj): - raise ValueError(f"Provided path: '{path_or_fileobj}' is not a file on the local file system") - elif not isinstance(self.path_or_fileobj, (io.BufferedIOBase, bytes)): - # ^^ Inspired from: https://stackoverflow.com/questions/44584829/how-to-determine-if-file-is-opened-in-binary-or-text-mode - raise ValueError( - "path_or_fileobj must be either an instance of str, bytes or" - " io.BufferedIOBase. If you passed a file-like object, make sure it is" - " in binary mode." - ) - if isinstance(self.path_or_fileobj, io.BufferedIOBase): - try: - self.path_or_fileobj.tell() - self.path_or_fileobj.seek(0, os.SEEK_CUR) - except (OSError, AttributeError) as exc: - raise ValueError( - "path_or_fileobj is a file-like object but does not implement seek() and tell()" - ) from exc - - # Compute "upload_info" attribute - if isinstance(self.path_or_fileobj, str): - self.upload_info = UploadInfo.from_path(self.path_or_fileobj) - elif isinstance(self.path_or_fileobj, bytes): - self.upload_info = UploadInfo.from_bytes(self.path_or_fileobj) - else: - self.upload_info = UploadInfo.from_fileobj(self.path_or_fileobj) - - @contextmanager - def as_file(self, with_tqdm: bool = False) -> Iterator[BinaryIO]: - """ - A context manager that yields a file-like object allowing to read the underlying - data behind `path_or_fileobj`. - - Args: - with_tqdm (`bool`, *optional*, defaults to `False`): - If True, iterating over the file object will display a progress bar. Only - works if the file-like object is a path to a file. Pure bytes and buffers - are not supported. - - Example: - - ```python - >>> operation = CommitOperationAdd( - ... path_in_repo="remote/dir/weights.h5", - ... path_or_fileobj="./local/weights.h5", - ... ) - CommitOperationAdd(path_in_repo='remote/dir/weights.h5', path_or_fileobj='./local/weights.h5') - - >>> with operation.as_file() as file: - ... content = file.read() - - >>> with operation.as_file(with_tqdm=True) as file: - ... while True: - ... data = file.read(1024) - ... if not data: - ... break - config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s] - - >>> with operation.as_file(with_tqdm=True) as file: - ... requests.put(..., data=file) - config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s] - ``` - """ - if isinstance(self.path_or_fileobj, str) or isinstance(self.path_or_fileobj, Path): - if with_tqdm: - with tqdm_stream_file(self.path_or_fileobj) as file: - yield file - else: - with open(self.path_or_fileobj, "rb") as file: - yield file - elif isinstance(self.path_or_fileobj, bytes): - yield io.BytesIO(self.path_or_fileobj) - elif isinstance(self.path_or_fileobj, io.BufferedIOBase): - prev_pos = self.path_or_fileobj.tell() - yield self.path_or_fileobj - self.path_or_fileobj.seek(prev_pos, io.SEEK_SET) - - def b64content(self) -> bytes: - """ - The base64-encoded content of `path_or_fileobj` - - Returns: `bytes` - """ - with self.as_file() as file: - return base64.b64encode(file.read()) - - @property - def _local_oid(self) -> Optional[str]: - """Return the OID of the local file. - - This OID is then compared to `self._remote_oid` to check if the file has changed compared to the remote one. - If the file did not change, we won't upload it again to prevent empty commits. - - For LFS files, the OID corresponds to the SHA256 of the file content (used a LFS ref). - For regular files, the OID corresponds to the SHA1 of the file content. - Note: this is slightly different to git OID computation since the oid of an LFS file is usually the git-SHA1 of the - pointer file content (not the actual file content). However, using the SHA256 is enough to detect changes - and more convenient client-side. - """ - if self._upload_mode is None: - return None - elif self._upload_mode == "lfs": - return self.upload_info.sha256.hex() - else: - # Regular file => compute sha1 - # => no need to read by chunk since the file is guaranteed to be <=5MB. - with self.as_file() as file: - return sha.git_hash(file.read()) - - -def _validate_path_in_repo(path_in_repo: str) -> str: - # Validate `path_in_repo` value to prevent a server-side issue - if path_in_repo.startswith("/"): - path_in_repo = path_in_repo[1:] - if path_in_repo == "." or path_in_repo == ".." or path_in_repo.startswith("../"): - raise ValueError(f"Invalid `path_in_repo` in CommitOperation: '{path_in_repo}'") - if path_in_repo.startswith("./"): - path_in_repo = path_in_repo[2:] - for forbidden in FORBIDDEN_FOLDERS: - if any(part == forbidden for part in path_in_repo.split("/")): - raise ValueError( - f"Invalid `path_in_repo` in CommitOperation: cannot update files under a '{forbidden}/' folder (path:" - f" '{path_in_repo}')." - ) - return path_in_repo - - -CommitOperation = Union[CommitOperationAdd, CommitOperationCopy, CommitOperationDelete] - - -def _warn_on_overwriting_operations(operations: List[CommitOperation]) -> None: - """ - Warn user when a list of operations is expected to overwrite itself in a single - commit. - - Rules: - - If a filepath is updated by multiple `CommitOperationAdd` operations, a warning - message is triggered. - - If a filepath is updated at least once by a `CommitOperationAdd` and then deleted - by a `CommitOperationDelete`, a warning is triggered. - - If a `CommitOperationDelete` deletes a filepath that is then updated by a - `CommitOperationAdd`, no warning is triggered. This is usually useless (no need to - delete before upload) but can happen if a user deletes an entire folder and then - add new files to it. - """ - nb_additions_per_path: Dict[str, int] = defaultdict(int) - for operation in operations: - path_in_repo = operation.path_in_repo - if isinstance(operation, CommitOperationAdd): - if nb_additions_per_path[path_in_repo] > 0: - warnings.warn( - "About to update multiple times the same file in the same commit:" - f" '{path_in_repo}'. This can cause undesired inconsistencies in" - " your repo." - ) - nb_additions_per_path[path_in_repo] += 1 - for parent in PurePosixPath(path_in_repo).parents: - # Also keep track of number of updated files per folder - # => warns if deleting a folder overwrite some contained files - nb_additions_per_path[str(parent)] += 1 - if isinstance(operation, CommitOperationDelete): - if nb_additions_per_path[str(PurePosixPath(path_in_repo))] > 0: - if operation.is_folder: - warnings.warn( - "About to delete a folder containing files that have just been" - f" updated within the same commit: '{path_in_repo}'. This can" - " cause undesired inconsistencies in your repo." - ) - else: - warnings.warn( - "About to delete a file that have just been updated within the" - f" same commit: '{path_in_repo}'. This can cause undesired" - " inconsistencies in your repo." - ) - - -@validate_hf_hub_args -def _upload_files( - *, - additions: List[CommitOperationAdd], - repo_type: str, - repo_id: str, - headers: Dict[str, str], - endpoint: Optional[str] = None, - num_threads: int = 5, - revision: Optional[str] = None, - create_pr: Optional[bool] = None, -): - """ - Negotiates per-file transfer (LFS vs Xet) and uploads in batches. - """ - xet_additions: List[CommitOperationAdd] = [] - lfs_actions: List[Dict] = [] - lfs_oid2addop: Dict[str, CommitOperationAdd] = {} - - for chunk in chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES): - chunk_list = [op for op in chunk] - - transfers: List[str] = ["basic", "multipart"] - has_buffered_io_data = any(isinstance(op.path_or_fileobj, io.BufferedIOBase) for op in chunk_list) - if is_xet_available(): - if not has_buffered_io_data: - transfers.append("xet") - else: - logger.warning( - "Uploading files as a binary IO buffer is not supported by Xet Storage. " - "Falling back to HTTP upload." - ) - - actions_chunk, errors_chunk, chosen_transfer = post_lfs_batch_info( - upload_infos=[op.upload_info for op in chunk_list], - repo_id=repo_id, - repo_type=repo_type, - revision=revision, - endpoint=endpoint, - headers=headers, - token=None, # already passed in 'headers' - transfers=transfers, - ) - if errors_chunk: - message = "\n".join( - [ - f"Encountered error for file with OID {err.get('oid')}: `{err.get('error', {}).get('message')}" - for err in errors_chunk - ] - ) - raise ValueError(f"LFS batch API returned errors:\n{message}") - - # If server returns a transfer we didn't offer (e.g "xet" while uploading from BytesIO), - # fall back to LFS for this chunk. - if chosen_transfer == "xet" and ("xet" in transfers): - xet_additions.extend(chunk_list) - else: - lfs_actions.extend(actions_chunk) - for op in chunk_list: - lfs_oid2addop[op.upload_info.sha256.hex()] = op - - if len(lfs_actions) > 0: - _upload_lfs_files( - actions=lfs_actions, - oid2addop=lfs_oid2addop, - headers=headers, - endpoint=endpoint, - num_threads=num_threads, - ) - - if len(xet_additions) > 0: - _upload_xet_files( - additions=xet_additions, - repo_type=repo_type, - repo_id=repo_id, - headers=headers, - endpoint=endpoint, - revision=revision, - create_pr=create_pr, - ) - - -@validate_hf_hub_args -def _upload_lfs_files( - *, - actions: List[Dict], - oid2addop: Dict[str, CommitOperationAdd], - headers: Dict[str, str], - endpoint: Optional[str] = None, - num_threads: int = 5, -): - """ - Uploads the content of `additions` to the Hub using the large file storage protocol. - - Relevant external documentation: - - LFS Batch API: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md - - Args: - actions (`List[Dict]`): - LFS batch actions returned by the server. - oid2addop (`Dict[str, CommitOperationAdd]`): - A dictionary mapping the OID of the file to the corresponding `CommitOperationAdd` object. - headers (`Dict[str, str]`): - Headers to use for the request, including authorization headers and user agent. - endpoint (`str`, *optional*): - The endpoint to use for the request. Defaults to `constants.ENDPOINT`. - num_threads (`int`, *optional*): - The number of concurrent threads to use when uploading. Defaults to 5. - - Raises: - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) - If an upload failed for any reason - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - headers (`Dict[str, str]`): - Headers to use for the request, including authorization headers and user agent. - num_threads (`int`, *optional*): - The number of concurrent threads to use when uploading. Defaults to 5. - revision (`str`, *optional*): - The git revision to upload to. - - Raises: - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) - If an upload failed for any reason - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If the server returns malformed responses - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - If the LFS batch endpoint returned an HTTP error. - """ - # Filter out files already present upstream - filtered_actions = [] - for action in actions: - if action.get("actions") is None: - logger.debug( - f"Content of file {oid2addop[action['oid']].path_in_repo} is already present upstream - skipping upload." - ) - else: - filtered_actions.append(action) - - # Upload according to server-provided actions - def _wrapped_lfs_upload(batch_action) -> None: - try: - operation = oid2addop[batch_action["oid"]] - lfs_upload(operation=operation, lfs_batch_action=batch_action, headers=headers, endpoint=endpoint) - except Exception as exc: - raise RuntimeError(f"Error while uploading '{operation.path_in_repo}' to the Hub.") from exc - - if constants.HF_HUB_ENABLE_HF_TRANSFER: - logger.debug(f"Uploading {len(filtered_actions)} LFS files to the Hub using `hf_transfer`.") - for action in hf_tqdm(filtered_actions, name="huggingface_hub.lfs_upload"): - _wrapped_lfs_upload(action) - elif len(filtered_actions) == 1: - logger.debug("Uploading 1 LFS file to the Hub") - _wrapped_lfs_upload(filtered_actions[0]) - else: - logger.debug( - f"Uploading {len(filtered_actions)} LFS files to the Hub using up to {num_threads} threads concurrently" - ) - thread_map( - _wrapped_lfs_upload, - filtered_actions, - desc=f"Upload {len(filtered_actions)} LFS files", - max_workers=num_threads, - tqdm_class=hf_tqdm, - ) - - -@validate_hf_hub_args -def _upload_xet_files( - *, - additions: List[CommitOperationAdd], - repo_type: str, - repo_id: str, - headers: Dict[str, str], - endpoint: Optional[str] = None, - revision: Optional[str] = None, - create_pr: Optional[bool] = None, -): - """ - Uploads the content of `additions` to the Hub using the xet storage protocol. - This chunks the files and deduplicates the chunks before uploading them to xetcas storage. - - Args: - additions (`List` of `CommitOperationAdd`): - The files to be uploaded. - repo_type (`str`): - Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - headers (`Dict[str, str]`): - Headers to use for the request, including authorization headers and user agent. - endpoint: (`str`, *optional*): - The endpoint to use for the xetcas service. Defaults to `constants.ENDPOINT`. - revision (`str`, *optional*): - The git revision to upload to. - create_pr (`bool`, *optional*): - Whether or not to create a Pull Request with that commit. - - Raises: - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) - If an upload failed for any reason. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If the server returns malformed responses or if the user is unauthorized to upload to xet storage. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - If the LFS batch endpoint returned an HTTP error. - - **How it works:** - The file download system uses Xet storage, which is a content-addressable storage system that breaks files into chunks - for efficient storage and transfer. - - `hf_xet.upload_files` manages uploading files by: - - Taking a list of file paths to upload - - Breaking files into smaller chunks for efficient storage - - Avoiding duplicate storage by recognizing identical chunks across files - - Connecting to a storage server (CAS server) that manages these chunks - - The upload process works like this: - 1. Create a local folder at ~/.cache/huggingface/xet/chunk-cache to store file chunks for reuse. - 2. Process files in parallel (up to 8 files at once): - 2.1. Read the file content. - 2.2. Split the file content into smaller chunks based on content patterns: each chunk gets a unique ID based on what's in it. - 2.3. For each chunk: - - Check if it already exists in storage. - - Skip uploading chunks that already exist. - 2.4. Group chunks into larger blocks for efficient transfer. - 2.5. Upload these blocks to the storage server. - 2.6. Create and upload information about how the file is structured. - 3. Return reference files that contain information about the uploaded files, which can be used later to download them. - """ - if len(additions) == 0: - return - - # at this point, we know that hf_xet is installed - from hf_xet import upload_bytes, upload_files - - from .utils._xet_progress_reporting import XetProgressReporter - - try: - xet_connection_info = fetch_xet_connection_info_from_repo_info( - token_type=XetTokenType.WRITE, - repo_id=repo_id, - repo_type=repo_type, - revision=revision, - headers=headers, - endpoint=endpoint, - params={"create_pr": "1"} if create_pr else None, - ) - except HfHubHTTPError as e: - if e.response.status_code == 401: - raise XetAuthorizationError( - f"You are unauthorized to upload to xet storage for {repo_type}/{repo_id}. " - f"Please check that you have configured your access token with write access to the repo." - ) from e - raise - - xet_endpoint = xet_connection_info.endpoint - access_token_info = (xet_connection_info.access_token, xet_connection_info.expiration_unix_epoch) - - def token_refresher() -> Tuple[str, int]: - new_xet_connection = fetch_xet_connection_info_from_repo_info( - token_type=XetTokenType.WRITE, - repo_id=repo_id, - repo_type=repo_type, - revision=revision, - headers=headers, - endpoint=endpoint, - params={"create_pr": "1"} if create_pr else None, - ) - if new_xet_connection is None: - raise XetRefreshTokenError("Failed to refresh xet token") - return new_xet_connection.access_token, new_xet_connection.expiration_unix_epoch - - if not are_progress_bars_disabled(): - progress = XetProgressReporter() - progress_callback = progress.update_progress - else: - progress, progress_callback = None, None - - try: - all_bytes_ops = [op for op in additions if isinstance(op.path_or_fileobj, bytes)] - all_paths_ops = [op for op in additions if isinstance(op.path_or_fileobj, (str, Path))] - - if len(all_paths_ops) > 0: - all_paths = [str(op.path_or_fileobj) for op in all_paths_ops] - upload_files( - all_paths, - xet_endpoint, - access_token_info, - token_refresher, - progress_callback, - repo_type, - ) - - if len(all_bytes_ops) > 0: - all_bytes = [op.path_or_fileobj for op in all_bytes_ops] - upload_bytes( - all_bytes, - xet_endpoint, - access_token_info, - token_refresher, - progress_callback, - repo_type, - ) - - finally: - if progress is not None: - progress.close(False) - - return - - -def _validate_preupload_info(preupload_info: dict): - files = preupload_info.get("files") - if not isinstance(files, list): - raise ValueError("preupload_info is improperly formatted") - for file_info in files: - if not ( - isinstance(file_info, dict) - and isinstance(file_info.get("path"), str) - and isinstance(file_info.get("uploadMode"), str) - and (file_info["uploadMode"] in ("lfs", "regular")) - ): - raise ValueError("preupload_info is improperly formatted:") - return preupload_info - - -@validate_hf_hub_args -def _fetch_upload_modes( - additions: Iterable[CommitOperationAdd], - repo_type: str, - repo_id: str, - headers: Dict[str, str], - revision: str, - endpoint: Optional[str] = None, - create_pr: bool = False, - gitignore_content: Optional[str] = None, -) -> None: - """ - Requests the Hub "preupload" endpoint to determine whether each input file should be uploaded as a regular git blob, - as a git LFS blob, or as a XET file. Input `additions` are mutated in-place with the upload mode. - - Args: - additions (`Iterable` of :class:`CommitOperationAdd`): - Iterable of :class:`CommitOperationAdd` describing the files to - upload to the Hub. - repo_type (`str`): - Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - headers (`Dict[str, str]`): - Headers to use for the request, including authorization headers and user agent. - revision (`str`): - The git revision to upload the files to. Can be any valid git revision. - gitignore_content (`str`, *optional*): - The content of the `.gitignore` file to know which files should be ignored. The order of priority - is to first check if `gitignore_content` is passed, then check if the `.gitignore` file is present - in the list of files to commit and finally default to the `.gitignore` file already hosted on the Hub - (if any). - Raises: - [`~utils.HfHubHTTPError`] - If the Hub API returned an error. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If the Hub API response is improperly formatted. - """ - endpoint = endpoint if endpoint is not None else constants.ENDPOINT - - # Fetch upload mode (LFS or regular) chunk by chunk. - upload_modes: Dict[str, UploadMode] = {} - should_ignore_info: Dict[str, bool] = {} - oid_info: Dict[str, Optional[str]] = {} - - for chunk in chunk_iterable(additions, 256): - payload: Dict = { - "files": [ - { - "path": op.path_in_repo, - "sample": base64.b64encode(op.upload_info.sample).decode("ascii"), - "size": op.upload_info.size, - } - for op in chunk - ] - } - if gitignore_content is not None: - payload["gitIgnore"] = gitignore_content - - resp = get_session().post( - f"{endpoint}/api/{repo_type}s/{repo_id}/preupload/{revision}", - json=payload, - headers=headers, - params={"create_pr": "1"} if create_pr else None, - ) - hf_raise_for_status(resp) - preupload_info = _validate_preupload_info(resp.json()) - upload_modes.update(**{file["path"]: file["uploadMode"] for file in preupload_info["files"]}) - should_ignore_info.update(**{file["path"]: file["shouldIgnore"] for file in preupload_info["files"]}) - oid_info.update(**{file["path"]: file.get("oid") for file in preupload_info["files"]}) - - # Set upload mode for each addition operation - for addition in additions: - addition._upload_mode = upload_modes[addition.path_in_repo] - addition._should_ignore = should_ignore_info[addition.path_in_repo] - addition._remote_oid = oid_info[addition.path_in_repo] - - # Empty files cannot be uploaded as LFS (S3 would fail with a 501 Not Implemented) - # => empty files are uploaded as "regular" to still allow users to commit them. - for addition in additions: - if addition.upload_info.size == 0: - addition._upload_mode = "regular" - - -@validate_hf_hub_args -def _fetch_files_to_copy( - copies: Iterable[CommitOperationCopy], - repo_type: str, - repo_id: str, - headers: Dict[str, str], - revision: str, - endpoint: Optional[str] = None, -) -> Dict[Tuple[str, Optional[str]], Union["RepoFile", bytes]]: - """ - Fetch information about the files to copy. - - For LFS files, we only need their metadata (file size and sha256) while for regular files - we need to download the raw content from the Hub. - - Args: - copies (`Iterable` of :class:`CommitOperationCopy`): - Iterable of :class:`CommitOperationCopy` describing the files to - copy on the Hub. - repo_type (`str`): - Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - headers (`Dict[str, str]`): - Headers to use for the request, including authorization headers and user agent. - revision (`str`): - The git revision to upload the files to. Can be any valid git revision. - - Returns: `Dict[Tuple[str, Optional[str]], Union[RepoFile, bytes]]]` - Key is the file path and revision of the file to copy. - Value is the raw content as bytes (for regular files) or the file information as a RepoFile (for LFS files). - - Raises: - [`~utils.HfHubHTTPError`] - If the Hub API returned an error. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If the Hub API response is improperly formatted. - """ - from .hf_api import HfApi, RepoFolder - - hf_api = HfApi(endpoint=endpoint, headers=headers) - files_to_copy: Dict[Tuple[str, Optional[str]], Union["RepoFile", bytes]] = {} - # Store (path, revision) -> oid mapping - oid_info: Dict[Tuple[str, Optional[str]], Optional[str]] = {} - # 1. Fetch OIDs for destination paths in batches. - dest_paths = [op.path_in_repo for op in copies] - for offset in range(0, len(dest_paths), FETCH_LFS_BATCH_SIZE): - dest_repo_files = hf_api.get_paths_info( - repo_id=repo_id, - paths=dest_paths[offset : offset + FETCH_LFS_BATCH_SIZE], - revision=revision, - repo_type=repo_type, - ) - for file in dest_repo_files: - if not isinstance(file, RepoFolder): - oid_info[(file.path, revision)] = file.blob_id - - # 2. Group by source revision and fetch source file info in batches. - for src_revision, operations in groupby(copies, key=lambda op: op.src_revision): - operations = list(operations) # type: ignore - src_paths = [op.src_path_in_repo for op in operations] - for offset in range(0, len(src_paths), FETCH_LFS_BATCH_SIZE): - src_repo_files = hf_api.get_paths_info( - repo_id=repo_id, - paths=src_paths[offset : offset + FETCH_LFS_BATCH_SIZE], - revision=src_revision or revision, - repo_type=repo_type, - ) - - for src_repo_file in src_repo_files: - if isinstance(src_repo_file, RepoFolder): - raise NotImplementedError("Copying a folder is not implemented.") - oid_info[(src_repo_file.path, src_revision)] = src_repo_file.blob_id - # If it's an LFS file, store the RepoFile object. Otherwise, download raw bytes. - if src_repo_file.lfs: - files_to_copy[(src_repo_file.path, src_revision)] = src_repo_file - else: - # TODO: (optimization) download regular files to copy concurrently - url = hf_hub_url( - endpoint=endpoint, - repo_type=repo_type, - repo_id=repo_id, - revision=src_revision or revision, - filename=src_repo_file.path, - ) - response = get_session().get(url, headers=headers) - hf_raise_for_status(response) - files_to_copy[(src_repo_file.path, src_revision)] = response.content - # 3. Ensure all operations found a corresponding file in the Hub - # and track src/dest OIDs for each operation. - for operation in operations: - if (operation.src_path_in_repo, src_revision) not in files_to_copy: - raise EntryNotFoundError( - f"Cannot copy {operation.src_path_in_repo} at revision " - f"{src_revision or revision}: file is missing on repo." - ) - operation._src_oid = oid_info.get((operation.src_path_in_repo, operation.src_revision)) - operation._dest_oid = oid_info.get((operation.path_in_repo, revision)) - return files_to_copy - - -def _prepare_commit_payload( - operations: Iterable[CommitOperation], - files_to_copy: Dict[Tuple[str, Optional[str]], Union["RepoFile", bytes]], - commit_message: str, - commit_description: Optional[str] = None, - parent_commit: Optional[str] = None, -) -> Iterable[Dict[str, Any]]: - """ - Builds the payload to POST to the `/commit` API of the Hub. - - Payload is returned as an iterator so that it can be streamed as a ndjson in the - POST request. - - For more information, see: - - https://github.com/huggingface/huggingface_hub/issues/1085#issuecomment-1265208073 - - http://ndjson.org/ - """ - commit_description = commit_description if commit_description is not None else "" - - # 1. Send a header item with the commit metadata - header_value = {"summary": commit_message, "description": commit_description} - if parent_commit is not None: - header_value["parentCommit"] = parent_commit - yield {"key": "header", "value": header_value} - - nb_ignored_files = 0 - - # 2. Send operations, one per line - for operation in operations: - # Skip ignored files - if isinstance(operation, CommitOperationAdd) and operation._should_ignore: - logger.debug(f"Skipping file '{operation.path_in_repo}' in commit (ignored by gitignore file).") - nb_ignored_files += 1 - continue - - # 2.a. Case adding a regular file - if isinstance(operation, CommitOperationAdd) and operation._upload_mode == "regular": - yield { - "key": "file", - "value": { - "content": operation.b64content().decode(), - "path": operation.path_in_repo, - "encoding": "base64", - }, - } - # 2.b. Case adding an LFS file - elif isinstance(operation, CommitOperationAdd) and operation._upload_mode == "lfs": - yield { - "key": "lfsFile", - "value": { - "path": operation.path_in_repo, - "algo": "sha256", - "oid": operation.upload_info.sha256.hex(), - "size": operation.upload_info.size, - }, - } - # 2.c. Case deleting a file or folder - elif isinstance(operation, CommitOperationDelete): - yield { - "key": "deletedFolder" if operation.is_folder else "deletedFile", - "value": {"path": operation.path_in_repo}, - } - # 2.d. Case copying a file or folder - elif isinstance(operation, CommitOperationCopy): - file_to_copy = files_to_copy[(operation.src_path_in_repo, operation.src_revision)] - if isinstance(file_to_copy, bytes): - yield { - "key": "file", - "value": { - "content": base64.b64encode(file_to_copy).decode(), - "path": operation.path_in_repo, - "encoding": "base64", - }, - } - elif file_to_copy.lfs: - yield { - "key": "lfsFile", - "value": { - "path": operation.path_in_repo, - "algo": "sha256", - "oid": file_to_copy.lfs.sha256, - }, - } - else: - raise ValueError( - "Malformed files_to_copy (should be raw file content as bytes or RepoFile objects with LFS info." - ) - # 2.e. Never expected to happen - else: - raise ValueError( - f"Unknown operation to commit. Operation: {operation}. Upload mode:" - f" {getattr(operation, '_upload_mode', None)}" - ) - - if nb_ignored_files > 0: - logger.info(f"Skipped {nb_ignored_files} file(s) in commit (ignored by gitignore file).") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_commit_scheduler.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_commit_scheduler.py deleted file mode 100644 index 1bc8db6a8ade4d2253dd241a66c86def5dac2733..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_commit_scheduler.py +++ /dev/null @@ -1,350 +0,0 @@ -import atexit -import logging -import os -import time -from concurrent.futures import Future -from dataclasses import dataclass -from io import SEEK_END, SEEK_SET, BytesIO -from pathlib import Path -from threading import Lock, Thread -from typing import Dict, List, Optional, Union - -from .hf_api import DEFAULT_IGNORE_PATTERNS, CommitInfo, CommitOperationAdd, HfApi -from .utils import filter_repo_objects - - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class _FileToUpload: - """Temporary dataclass to store info about files to upload. Not meant to be used directly.""" - - local_path: Path - path_in_repo: str - size_limit: int - last_modified: float - - -class CommitScheduler: - """ - Scheduler to upload a local folder to the Hub at regular intervals (e.g. push to hub every 5 minutes). - - The recommended way to use the scheduler is to use it as a context manager. This ensures that the scheduler is - properly stopped and the last commit is triggered when the script ends. The scheduler can also be stopped manually - with the `stop` method. Checkout the [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#scheduled-uploads) - to learn more about how to use it. - - Args: - repo_id (`str`): - The id of the repo to commit to. - folder_path (`str` or `Path`): - Path to the local folder to upload regularly. - every (`int` or `float`, *optional*): - The number of minutes between each commit. Defaults to 5 minutes. - path_in_repo (`str`, *optional*): - Relative path of the directory in the repo, for example: `"checkpoints/"`. Defaults to the root folder - of the repository. - repo_type (`str`, *optional*): - The type of the repo to commit to. Defaults to `model`. - revision (`str`, *optional*): - The revision of the repo to commit to. Defaults to `main`. - private (`bool`, *optional*): - Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists. - token (`str`, *optional*): - The token to use to commit to the repo. Defaults to the token saved on the machine. - allow_patterns (`List[str]` or `str`, *optional*): - If provided, only files matching at least one pattern are uploaded. - ignore_patterns (`List[str]` or `str`, *optional*): - If provided, files matching any of the patterns are not uploaded. - squash_history (`bool`, *optional*): - Whether to squash the history of the repo after each commit. Defaults to `False`. Squashing commits is - useful to avoid degraded performances on the repo when it grows too large. - hf_api (`HfApi`, *optional*): - The [`HfApi`] client to use to commit to the Hub. Can be set with custom settings (user agent, token,...). - - Example: - ```py - >>> from pathlib import Path - >>> from huggingface_hub import CommitScheduler - - # Scheduler uploads every 10 minutes - >>> csv_path = Path("watched_folder/data.csv") - >>> CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path=csv_path.parent, every=10) - - >>> with csv_path.open("a") as f: - ... f.write("first line") - - # Some time later (...) - >>> with csv_path.open("a") as f: - ... f.write("second line") - ``` - - Example using a context manager: - ```py - >>> from pathlib import Path - >>> from huggingface_hub import CommitScheduler - - >>> with CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path="watched_folder", every=10) as scheduler: - ... csv_path = Path("watched_folder/data.csv") - ... with csv_path.open("a") as f: - ... f.write("first line") - ... (...) - ... with csv_path.open("a") as f: - ... f.write("second line") - - # Scheduler is now stopped and last commit have been triggered - ``` - """ - - def __init__( - self, - *, - repo_id: str, - folder_path: Union[str, Path], - every: Union[int, float] = 5, - path_in_repo: Optional[str] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - private: Optional[bool] = None, - token: Optional[str] = None, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - squash_history: bool = False, - hf_api: Optional["HfApi"] = None, - ) -> None: - self.api = hf_api or HfApi(token=token) - - # Folder - self.folder_path = Path(folder_path).expanduser().resolve() - self.path_in_repo = path_in_repo or "" - self.allow_patterns = allow_patterns - - if ignore_patterns is None: - ignore_patterns = [] - elif isinstance(ignore_patterns, str): - ignore_patterns = [ignore_patterns] - self.ignore_patterns = ignore_patterns + DEFAULT_IGNORE_PATTERNS - - if self.folder_path.is_file(): - raise ValueError(f"'folder_path' must be a directory, not a file: '{self.folder_path}'.") - self.folder_path.mkdir(parents=True, exist_ok=True) - - # Repository - repo_url = self.api.create_repo(repo_id=repo_id, private=private, repo_type=repo_type, exist_ok=True) - self.repo_id = repo_url.repo_id - self.repo_type = repo_type - self.revision = revision - self.token = token - - # Keep track of already uploaded files - self.last_uploaded: Dict[Path, float] = {} # key is local path, value is timestamp - - # Scheduler - if not every > 0: - raise ValueError(f"'every' must be a positive integer, not '{every}'.") - self.lock = Lock() - self.every = every - self.squash_history = squash_history - - logger.info(f"Scheduled job to push '{self.folder_path}' to '{self.repo_id}' every {self.every} minutes.") - self._scheduler_thread = Thread(target=self._run_scheduler, daemon=True) - self._scheduler_thread.start() - atexit.register(self._push_to_hub) - - self.__stopped = False - - def stop(self) -> None: - """Stop the scheduler. - - A stopped scheduler cannot be restarted. Mostly for tests purposes. - """ - self.__stopped = True - - def __enter__(self) -> "CommitScheduler": - return self - - def __exit__(self, exc_type, exc_value, traceback) -> None: - # Upload last changes before exiting - self.trigger().result() - self.stop() - return - - def _run_scheduler(self) -> None: - """Dumb thread waiting between each scheduled push to Hub.""" - while True: - self.last_future = self.trigger() - time.sleep(self.every * 60) - if self.__stopped: - break - - def trigger(self) -> Future: - """Trigger a `push_to_hub` and return a future. - - This method is automatically called every `every` minutes. You can also call it manually to trigger a commit - immediately, without waiting for the next scheduled commit. - """ - return self.api.run_as_future(self._push_to_hub) - - def _push_to_hub(self) -> Optional[CommitInfo]: - if self.__stopped: # If stopped, already scheduled commits are ignored - return None - - logger.info("(Background) scheduled commit triggered.") - try: - value = self.push_to_hub() - if self.squash_history: - logger.info("(Background) squashing repo history.") - self.api.super_squash_history(repo_id=self.repo_id, repo_type=self.repo_type, branch=self.revision) - return value - except Exception as e: - logger.error(f"Error while pushing to Hub: {e}") # Depending on the setup, error might be silenced - raise - - def push_to_hub(self) -> Optional[CommitInfo]: - """ - Push folder to the Hub and return the commit info. - - > [!WARNING] - > This method is not meant to be called directly. It is run in the background by the scheduler, respecting a - > queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency - > issues. - - The default behavior of `push_to_hub` is to assume an append-only folder. It lists all files in the folder and - uploads only changed files. If no changes are found, the method returns without committing anything. If you want - to change this behavior, you can inherit from [`CommitScheduler`] and override this method. This can be useful - for example to compress data together in a single file before committing. For more details and examples, check - out our [integration guide](https://huggingface.co/docs/huggingface_hub/main/en/guides/upload#scheduled-uploads). - """ - # Check files to upload (with lock) - with self.lock: - logger.debug("Listing files to upload for scheduled commit.") - - # List files from folder (taken from `_prepare_upload_folder_additions`) - relpath_to_abspath = { - path.relative_to(self.folder_path).as_posix(): path - for path in sorted(self.folder_path.glob("**/*")) # sorted to be deterministic - if path.is_file() - } - prefix = f"{self.path_in_repo.strip('/')}/" if self.path_in_repo else "" - - # Filter with pattern + filter out unchanged files + retrieve current file size - files_to_upload: List[_FileToUpload] = [] - for relpath in filter_repo_objects( - relpath_to_abspath.keys(), allow_patterns=self.allow_patterns, ignore_patterns=self.ignore_patterns - ): - local_path = relpath_to_abspath[relpath] - stat = local_path.stat() - if self.last_uploaded.get(local_path) is None or self.last_uploaded[local_path] != stat.st_mtime: - files_to_upload.append( - _FileToUpload( - local_path=local_path, - path_in_repo=prefix + relpath, - size_limit=stat.st_size, - last_modified=stat.st_mtime, - ) - ) - - # Return if nothing to upload - if len(files_to_upload) == 0: - logger.debug("Dropping schedule commit: no changed file to upload.") - return None - - # Convert `_FileToUpload` as `CommitOperationAdd` (=> compute file shas + limit to file size) - logger.debug("Removing unchanged files since previous scheduled commit.") - add_operations = [ - CommitOperationAdd( - # Cap the file to its current size, even if the user append data to it while a scheduled commit is happening - path_or_fileobj=PartialFileIO(file_to_upload.local_path, size_limit=file_to_upload.size_limit), - path_in_repo=file_to_upload.path_in_repo, - ) - for file_to_upload in files_to_upload - ] - - # Upload files (append mode expected - no need for lock) - logger.debug("Uploading files for scheduled commit.") - commit_info = self.api.create_commit( - repo_id=self.repo_id, - repo_type=self.repo_type, - operations=add_operations, - commit_message="Scheduled Commit", - revision=self.revision, - ) - - # Successful commit: keep track of the latest "last_modified" for each file - for file in files_to_upload: - self.last_uploaded[file.local_path] = file.last_modified - return commit_info - - -class PartialFileIO(BytesIO): - """A file-like object that reads only the first part of a file. - - Useful to upload a file to the Hub when the user might still be appending data to it. Only the first part of the - file is uploaded (i.e. the part that was available when the filesystem was first scanned). - - In practice, only used internally by the CommitScheduler to regularly push a folder to the Hub with minimal - disturbance for the user. The object is passed to `CommitOperationAdd`. - - Only supports `read`, `tell` and `seek` methods. - - Args: - file_path (`str` or `Path`): - Path to the file to read. - size_limit (`int`): - The maximum number of bytes to read from the file. If the file is larger than this, only the first part - will be read (and uploaded). - """ - - def __init__(self, file_path: Union[str, Path], size_limit: int) -> None: - self._file_path = Path(file_path) - self._file = self._file_path.open("rb") - self._size_limit = min(size_limit, os.fstat(self._file.fileno()).st_size) - - def __del__(self) -> None: - self._file.close() - return super().__del__() - - def __repr__(self) -> str: - return f"" - - def __len__(self) -> int: - return self._size_limit - - def __getattribute__(self, name: str): - if name.startswith("_") or name in ("read", "tell", "seek"): # only 3 public methods supported - return super().__getattribute__(name) - raise NotImplementedError(f"PartialFileIO does not support '{name}'.") - - def tell(self) -> int: - """Return the current file position.""" - return self._file.tell() - - def seek(self, __offset: int, __whence: int = SEEK_SET) -> int: - """Change the stream position to the given offset. - - Behavior is the same as a regular file, except that the position is capped to the size limit. - """ - if __whence == SEEK_END: - # SEEK_END => set from the truncated end - __offset = len(self) + __offset - __whence = SEEK_SET - - pos = self._file.seek(__offset, __whence) - if pos > self._size_limit: - return self._file.seek(self._size_limit) - return pos - - def read(self, __size: Optional[int] = -1) -> bytes: - """Read at most `__size` bytes from the file. - - Behavior is the same as a regular file, except that it is capped to the size limit. - """ - current = self._file.tell() - if __size is None or __size < 0: - # Read until file limit - truncated_size = self._size_limit - current - else: - # Read until file limit or __size - truncated_size = min(__size, self._size_limit - current) - return self._file.read(truncated_size) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_inference_endpoints.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_inference_endpoints.py deleted file mode 100644 index 37f772bfbe28013ff5329d0a19a438706d50a19c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_inference_endpoints.py +++ /dev/null @@ -1,413 +0,0 @@ -import time -from dataclasses import dataclass, field -from datetime import datetime -from enum import Enum -from typing import TYPE_CHECKING, Dict, Optional, Union - -from huggingface_hub.errors import InferenceEndpointError, InferenceEndpointTimeoutError - -from .utils import get_session, logging, parse_datetime - - -if TYPE_CHECKING: - from .hf_api import HfApi - from .inference._client import InferenceClient - from .inference._generated._async_client import AsyncInferenceClient - -logger = logging.get_logger(__name__) - - -class InferenceEndpointStatus(str, Enum): - PENDING = "pending" - INITIALIZING = "initializing" - UPDATING = "updating" - UPDATE_FAILED = "updateFailed" - RUNNING = "running" - PAUSED = "paused" - FAILED = "failed" - SCALED_TO_ZERO = "scaledToZero" - - -class InferenceEndpointType(str, Enum): - PUBlIC = "public" - PROTECTED = "protected" - PRIVATE = "private" - - -@dataclass -class InferenceEndpoint: - """ - Contains information about a deployed Inference Endpoint. - - Args: - name (`str`): - The unique name of the Inference Endpoint. - namespace (`str`): - The namespace where the Inference Endpoint is located. - repository (`str`): - The name of the model repository deployed on this Inference Endpoint. - status ([`InferenceEndpointStatus`]): - The current status of the Inference Endpoint. - url (`str`, *optional*): - The URL of the Inference Endpoint, if available. Only a deployed Inference Endpoint will have a URL. - framework (`str`): - The machine learning framework used for the model. - revision (`str`): - The specific model revision deployed on the Inference Endpoint. - task (`str`): - The task associated with the deployed model. - created_at (`datetime.datetime`): - The timestamp when the Inference Endpoint was created. - updated_at (`datetime.datetime`): - The timestamp of the last update of the Inference Endpoint. - type ([`InferenceEndpointType`]): - The type of the Inference Endpoint (public, protected, private). - raw (`Dict`): - The raw dictionary data returned from the API. - token (`str` or `bool`, *optional*): - Authentication token for the Inference Endpoint, if set when requesting the API. Will default to the - locally saved token if not provided. Pass `token=False` if you don't want to send your token to the server. - - Example: - ```python - >>> from huggingface_hub import get_inference_endpoint - >>> endpoint = get_inference_endpoint("my-text-to-image") - >>> endpoint - InferenceEndpoint(name='my-text-to-image', ...) - - # Get status - >>> endpoint.status - 'running' - >>> endpoint.url - 'https://my-text-to-image.region.vendor.endpoints.huggingface.cloud' - - # Run inference - >>> endpoint.client.text_to_image(...) - - # Pause endpoint to save $$$ - >>> endpoint.pause() - - # ... - # Resume and wait for deployment - >>> endpoint.resume() - >>> endpoint.wait() - >>> endpoint.client.text_to_image(...) - ``` - """ - - # Field in __repr__ - name: str = field(init=False) - namespace: str - repository: str = field(init=False) - status: InferenceEndpointStatus = field(init=False) - health_route: str = field(init=False) - url: Optional[str] = field(init=False) - - # Other fields - framework: str = field(repr=False, init=False) - revision: str = field(repr=False, init=False) - task: str = field(repr=False, init=False) - created_at: datetime = field(repr=False, init=False) - updated_at: datetime = field(repr=False, init=False) - type: InferenceEndpointType = field(repr=False, init=False) - - # Raw dict from the API - raw: Dict = field(repr=False) - - # Internal fields - _token: Union[str, bool, None] = field(repr=False, compare=False) - _api: "HfApi" = field(repr=False, compare=False) - - @classmethod - def from_raw( - cls, raw: Dict, namespace: str, token: Union[str, bool, None] = None, api: Optional["HfApi"] = None - ) -> "InferenceEndpoint": - """Initialize object from raw dictionary.""" - if api is None: - from .hf_api import HfApi - - api = HfApi() - if token is None: - token = api.token - - # All other fields are populated in __post_init__ - return cls(raw=raw, namespace=namespace, _token=token, _api=api) - - def __post_init__(self) -> None: - """Populate fields from raw dictionary.""" - self._populate_from_raw() - - @property - def client(self) -> "InferenceClient": - """Returns a client to make predictions on this Inference Endpoint. - - Returns: - [`InferenceClient`]: an inference client pointing to the deployed endpoint. - - Raises: - [`InferenceEndpointError`]: If the Inference Endpoint is not yet deployed. - """ - if self.url is None: - raise InferenceEndpointError( - "Cannot create a client for this Inference Endpoint as it is not yet deployed. " - "Please wait for the Inference Endpoint to be deployed using `endpoint.wait()` and try again." - ) - from .inference._client import InferenceClient - - return InferenceClient( - model=self.url, - token=self._token, # type: ignore[arg-type] # boolean token shouldn't be possible. In practice it's ok. - ) - - @property - def async_client(self) -> "AsyncInferenceClient": - """Returns a client to make predictions on this Inference Endpoint. - - Returns: - [`AsyncInferenceClient`]: an asyncio-compatible inference client pointing to the deployed endpoint. - - Raises: - [`InferenceEndpointError`]: If the Inference Endpoint is not yet deployed. - """ - if self.url is None: - raise InferenceEndpointError( - "Cannot create a client for this Inference Endpoint as it is not yet deployed. " - "Please wait for the Inference Endpoint to be deployed using `endpoint.wait()` and try again." - ) - from .inference._generated._async_client import AsyncInferenceClient - - return AsyncInferenceClient( - model=self.url, - token=self._token, # type: ignore[arg-type] # boolean token shouldn't be possible. In practice it's ok. - ) - - def wait(self, timeout: Optional[int] = None, refresh_every: int = 5) -> "InferenceEndpoint": - """Wait for the Inference Endpoint to be deployed. - - Information from the server will be fetched every 1s. If the Inference Endpoint is not deployed after `timeout` - seconds, a [`InferenceEndpointTimeoutError`] will be raised. The [`InferenceEndpoint`] will be mutated in place with the latest - data. - - Args: - timeout (`int`, *optional*): - The maximum time to wait for the Inference Endpoint to be deployed, in seconds. If `None`, will wait - indefinitely. - refresh_every (`int`, *optional*): - The time to wait between each fetch of the Inference Endpoint status, in seconds. Defaults to 5s. - - Returns: - [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. - - Raises: - [`InferenceEndpointError`] - If the Inference Endpoint ended up in a failed state. - [`InferenceEndpointTimeoutError`] - If the Inference Endpoint is not deployed after `timeout` seconds. - """ - if timeout is not None and timeout < 0: - raise ValueError("`timeout` cannot be negative.") - if refresh_every <= 0: - raise ValueError("`refresh_every` must be positive.") - - start = time.time() - while True: - if self.status == InferenceEndpointStatus.FAILED: - raise InferenceEndpointError( - f"Inference Endpoint {self.name} failed to deploy. Please check the logs for more information." - ) - if self.status == InferenceEndpointStatus.UPDATE_FAILED: - raise InferenceEndpointError( - f"Inference Endpoint {self.name} failed to update. Please check the logs for more information." - ) - if self.status == InferenceEndpointStatus.RUNNING and self.url is not None: - # Verify the endpoint is actually reachable - _health_url = f"{self.url.rstrip('/')}/{self.health_route.lstrip('/')}" - response = get_session().get(_health_url, headers=self._api._build_hf_headers(token=self._token)) - if response.status_code == 200: - logger.info("Inference Endpoint is ready to be used.") - return self - - if timeout is not None: - if time.time() - start > timeout: - raise InferenceEndpointTimeoutError("Timeout while waiting for Inference Endpoint to be deployed.") - logger.info(f"Inference Endpoint is not deployed yet ({self.status}). Waiting {refresh_every}s...") - time.sleep(refresh_every) - self.fetch() - - def fetch(self) -> "InferenceEndpoint": - """Fetch latest information about the Inference Endpoint. - - Returns: - [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. - """ - obj = self._api.get_inference_endpoint(name=self.name, namespace=self.namespace, token=self._token) # type: ignore [arg-type] - self.raw = obj.raw - self._populate_from_raw() - return self - - def update( - self, - *, - # Compute update - accelerator: Optional[str] = None, - instance_size: Optional[str] = None, - instance_type: Optional[str] = None, - min_replica: Optional[int] = None, - max_replica: Optional[int] = None, - scale_to_zero_timeout: Optional[int] = None, - # Model update - repository: Optional[str] = None, - framework: Optional[str] = None, - revision: Optional[str] = None, - task: Optional[str] = None, - custom_image: Optional[Dict] = None, - secrets: Optional[Dict[str, str]] = None, - ) -> "InferenceEndpoint": - """Update the Inference Endpoint. - - This method allows the update of either the compute configuration, the deployed model, or both. All arguments are - optional but at least one must be provided. - - This is an alias for [`HfApi.update_inference_endpoint`]. The current object is mutated in place with the - latest data from the server. - - Args: - accelerator (`str`, *optional*): - The hardware accelerator to be used for inference (e.g. `"cpu"`). - instance_size (`str`, *optional*): - The size or type of the instance to be used for hosting the model (e.g. `"x4"`). - instance_type (`str`, *optional*): - The cloud instance type where the Inference Endpoint will be deployed (e.g. `"intel-icl"`). - min_replica (`int`, *optional*): - The minimum number of replicas (instances) to keep running for the Inference Endpoint. - max_replica (`int`, *optional*): - The maximum number of replicas (instances) to scale to for the Inference Endpoint. - scale_to_zero_timeout (`int`, *optional*): - The duration in minutes before an inactive endpoint is scaled to zero. - - repository (`str`, *optional*): - The name of the model repository associated with the Inference Endpoint (e.g. `"gpt2"`). - framework (`str`, *optional*): - The machine learning framework used for the model (e.g. `"custom"`). - revision (`str`, *optional*): - The specific model revision to deploy on the Inference Endpoint (e.g. `"6c0e6080953db56375760c0471a8c5f2929baf11"`). - task (`str`, *optional*): - The task on which to deploy the model (e.g. `"text-classification"`). - custom_image (`Dict`, *optional*): - A custom Docker image to use for the Inference Endpoint. This is useful if you want to deploy an - Inference Endpoint running on the `text-generation-inference` (TGI) framework (see examples). - secrets (`Dict[str, str]`, *optional*): - Secret values to inject in the container environment. - Returns: - [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. - """ - # Make API call - obj = self._api.update_inference_endpoint( - name=self.name, - namespace=self.namespace, - accelerator=accelerator, - instance_size=instance_size, - instance_type=instance_type, - min_replica=min_replica, - max_replica=max_replica, - scale_to_zero_timeout=scale_to_zero_timeout, - repository=repository, - framework=framework, - revision=revision, - task=task, - custom_image=custom_image, - secrets=secrets, - token=self._token, # type: ignore [arg-type] - ) - - # Mutate current object - self.raw = obj.raw - self._populate_from_raw() - return self - - def pause(self) -> "InferenceEndpoint": - """Pause the Inference Endpoint. - - A paused Inference Endpoint will not be charged. It can be resumed at any time using [`InferenceEndpoint.resume`]. - This is different than scaling the Inference Endpoint to zero with [`InferenceEndpoint.scale_to_zero`], which - would be automatically restarted when a request is made to it. - - This is an alias for [`HfApi.pause_inference_endpoint`]. The current object is mutated in place with the - latest data from the server. - - Returns: - [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. - """ - obj = self._api.pause_inference_endpoint(name=self.name, namespace=self.namespace, token=self._token) # type: ignore [arg-type] - self.raw = obj.raw - self._populate_from_raw() - return self - - def resume(self, running_ok: bool = True) -> "InferenceEndpoint": - """Resume the Inference Endpoint. - - This is an alias for [`HfApi.resume_inference_endpoint`]. The current object is mutated in place with the - latest data from the server. - - Args: - running_ok (`bool`, *optional*): - If `True`, the method will not raise an error if the Inference Endpoint is already running. Defaults to - `True`. - - Returns: - [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. - """ - obj = self._api.resume_inference_endpoint( - name=self.name, namespace=self.namespace, running_ok=running_ok, token=self._token - ) # type: ignore [arg-type] - self.raw = obj.raw - self._populate_from_raw() - return self - - def scale_to_zero(self) -> "InferenceEndpoint": - """Scale Inference Endpoint to zero. - - An Inference Endpoint scaled to zero will not be charged. It will be resume on the next request to it, with a - cold start delay. This is different than pausing the Inference Endpoint with [`InferenceEndpoint.pause`], which - would require a manual resume with [`InferenceEndpoint.resume`]. - - This is an alias for [`HfApi.scale_to_zero_inference_endpoint`]. The current object is mutated in place with the - latest data from the server. - - Returns: - [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. - """ - obj = self._api.scale_to_zero_inference_endpoint(name=self.name, namespace=self.namespace, token=self._token) # type: ignore [arg-type] - self.raw = obj.raw - self._populate_from_raw() - return self - - def delete(self) -> None: - """Delete the Inference Endpoint. - - This operation is not reversible. If you don't want to be charged for an Inference Endpoint, it is preferable - to pause it with [`InferenceEndpoint.pause`] or scale it to zero with [`InferenceEndpoint.scale_to_zero`]. - - This is an alias for [`HfApi.delete_inference_endpoint`]. - """ - self._api.delete_inference_endpoint(name=self.name, namespace=self.namespace, token=self._token) # type: ignore [arg-type] - - def _populate_from_raw(self) -> None: - """Populate fields from raw dictionary. - - Called in __post_init__ + each time the Inference Endpoint is updated. - """ - # Repr fields - self.name = self.raw["name"] - self.repository = self.raw["model"]["repository"] - self.status = self.raw["status"]["state"] - self.url = self.raw["status"].get("url") - self.health_route = self.raw["healthRoute"] - - # Other fields - self.framework = self.raw["model"]["framework"] - self.revision = self.raw["model"]["revision"] - self.task = self.raw["model"]["task"] - self.created_at = parse_datetime(self.raw["status"]["createdAt"]) - self.updated_at = parse_datetime(self.raw["status"]["updatedAt"]) - self.type = self.raw["type"] diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_jobs_api.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_jobs_api.py deleted file mode 100644 index 00177a008c177d5484fb9069d0684bbf416e9289..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_jobs_api.py +++ /dev/null @@ -1,301 +0,0 @@ -# coding=utf-8 -# Copyright 2025-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from dataclasses import dataclass -from datetime import datetime -from enum import Enum -from typing import Any, Dict, List, Optional, Union - -from huggingface_hub import constants -from huggingface_hub._space_api import SpaceHardware -from huggingface_hub.utils._datetime import parse_datetime - - -class JobStage(str, Enum): - """ - Enumeration of possible stage of a Job on the Hub. - - Value can be compared to a string: - ```py - assert JobStage.COMPLETED == "COMPLETED" - ``` - Possible values are: `COMPLETED`, `CANCELED`, `ERROR`, `DELETED`, `RUNNING`. - Taken from https://github.com/huggingface/moon-landing/blob/main/server/job_types/JobInfo.ts#L61 (private url). - """ - - # Copied from moon-landing > server > lib > Job.ts - COMPLETED = "COMPLETED" - CANCELED = "CANCELED" - ERROR = "ERROR" - DELETED = "DELETED" - RUNNING = "RUNNING" - - -@dataclass -class JobStatus: - stage: JobStage - message: Optional[str] - - -@dataclass -class JobOwner: - id: str - name: str - type: str - - -@dataclass -class JobInfo: - """ - Contains information about a Job. - - Args: - id (`str`): - Job ID. - created_at (`datetime` or `None`): - When the Job was created. - docker_image (`str` or `None`): - The Docker image from Docker Hub used for the Job. - Can be None if space_id is present instead. - space_id (`str` or `None`): - The Docker image from Hugging Face Spaces used for the Job. - Can be None if docker_image is present instead. - command (`List[str]` or `None`): - Command of the Job, e.g. `["python", "-c", "print('hello world')"]` - arguments (`List[str]` or `None`): - Arguments passed to the command - environment (`Dict[str]` or `None`): - Environment variables of the Job as a dictionary. - secrets (`Dict[str]` or `None`): - Secret environment variables of the Job (encrypted). - flavor (`str` or `None`): - Flavor for the hardware, as in Hugging Face Spaces. See [`SpaceHardware`] for possible values. - E.g. `"cpu-basic"`. - status: (`JobStatus` or `None`): - Status of the Job, e.g. `JobStatus(stage="RUNNING", message=None)` - See [`JobStage`] for possible stage values. - owner: (`JobOwner` or `None`): - Owner of the Job, e.g. `JobOwner(id="5e9ecfc04957053f60648a3e", name="lhoestq", type="user")` - - Example: - - ```python - >>> from huggingface_hub import run_job - >>> job = run_job( - ... image="python:3.12", - ... command=["python", "-c", "print('Hello from the cloud!')"] - ... ) - >>> job - JobInfo(id='687fb701029421ae5549d998', created_at=datetime.datetime(2025, 7, 22, 16, 6, 25, 79000, tzinfo=datetime.timezone.utc), docker_image='python:3.12', space_id=None, command=['python', '-c', "print('Hello from the cloud!')"], arguments=[], environment={}, secrets={}, flavor='cpu-basic', status=JobStatus(stage='RUNNING', message=None), owner=JobOwner(id='5e9ecfc04957053f60648a3e', name='lhoestq', type='user'), endpoint='https://huggingface.co', url='https://huggingface.co/jobs/lhoestq/687fb701029421ae5549d998') - >>> job.id - '687fb701029421ae5549d998' - >>> job.url - 'https://huggingface.co/jobs/lhoestq/687fb701029421ae5549d998' - >>> job.status.stage - 'RUNNING' - ``` - """ - - id: str - created_at: Optional[datetime] - docker_image: Optional[str] - space_id: Optional[str] - command: Optional[List[str]] - arguments: Optional[List[str]] - environment: Optional[Dict[str, Any]] - secrets: Optional[Dict[str, Any]] - flavor: Optional[SpaceHardware] - status: JobStatus - owner: JobOwner - - # Inferred fields - endpoint: str - url: str - - def __init__(self, **kwargs) -> None: - self.id = kwargs["id"] - created_at = kwargs.get("createdAt") or kwargs.get("created_at") - self.created_at = parse_datetime(created_at) if created_at else None - self.docker_image = kwargs.get("dockerImage") or kwargs.get("docker_image") - self.space_id = kwargs.get("spaceId") or kwargs.get("space_id") - owner = kwargs.get("owner", {}) - self.owner = JobOwner(id=owner["id"], name=owner["name"], type=owner["type"]) - self.command = kwargs.get("command") - self.arguments = kwargs.get("arguments") - self.environment = kwargs.get("environment") - self.secrets = kwargs.get("secrets") - self.flavor = kwargs.get("flavor") - status = kwargs.get("status", {}) - self.status = JobStatus(stage=status["stage"], message=status.get("message")) - - # Inferred fields - self.endpoint = kwargs.get("endpoint", constants.ENDPOINT) - self.url = f"{self.endpoint}/jobs/{self.owner.name}/{self.id}" - - -@dataclass -class JobSpec: - docker_image: Optional[str] - space_id: Optional[str] - command: Optional[List[str]] - arguments: Optional[List[str]] - environment: Optional[Dict[str, Any]] - secrets: Optional[Dict[str, Any]] - flavor: Optional[SpaceHardware] - timeout: Optional[int] - tags: Optional[List[str]] - arch: Optional[str] - - def __init__(self, **kwargs) -> None: - self.docker_image = kwargs.get("dockerImage") or kwargs.get("docker_image") - self.space_id = kwargs.get("spaceId") or kwargs.get("space_id") - self.command = kwargs.get("command") - self.arguments = kwargs.get("arguments") - self.environment = kwargs.get("environment") - self.secrets = kwargs.get("secrets") - self.flavor = kwargs.get("flavor") - self.timeout = kwargs.get("timeout") - self.tags = kwargs.get("tags") - self.arch = kwargs.get("arch") - - -@dataclass -class LastJobInfo: - id: str - at: datetime - - def __init__(self, **kwargs) -> None: - self.id = kwargs["id"] - self.at = parse_datetime(kwargs["at"]) - - -@dataclass -class ScheduledJobStatus: - last_job: Optional[LastJobInfo] - next_job_run_at: Optional[datetime] - - def __init__(self, **kwargs) -> None: - last_job = kwargs.get("lastJob") or kwargs.get("last_job") - self.last_job = LastJobInfo(**last_job) if last_job else None - next_job_run_at = kwargs.get("nextJobRunAt") or kwargs.get("next_job_run_at") - self.next_job_run_at = parse_datetime(str(next_job_run_at)) if next_job_run_at else None - - -@dataclass -class ScheduledJobInfo: - """ - Contains information about a Job. - - Args: - id (`str`): - Scheduled Job ID. - created_at (`datetime` or `None`): - When the scheduled Job was created. - tags (`List[str]` or `None`): - The tags of the scheduled Job. - schedule (`str` or `None`): - One of "@annually", "@yearly", "@monthly", "@weekly", "@daily", "@hourly", or a - CRON schedule expression (e.g., '0 9 * * 1' for 9 AM every Monday). - suspend (`bool` or `None`): - Whether the scheduled job is suspended (paused). - concurrency (`bool` or `None`): - Whether multiple instances of this Job can run concurrently. - status (`ScheduledJobStatus` or `None`): - Status of the scheduled Job. - owner: (`JobOwner` or `None`): - Owner of the scheduled Job, e.g. `JobOwner(id="5e9ecfc04957053f60648a3e", name="lhoestq", type="user")` - job_spec: (`JobSpec` or `None`): - Specifications of the Job. - - Example: - - ```python - >>> from huggingface_hub import run_job - >>> scheduled_job = create_scheduled_job( - ... image="python:3.12", - ... command=["python", "-c", "print('Hello from the cloud!')"], - ... schedule="@hourly", - ... ) - >>> scheduled_job.id - '687fb701029421ae5549d999' - >>> scheduled_job.status.next_job_run_at - datetime.datetime(2025, 7, 22, 17, 6, 25, 79000, tzinfo=datetime.timezone.utc) - ``` - """ - - id: str - created_at: Optional[datetime] - job_spec: JobSpec - schedule: Optional[str] - suspend: Optional[bool] - concurrency: Optional[bool] - status: ScheduledJobStatus - owner: JobOwner - - def __init__(self, **kwargs) -> None: - self.id = kwargs["id"] - created_at = kwargs.get("createdAt") or kwargs.get("created_at") - self.created_at = parse_datetime(created_at) if created_at else None - self.job_spec = JobSpec(**(kwargs.get("job_spec") or kwargs.get("jobSpec", {}))) - self.schedule = kwargs.get("schedule") - self.suspend = kwargs.get("suspend") - self.concurrency = kwargs.get("concurrency") - status = kwargs.get("status", {}) - self.status = ScheduledJobStatus( - last_job=status.get("last_job") or status.get("lastJob"), - next_job_run_at=status.get("next_job_run_at") or status.get("nextJobRunAt"), - ) - owner = kwargs.get("owner", {}) - self.owner = JobOwner(id=owner["id"], name=owner["name"], type=owner["type"]) - - -def _create_job_spec( - *, - image: str, - command: List[str], - env: Optional[Dict[str, Any]], - secrets: Optional[Dict[str, Any]], - flavor: Optional[SpaceHardware], - timeout: Optional[Union[int, float, str]], -) -> Dict[str, Any]: - # prepare job spec to send to HF Jobs API - job_spec: Dict[str, Any] = { - "command": command, - "arguments": [], - "environment": env or {}, - "flavor": flavor or SpaceHardware.CPU_BASIC, - } - # secrets are optional - if secrets: - job_spec["secrets"] = secrets - # timeout is optional - if timeout: - time_units_factors = {"s": 1, "m": 60, "h": 3600, "d": 3600 * 24} - if isinstance(timeout, str) and timeout[-1] in time_units_factors: - job_spec["timeoutSeconds"] = int(float(timeout[:-1]) * time_units_factors[timeout[-1]]) - else: - job_spec["timeoutSeconds"] = int(timeout) - # input is either from docker hub or from HF spaces - for prefix in ( - "https://huggingface.co/spaces/", - "https://hf.co/spaces/", - "huggingface.co/spaces/", - "hf.co/spaces/", - ): - if image.startswith(prefix): - job_spec["spaceId"] = image[len(prefix) :] - break - else: - job_spec["dockerImage"] = image - return job_spec diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_local_folder.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_local_folder.py deleted file mode 100644 index 37f6c32a760ecf03794c129735fe2e15516952d1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_local_folder.py +++ /dev/null @@ -1,447 +0,0 @@ -# coding=utf-8 -# Copyright 2024-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains utilities to handle the `../.cache/huggingface` folder in local directories. - -First discussed in https://github.com/huggingface/huggingface_hub/issues/1738 to store -download metadata when downloading files from the hub to a local directory (without -using the cache). - -./.cache/huggingface folder structure: -[4.0K] data -├── [4.0K] .cache -│ └── [4.0K] huggingface -│ └── [4.0K] download -│ ├── [ 16] file.parquet.metadata -│ ├── [ 16] file.txt.metadata -│ └── [4.0K] folder -│ └── [ 16] file.parquet.metadata -│ -├── [6.5G] file.parquet -├── [1.5K] file.txt -└── [4.0K] folder - └── [ 16] file.parquet - - -Download metadata file structure: -``` -# file.txt.metadata -11c5a3d5811f50298f278a704980280950aedb10 -a16a55fda99d2f2e7b69cce5cf93ff4ad3049930 -1712656091.123 - -# file.parquet.metadata -11c5a3d5811f50298f278a704980280950aedb10 -7c5d3f4b8b76583b422fcb9189ad6c89d5d97a094541ce8932dce3ecabde1421 -1712656091.123 -} -``` -""" - -import base64 -import hashlib -import logging -import os -import time -from dataclasses import dataclass -from pathlib import Path -from typing import Optional - -from .utils import WeakFileLock - - -logger = logging.getLogger(__name__) - - -@dataclass -class LocalDownloadFilePaths: - """ - Paths to the files related to a download process in a local dir. - - Returned by [`get_local_download_paths`]. - - Attributes: - file_path (`Path`): - Path where the file will be saved. - lock_path (`Path`): - Path to the lock file used to ensure atomicity when reading/writing metadata. - metadata_path (`Path`): - Path to the metadata file. - """ - - file_path: Path - lock_path: Path - metadata_path: Path - - def incomplete_path(self, etag: str) -> Path: - """Return the path where a file will be temporarily downloaded before being moved to `file_path`.""" - path = self.metadata_path.parent / f"{_short_hash(self.metadata_path.name)}.{etag}.incomplete" - resolved_path = str(path.resolve()) - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix. - if os.name == "nt" and len(resolved_path) > 255 and not resolved_path.startswith("\\\\?\\"): - path = Path("\\\\?\\" + resolved_path) - return path - - -@dataclass(frozen=True) -class LocalUploadFilePaths: - """ - Paths to the files related to an upload process in a local dir. - - Returned by [`get_local_upload_paths`]. - - Attributes: - path_in_repo (`str`): - Path of the file in the repo. - file_path (`Path`): - Path where the file will be saved. - lock_path (`Path`): - Path to the lock file used to ensure atomicity when reading/writing metadata. - metadata_path (`Path`): - Path to the metadata file. - """ - - path_in_repo: str - file_path: Path - lock_path: Path - metadata_path: Path - - -@dataclass -class LocalDownloadFileMetadata: - """ - Metadata about a file in the local directory related to a download process. - - Attributes: - filename (`str`): - Path of the file in the repo. - commit_hash (`str`): - Commit hash of the file in the repo. - etag (`str`): - ETag of the file in the repo. Used to check if the file has changed. - For LFS files, this is the sha256 of the file. For regular files, it corresponds to the git hash. - timestamp (`int`): - Unix timestamp of when the metadata was saved i.e. when the metadata was accurate. - """ - - filename: str - commit_hash: str - etag: str - timestamp: float - - -@dataclass -class LocalUploadFileMetadata: - """ - Metadata about a file in the local directory related to an upload process. - """ - - size: int - - # Default values correspond to "we don't know yet" - timestamp: Optional[float] = None - should_ignore: Optional[bool] = None - sha256: Optional[str] = None - upload_mode: Optional[str] = None - remote_oid: Optional[str] = None - is_uploaded: bool = False - is_committed: bool = False - - def save(self, paths: LocalUploadFilePaths) -> None: - """Save the metadata to disk.""" - with WeakFileLock(paths.lock_path): - with paths.metadata_path.open("w") as f: - new_timestamp = time.time() - f.write(str(new_timestamp) + "\n") - - f.write(str(self.size)) # never None - f.write("\n") - - if self.should_ignore is not None: - f.write(str(int(self.should_ignore))) - f.write("\n") - - if self.sha256 is not None: - f.write(self.sha256) - f.write("\n") - - if self.upload_mode is not None: - f.write(self.upload_mode) - f.write("\n") - - if self.remote_oid is not None: - f.write(self.remote_oid) - f.write("\n") - - f.write(str(int(self.is_uploaded)) + "\n") - f.write(str(int(self.is_committed)) + "\n") - - self.timestamp = new_timestamp - - -def get_local_download_paths(local_dir: Path, filename: str) -> LocalDownloadFilePaths: - """Compute paths to the files related to a download process. - - Folders containing the paths are all guaranteed to exist. - - Args: - local_dir (`Path`): - Path to the local directory in which files are downloaded. - filename (`str`): - Path of the file in the repo. - - Return: - [`LocalDownloadFilePaths`]: the paths to the files (file_path, lock_path, metadata_path, incomplete_path). - """ - # filename is the path in the Hub repository (separated by '/') - # make sure to have a cross platform transcription - sanitized_filename = os.path.join(*filename.split("/")) - if os.name == "nt": - if sanitized_filename.startswith("..\\") or "\\..\\" in sanitized_filename: - raise ValueError( - f"Invalid filename: cannot handle filename '{sanitized_filename}' on Windows. Please ask the repository" - " owner to rename this file." - ) - file_path = local_dir / sanitized_filename - metadata_path = _huggingface_dir(local_dir) / "download" / f"{sanitized_filename}.metadata" - lock_path = metadata_path.with_suffix(".lock") - - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix - if os.name == "nt": - if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255: - file_path = Path("\\\\?\\" + os.path.abspath(file_path)) - lock_path = Path("\\\\?\\" + os.path.abspath(lock_path)) - metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path)) - - file_path.parent.mkdir(parents=True, exist_ok=True) - metadata_path.parent.mkdir(parents=True, exist_ok=True) - return LocalDownloadFilePaths(file_path=file_path, lock_path=lock_path, metadata_path=metadata_path) - - -def get_local_upload_paths(local_dir: Path, filename: str) -> LocalUploadFilePaths: - """Compute paths to the files related to an upload process. - - Folders containing the paths are all guaranteed to exist. - - Args: - local_dir (`Path`): - Path to the local directory that is uploaded. - filename (`str`): - Path of the file in the repo. - - Return: - [`LocalUploadFilePaths`]: the paths to the files (file_path, lock_path, metadata_path). - """ - # filename is the path in the Hub repository (separated by '/') - # make sure to have a cross platform transcription - sanitized_filename = os.path.join(*filename.split("/")) - if os.name == "nt": - if sanitized_filename.startswith("..\\") or "\\..\\" in sanitized_filename: - raise ValueError( - f"Invalid filename: cannot handle filename '{sanitized_filename}' on Windows. Please ask the repository" - " owner to rename this file." - ) - file_path = local_dir / sanitized_filename - metadata_path = _huggingface_dir(local_dir) / "upload" / f"{sanitized_filename}.metadata" - lock_path = metadata_path.with_suffix(".lock") - - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix - if os.name == "nt": - if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255: - file_path = Path("\\\\?\\" + os.path.abspath(file_path)) - lock_path = Path("\\\\?\\" + os.path.abspath(lock_path)) - metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path)) - - file_path.parent.mkdir(parents=True, exist_ok=True) - metadata_path.parent.mkdir(parents=True, exist_ok=True) - return LocalUploadFilePaths( - path_in_repo=filename, file_path=file_path, lock_path=lock_path, metadata_path=metadata_path - ) - - -def read_download_metadata(local_dir: Path, filename: str) -> Optional[LocalDownloadFileMetadata]: - """Read metadata about a file in the local directory related to a download process. - - Args: - local_dir (`Path`): - Path to the local directory in which files are downloaded. - filename (`str`): - Path of the file in the repo. - - Return: - `[LocalDownloadFileMetadata]` or `None`: the metadata if it exists, `None` otherwise. - """ - paths = get_local_download_paths(local_dir, filename) - with WeakFileLock(paths.lock_path): - if paths.metadata_path.exists(): - try: - with paths.metadata_path.open() as f: - commit_hash = f.readline().strip() - etag = f.readline().strip() - timestamp = float(f.readline().strip()) - metadata = LocalDownloadFileMetadata( - filename=filename, - commit_hash=commit_hash, - etag=etag, - timestamp=timestamp, - ) - except Exception as e: - # remove the metadata file if it is corrupted / not the right format - logger.warning( - f"Invalid metadata file {paths.metadata_path}: {e}. Removing it from disk and continue." - ) - try: - paths.metadata_path.unlink() - except Exception as e: - logger.warning(f"Could not remove corrupted metadata file {paths.metadata_path}: {e}") - - try: - # check if the file exists and hasn't been modified since the metadata was saved - stat = paths.file_path.stat() - if ( - stat.st_mtime - 1 <= metadata.timestamp - ): # allow 1s difference as stat.st_mtime might not be precise - return metadata - logger.info(f"Ignored metadata for '{filename}' (outdated). Will re-compute hash.") - except FileNotFoundError: - # file does not exist => metadata is outdated - return None - return None - - -def read_upload_metadata(local_dir: Path, filename: str) -> LocalUploadFileMetadata: - """Read metadata about a file in the local directory related to an upload process. - - TODO: factorize logic with `read_download_metadata`. - - Args: - local_dir (`Path`): - Path to the local directory in which files are downloaded. - filename (`str`): - Path of the file in the repo. - - Return: - `[LocalUploadFileMetadata]` or `None`: the metadata if it exists, `None` otherwise. - """ - paths = get_local_upload_paths(local_dir, filename) - with WeakFileLock(paths.lock_path): - if paths.metadata_path.exists(): - try: - with paths.metadata_path.open() as f: - timestamp = float(f.readline().strip()) - - size = int(f.readline().strip()) # never None - - _should_ignore = f.readline().strip() - should_ignore = None if _should_ignore == "" else bool(int(_should_ignore)) - - _sha256 = f.readline().strip() - sha256 = None if _sha256 == "" else _sha256 - - _upload_mode = f.readline().strip() - upload_mode = None if _upload_mode == "" else _upload_mode - if upload_mode not in (None, "regular", "lfs"): - raise ValueError(f"Invalid upload mode in metadata {paths.path_in_repo}: {upload_mode}") - - _remote_oid = f.readline().strip() - remote_oid = None if _remote_oid == "" else _remote_oid - - is_uploaded = bool(int(f.readline().strip())) - is_committed = bool(int(f.readline().strip())) - - metadata = LocalUploadFileMetadata( - timestamp=timestamp, - size=size, - should_ignore=should_ignore, - sha256=sha256, - upload_mode=upload_mode, - remote_oid=remote_oid, - is_uploaded=is_uploaded, - is_committed=is_committed, - ) - except Exception as e: - # remove the metadata file if it is corrupted / not the right format - logger.warning( - f"Invalid metadata file {paths.metadata_path}: {e}. Removing it from disk and continue." - ) - try: - paths.metadata_path.unlink() - except Exception as e: - logger.warning(f"Could not remove corrupted metadata file {paths.metadata_path}: {e}") - - # TODO: can we do better? - if ( - metadata.timestamp is not None - and metadata.is_uploaded # file was uploaded - and not metadata.is_committed # but not committed - and time.time() - metadata.timestamp > 20 * 3600 # and it's been more than 20 hours - ): # => we consider it as garbage-collected by S3 - metadata.is_uploaded = False - - # check if the file exists and hasn't been modified since the metadata was saved - try: - if metadata.timestamp is not None and paths.file_path.stat().st_mtime <= metadata.timestamp: - return metadata - logger.info(f"Ignored metadata for '{filename}' (outdated). Will re-compute hash.") - except FileNotFoundError: - # file does not exist => metadata is outdated - pass - - # empty metadata => we don't know anything expect its size - return LocalUploadFileMetadata(size=paths.file_path.stat().st_size) - - -def write_download_metadata(local_dir: Path, filename: str, commit_hash: str, etag: str) -> None: - """Write metadata about a file in the local directory related to a download process. - - Args: - local_dir (`Path`): - Path to the local directory in which files are downloaded. - """ - paths = get_local_download_paths(local_dir, filename) - with WeakFileLock(paths.lock_path): - with paths.metadata_path.open("w") as f: - f.write(f"{commit_hash}\n{etag}\n{time.time()}\n") - - -def _huggingface_dir(local_dir: Path) -> Path: - """Return the path to the `.cache/huggingface` directory in a local directory.""" - # Wrap in lru_cache to avoid overwriting the .gitignore file if called multiple times - path = local_dir / ".cache" / "huggingface" - path.mkdir(exist_ok=True, parents=True) - - # Create a .gitignore file in the .cache/huggingface directory if it doesn't exist - # Should be thread-safe enough like this. - gitignore = path / ".gitignore" - gitignore_lock = path / ".gitignore.lock" - if not gitignore.exists(): - try: - with WeakFileLock(gitignore_lock, timeout=0.1): - gitignore.write_text("*") - except IndexError: - pass - except OSError: # TimeoutError, FileNotFoundError, PermissionError, etc. - pass - try: - gitignore_lock.unlink() - except OSError: - pass - return path - - -def _short_hash(filename: str) -> str: - return base64.urlsafe_b64encode(hashlib.sha1(filename.encode()).digest()).decode() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_login.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_login.py deleted file mode 100644 index 8f721b68348fc3abeb2f90b6a756cb125ce19571..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_login.py +++ /dev/null @@ -1,514 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains methods to log in to the Hub.""" - -import os -import subprocess -from getpass import getpass -from pathlib import Path -from typing import Optional - -from . import constants -from .commands._cli_utils import ANSI -from .utils import ( - capture_output, - get_token, - is_google_colab, - is_notebook, - list_credential_helpers, - logging, - run_subprocess, - set_git_credential, - unset_git_credential, -) -from .utils._auth import ( - _get_token_by_name, - _get_token_from_environment, - _get_token_from_file, - _get_token_from_google_colab, - _save_stored_tokens, - _save_token, - get_stored_tokens, -) -from .utils._deprecation import _deprecate_arguments, _deprecate_positional_args - - -logger = logging.get_logger(__name__) - -_HF_LOGO_ASCII = """ - _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_| - _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _| - _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_| - _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _| - _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_| -""" - - -@_deprecate_arguments( - version="1.0", - deprecated_args="write_permission", - custom_message="Fine-grained tokens added complexity to the permissions, making it irrelevant to check if a token has 'write' access.", -) -@_deprecate_positional_args(version="1.0") -def login( - token: Optional[str] = None, - *, - add_to_git_credential: bool = False, - new_session: bool = True, - write_permission: bool = False, -) -> None: - """Login the machine to access the Hub. - - The `token` is persisted in cache and set as a git credential. Once done, the machine - is logged in and the access token will be available across all `huggingface_hub` - components. If `token` is not provided, it will be prompted to the user either with - a widget (in a notebook) or via the terminal. - - To log in from outside of a script, one can also use `hf auth login` which is - a cli command that wraps [`login`]. - - > [!TIP] - > [`login`] is a drop-in replacement method for [`notebook_login`] as it wraps and - > extends its capabilities. - - > [!TIP] - > When the token is not passed, [`login`] will automatically detect if the script runs - > in a notebook or not. However, this detection might not be accurate due to the - > variety of notebooks that exists nowadays. If that is the case, you can always force - > the UI by using [`notebook_login`] or [`interpreter_login`]. - - Args: - token (`str`, *optional*): - User access token to generate from https://huggingface.co/settings/token. - add_to_git_credential (`bool`, defaults to `False`): - If `True`, token will be set as git credential. If no git credential helper - is configured, a warning will be displayed to the user. If `token` is `None`, - the value of `add_to_git_credential` is ignored and will be prompted again - to the end user. - new_session (`bool`, defaults to `True`): - If `True`, will request a token even if one is already saved on the machine. - write_permission (`bool`): - Ignored and deprecated argument. - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If an organization token is passed. Only personal account tokens are valid - to log in. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If token is invalid. - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) - If running in a notebook but `ipywidgets` is not installed. - """ - if token is not None: - if not add_to_git_credential: - logger.info( - "The token has not been saved to the git credentials helper. Pass " - "`add_to_git_credential=True` in this function directly or " - "`--add-to-git-credential` if using via `hf`CLI if " - "you want to set the git credential as well." - ) - _login(token, add_to_git_credential=add_to_git_credential) - elif is_notebook(): - notebook_login(new_session=new_session) - else: - interpreter_login(new_session=new_session) - - -def logout(token_name: Optional[str] = None) -> None: - """Logout the machine from the Hub. - - Token is deleted from the machine and removed from git credential. - - Args: - token_name (`str`, *optional*): - Name of the access token to logout from. If `None`, will logout from all saved access tokens. - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError): - If the access token name is not found. - """ - if get_token() is None and not get_stored_tokens(): # No active token and no saved access tokens - logger.warning("Not logged in!") - return - if not token_name: - # Delete all saved access tokens and token - for file_path in (constants.HF_TOKEN_PATH, constants.HF_STORED_TOKENS_PATH): - try: - Path(file_path).unlink() - except FileNotFoundError: - pass - logger.info("Successfully logged out from all access tokens.") - else: - _logout_from_token(token_name) - logger.info(f"Successfully logged out from access token: {token_name}.") - - unset_git_credential() - - # Check if still logged in - if _get_token_from_google_colab() is not None: - raise EnvironmentError( - "You are automatically logged in using a Google Colab secret.\n" - "To log out, you must unset the `HF_TOKEN` secret in your Colab settings." - ) - if _get_token_from_environment() is not None: - raise EnvironmentError( - "Token has been deleted from your machine but you are still logged in.\n" - "To log out, you must clear out both `HF_TOKEN` and `HUGGING_FACE_HUB_TOKEN` environment variables." - ) - - -def auth_switch(token_name: str, add_to_git_credential: bool = False) -> None: - """Switch to a different access token. - - Args: - token_name (`str`): - Name of the access token to switch to. - add_to_git_credential (`bool`, defaults to `False`): - If `True`, token will be set as git credential. If no git credential helper - is configured, a warning will be displayed to the user. If `token` is `None`, - the value of `add_to_git_credential` is ignored and will be prompted again - to the end user. - - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError): - If the access token name is not found. - """ - token = _get_token_by_name(token_name) - if not token: - raise ValueError(f"Access token {token_name} not found in {constants.HF_STORED_TOKENS_PATH}") - # Write token to HF_TOKEN_PATH - _set_active_token(token_name, add_to_git_credential) - logger.info(f"The current active token is: {token_name}") - token_from_environment = _get_token_from_environment() - if token_from_environment is not None and token_from_environment != token: - logger.warning( - "The environment variable `HF_TOKEN` is set and will override the access token you've just switched to." - ) - - -def auth_list() -> None: - """List all stored access tokens.""" - tokens = get_stored_tokens() - - if not tokens: - logger.info("No access tokens found.") - return - # Find current token - current_token = get_token() - current_token_name = None - for token_name in tokens: - if tokens.get(token_name) == current_token: - current_token_name = token_name - # Print header - max_offset = max(len("token"), max(len(token) for token in tokens)) + 2 - print(f" {{:<{max_offset}}}| {{:<15}}".format("name", "token")) - print("-" * (max_offset + 2) + "|" + "-" * 15) - - # Print saved access tokens - for token_name in tokens: - token = tokens.get(token_name, "") - masked_token = f"{token[:3]}****{token[-4:]}" if token != "" else token - is_current = "*" if token == current_token else " " - - print(f"{is_current} {{:<{max_offset}}}| {{:<15}}".format(token_name, masked_token)) - - if _get_token_from_environment(): - logger.warning( - "\nNote: Environment variable `HF_TOKEN` is set and is the current active token independently from the stored tokens listed above." - ) - elif current_token_name is None: - logger.warning( - "\nNote: No active token is set and no environment variable `HF_TOKEN` is found. Use `hf auth login` to log in." - ) - - -### -# Interpreter-based login (text) -### - - -@_deprecate_arguments( - version="1.0", - deprecated_args="write_permission", - custom_message="Fine-grained tokens added complexity to the permissions, making it irrelevant to check if a token has 'write' access.", -) -@_deprecate_positional_args(version="1.0") -def interpreter_login(*, new_session: bool = True, write_permission: bool = False) -> None: - """ - Displays a prompt to log in to the HF website and store the token. - - This is equivalent to [`login`] without passing a token when not run in a notebook. - [`interpreter_login`] is useful if you want to force the use of the terminal prompt - instead of a notebook widget. - - For more details, see [`login`]. - - Args: - new_session (`bool`, defaults to `True`): - If `True`, will request a token even if one is already saved on the machine. - write_permission (`bool`): - Ignored and deprecated argument. - """ - if not new_session and get_token() is not None: - logger.info("User is already logged in.") - return - - from .commands.delete_cache import _ask_for_confirmation_no_tui - - print(_HF_LOGO_ASCII) - if get_token() is not None: - logger.info( - " A token is already saved on your machine. Run `hf auth whoami`" - " to get more information or `hf auth logout` if you want" - " to log out." - ) - logger.info(" Setting a new token will erase the existing one.") - - logger.info( - " To log in, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens ." - ) - if os.name == "nt": - logger.info("Token can be pasted using 'Right-Click'.") - token = getpass("Enter your token (input will not be visible): ") - add_to_git_credential = _ask_for_confirmation_no_tui("Add token as git credential?") - - _login(token=token, add_to_git_credential=add_to_git_credential) - - -### -# Notebook-based login (widget) -### - -NOTEBOOK_LOGIN_PASSWORD_HTML = """

Immediately click login after typing your password or -it might be stored in plain text in this notebook file.
""" - - -NOTEBOOK_LOGIN_TOKEN_HTML_START = """

Copy a token from your Hugging Face -tokens page and paste it below.
Immediately click login after copying -your token or it might be stored in plain text in this notebook file.
""" - - -NOTEBOOK_LOGIN_TOKEN_HTML_END = """ -Pro Tip: If you don't already have one, you can create a dedicated -'notebooks' token with 'write' access, that you can then easily reuse for all -notebooks. """ - - -@_deprecate_arguments( - version="1.0", - deprecated_args="write_permission", - custom_message="Fine-grained tokens added complexity to the permissions, making it irrelevant to check if a token has 'write' access.", -) -@_deprecate_positional_args(version="1.0") -def notebook_login(*, new_session: bool = True, write_permission: bool = False) -> None: - """ - Displays a widget to log in to the HF website and store the token. - - This is equivalent to [`login`] without passing a token when run in a notebook. - [`notebook_login`] is useful if you want to force the use of the notebook widget - instead of a prompt in the terminal. - - For more details, see [`login`]. - - Args: - new_session (`bool`, defaults to `True`): - If `True`, will request a token even if one is already saved on the machine. - write_permission (`bool`): - Ignored and deprecated argument. - """ - try: - import ipywidgets.widgets as widgets # type: ignore - from IPython.display import display # type: ignore - except ImportError: - raise ImportError( - "The `notebook_login` function can only be used in a notebook (Jupyter or" - " Colab) and you need the `ipywidgets` module: `pip install ipywidgets`." - ) - if not new_session and get_token() is not None: - logger.info("User is already logged in.") - return - - box_layout = widgets.Layout(display="flex", flex_flow="column", align_items="center", width="50%") - - token_widget = widgets.Password(description="Token:") - git_checkbox_widget = widgets.Checkbox(value=True, description="Add token as git credential?") - token_finish_button = widgets.Button(description="Login") - - login_token_widget = widgets.VBox( - [ - widgets.HTML(NOTEBOOK_LOGIN_TOKEN_HTML_START), - token_widget, - git_checkbox_widget, - token_finish_button, - widgets.HTML(NOTEBOOK_LOGIN_TOKEN_HTML_END), - ], - layout=box_layout, - ) - display(login_token_widget) - - # On click events - def login_token_event(t): - """Event handler for the login button.""" - token = token_widget.value - add_to_git_credential = git_checkbox_widget.value - # Erase token and clear value to make sure it's not saved in the notebook. - token_widget.value = "" - # Hide inputs - login_token_widget.children = [widgets.Label("Connecting...")] - try: - with capture_output() as captured: - _login(token, add_to_git_credential=add_to_git_credential) - message = captured.getvalue() - except Exception as error: - message = str(error) - # Print result (success message or error) - login_token_widget.children = [widgets.Label(line) for line in message.split("\n") if line.strip()] - - token_finish_button.on_click(login_token_event) - - -### -# Login private helpers -### - - -def _login( - token: str, - add_to_git_credential: bool, -) -> None: - from .hf_api import whoami # avoid circular import - - if token.startswith("api_org"): - raise ValueError("You must use your personal account token, not an organization token.") - - token_info = whoami(token) - permission = token_info["auth"]["accessToken"]["role"] - logger.info(f"Token is valid (permission: {permission}).") - - token_name = token_info["auth"]["accessToken"]["displayName"] - # Store token locally - _save_token(token=token, token_name=token_name) - # Set active token - _set_active_token(token_name=token_name, add_to_git_credential=add_to_git_credential) - logger.info("Login successful.") - if _get_token_from_environment(): - logger.warning( - "Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured." - ) - else: - logger.info(f"The current active token is: `{token_name}`") - - -def _logout_from_token(token_name: str) -> None: - """Logout from a specific access token. - - Args: - token_name (`str`): - The name of the access token to logout from. - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError): - If the access token name is not found. - """ - stored_tokens = get_stored_tokens() - # If there is no access tokens saved or the access token name is not found, do nothing - if not stored_tokens or token_name not in stored_tokens: - return - - token = stored_tokens.pop(token_name) - _save_stored_tokens(stored_tokens) - - if token == _get_token_from_file(): - logger.warning(f"Active token '{token_name}' has been deleted.") - Path(constants.HF_TOKEN_PATH).unlink(missing_ok=True) - - -def _set_active_token( - token_name: str, - add_to_git_credential: bool, -) -> None: - """Set the active access token. - - Args: - token_name (`str`): - The name of the token to set as active. - """ - token = _get_token_by_name(token_name) - if not token: - raise ValueError(f"Token {token_name} not found in {constants.HF_STORED_TOKENS_PATH}") - if add_to_git_credential: - if _is_git_credential_helper_configured(): - set_git_credential(token) - logger.info( - "Your token has been saved in your configured git credential helpers" - + f" ({','.join(list_credential_helpers())})." - ) - else: - logger.warning("Token has not been saved to git credential helper.") - # Write token to HF_TOKEN_PATH - path = Path(constants.HF_TOKEN_PATH) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(token) - logger.info(f"Your token has been saved to {constants.HF_TOKEN_PATH}") - - -def _is_git_credential_helper_configured() -> bool: - """Check if a git credential helper is configured. - - Warns user if not the case (except for Google Colab where "store" is set by default - by `huggingface_hub`). - """ - helpers = list_credential_helpers() - if len(helpers) > 0: - return True # Do not warn: at least 1 helper is set - - # Only in Google Colab to avoid the warning message - # See https://github.com/huggingface/huggingface_hub/issues/1043#issuecomment-1247010710 - if is_google_colab(): - _set_store_as_git_credential_helper_globally() - return True # Do not warn: "store" is used by default in Google Colab - - # Otherwise, warn user - print( - ANSI.red( - "Cannot authenticate through git-credential as no helper is defined on your" - " machine.\nYou might have to re-authenticate when pushing to the Hugging" - " Face Hub.\nRun the following command in your terminal in case you want to" - " set the 'store' credential helper as default.\n\ngit config --global" - " credential.helper store\n\nRead" - " https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more" - " details." - ) - ) - return False - - -def _set_store_as_git_credential_helper_globally() -> None: - """Set globally the credential.helper to `store`. - - To be used only in Google Colab as we assume the user doesn't care about the git - credential config. It is the only particular case where we don't want to display the - warning message in [`notebook_login()`]. - - Related: - - https://github.com/huggingface/huggingface_hub/issues/1043 - - https://github.com/huggingface/huggingface_hub/issues/1051 - - https://git-scm.com/docs/git-credential-store - """ - try: - run_subprocess("git config --global credential.helper store") - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_oauth.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_oauth.py deleted file mode 100644 index 9f8eb607962bc18fec348fed18ce269524983e23..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_oauth.py +++ /dev/null @@ -1,460 +0,0 @@ -import datetime -import hashlib -import logging -import os -import time -import urllib.parse -import warnings -from dataclasses import dataclass -from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union - -from . import constants -from .hf_api import whoami -from .utils import experimental, get_token - - -logger = logging.getLogger(__name__) - -if TYPE_CHECKING: - import fastapi - - -@dataclass -class OAuthOrgInfo: - """ - Information about an organization linked to a user logged in with OAuth. - - Attributes: - sub (`str`): - Unique identifier for the org. OpenID Connect field. - name (`str`): - The org's full name. OpenID Connect field. - preferred_username (`str`): - The org's username. OpenID Connect field. - picture (`str`): - The org's profile picture URL. OpenID Connect field. - is_enterprise (`bool`): - Whether the org is an enterprise org. Hugging Face field. - can_pay (`Optional[bool]`, *optional*): - Whether the org has a payment method set up. Hugging Face field. - role_in_org (`Optional[str]`, *optional*): - The user's role in the org. Hugging Face field. - security_restrictions (`Optional[List[Literal["ip", "token-policy", "mfa", "sso"]]]`, *optional*): - Array of security restrictions that the user hasn't completed for this org. Possible values: "ip", "token-policy", "mfa", "sso". Hugging Face field. - """ - - sub: str - name: str - preferred_username: str - picture: str - is_enterprise: bool - can_pay: Optional[bool] = None - role_in_org: Optional[str] = None - security_restrictions: Optional[List[Literal["ip", "token-policy", "mfa", "sso"]]] = None - - -@dataclass -class OAuthUserInfo: - """ - Information about a user logged in with OAuth. - - Attributes: - sub (`str`): - Unique identifier for the user, even in case of rename. OpenID Connect field. - name (`str`): - The user's full name. OpenID Connect field. - preferred_username (`str`): - The user's username. OpenID Connect field. - email_verified (`Optional[bool]`, *optional*): - Indicates if the user's email is verified. OpenID Connect field. - email (`Optional[str]`, *optional*): - The user's email address. OpenID Connect field. - picture (`str`): - The user's profile picture URL. OpenID Connect field. - profile (`str`): - The user's profile URL. OpenID Connect field. - website (`Optional[str]`, *optional*): - The user's website URL. OpenID Connect field. - is_pro (`bool`): - Whether the user is a pro user. Hugging Face field. - can_pay (`Optional[bool]`, *optional*): - Whether the user has a payment method set up. Hugging Face field. - orgs (`Optional[List[OrgInfo]]`, *optional*): - List of organizations the user is part of. Hugging Face field. - """ - - sub: str - name: str - preferred_username: str - email_verified: Optional[bool] - email: Optional[str] - picture: str - profile: str - website: Optional[str] - is_pro: bool - can_pay: Optional[bool] - orgs: Optional[List[OAuthOrgInfo]] - - -@dataclass -class OAuthInfo: - """ - Information about the OAuth login. - - Attributes: - access_token (`str`): - The access token. - access_token_expires_at (`datetime.datetime`): - The expiration date of the access token. - user_info ([`OAuthUserInfo`]): - The user information. - state (`str`, *optional*): - State passed to the OAuth provider in the original request to the OAuth provider. - scope (`str`): - Granted scope. - """ - - access_token: str - access_token_expires_at: datetime.datetime - user_info: OAuthUserInfo - state: Optional[str] - scope: str - - -@experimental -def attach_huggingface_oauth(app: "fastapi.FastAPI", route_prefix: str = "/"): - """ - Add OAuth endpoints to a FastAPI app to enable OAuth login with Hugging Face. - - How to use: - - Call this method on your FastAPI app to add the OAuth endpoints. - - Inside your route handlers, call `parse_huggingface_oauth(request)` to retrieve the OAuth info. - - If user is logged in, an [`OAuthInfo`] object is returned with the user's info. If not, `None` is returned. - - In your app, make sure to add links to `/oauth/huggingface/login` and `/oauth/huggingface/logout` for the user to log in and out. - - Example: - ```py - from huggingface_hub import attach_huggingface_oauth, parse_huggingface_oauth - - # Create a FastAPI app - app = FastAPI() - - # Add OAuth endpoints to the FastAPI app - attach_huggingface_oauth(app) - - # Add a route that greets the user if they are logged in - @app.get("/") - def greet_json(request: Request): - # Retrieve the OAuth info from the request - oauth_info = parse_huggingface_oauth(request) # e.g. OAuthInfo dataclass - if oauth_info is None: - return {"msg": "Not logged in!"} - return {"msg": f"Hello, {oauth_info.user_info.preferred_username}!"} - ``` - """ - # TODO: handle generic case (handling OAuth in a non-Space environment with custom dev values) (low priority) - - # Add SessionMiddleware to the FastAPI app to store the OAuth info in the session. - # Session Middleware requires a secret key to sign the cookies. Let's use a hash - # of the OAuth secret key to make it unique to the Space + updated in case OAuth - # config gets updated. When ran locally, we use an empty string as a secret key. - try: - from starlette.middleware.sessions import SessionMiddleware - except ImportError as e: - raise ImportError( - "Cannot initialize OAuth to due a missing library. Please run `pip install huggingface_hub[oauth]` or add " - "`huggingface_hub[oauth]` to your requirements.txt file in order to install the required dependencies." - ) from e - session_secret = (constants.OAUTH_CLIENT_SECRET or "") + "-v1" - app.add_middleware( - SessionMiddleware, # type: ignore[arg-type] - secret_key=hashlib.sha256(session_secret.encode()).hexdigest(), - same_site="none", - https_only=True, - ) # type: ignore - - # Add OAuth endpoints to the FastAPI app: - # - {route_prefix}/oauth/huggingface/login - # - {route_prefix}/oauth/huggingface/callback - # - {route_prefix}/oauth/huggingface/logout - # If the app is running in a Space, OAuth is enabled normally. - # Otherwise, we mock the endpoints to make the user log in with a fake user profile - without any calls to hf.co. - route_prefix = route_prefix.strip("/") - if os.getenv("SPACE_ID") is not None: - logger.info("OAuth is enabled in the Space. Adding OAuth routes.") - _add_oauth_routes(app, route_prefix=route_prefix) - else: - logger.info("App is not running in a Space. Adding mocked OAuth routes.") - _add_mocked_oauth_routes(app, route_prefix=route_prefix) - - -def parse_huggingface_oauth(request: "fastapi.Request") -> Optional[OAuthInfo]: - """ - Returns the information from a logged in user as a [`OAuthInfo`] object. - - For flexibility and future-proofing, this method is very lax in its parsing and does not raise errors. - Missing fields are set to `None` without a warning. - - Return `None`, if the user is not logged in (no info in session cookie). - - See [`attach_huggingface_oauth`] for an example on how to use this method. - """ - if "oauth_info" not in request.session: - logger.debug("No OAuth info in session.") - return None - - logger.debug("Parsing OAuth info from session.") - oauth_data = request.session["oauth_info"] - user_data = oauth_data.get("userinfo", {}) - orgs_data = user_data.get("orgs", []) - - orgs = ( - [ - OAuthOrgInfo( - sub=org.get("sub"), - name=org.get("name"), - preferred_username=org.get("preferred_username"), - picture=org.get("picture"), - is_enterprise=org.get("isEnterprise"), - can_pay=org.get("canPay"), - role_in_org=org.get("roleInOrg"), - security_restrictions=org.get("securityRestrictions"), - ) - for org in orgs_data - ] - if orgs_data - else None - ) - - user_info = OAuthUserInfo( - sub=user_data.get("sub"), - name=user_data.get("name"), - preferred_username=user_data.get("preferred_username"), - email_verified=user_data.get("email_verified"), - email=user_data.get("email"), - picture=user_data.get("picture"), - profile=user_data.get("profile"), - website=user_data.get("website"), - is_pro=user_data.get("isPro"), - can_pay=user_data.get("canPay"), - orgs=orgs, - ) - - return OAuthInfo( - access_token=oauth_data.get("access_token"), - access_token_expires_at=datetime.datetime.fromtimestamp(oauth_data.get("expires_at")), - user_info=user_info, - state=oauth_data.get("state"), - scope=oauth_data.get("scope"), - ) - - -def _add_oauth_routes(app: "fastapi.FastAPI", route_prefix: str) -> None: - """Add OAuth routes to the FastAPI app (login, callback handler and logout).""" - try: - import fastapi - from authlib.integrations.base_client.errors import MismatchingStateError - from authlib.integrations.starlette_client import OAuth - from fastapi.responses import RedirectResponse - except ImportError as e: - raise ImportError( - "Cannot initialize OAuth to due a missing library. Please run `pip install huggingface_hub[oauth]` or add " - "`huggingface_hub[oauth]` to your requirements.txt file." - ) from e - - # Check environment variables - msg = ( - "OAuth is required but '{}' environment variable is not set. Make sure you've enabled OAuth in your Space by" - " setting `hf_oauth: true` in the Space metadata." - ) - if constants.OAUTH_CLIENT_ID is None: - raise ValueError(msg.format("OAUTH_CLIENT_ID")) - if constants.OAUTH_CLIENT_SECRET is None: - raise ValueError(msg.format("OAUTH_CLIENT_SECRET")) - if constants.OAUTH_SCOPES is None: - raise ValueError(msg.format("OAUTH_SCOPES")) - if constants.OPENID_PROVIDER_URL is None: - raise ValueError(msg.format("OPENID_PROVIDER_URL")) - - # Register OAuth server - oauth = OAuth() - oauth.register( - name="huggingface", - client_id=constants.OAUTH_CLIENT_ID, - client_secret=constants.OAUTH_CLIENT_SECRET, - client_kwargs={"scope": constants.OAUTH_SCOPES}, - server_metadata_url=constants.OPENID_PROVIDER_URL + "/.well-known/openid-configuration", - ) - - login_uri, callback_uri, logout_uri = _get_oauth_uris(route_prefix) - - # Register OAuth endpoints - @app.get(login_uri) - async def oauth_login(request: fastapi.Request) -> RedirectResponse: - """Endpoint that redirects to HF OAuth page.""" - redirect_uri = _generate_redirect_uri(request) - return await oauth.huggingface.authorize_redirect(request, redirect_uri) # type: ignore - - @app.get(callback_uri) - async def oauth_redirect_callback(request: fastapi.Request) -> RedirectResponse: - """Endpoint that handles the OAuth callback.""" - try: - oauth_info = await oauth.huggingface.authorize_access_token(request) # type: ignore - except MismatchingStateError: - # Parse query params - nb_redirects = int(request.query_params.get("_nb_redirects", 0)) - target_url = request.query_params.get("_target_url") - - # Build redirect URI with the same query params as before and bump nb_redirects count - query_params: Dict[str, Union[int, str]] = {"_nb_redirects": nb_redirects + 1} - if target_url: - query_params["_target_url"] = target_url - - redirect_uri = f"{login_uri}?{urllib.parse.urlencode(query_params)}" - - # If the user is redirected more than 3 times, it is very likely that the cookie is not working properly. - # (e.g. browser is blocking third-party cookies in iframe). In this case, redirect the user in the - # non-iframe view. - if nb_redirects > constants.OAUTH_MAX_REDIRECTS: - host = os.environ.get("SPACE_HOST") - if host is None: # cannot happen in a Space - raise RuntimeError( - "App is not running in a Space (SPACE_HOST environment variable is not set). Cannot redirect to non-iframe view." - ) from None - host_url = "https://" + host.rstrip("/") - return RedirectResponse(host_url + redirect_uri) - - # Redirect the user to the login page again - return RedirectResponse(redirect_uri) - - # OAuth login worked => store the user info in the session and redirect - logger.debug("Successfully logged in with OAuth. Storing user info in session.") - request.session["oauth_info"] = oauth_info - return RedirectResponse(_get_redirect_target(request)) - - @app.get(logout_uri) - async def oauth_logout(request: fastapi.Request) -> RedirectResponse: - """Endpoint that logs out the user (e.g. delete info from cookie session).""" - logger.debug("Logged out with OAuth. Removing user info from session.") - request.session.pop("oauth_info", None) - return RedirectResponse(_get_redirect_target(request)) - - -def _add_mocked_oauth_routes(app: "fastapi.FastAPI", route_prefix: str = "/") -> None: - """Add fake oauth routes if app is run locally and OAuth is enabled. - - Using OAuth will have the same behavior as in a Space but instead of authenticating with HF, a mocked user profile - is added to the session. - """ - try: - import fastapi - from fastapi.responses import RedirectResponse - from starlette.datastructures import URL - except ImportError as e: - raise ImportError( - "Cannot initialize OAuth to due a missing library. Please run `pip install huggingface_hub[oauth]` or add " - "`huggingface_hub[oauth]` to your requirements.txt file." - ) from e - - warnings.warn( - "OAuth is not supported outside of a Space environment. To help you debug your app locally, the oauth endpoints" - " are mocked to return your profile and token. To make it work, your machine must be logged in to Huggingface." - ) - mocked_oauth_info = _get_mocked_oauth_info() - - login_uri, callback_uri, logout_uri = _get_oauth_uris(route_prefix) - - # Define OAuth routes - @app.get(login_uri) - async def oauth_login(request: fastapi.Request) -> RedirectResponse: - """Fake endpoint that redirects to HF OAuth page.""" - # Define target (where to redirect after login) - redirect_uri = _generate_redirect_uri(request) - return RedirectResponse(callback_uri + "?" + urllib.parse.urlencode({"_target_url": redirect_uri})) - - @app.get(callback_uri) - async def oauth_redirect_callback(request: fastapi.Request) -> RedirectResponse: - """Endpoint that handles the OAuth callback.""" - request.session["oauth_info"] = mocked_oauth_info - return RedirectResponse(_get_redirect_target(request)) - - @app.get(logout_uri) - async def oauth_logout(request: fastapi.Request) -> RedirectResponse: - """Endpoint that logs out the user (e.g. delete cookie session).""" - request.session.pop("oauth_info", None) - logout_url = URL("/").include_query_params(**request.query_params) - return RedirectResponse(url=logout_url, status_code=302) # see https://github.com/gradio-app/gradio/pull/9659 - - -def _generate_redirect_uri(request: "fastapi.Request") -> str: - if "_target_url" in request.query_params: - # if `_target_url` already in query params => respect it - target = request.query_params["_target_url"] - else: - # otherwise => keep query params - target = "/?" + urllib.parse.urlencode(request.query_params) - - redirect_uri = request.url_for("oauth_redirect_callback").include_query_params(_target_url=target) - redirect_uri_as_str = str(redirect_uri) - if redirect_uri.netloc.endswith(".hf.space"): - # In Space, FastAPI redirect as http but we want https - redirect_uri_as_str = redirect_uri_as_str.replace("http://", "https://") - return redirect_uri_as_str - - -def _get_redirect_target(request: "fastapi.Request", default_target: str = "/") -> str: - return request.query_params.get("_target_url", default_target) - - -def _get_mocked_oauth_info() -> Dict: - token = get_token() - if token is None: - raise ValueError( - "Your machine must be logged in to HF to debug an OAuth app locally. Please" - " run `hf auth login` or set `HF_TOKEN` as environment variable " - "with one of your access token. You can generate a new token in your " - "settings page (https://huggingface.co/settings/tokens)." - ) - - user = whoami() - if user["type"] != "user": - raise ValueError( - "Your machine is not logged in with a personal account. Please use a " - "personal access token. You can generate a new token in your settings page" - " (https://huggingface.co/settings/tokens)." - ) - - return { - "access_token": token, - "token_type": "bearer", - "expires_in": 8 * 60 * 60, # 8 hours - "id_token": "FOOBAR", - "scope": "openid profile", - "refresh_token": "hf_oauth__refresh_token", - "expires_at": int(time.time()) + 8 * 60 * 60, # 8 hours - "userinfo": { - "sub": "0123456789", - "name": user["fullname"], - "preferred_username": user["name"], - "profile": f"https://huggingface.co/{user['name']}", - "picture": user["avatarUrl"], - "website": "", - "aud": "00000000-0000-0000-0000-000000000000", - "auth_time": 1691672844, - "nonce": "aaaaaaaaaaaaaaaaaaa", - "iat": 1691672844, - "exp": 1691676444, - "iss": "https://huggingface.co", - }, - } - - -def _get_oauth_uris(route_prefix: str = "/") -> Tuple[str, str, str]: - route_prefix = route_prefix.strip("/") - if route_prefix: - route_prefix = f"/{route_prefix}" - return ( - f"{route_prefix}/oauth/huggingface/login", - f"{route_prefix}/oauth/huggingface/callback", - f"{route_prefix}/oauth/huggingface/logout", - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_snapshot_download.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_snapshot_download.py deleted file mode 100644 index 0db8a29f7e65a4841590d033f6b7b51d46647bf0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_snapshot_download.py +++ /dev/null @@ -1,343 +0,0 @@ -import os -from pathlib import Path -from typing import Dict, Iterable, List, Literal, Optional, Type, Union - -import requests -from tqdm.auto import tqdm as base_tqdm -from tqdm.contrib.concurrent import thread_map - -from . import constants -from .errors import ( - GatedRepoError, - HfHubHTTPError, - LocalEntryNotFoundError, - RepositoryNotFoundError, - RevisionNotFoundError, -) -from .file_download import REGEX_COMMIT_HASH, hf_hub_download, repo_folder_name -from .hf_api import DatasetInfo, HfApi, ModelInfo, RepoFile, SpaceInfo -from .utils import OfflineModeIsEnabled, filter_repo_objects, logging, validate_hf_hub_args -from .utils import tqdm as hf_tqdm - - -logger = logging.get_logger(__name__) - -VERY_LARGE_REPO_THRESHOLD = 50000 # After this limit, we don't consider `repo_info.siblings` to be reliable enough - - -@validate_hf_hub_args -def snapshot_download( - repo_id: str, - *, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - cache_dir: Union[str, Path, None] = None, - local_dir: Union[str, Path, None] = None, - library_name: Optional[str] = None, - library_version: Optional[str] = None, - user_agent: Optional[Union[Dict, str]] = None, - proxies: Optional[Dict] = None, - etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, - force_download: bool = False, - token: Optional[Union[bool, str]] = None, - local_files_only: bool = False, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - max_workers: int = 8, - tqdm_class: Optional[Type[base_tqdm]] = None, - headers: Optional[Dict[str, str]] = None, - endpoint: Optional[str] = None, - # Deprecated args - local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto", - resume_download: Optional[bool] = None, -) -> str: - """Download repo files. - - Download a whole snapshot of a repo's files at the specified revision. This is useful when you want all files from - a repo, because you don't know which ones you will need a priori. All files are nested inside a folder in order - to keep their actual filename relative to that folder. You can also filter which files to download using - `allow_patterns` and `ignore_patterns`. - - If `local_dir` is provided, the file structure from the repo will be replicated in this location. When using this - option, the `cache_dir` will not be used and a `.cache/huggingface/` folder will be created at the root of `local_dir` - to store some metadata related to the downloaded files. While this mechanism is not as robust as the main - cache-system, it's optimized for regularly pulling the latest version of a repository. - - An alternative would be to clone the repo but this requires git and git-lfs to be installed and properly - configured. It is also not possible to filter which files to download when cloning a repository using git. - - Args: - repo_id (`str`): - A user or an organization name and a repo name separated by a `/`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if downloading from a dataset or space, - `None` or `"model"` if downloading from a model. Default is `None`. - revision (`str`, *optional*): - An optional Git revision id which can be a branch name, a tag, or a - commit hash. - cache_dir (`str`, `Path`, *optional*): - Path to the folder where cached files are stored. - local_dir (`str` or `Path`, *optional*): - If provided, the downloaded files will be placed under this directory. - library_name (`str`, *optional*): - The name of the library to which the object corresponds. - library_version (`str`, *optional*): - The version of the library. - user_agent (`str`, `dict`, *optional*): - The user-agent info in the form of a dictionary or a string. - proxies (`dict`, *optional*): - Dictionary mapping protocol to the URL of the proxy passed to - `requests.request`. - etag_timeout (`float`, *optional*, defaults to `10`): - When fetching ETag, how many seconds to wait for the server to send - data before giving up which is passed to `requests.request`. - force_download (`bool`, *optional*, defaults to `False`): - Whether the file should be downloaded even if it already exists in the local cache. - token (`str`, `bool`, *optional*): - A token to be used for the download. - - If `True`, the token is read from the HuggingFace config - folder. - - If a string, it's used as the authentication token. - headers (`dict`, *optional*): - Additional headers to include in the request. Those headers take precedence over the others. - local_files_only (`bool`, *optional*, defaults to `False`): - If `True`, avoid downloading the file and return the path to the - local cached file if it exists. - allow_patterns (`List[str]` or `str`, *optional*): - If provided, only files matching at least one pattern are downloaded. - ignore_patterns (`List[str]` or `str`, *optional*): - If provided, files matching any of the patterns are not downloaded. - max_workers (`int`, *optional*): - Number of concurrent threads to download files (1 thread = 1 file download). - Defaults to 8. - tqdm_class (`tqdm`, *optional*): - If provided, overwrites the default behavior for the progress bar. Passed - argument must inherit from `tqdm.auto.tqdm` or at least mimic its behavior. - Note that the `tqdm_class` is not passed to each individual download. - Defaults to the custom HF progress bar that can be disabled by setting - `HF_HUB_DISABLE_PROGRESS_BARS` environment variable. - - Returns: - `str`: folder path of the repo snapshot. - - Raises: - [`~utils.RepositoryNotFoundError`] - If the repository to download from cannot be found. This may be because it doesn't exist, - or because it is set to `private` and you do not have access. - [`~utils.RevisionNotFoundError`] - If the revision to download from cannot be found. - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) - If `token=True` and the token cannot be found. - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if - ETag cannot be determined. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - if some parameter value is invalid. - """ - if cache_dir is None: - cache_dir = constants.HF_HUB_CACHE - if revision is None: - revision = constants.DEFAULT_REVISION - if isinstance(cache_dir, Path): - cache_dir = str(cache_dir) - - if repo_type is None: - repo_type = "model" - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES)}") - - storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type)) - - api = HfApi( - library_name=library_name, - library_version=library_version, - user_agent=user_agent, - endpoint=endpoint, - headers=headers, - token=token, - ) - - repo_info: Union[ModelInfo, DatasetInfo, SpaceInfo, None] = None - api_call_error: Optional[Exception] = None - if not local_files_only: - # try/except logic to handle different errors => taken from `hf_hub_download` - try: - # if we have internet connection we want to list files to download - repo_info = api.repo_info(repo_id=repo_id, repo_type=repo_type, revision=revision) - except (requests.exceptions.SSLError, requests.exceptions.ProxyError): - # Actually raise for those subclasses of ConnectionError - raise - except ( - requests.exceptions.ConnectionError, - requests.exceptions.Timeout, - OfflineModeIsEnabled, - ) as error: - # Internet connection is down - # => will try to use local files only - api_call_error = error - pass - except RevisionNotFoundError: - # The repo was found but the revision doesn't exist on the Hub (never existed or got deleted) - raise - except requests.HTTPError as error: - # Multiple reasons for an http error: - # - Repository is private and invalid/missing token sent - # - Repository is gated and invalid/missing token sent - # - Hub is down (error 500 or 504) - # => let's switch to 'local_files_only=True' to check if the files are already cached. - # (if it's not the case, the error will be re-raised) - api_call_error = error - pass - - # At this stage, if `repo_info` is None it means either: - # - internet connection is down - # - internet connection is deactivated (local_files_only=True or HF_HUB_OFFLINE=True) - # - repo is private/gated and invalid/missing token sent - # - Hub is down - # => let's look if we can find the appropriate folder in the cache: - # - if the specified revision is a commit hash, look inside "snapshots". - # - f the specified revision is a branch or tag, look inside "refs". - # => if local_dir is not None, we will return the path to the local folder if it exists. - if repo_info is None: - # Try to get which commit hash corresponds to the specified revision - commit_hash = None - if REGEX_COMMIT_HASH.match(revision): - commit_hash = revision - else: - ref_path = os.path.join(storage_folder, "refs", revision) - if os.path.exists(ref_path): - # retrieve commit_hash from refs file - with open(ref_path) as f: - commit_hash = f.read() - - # Try to locate snapshot folder for this commit hash - if commit_hash is not None and local_dir is None: - snapshot_folder = os.path.join(storage_folder, "snapshots", commit_hash) - if os.path.exists(snapshot_folder): - # Snapshot folder exists => let's return it - # (but we can't check if all the files are actually there) - return snapshot_folder - - # If local_dir is not None, return it if it exists and is not empty - if local_dir is not None: - local_dir = Path(local_dir) - if local_dir.is_dir() and any(local_dir.iterdir()): - logger.warning( - f"Returning existing local_dir `{local_dir}` as remote repo cannot be accessed in `snapshot_download` ({api_call_error})." - ) - return str(local_dir.resolve()) - # If we couldn't find the appropriate folder on disk, raise an error. - if local_files_only: - raise LocalEntryNotFoundError( - "Cannot find an appropriate cached snapshot folder for the specified revision on the local disk and " - "outgoing traffic has been disabled. To enable repo look-ups and downloads online, pass " - "'local_files_only=False' as input." - ) - elif isinstance(api_call_error, OfflineModeIsEnabled): - raise LocalEntryNotFoundError( - "Cannot find an appropriate cached snapshot folder for the specified revision on the local disk and " - "outgoing traffic has been disabled. To enable repo look-ups and downloads online, set " - "'HF_HUB_OFFLINE=0' as environment variable." - ) from api_call_error - elif isinstance(api_call_error, (RepositoryNotFoundError, GatedRepoError)) or ( - isinstance(api_call_error, HfHubHTTPError) and api_call_error.response.status_code == 401 - ): - # Repo not found, gated, or specific authentication error => let's raise the actual error - raise api_call_error - else: - # Otherwise: most likely a connection issue or Hub downtime => let's warn the user - raise LocalEntryNotFoundError( - "An error happened while trying to locate the files on the Hub and we cannot find the appropriate" - " snapshot folder for the specified revision on the local disk. Please check your internet connection" - " and try again." - ) from api_call_error - - # At this stage, internet connection is up and running - # => let's download the files! - assert repo_info.sha is not None, "Repo info returned from server must have a revision sha." - - # Corner case: on very large repos, the siblings list in `repo_info` might not contain all files. - # In that case, we need to use the `list_repo_tree` method to prevent caching issues. - repo_files: Iterable[str] = [f.rfilename for f in repo_info.siblings] if repo_info.siblings is not None else [] - unreliable_nb_files = ( - repo_info.siblings is None - or len(repo_info.siblings) == 0 - or len(repo_info.siblings) > VERY_LARGE_REPO_THRESHOLD - ) - if unreliable_nb_files: - logger.info( - "Number of files in the repo is unreliable. Using `list_repo_tree` to ensure all files are listed." - ) - repo_files = ( - f.rfilename - for f in api.list_repo_tree(repo_id=repo_id, recursive=True, revision=revision, repo_type=repo_type) - if isinstance(f, RepoFile) - ) - - filtered_repo_files: Iterable[str] = filter_repo_objects( - items=repo_files, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - ) - - if not unreliable_nb_files: - filtered_repo_files = list(filtered_repo_files) - tqdm_desc = f"Fetching {len(filtered_repo_files)} files" - else: - tqdm_desc = "Fetching ... files" - - commit_hash = repo_info.sha - snapshot_folder = os.path.join(storage_folder, "snapshots", commit_hash) - # if passed revision is not identical to commit_hash - # then revision has to be a branch name or tag name. - # In that case store a ref. - if revision != commit_hash: - ref_path = os.path.join(storage_folder, "refs", revision) - try: - os.makedirs(os.path.dirname(ref_path), exist_ok=True) - with open(ref_path, "w") as f: - f.write(commit_hash) - except OSError as e: - logger.warning(f"Ignored error while writing commit hash to {ref_path}: {e}.") - - # we pass the commit_hash to hf_hub_download - # so no network call happens if we already - # have the file locally. - def _inner_hf_hub_download(repo_file: str): - return hf_hub_download( - repo_id, - filename=repo_file, - repo_type=repo_type, - revision=commit_hash, - endpoint=endpoint, - cache_dir=cache_dir, - local_dir=local_dir, - local_dir_use_symlinks=local_dir_use_symlinks, - library_name=library_name, - library_version=library_version, - user_agent=user_agent, - proxies=proxies, - etag_timeout=etag_timeout, - resume_download=resume_download, - force_download=force_download, - token=token, - headers=headers, - ) - - if constants.HF_HUB_ENABLE_HF_TRANSFER: - # when using hf_transfer we don't want extra parallelism - # from the one hf_transfer provides - for file in filtered_repo_files: - _inner_hf_hub_download(file) - else: - thread_map( - _inner_hf_hub_download, - filtered_repo_files, - desc=tqdm_desc, - max_workers=max_workers, - # User can use its own tqdm class or the default one from `huggingface_hub.utils` - tqdm_class=tqdm_class or hf_tqdm, - ) - - if local_dir is not None: - return str(os.path.realpath(local_dir)) - return snapshot_folder diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_space_api.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_space_api.py deleted file mode 100644 index 05fccfbc1ebdfc14840a88751914b8fc0d1a498d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_space_api.py +++ /dev/null @@ -1,168 +0,0 @@ -# coding=utf-8 -# Copyright 2019-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from dataclasses import dataclass -from datetime import datetime -from enum import Enum -from typing import Dict, Optional - -from huggingface_hub.utils import parse_datetime - - -class SpaceStage(str, Enum): - """ - Enumeration of possible stage of a Space on the Hub. - - Value can be compared to a string: - ```py - assert SpaceStage.BUILDING == "BUILDING" - ``` - - Taken from https://github.com/huggingface/moon-landing/blob/main/server/repo_types/SpaceInfo.ts#L61 (private url). - """ - - # Copied from moon-landing > server > repo_types > SpaceInfo.ts (private repo) - NO_APP_FILE = "NO_APP_FILE" - CONFIG_ERROR = "CONFIG_ERROR" - BUILDING = "BUILDING" - BUILD_ERROR = "BUILD_ERROR" - RUNNING = "RUNNING" - RUNNING_BUILDING = "RUNNING_BUILDING" - RUNTIME_ERROR = "RUNTIME_ERROR" - DELETING = "DELETING" - STOPPED = "STOPPED" - PAUSED = "PAUSED" - - -class SpaceHardware(str, Enum): - """ - Enumeration of hardwares available to run your Space on the Hub. - - Value can be compared to a string: - ```py - assert SpaceHardware.CPU_BASIC == "cpu-basic" - ``` - - Taken from https://github.com/huggingface-internal/moon-landing/blob/main/server/repo_types/SpaceHardwareFlavor.ts (private url). - """ - - # CPU - CPU_BASIC = "cpu-basic" - CPU_UPGRADE = "cpu-upgrade" - CPU_XL = "cpu-xl" - - # ZeroGPU - ZERO_A10G = "zero-a10g" - - # GPU - T4_SMALL = "t4-small" - T4_MEDIUM = "t4-medium" - L4X1 = "l4x1" - L4X4 = "l4x4" - L40SX1 = "l40sx1" - L40SX4 = "l40sx4" - L40SX8 = "l40sx8" - A10G_SMALL = "a10g-small" - A10G_LARGE = "a10g-large" - A10G_LARGEX2 = "a10g-largex2" - A10G_LARGEX4 = "a10g-largex4" - A100_LARGE = "a100-large" - H100 = "h100" - H100X8 = "h100x8" - - -class SpaceStorage(str, Enum): - """ - Enumeration of persistent storage available for your Space on the Hub. - - Value can be compared to a string: - ```py - assert SpaceStorage.SMALL == "small" - ``` - - Taken from https://github.com/huggingface/moon-landing/blob/main/server/repo_types/SpaceHardwareFlavor.ts#L24 (private url). - """ - - SMALL = "small" - MEDIUM = "medium" - LARGE = "large" - - -@dataclass -class SpaceRuntime: - """ - Contains information about the current runtime of a Space. - - Args: - stage (`str`): - Current stage of the space. Example: RUNNING. - hardware (`str` or `None`): - Current hardware of the space. Example: "cpu-basic". Can be `None` if Space - is `BUILDING` for the first time. - requested_hardware (`str` or `None`): - Requested hardware. Can be different than `hardware` especially if the request - has just been made. Example: "t4-medium". Can be `None` if no hardware has - been requested yet. - sleep_time (`int` or `None`): - Number of seconds the Space will be kept alive after the last request. By default (if value is `None`), the - Space will never go to sleep if it's running on an upgraded hardware, while it will go to sleep after 48 - hours on a free 'cpu-basic' hardware. For more details, see https://huggingface.co/docs/hub/spaces-gpus#sleep-time. - raw (`dict`): - Raw response from the server. Contains more information about the Space - runtime like number of replicas, number of cpu, memory size,... - """ - - stage: SpaceStage - hardware: Optional[SpaceHardware] - requested_hardware: Optional[SpaceHardware] - sleep_time: Optional[int] - storage: Optional[SpaceStorage] - raw: Dict - - def __init__(self, data: Dict) -> None: - self.stage = data["stage"] - self.hardware = data.get("hardware", {}).get("current") - self.requested_hardware = data.get("hardware", {}).get("requested") - self.sleep_time = data.get("gcTimeout") - self.storage = data.get("storage") - self.raw = data - - -@dataclass -class SpaceVariable: - """ - Contains information about the current variables of a Space. - - Args: - key (`str`): - Variable key. Example: `"MODEL_REPO_ID"` - value (`str`): - Variable value. Example: `"the_model_repo_id"`. - description (`str` or None): - Description of the variable. Example: `"Model Repo ID of the implemented model"`. - updatedAt (`datetime` or None): - datetime of the last update of the variable (if the variable has been updated at least once). - """ - - key: str - value: str - description: Optional[str] - updated_at: Optional[datetime] - - def __init__(self, key: str, values: Dict) -> None: - self.key = key - self.value = values["value"] - self.description = values.get("description") - updated_at = values.get("updatedAt") - self.updated_at = parse_datetime(updated_at) if updated_at is not None else None diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_tensorboard_logger.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_tensorboard_logger.py deleted file mode 100644 index 4d9581d8ee127436ec1e1d585ed0426422a66131..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_tensorboard_logger.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2023 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains a logger to push training logs to the Hub, using Tensorboard.""" - -from pathlib import Path -from typing import List, Optional, Union - -from ._commit_scheduler import CommitScheduler -from .errors import EntryNotFoundError -from .repocard import ModelCard -from .utils import experimental - - -# Depending on user's setup, SummaryWriter can come either from 'tensorboardX' -# or from 'torch.utils.tensorboard'. Both are compatible so let's try to load -# from either of them. -try: - from tensorboardX import SummaryWriter as _RuntimeSummaryWriter - - is_summary_writer_available = True -except ImportError: - try: - from torch.utils.tensorboard import SummaryWriter as _RuntimeSummaryWriter - - is_summary_writer_available = True - except ImportError: - # Dummy class to avoid failing at import. Will raise on instance creation. - class _DummySummaryWriter: - pass - - _RuntimeSummaryWriter = _DummySummaryWriter # type: ignore[assignment] - is_summary_writer_available = False - - -class HFSummaryWriter(_RuntimeSummaryWriter): - """ - Wrapper around the tensorboard's `SummaryWriter` to push training logs to the Hub. - - Data is logged locally and then pushed to the Hub asynchronously. Pushing data to the Hub is done in a separate - thread to avoid blocking the training script. In particular, if the upload fails for any reason (e.g. a connection - issue), the main script will not be interrupted. Data is automatically pushed to the Hub every `commit_every` - minutes (default to every 5 minutes). - - > [!WARNING] - > `HFSummaryWriter` is experimental. Its API is subject to change in the future without prior notice. - - Args: - repo_id (`str`): - The id of the repo to which the logs will be pushed. - logdir (`str`, *optional*): - The directory where the logs will be written. If not specified, a local directory will be created by the - underlying `SummaryWriter` object. - commit_every (`int` or `float`, *optional*): - The frequency (in minutes) at which the logs will be pushed to the Hub. Defaults to 5 minutes. - squash_history (`bool`, *optional*): - Whether to squash the history of the repo after each commit. Defaults to `False`. Squashing commits is - useful to avoid degraded performances on the repo when it grows too large. - repo_type (`str`, *optional*): - The type of the repo to which the logs will be pushed. Defaults to "model". - repo_revision (`str`, *optional*): - The revision of the repo to which the logs will be pushed. Defaults to "main". - repo_private (`bool`, *optional*): - Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists. - path_in_repo (`str`, *optional*): - The path to the folder in the repo where the logs will be pushed. Defaults to "tensorboard/". - repo_allow_patterns (`List[str]` or `str`, *optional*): - A list of patterns to include in the upload. Defaults to `"*.tfevents.*"`. Check out the - [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-folder) for more details. - repo_ignore_patterns (`List[str]` or `str`, *optional*): - A list of patterns to exclude in the upload. Check out the - [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-folder) for more details. - token (`str`, *optional*): - Authentication token. Will default to the stored token. See https://huggingface.co/settings/token for more - details - kwargs: - Additional keyword arguments passed to `SummaryWriter`. - - Examples: - ```diff - # Taken from https://pytorch.org/docs/stable/tensorboard.html - - from torch.utils.tensorboard import SummaryWriter - + from huggingface_hub import HFSummaryWriter - - import numpy as np - - - writer = SummaryWriter() - + writer = HFSummaryWriter(repo_id="username/my-trained-model") - - for n_iter in range(100): - writer.add_scalar('Loss/train', np.random.random(), n_iter) - writer.add_scalar('Loss/test', np.random.random(), n_iter) - writer.add_scalar('Accuracy/train', np.random.random(), n_iter) - writer.add_scalar('Accuracy/test', np.random.random(), n_iter) - ``` - - ```py - >>> from huggingface_hub import HFSummaryWriter - - # Logs are automatically pushed every 15 minutes (5 by default) + when exiting the context manager - >>> with HFSummaryWriter(repo_id="test_hf_logger", commit_every=15) as logger: - ... logger.add_scalar("a", 1) - ... logger.add_scalar("b", 2) - ``` - """ - - @experimental - def __new__(cls, *args, **kwargs) -> "HFSummaryWriter": - if not is_summary_writer_available: - raise ImportError( - "You must have `tensorboard` installed to use `HFSummaryWriter`. Please run `pip install --upgrade" - " tensorboardX` first." - ) - return super().__new__(cls) - - def __init__( - self, - repo_id: str, - *, - logdir: Optional[str] = None, - commit_every: Union[int, float] = 5, - squash_history: bool = False, - repo_type: Optional[str] = None, - repo_revision: Optional[str] = None, - repo_private: Optional[bool] = None, - path_in_repo: Optional[str] = "tensorboard", - repo_allow_patterns: Optional[Union[List[str], str]] = "*.tfevents.*", - repo_ignore_patterns: Optional[Union[List[str], str]] = None, - token: Optional[str] = None, - **kwargs, - ): - # Initialize SummaryWriter - super().__init__(logdir=logdir, **kwargs) - - # Check logdir has been correctly initialized and fail early otherwise. In practice, SummaryWriter takes care of it. - if not isinstance(self.logdir, str): - raise ValueError(f"`self.logdir` must be a string. Got '{self.logdir}' of type {type(self.logdir)}.") - - # Append logdir name to `path_in_repo` - if path_in_repo is None or path_in_repo == "": - path_in_repo = Path(self.logdir).name - else: - path_in_repo = path_in_repo.strip("/") + "/" + Path(self.logdir).name - - # Initialize scheduler - self.scheduler = CommitScheduler( - folder_path=self.logdir, - path_in_repo=path_in_repo, - repo_id=repo_id, - repo_type=repo_type, - revision=repo_revision, - private=repo_private, - token=token, - allow_patterns=repo_allow_patterns, - ignore_patterns=repo_ignore_patterns, - every=commit_every, - squash_history=squash_history, - ) - - # Exposing some high-level info at root level - self.repo_id = self.scheduler.repo_id - self.repo_type = self.scheduler.repo_type - self.repo_revision = self.scheduler.revision - - # Add `hf-summary-writer` tag to the model card metadata - try: - card = ModelCard.load(repo_id_or_path=self.repo_id, repo_type=self.repo_type) - except EntryNotFoundError: - card = ModelCard("") - tags = card.data.get("tags", []) - if "hf-summary-writer" not in tags: - tags.append("hf-summary-writer") - card.data["tags"] = tags - card.push_to_hub(repo_id=self.repo_id, repo_type=self.repo_type) - - def __exit__(self, exc_type, exc_val, exc_tb): - """Push to hub in a non-blocking way when exiting the logger's context manager.""" - super().__exit__(exc_type, exc_val, exc_tb) - future = self.scheduler.trigger() - future.result() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_upload_large_folder.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_upload_large_folder.py deleted file mode 100644 index 1ccbc07d39d3d03e9bb8c39f1bb16aa2ca4ab41f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_upload_large_folder.py +++ /dev/null @@ -1,755 +0,0 @@ -# coding=utf-8 -# Copyright 2024-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import enum -import logging -import os -import queue -import shutil -import sys -import threading -import time -import traceback -from datetime import datetime -from pathlib import Path -from threading import Lock -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union -from urllib.parse import quote - -from . import constants -from ._commit_api import CommitOperationAdd, UploadInfo, _fetch_upload_modes -from ._local_folder import LocalUploadFileMetadata, LocalUploadFilePaths, get_local_upload_paths, read_upload_metadata -from .constants import DEFAULT_REVISION, REPO_TYPES -from .utils import DEFAULT_IGNORE_PATTERNS, filter_repo_objects, tqdm -from .utils._cache_manager import _format_size -from .utils._runtime import is_xet_available -from .utils.sha import sha_fileobj - - -if TYPE_CHECKING: - from .hf_api import HfApi - -logger = logging.getLogger(__name__) - -WAITING_TIME_IF_NO_TASKS = 10 # seconds -MAX_NB_FILES_FETCH_UPLOAD_MODE = 100 -COMMIT_SIZE_SCALE: List[int] = [20, 50, 75, 100, 125, 200, 250, 400, 600, 1000] - -UPLOAD_BATCH_SIZE_XET = 256 # Max 256 files per upload batch for XET-enabled repos -UPLOAD_BATCH_SIZE_LFS = 1 # Otherwise, batches of 1 for regular LFS upload - -# Repository limits (from https://huggingface.co/docs/hub/repositories-recommendations) -MAX_FILES_PER_REPO = 100_000 # Recommended maximum number of files per repository -MAX_FILES_PER_FOLDER = 10_000 # Recommended maximum number of files per folder -MAX_FILE_SIZE_GB = 50 # Hard limit for individual file size -RECOMMENDED_FILE_SIZE_GB = 20 # Recommended maximum for individual file size - - -def _validate_upload_limits(paths_list: List[LocalUploadFilePaths]) -> None: - """ - Validate upload against repository limits and warn about potential issues. - - Args: - paths_list: List of file paths to be uploaded - - Warns about: - - Too many files in the repository (>100k) - - Too many entries (files or subdirectories) in a single folder (>10k) - - Files exceeding size limits (>20GB recommended, >50GB hard limit) - """ - logger.info("Running validation checks on files to upload...") - - # Check 1: Total file count - if len(paths_list) > MAX_FILES_PER_REPO: - logger.warning( - f"You are about to upload {len(paths_list):,} files. " - f"This exceeds the recommended limit of {MAX_FILES_PER_REPO:,} files per repository.\n" - f"Consider:\n" - f" - Splitting your data into multiple repositories\n" - f" - Using fewer, larger files (e.g., parquet files)\n" - f" - See: https://huggingface.co/docs/hub/repositories-recommendations" - ) - - # Check 2: Files and subdirectories per folder - # Track immediate children (files and subdirs) for each folder - from collections import defaultdict - - entries_per_folder: Dict[str, Any] = defaultdict(lambda: {"files": 0, "subdirs": set()}) - - for paths in paths_list: - path = Path(paths.path_in_repo) - parts = path.parts - - # Count this file in its immediate parent directory - parent = str(path.parent) if str(path.parent) != "." else "." - entries_per_folder[parent]["files"] += 1 - - # Track immediate subdirectories for each parent folder - # Walk through the path components to track parent-child relationships - for i, child in enumerate(parts[:-1]): - parent = "." if i == 0 else "/".join(parts[:i]) - entries_per_folder[parent]["subdirs"].add(child) - - # Check limits for each folder - for folder, data in entries_per_folder.items(): - file_count = data["files"] - subdir_count = len(data["subdirs"]) - total_entries = file_count + subdir_count - - if total_entries > MAX_FILES_PER_FOLDER: - folder_display = "root" if folder == "." else folder - logger.warning( - f"Folder '{folder_display}' contains {total_entries:,} entries " - f"({file_count:,} files and {subdir_count:,} subdirectories). " - f"This exceeds the recommended {MAX_FILES_PER_FOLDER:,} entries per folder.\n" - "Consider reorganising into sub-folders." - ) - - # Check 3: File sizes - large_files = [] - very_large_files = [] - - for paths in paths_list: - size = paths.file_path.stat().st_size - size_gb = size / 1_000_000_000 # Use decimal GB as per Hub limits - - if size_gb > MAX_FILE_SIZE_GB: - very_large_files.append((paths.path_in_repo, size_gb)) - elif size_gb > RECOMMENDED_FILE_SIZE_GB: - large_files.append((paths.path_in_repo, size_gb)) - - # Warn about very large files (>50GB) - if very_large_files: - files_str = "\n - ".join(f"{path}: {size:.1f}GB" for path, size in very_large_files[:5]) - more_str = f"\n ... and {len(very_large_files) - 5} more files" if len(very_large_files) > 5 else "" - logger.warning( - f"Found {len(very_large_files)} files exceeding the {MAX_FILE_SIZE_GB}GB hard limit:\n" - f" - {files_str}{more_str}\n" - f"These files may fail to upload. Consider splitting them into smaller chunks." - ) - - # Warn about large files (>20GB) - if large_files: - files_str = "\n - ".join(f"{path}: {size:.1f}GB" for path, size in large_files[:5]) - more_str = f"\n ... and {len(large_files) - 5} more files" if len(large_files) > 5 else "" - logger.warning( - f"Found {len(large_files)} files larger than {RECOMMENDED_FILE_SIZE_GB}GB (recommended limit):\n" - f" - {files_str}{more_str}\n" - f"Large files may slow down loading and processing." - ) - - logger.info("Validation checks complete.") - - -def upload_large_folder_internal( - api: "HfApi", - repo_id: str, - folder_path: Union[str, Path], - *, - repo_type: str, # Repo type is required! - revision: Optional[str] = None, - private: Optional[bool] = None, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - num_workers: Optional[int] = None, - print_report: bool = True, - print_report_every: int = 60, -): - """Upload a large folder to the Hub in the most resilient way possible. - - See [`HfApi.upload_large_folder`] for the full documentation. - """ - # 1. Check args and setup - if repo_type is None: - raise ValueError( - "For large uploads, `repo_type` is explicitly required. Please set it to `model`, `dataset` or `space`." - " If you are using the CLI, pass it as `--repo-type=model`." - ) - if repo_type not in REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}") - if revision is None: - revision = DEFAULT_REVISION - - folder_path = Path(folder_path).expanduser().resolve() - if not folder_path.is_dir(): - raise ValueError(f"Provided path: '{folder_path}' is not a directory") - - if ignore_patterns is None: - ignore_patterns = [] - elif isinstance(ignore_patterns, str): - ignore_patterns = [ignore_patterns] - ignore_patterns += DEFAULT_IGNORE_PATTERNS - - if num_workers is None: - nb_cores = os.cpu_count() or 1 - num_workers = max(nb_cores - 2, 2) # Use all but 2 cores, or at least 2 cores - - # 2. Create repo if missing - repo_url = api.create_repo(repo_id=repo_id, repo_type=repo_type, private=private, exist_ok=True) - logger.info(f"Repo created: {repo_url}") - repo_id = repo_url.repo_id - # 2.1 Check if xet is enabled to set batch file upload size - is_xet_enabled = ( - is_xet_available() - and api.repo_info( - repo_id=repo_id, - repo_type=repo_type, - revision=revision, - expand="xetEnabled", - ).xet_enabled - ) - upload_batch_size = UPLOAD_BATCH_SIZE_XET if is_xet_enabled else UPLOAD_BATCH_SIZE_LFS - - # 3. List files to upload - filtered_paths_list = filter_repo_objects( - (path.relative_to(folder_path).as_posix() for path in folder_path.glob("**/*") if path.is_file()), - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - ) - paths_list = [get_local_upload_paths(folder_path, relpath) for relpath in filtered_paths_list] - logger.info(f"Found {len(paths_list)} candidate files to upload") - - # Validate upload against repository limits - _validate_upload_limits(paths_list) - - logger.info("Starting upload...") - - # Read metadata for each file - items = [ - (paths, read_upload_metadata(folder_path, paths.path_in_repo)) - for paths in tqdm(paths_list, desc="Recovering from metadata files") - ] - - # 4. Start workers - status = LargeUploadStatus(items, upload_batch_size) - threads = [ - threading.Thread( - target=_worker_job, - kwargs={ - "status": status, - "api": api, - "repo_id": repo_id, - "repo_type": repo_type, - "revision": revision, - }, - ) - for _ in range(num_workers) - ] - - for thread in threads: - thread.start() - - # 5. Print regular reports - if print_report: - print("\n\n" + status.current_report()) - last_report_ts = time.time() - while True: - time.sleep(1) - if time.time() - last_report_ts >= print_report_every: - if print_report: - _print_overwrite(status.current_report()) - last_report_ts = time.time() - if status.is_done(): - logging.info("Is done: exiting main loop") - break - - for thread in threads: - thread.join() - - logger.info(status.current_report()) - logging.info("Upload is complete!") - - -#################### -# Logic to manage workers and synchronize tasks -#################### - - -class WorkerJob(enum.Enum): - SHA256 = enum.auto() - GET_UPLOAD_MODE = enum.auto() - PREUPLOAD_LFS = enum.auto() - COMMIT = enum.auto() - WAIT = enum.auto() # if no tasks are available but we don't want to exit - - -JOB_ITEM_T = Tuple[LocalUploadFilePaths, LocalUploadFileMetadata] - - -class LargeUploadStatus: - """Contains information, queues and tasks for a large upload process.""" - - def __init__(self, items: List[JOB_ITEM_T], upload_batch_size: int = 1): - self.items = items - self.queue_sha256: "queue.Queue[JOB_ITEM_T]" = queue.Queue() - self.queue_get_upload_mode: "queue.Queue[JOB_ITEM_T]" = queue.Queue() - self.queue_preupload_lfs: "queue.Queue[JOB_ITEM_T]" = queue.Queue() - self.queue_commit: "queue.Queue[JOB_ITEM_T]" = queue.Queue() - self.lock = Lock() - - self.nb_workers_sha256: int = 0 - self.nb_workers_get_upload_mode: int = 0 - self.nb_workers_preupload_lfs: int = 0 - self.upload_batch_size: int = upload_batch_size - self.nb_workers_commit: int = 0 - self.nb_workers_waiting: int = 0 - self.last_commit_attempt: Optional[float] = None - - self._started_at = datetime.now() - self._chunk_idx: int = 1 - self._chunk_lock: Lock = Lock() - - # Setup queues - for item in self.items: - paths, metadata = item - if metadata.sha256 is None: - self.queue_sha256.put(item) - elif metadata.upload_mode is None: - self.queue_get_upload_mode.put(item) - elif metadata.upload_mode == "lfs" and not metadata.is_uploaded: - self.queue_preupload_lfs.put(item) - elif not metadata.is_committed: - self.queue_commit.put(item) - else: - logger.debug(f"Skipping file {paths.path_in_repo} (already uploaded and committed)") - - def target_chunk(self) -> int: - with self._chunk_lock: - return COMMIT_SIZE_SCALE[self._chunk_idx] - - def update_chunk(self, success: bool, nb_items: int, duration: float) -> None: - with self._chunk_lock: - if not success: - logger.warning(f"Failed to commit {nb_items} files at once. Will retry with less files in next batch.") - self._chunk_idx -= 1 - elif nb_items >= COMMIT_SIZE_SCALE[self._chunk_idx] and duration < 40: - logger.info(f"Successfully committed {nb_items} at once. Increasing the limit for next batch.") - self._chunk_idx += 1 - - self._chunk_idx = max(0, min(self._chunk_idx, len(COMMIT_SIZE_SCALE) - 1)) - - def current_report(self) -> str: - """Generate a report of the current status of the large upload.""" - nb_hashed = 0 - size_hashed = 0 - nb_preuploaded = 0 - nb_lfs = 0 - nb_lfs_unsure = 0 - size_preuploaded = 0 - nb_committed = 0 - size_committed = 0 - total_size = 0 - ignored_files = 0 - total_files = 0 - - with self.lock: - for _, metadata in self.items: - if metadata.should_ignore: - ignored_files += 1 - continue - total_size += metadata.size - total_files += 1 - if metadata.sha256 is not None: - nb_hashed += 1 - size_hashed += metadata.size - if metadata.upload_mode == "lfs": - nb_lfs += 1 - if metadata.upload_mode is None: - nb_lfs_unsure += 1 - if metadata.is_uploaded: - nb_preuploaded += 1 - size_preuploaded += metadata.size - if metadata.is_committed: - nb_committed += 1 - size_committed += metadata.size - total_size_str = _format_size(total_size) - - now = datetime.now() - now_str = now.strftime("%Y-%m-%d %H:%M:%S") - elapsed = now - self._started_at - elapsed_str = str(elapsed).split(".")[0] # remove milliseconds - - message = "\n" + "-" * 10 - message += f" {now_str} ({elapsed_str}) " - message += "-" * 10 + "\n" - - message += "Files: " - message += f"hashed {nb_hashed}/{total_files} ({_format_size(size_hashed)}/{total_size_str}) | " - message += f"pre-uploaded: {nb_preuploaded}/{nb_lfs} ({_format_size(size_preuploaded)}/{total_size_str})" - if nb_lfs_unsure > 0: - message += f" (+{nb_lfs_unsure} unsure)" - message += f" | committed: {nb_committed}/{total_files} ({_format_size(size_committed)}/{total_size_str})" - message += f" | ignored: {ignored_files}\n" - - message += "Workers: " - message += f"hashing: {self.nb_workers_sha256} | " - message += f"get upload mode: {self.nb_workers_get_upload_mode} | " - message += f"pre-uploading: {self.nb_workers_preupload_lfs} | " - message += f"committing: {self.nb_workers_commit} | " - message += f"waiting: {self.nb_workers_waiting}\n" - message += "-" * 51 - - return message - - def is_done(self) -> bool: - with self.lock: - return all(metadata.is_committed or metadata.should_ignore for _, metadata in self.items) - - -def _worker_job( - status: LargeUploadStatus, - api: "HfApi", - repo_id: str, - repo_type: str, - revision: str, -): - """ - Main process for a worker. The worker will perform tasks based on the priority list until all files are uploaded - and committed. If no tasks are available, the worker will wait for 10 seconds before checking again. - - If a task fails for any reason, the item(s) are put back in the queue for another worker to pick up. - - Read `upload_large_folder` docstring for more information on how tasks are prioritized. - """ - while True: - next_job: Optional[Tuple[WorkerJob, List[JOB_ITEM_T]]] = None - - # Determine next task - next_job = _determine_next_job(status) - if next_job is None: - return - job, items = next_job - - # Perform task - if job == WorkerJob.SHA256: - item = items[0] # single item - try: - _compute_sha256(item) - status.queue_get_upload_mode.put(item) - except KeyboardInterrupt: - raise - except Exception as e: - logger.error(f"Failed to compute sha256: {e}") - traceback.format_exc() - status.queue_sha256.put(item) - - with status.lock: - status.nb_workers_sha256 -= 1 - - elif job == WorkerJob.GET_UPLOAD_MODE: - try: - _get_upload_mode(items, api=api, repo_id=repo_id, repo_type=repo_type, revision=revision) - except KeyboardInterrupt: - raise - except Exception as e: - logger.error(f"Failed to get upload mode: {e}") - traceback.format_exc() - - # Items are either: - # - dropped (if should_ignore) - # - put in LFS queue (if LFS) - # - put in commit queue (if regular) - # - or put back (if error occurred). - for item in items: - _, metadata = item - if metadata.should_ignore: - continue - if metadata.upload_mode == "lfs": - status.queue_preupload_lfs.put(item) - elif metadata.upload_mode == "regular": - status.queue_commit.put(item) - else: - status.queue_get_upload_mode.put(item) - - with status.lock: - status.nb_workers_get_upload_mode -= 1 - - elif job == WorkerJob.PREUPLOAD_LFS: - try: - _preupload_lfs(items, api=api, repo_id=repo_id, repo_type=repo_type, revision=revision) - for item in items: - status.queue_commit.put(item) - except KeyboardInterrupt: - raise - except Exception as e: - logger.error(f"Failed to preupload LFS: {e}") - traceback.format_exc() - for item in items: - status.queue_preupload_lfs.put(item) - - with status.lock: - status.nb_workers_preupload_lfs -= 1 - - elif job == WorkerJob.COMMIT: - start_ts = time.time() - success = True - try: - _commit(items, api=api, repo_id=repo_id, repo_type=repo_type, revision=revision) - except KeyboardInterrupt: - raise - except Exception as e: - logger.error(f"Failed to commit: {e}") - traceback.format_exc() - for item in items: - status.queue_commit.put(item) - success = False - duration = time.time() - start_ts - status.update_chunk(success, len(items), duration) - with status.lock: - status.last_commit_attempt = time.time() - status.nb_workers_commit -= 1 - - elif job == WorkerJob.WAIT: - time.sleep(WAITING_TIME_IF_NO_TASKS) - with status.lock: - status.nb_workers_waiting -= 1 - - -def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob, List[JOB_ITEM_T]]]: - with status.lock: - # 1. Commit if more than 5 minutes since last commit attempt (and at least 1 file) - if ( - status.nb_workers_commit == 0 - and status.queue_commit.qsize() > 0 - and status.last_commit_attempt is not None - and time.time() - status.last_commit_attempt > 5 * 60 - ): - status.nb_workers_commit += 1 - logger.debug("Job: commit (more than 5 minutes since last commit attempt)") - return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk())) - - # 2. Commit if at least 100 files are ready to commit - elif status.nb_workers_commit == 0 and status.queue_commit.qsize() >= 150: - status.nb_workers_commit += 1 - logger.debug("Job: commit (>100 files ready)") - return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk())) - - # 3. Get upload mode if at least 100 files - elif status.queue_get_upload_mode.qsize() >= MAX_NB_FILES_FETCH_UPLOAD_MODE: - status.nb_workers_get_upload_mode += 1 - logger.debug(f"Job: get upload mode (>{MAX_NB_FILES_FETCH_UPLOAD_MODE} files ready)") - return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode, MAX_NB_FILES_FETCH_UPLOAD_MODE)) - - # 4. Preupload LFS file if at least `status.upload_batch_size` files and no worker is preuploading LFS - elif status.queue_preupload_lfs.qsize() >= status.upload_batch_size and status.nb_workers_preupload_lfs == 0: - status.nb_workers_preupload_lfs += 1 - logger.debug("Job: preupload LFS (no other worker preuploading LFS)") - return (WorkerJob.PREUPLOAD_LFS, _get_n(status.queue_preupload_lfs, status.upload_batch_size)) - - # 5. Compute sha256 if at least 1 file and no worker is computing sha256 - elif status.queue_sha256.qsize() > 0 and status.nb_workers_sha256 == 0: - status.nb_workers_sha256 += 1 - logger.debug("Job: sha256 (no other worker computing sha256)") - return (WorkerJob.SHA256, _get_one(status.queue_sha256)) - - # 6. Get upload mode if at least 1 file and no worker is getting upload mode - elif status.queue_get_upload_mode.qsize() > 0 and status.nb_workers_get_upload_mode == 0: - status.nb_workers_get_upload_mode += 1 - logger.debug("Job: get upload mode (no other worker getting upload mode)") - return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode, MAX_NB_FILES_FETCH_UPLOAD_MODE)) - - # 7. Preupload LFS file if at least `status.upload_batch_size` files - # Skip if hf_transfer is enabled and there is already a worker preuploading LFS - elif status.queue_preupload_lfs.qsize() >= status.upload_batch_size and ( - status.nb_workers_preupload_lfs == 0 or not constants.HF_HUB_ENABLE_HF_TRANSFER - ): - status.nb_workers_preupload_lfs += 1 - logger.debug("Job: preupload LFS") - return (WorkerJob.PREUPLOAD_LFS, _get_n(status.queue_preupload_lfs, status.upload_batch_size)) - - # 8. Compute sha256 if at least 1 file - elif status.queue_sha256.qsize() > 0: - status.nb_workers_sha256 += 1 - logger.debug("Job: sha256") - return (WorkerJob.SHA256, _get_one(status.queue_sha256)) - - # 9. Get upload mode if at least 1 file - elif status.queue_get_upload_mode.qsize() > 0: - status.nb_workers_get_upload_mode += 1 - logger.debug("Job: get upload mode") - return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode, MAX_NB_FILES_FETCH_UPLOAD_MODE)) - - # 10. Preupload LFS file if at least 1 file - elif status.queue_preupload_lfs.qsize() > 0: - status.nb_workers_preupload_lfs += 1 - logger.debug("Job: preupload LFS") - return (WorkerJob.PREUPLOAD_LFS, _get_n(status.queue_preupload_lfs, status.upload_batch_size)) - - # 11. Commit if at least 1 file and 1 min since last commit attempt - elif ( - status.nb_workers_commit == 0 - and status.queue_commit.qsize() > 0 - and status.last_commit_attempt is not None - and time.time() - status.last_commit_attempt > 1 * 60 - ): - status.nb_workers_commit += 1 - logger.debug("Job: commit (1 min since last commit attempt)") - return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk())) - - # 12. Commit if at least 1 file all other queues are empty and all workers are waiting - # e.g. when it's the last commit - elif ( - status.nb_workers_commit == 0 - and status.queue_commit.qsize() > 0 - and status.queue_sha256.qsize() == 0 - and status.queue_get_upload_mode.qsize() == 0 - and status.queue_preupload_lfs.qsize() == 0 - and status.nb_workers_sha256 == 0 - and status.nb_workers_get_upload_mode == 0 - and status.nb_workers_preupload_lfs == 0 - ): - status.nb_workers_commit += 1 - logger.debug("Job: commit") - return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk())) - - # 13. If all queues are empty, exit - elif all(metadata.is_committed or metadata.should_ignore for _, metadata in status.items): - logger.info("All files have been processed! Exiting worker.") - return None - - # 14. If no task is available, wait - else: - status.nb_workers_waiting += 1 - logger.debug(f"No task available, waiting... ({WAITING_TIME_IF_NO_TASKS}s)") - return (WorkerJob.WAIT, []) - - -#################### -# Atomic jobs (sha256, get_upload_mode, preupload_lfs, commit) -#################### - - -def _compute_sha256(item: JOB_ITEM_T) -> None: - """Compute sha256 of a file and save it in metadata.""" - paths, metadata = item - if metadata.sha256 is None: - with paths.file_path.open("rb") as f: - metadata.sha256 = sha_fileobj(f).hex() - metadata.save(paths) - - -def _get_upload_mode(items: List[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None: - """Get upload mode for each file and update metadata. - - Also receive info if the file should be ignored. - """ - additions = [_build_hacky_operation(item) for item in items] - _fetch_upload_modes( - additions=additions, - repo_type=repo_type, - repo_id=repo_id, - headers=api._build_hf_headers(), - revision=quote(revision, safe=""), - endpoint=api.endpoint, - ) - for item, addition in zip(items, additions): - paths, metadata = item - metadata.upload_mode = addition._upload_mode - metadata.should_ignore = addition._should_ignore - metadata.remote_oid = addition._remote_oid - metadata.save(paths) - - -def _preupload_lfs(items: List[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None: - """Preupload LFS files and update metadata.""" - additions = [_build_hacky_operation(item) for item in items] - api.preupload_lfs_files( - repo_id=repo_id, - repo_type=repo_type, - revision=revision, - additions=additions, - ) - - for paths, metadata in items: - metadata.is_uploaded = True - metadata.save(paths) - - -def _commit(items: List[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None: - """Commit files to the repo.""" - additions = [_build_hacky_operation(item) for item in items] - api.create_commit( - repo_id=repo_id, - repo_type=repo_type, - revision=revision, - operations=additions, - commit_message="Add files using upload-large-folder tool", - ) - for paths, metadata in items: - metadata.is_committed = True - metadata.save(paths) - - -#################### -# Hacks with CommitOperationAdd to bypass checks/sha256 calculation -#################### - - -class HackyCommitOperationAdd(CommitOperationAdd): - def __post_init__(self) -> None: - if isinstance(self.path_or_fileobj, Path): - self.path_or_fileobj = str(self.path_or_fileobj) - - -def _build_hacky_operation(item: JOB_ITEM_T) -> HackyCommitOperationAdd: - paths, metadata = item - operation = HackyCommitOperationAdd(path_in_repo=paths.path_in_repo, path_or_fileobj=paths.file_path) - with paths.file_path.open("rb") as file: - sample = file.peek(512)[:512] - if metadata.sha256 is None: - raise ValueError("sha256 must have been computed by now!") - operation.upload_info = UploadInfo(sha256=bytes.fromhex(metadata.sha256), size=metadata.size, sample=sample) - operation._upload_mode = metadata.upload_mode # type: ignore[assignment] - operation._should_ignore = metadata.should_ignore - operation._remote_oid = metadata.remote_oid - return operation - - -#################### -# Misc helpers -#################### - - -def _get_one(queue: "queue.Queue[JOB_ITEM_T]") -> List[JOB_ITEM_T]: - return [queue.get()] - - -def _get_n(queue: "queue.Queue[JOB_ITEM_T]", n: int) -> List[JOB_ITEM_T]: - return [queue.get() for _ in range(min(queue.qsize(), n))] - - -def _print_overwrite(report: str) -> None: - """Print a report, overwriting the previous lines. - - Since tqdm in using `sys.stderr` to (re-)write progress bars, we need to use `sys.stdout` - to print the report. - - Note: works well only if no other process is writing to `sys.stdout`! - """ - report += "\n" - # Get terminal width - terminal_width = shutil.get_terminal_size().columns - - # Count number of lines that should be cleared - nb_lines = sum(len(line) // terminal_width + 1 for line in report.splitlines()) - - # Clear previous lines based on the number of lines in the report - for _ in range(nb_lines): - sys.stdout.write("\r\033[K") # Clear line - sys.stdout.write("\033[F") # Move cursor up one line - - # Print the new report, filling remaining space with whitespace - sys.stdout.write(report) - sys.stdout.write(" " * (terminal_width - len(report.splitlines()[-1]))) - sys.stdout.flush() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_webhooks_payload.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_webhooks_payload.py deleted file mode 100644 index 288f4b08b9428980e99ca06703442eab62fad277..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_webhooks_payload.py +++ /dev/null @@ -1,137 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains data structures to parse the webhooks payload.""" - -from typing import List, Literal, Optional - -from .utils import is_pydantic_available - - -if is_pydantic_available(): - from pydantic import BaseModel -else: - # Define a dummy BaseModel to avoid import errors when pydantic is not installed - # Import error will be raised when trying to use the class - - class BaseModel: # type: ignore [no-redef] - def __init__(self, *args, **kwargs) -> None: - raise ImportError( - "You must have `pydantic` installed to use `WebhookPayload`. This is an optional dependency that" - " should be installed separately. Please run `pip install --upgrade pydantic` and retry." - ) - - -# This is an adaptation of the ReportV3 interface implemented in moon-landing. V0, V1 and V2 have been ignored as they -# are not in used anymore. To keep in sync when format is updated in -# https://github.com/huggingface/moon-landing/blob/main/server/lib/HFWebhooks.ts (internal link). - - -WebhookEvent_T = Literal[ - "create", - "delete", - "move", - "update", -] -RepoChangeEvent_T = Literal[ - "add", - "move", - "remove", - "update", -] -RepoType_T = Literal[ - "dataset", - "model", - "space", -] -DiscussionStatus_T = Literal[ - "closed", - "draft", - "open", - "merged", -] -SupportedWebhookVersion = Literal[3] - - -class ObjectId(BaseModel): - id: str - - -class WebhookPayloadUrl(BaseModel): - web: str - api: Optional[str] = None - - -class WebhookPayloadMovedTo(BaseModel): - name: str - owner: ObjectId - - -class WebhookPayloadWebhook(ObjectId): - version: SupportedWebhookVersion - - -class WebhookPayloadEvent(BaseModel): - action: WebhookEvent_T - scope: str - - -class WebhookPayloadDiscussionChanges(BaseModel): - base: str - mergeCommitId: Optional[str] = None - - -class WebhookPayloadComment(ObjectId): - author: ObjectId - hidden: bool - content: Optional[str] = None - url: WebhookPayloadUrl - - -class WebhookPayloadDiscussion(ObjectId): - num: int - author: ObjectId - url: WebhookPayloadUrl - title: str - isPullRequest: bool - status: DiscussionStatus_T - changes: Optional[WebhookPayloadDiscussionChanges] = None - pinned: Optional[bool] = None - - -class WebhookPayloadRepo(ObjectId): - owner: ObjectId - head_sha: Optional[str] = None - name: str - private: bool - subdomain: Optional[str] = None - tags: Optional[List[str]] = None - type: Literal["dataset", "model", "space"] - url: WebhookPayloadUrl - - -class WebhookPayloadUpdatedRef(BaseModel): - ref: str - oldSha: Optional[str] = None - newSha: Optional[str] = None - - -class WebhookPayload(BaseModel): - event: WebhookPayloadEvent - repo: WebhookPayloadRepo - discussion: Optional[WebhookPayloadDiscussion] = None - comment: Optional[WebhookPayloadComment] = None - webhook: WebhookPayloadWebhook - movedTo: Optional[WebhookPayloadMovedTo] = None - updatedRefs: Optional[List[WebhookPayloadUpdatedRef]] = None diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_webhooks_server.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/_webhooks_server.py deleted file mode 100644 index a3668304553e13f9605a59ec623aceb5202a2488..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/_webhooks_server.py +++ /dev/null @@ -1,376 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains `WebhooksServer` and `webhook_endpoint` to create a webhook server easily.""" - -import atexit -import inspect -import os -from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, Dict, Optional - -from .utils import experimental, is_fastapi_available, is_gradio_available - - -if TYPE_CHECKING: - import gradio as gr - from fastapi import Request - -if is_fastapi_available(): - from fastapi import FastAPI, Request - from fastapi.responses import JSONResponse -else: - # Will fail at runtime if FastAPI is not available - FastAPI = Request = JSONResponse = None # type: ignore - - -_global_app: Optional["WebhooksServer"] = None -_is_local = os.environ.get("SPACE_ID") is None - - -@experimental -class WebhooksServer: - """ - The [`WebhooksServer`] class lets you create an instance of a Gradio app that can receive Huggingface webhooks. - These webhooks can be registered using the [`~WebhooksServer.add_webhook`] decorator. Webhook endpoints are added to - the app as a POST endpoint to the FastAPI router. Once all the webhooks are registered, the `launch` method has to be - called to start the app. - - It is recommended to accept [`WebhookPayload`] as the first argument of the webhook function. It is a Pydantic - model that contains all the information about the webhook event. The data will be parsed automatically for you. - - Check out the [webhooks guide](../guides/webhooks_server) for a step-by-step tutorial on how to setup your - WebhooksServer and deploy it on a Space. - - > [!WARNING] - > `WebhooksServer` is experimental. Its API is subject to change in the future. - - > [!WARNING] - > You must have `gradio` installed to use `WebhooksServer` (`pip install --upgrade gradio`). - - Args: - ui (`gradio.Blocks`, optional): - A Gradio UI instance to be used as the Space landing page. If `None`, a UI displaying instructions - about the configured webhooks is created. - webhook_secret (`str`, optional): - A secret key to verify incoming webhook requests. You can set this value to any secret you want as long as - you also configure it in your [webhooks settings panel](https://huggingface.co/settings/webhooks). You - can also set this value as the `WEBHOOK_SECRET` environment variable. If no secret is provided, the - webhook endpoints are opened without any security. - - Example: - - ```python - import gradio as gr - from huggingface_hub import WebhooksServer, WebhookPayload - - with gr.Blocks() as ui: - ... - - app = WebhooksServer(ui=ui, webhook_secret="my_secret_key") - - @app.add_webhook("/say_hello") - async def hello(payload: WebhookPayload): - return {"message": "hello"} - - app.launch() - ``` - """ - - def __new__(cls, *args, **kwargs) -> "WebhooksServer": - if not is_gradio_available(): - raise ImportError( - "You must have `gradio` installed to use `WebhooksServer`. Please run `pip install --upgrade gradio`" - " first." - ) - if not is_fastapi_available(): - raise ImportError( - "You must have `fastapi` installed to use `WebhooksServer`. Please run `pip install --upgrade fastapi`" - " first." - ) - return super().__new__(cls) - - def __init__( - self, - ui: Optional["gr.Blocks"] = None, - webhook_secret: Optional[str] = None, - ) -> None: - self._ui = ui - - self.webhook_secret = webhook_secret or os.getenv("WEBHOOK_SECRET") - self.registered_webhooks: Dict[str, Callable] = {} - _warn_on_empty_secret(self.webhook_secret) - - def add_webhook(self, path: Optional[str] = None) -> Callable: - """ - Decorator to add a webhook to the [`WebhooksServer`] server. - - Args: - path (`str`, optional): - The URL path to register the webhook function. If not provided, the function name will be used as the - path. In any case, all webhooks are registered under `/webhooks`. - - Raises: - ValueError: If the provided path is already registered as a webhook. - - Example: - ```python - from huggingface_hub import WebhooksServer, WebhookPayload - - app = WebhooksServer() - - @app.add_webhook - async def trigger_training(payload: WebhookPayload): - if payload.repo.type == "dataset" and payload.event.action == "update": - # Trigger a training job if a dataset is updated - ... - - app.launch() - ``` - """ - # Usage: directly as decorator. Example: `@app.add_webhook` - if callable(path): - # If path is a function, it means it was used as a decorator without arguments - return self.add_webhook()(path) - - # Usage: provide a path. Example: `@app.add_webhook(...)` - @wraps(FastAPI.post) - def _inner_post(*args, **kwargs): - func = args[0] - abs_path = f"/webhooks/{(path or func.__name__).strip('/')}" - if abs_path in self.registered_webhooks: - raise ValueError(f"Webhook {abs_path} already exists.") - self.registered_webhooks[abs_path] = func - - return _inner_post - - def launch(self, prevent_thread_lock: bool = False, **launch_kwargs: Any) -> None: - """Launch the Gradio app and register webhooks to the underlying FastAPI server. - - Input parameters are forwarded to Gradio when launching the app. - """ - ui = self._ui or self._get_default_ui() - - # Start Gradio App - # - as non-blocking so that webhooks can be added afterwards - # - as shared if launch locally (to debug webhooks) - launch_kwargs.setdefault("share", _is_local) - self.fastapi_app, _, _ = ui.launch(prevent_thread_lock=True, **launch_kwargs) - - # Register webhooks to FastAPI app - for path, func in self.registered_webhooks.items(): - # Add secret check if required - if self.webhook_secret is not None: - func = _wrap_webhook_to_check_secret(func, webhook_secret=self.webhook_secret) - - # Add route to FastAPI app - self.fastapi_app.post(path)(func) - - # Print instructions and block main thread - space_host = os.environ.get("SPACE_HOST") - url = "https://" + space_host if space_host is not None else (ui.share_url or ui.local_url) - if url is None: - raise ValueError("Cannot find the URL of the app. Please provide a valid `ui` or update `gradio` version.") - url = url.strip("/") - message = "\nWebhooks are correctly setup and ready to use:" - message += "\n" + "\n".join(f" - POST {url}{webhook}" for webhook in self.registered_webhooks) - message += "\nGo to https://huggingface.co/settings/webhooks to setup your webhooks." - print(message) - - if not prevent_thread_lock: - ui.block_thread() - - def _get_default_ui(self) -> "gr.Blocks": - """Default UI if not provided (lists webhooks and provides basic instructions).""" - import gradio as gr - - with gr.Blocks() as ui: - gr.Markdown("# This is an app to process 🤗 Webhooks") - gr.Markdown( - "Webhooks are a foundation for MLOps-related features. They allow you to listen for new changes on" - " specific repos or to all repos belonging to particular set of users/organizations (not just your" - " repos, but any repo). Check out this [guide](https://huggingface.co/docs/hub/webhooks) to get to" - " know more about webhooks on the Huggingface Hub." - ) - gr.Markdown( - f"{len(self.registered_webhooks)} webhook(s) are registered:" - + "\n\n" - + "\n ".join( - f"- [{webhook_path}]({_get_webhook_doc_url(webhook.__name__, webhook_path)})" - for webhook_path, webhook in self.registered_webhooks.items() - ) - ) - gr.Markdown( - "Go to https://huggingface.co/settings/webhooks to setup your webhooks." - + "\nYou app is running locally. Please look at the logs to check the full URL you need to set." - if _is_local - else ( - "\nThis app is running on a Space. You can find the corresponding URL in the options menu" - " (top-right) > 'Embed the Space'. The URL looks like 'https://{username}-{repo_name}.hf.space'." - ) - ) - return ui - - -@experimental -def webhook_endpoint(path: Optional[str] = None) -> Callable: - """Decorator to start a [`WebhooksServer`] and register the decorated function as a webhook endpoint. - - This is a helper to get started quickly. If you need more flexibility (custom landing page or webhook secret), - you can use [`WebhooksServer`] directly. You can register multiple webhook endpoints (to the same server) by using - this decorator multiple times. - - Check out the [webhooks guide](../guides/webhooks_server) for a step-by-step tutorial on how to setup your - server and deploy it on a Space. - - > [!WARNING] - > `webhook_endpoint` is experimental. Its API is subject to change in the future. - - > [!WARNING] - > You must have `gradio` installed to use `webhook_endpoint` (`pip install --upgrade gradio`). - - Args: - path (`str`, optional): - The URL path to register the webhook function. If not provided, the function name will be used as the path. - In any case, all webhooks are registered under `/webhooks`. - - Examples: - The default usage is to register a function as a webhook endpoint. The function name will be used as the path. - The server will be started automatically at exit (i.e. at the end of the script). - - ```python - from huggingface_hub import webhook_endpoint, WebhookPayload - - @webhook_endpoint - async def trigger_training(payload: WebhookPayload): - if payload.repo.type == "dataset" and payload.event.action == "update": - # Trigger a training job if a dataset is updated - ... - - # Server is automatically started at the end of the script. - ``` - - Advanced usage: register a function as a webhook endpoint and start the server manually. This is useful if you - are running it in a notebook. - - ```python - from huggingface_hub import webhook_endpoint, WebhookPayload - - @webhook_endpoint - async def trigger_training(payload: WebhookPayload): - if payload.repo.type == "dataset" and payload.event.action == "update": - # Trigger a training job if a dataset is updated - ... - - # Start the server manually - trigger_training.launch() - ``` - """ - if callable(path): - # If path is a function, it means it was used as a decorator without arguments - return webhook_endpoint()(path) - - @wraps(WebhooksServer.add_webhook) - def _inner(func: Callable) -> Callable: - app = _get_global_app() - app.add_webhook(path)(func) - if len(app.registered_webhooks) == 1: - # Register `app.launch` to run at exit (only once) - atexit.register(app.launch) - - @wraps(app.launch) - def _launch_now(): - # Run the app directly (without waiting atexit) - atexit.unregister(app.launch) - app.launch() - - func.launch = _launch_now # type: ignore - return func - - return _inner - - -def _get_global_app() -> WebhooksServer: - global _global_app - if _global_app is None: - _global_app = WebhooksServer() - return _global_app - - -def _warn_on_empty_secret(webhook_secret: Optional[str]) -> None: - if webhook_secret is None: - print("Webhook secret is not defined. This means your webhook endpoints will be open to everyone.") - print( - "To add a secret, set `WEBHOOK_SECRET` as environment variable or pass it at initialization: " - "\n\t`app = WebhooksServer(webhook_secret='my_secret', ...)`" - ) - print( - "For more details about webhook secrets, please refer to" - " https://huggingface.co/docs/hub/webhooks#webhook-secret." - ) - else: - print("Webhook secret is correctly defined.") - - -def _get_webhook_doc_url(webhook_name: str, webhook_path: str) -> str: - """Returns the anchor to a given webhook in the docs (experimental)""" - return "/docs#/default/" + webhook_name + webhook_path.replace("/", "_") + "_post" - - -def _wrap_webhook_to_check_secret(func: Callable, webhook_secret: str) -> Callable: - """Wraps a webhook function to check the webhook secret before calling the function. - - This is a hacky way to add the `request` parameter to the function signature. Since FastAPI based itself on route - parameters to inject the values to the function, we need to hack the function signature to retrieve the `Request` - object (and hence the headers). A far cleaner solution would be to use a middleware. However, since - `fastapi==0.90.1`, a middleware cannot be added once the app has started. And since the FastAPI app is started by - Gradio internals (and not by us), we cannot add a middleware. - - This method is called only when a secret has been defined by the user. If a request is sent without the - "x-webhook-secret", the function will return a 401 error (unauthorized). If the header is sent but is incorrect, - the function will return a 403 error (forbidden). - - Inspired by https://stackoverflow.com/a/33112180. - """ - initial_sig = inspect.signature(func) - - @wraps(func) - async def _protected_func(request: Request, **kwargs): - request_secret = request.headers.get("x-webhook-secret") - if request_secret is None: - return JSONResponse({"error": "x-webhook-secret header not set."}, status_code=401) - if request_secret != webhook_secret: - return JSONResponse({"error": "Invalid webhook secret."}, status_code=403) - - # Inject `request` in kwargs if required - if "request" in initial_sig.parameters: - kwargs["request"] = request - - # Handle both sync and async routes - if inspect.iscoroutinefunction(func): - return await func(**kwargs) - else: - return func(**kwargs) - - # Update signature to include request - if "request" not in initial_sig.parameters: - _protected_func.__signature__ = initial_sig.replace( # type: ignore - parameters=( - inspect.Parameter(name="request", kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Request), - ) - + tuple(initial_sig.parameters.values()) - ) - - # Return protected route - return _protected_func diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/__init__.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/__init__.py deleted file mode 100644 index 7a1a8d793b89e16e5fa46ec5d420ec96fe1d72fe..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from abc import ABC, abstractmethod -from argparse import _SubParsersAction - - -class BaseHuggingfaceCLICommand(ABC): - @staticmethod - @abstractmethod - def register_subcommand(parser: _SubParsersAction): - raise NotImplementedError() - - @abstractmethod - def run(self): - raise NotImplementedError() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/_cli_utils.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/_cli_utils.py deleted file mode 100644 index bd56ad6896db2a257323e022896940c0ba0d68d3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/_cli_utils.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2022 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains a utility for good-looking prints.""" - -import os -from typing import List, Union - - -class ANSI: - """ - Helper for en.wikipedia.org/wiki/ANSI_escape_code - """ - - _bold = "\u001b[1m" - _gray = "\u001b[90m" - _red = "\u001b[31m" - _reset = "\u001b[0m" - _yellow = "\u001b[33m" - - @classmethod - def bold(cls, s: str) -> str: - return cls._format(s, cls._bold) - - @classmethod - def gray(cls, s: str) -> str: - return cls._format(s, cls._gray) - - @classmethod - def red(cls, s: str) -> str: - return cls._format(s, cls._bold + cls._red) - - @classmethod - def yellow(cls, s: str) -> str: - return cls._format(s, cls._yellow) - - @classmethod - def _format(cls, s: str, code: str) -> str: - if os.environ.get("NO_COLOR"): - # See https://no-color.org/ - return s - return f"{code}{s}{cls._reset}" - - -def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str: - """ - Inspired by: - - - stackoverflow.com/a/8356620/593036 - - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data - """ - col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] - row_format = ("{{:{}}} " * len(headers)).format(*col_widths) - lines = [] - lines.append(row_format.format(*headers)) - lines.append(row_format.format(*["-" * w for w in col_widths])) - for row in rows: - lines.append(row_format.format(*row)) - return "\n".join(lines) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/auth.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/auth.py deleted file mode 100644 index bbf475a4f8785152b992b116a69b4b16293688f3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/auth.py +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains commands to authenticate to the Hugging Face Hub and interact with your repositories. - -Usage: - # login and save token locally. - hf auth login --token=hf_*** --add-to-git-credential - - # switch between tokens - hf auth switch - - # list all tokens - hf auth list - - # logout from all tokens - hf auth logout - - # check which account you are logged in as - hf auth whoami -""" - -from argparse import _SubParsersAction -from typing import List, Optional - -from requests.exceptions import HTTPError - -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.constants import ENDPOINT -from huggingface_hub.hf_api import HfApi - -from .._login import auth_list, auth_switch, login, logout -from ..utils import get_stored_tokens, get_token, logging -from ._cli_utils import ANSI - - -logger = logging.get_logger(__name__) - -try: - from InquirerPy import inquirer - from InquirerPy.base.control import Choice - - _inquirer_py_available = True -except ImportError: - _inquirer_py_available = False - - -class AuthCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - # Create the main 'auth' command - auth_parser = parser.add_parser("auth", help="Manage authentication (login, logout, etc.).") - auth_subparsers = auth_parser.add_subparsers(help="Authentication subcommands") - - # Show help if no subcommand is provided - auth_parser.set_defaults(func=lambda args: auth_parser.print_help()) - - # Add 'login' as a subcommand of 'auth' - login_parser = auth_subparsers.add_parser( - "login", help="Log in using a token from huggingface.co/settings/tokens" - ) - login_parser.add_argument( - "--token", - type=str, - help="Token generated from https://huggingface.co/settings/tokens", - ) - login_parser.add_argument( - "--add-to-git-credential", - action="store_true", - help="Optional: Save token to git credential helper.", - ) - login_parser.set_defaults(func=lambda args: AuthLogin(args)) - - # Add 'logout' as a subcommand of 'auth' - logout_parser = auth_subparsers.add_parser("logout", help="Log out") - logout_parser.add_argument( - "--token-name", - type=str, - help="Optional: Name of the access token to log out from.", - ) - logout_parser.set_defaults(func=lambda args: AuthLogout(args)) - - # Add 'whoami' as a subcommand of 'auth' - whoami_parser = auth_subparsers.add_parser( - "whoami", help="Find out which huggingface.co account you are logged in as." - ) - whoami_parser.set_defaults(func=lambda args: AuthWhoami(args)) - - # Existing subcommands - auth_switch_parser = auth_subparsers.add_parser("switch", help="Switch between access tokens") - auth_switch_parser.add_argument( - "--token-name", - type=str, - help="Optional: Name of the access token to switch to.", - ) - auth_switch_parser.add_argument( - "--add-to-git-credential", - action="store_true", - help="Optional: Save token to git credential helper.", - ) - auth_switch_parser.set_defaults(func=lambda args: AuthSwitch(args)) - - auth_list_parser = auth_subparsers.add_parser("list", help="List all stored access tokens") - auth_list_parser.set_defaults(func=lambda args: AuthList(args)) - - -class BaseAuthCommand: - def __init__(self, args): - self.args = args - self._api = HfApi() - - -class AuthLogin(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - login( - token=self.args.token, - add_to_git_credential=self.args.add_to_git_credential, - ) - - -class AuthLogout(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - logout(token_name=self.args.token_name) - - -class AuthSwitch(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - token_name = self.args.token_name - if token_name is None: - token_name = self._select_token_name() - - if token_name is None: - print("No token name provided. Aborting.") - exit() - auth_switch(token_name, add_to_git_credential=self.args.add_to_git_credential) - - def _select_token_name(self) -> Optional[str]: - token_names = list(get_stored_tokens().keys()) - - if not token_names: - logger.error("No stored tokens found. Please login first.") - return None - - if _inquirer_py_available: - return self._select_token_name_tui(token_names) - # if inquirer is not available, use a simpler terminal UI - print("Available stored tokens:") - for i, token_name in enumerate(token_names, 1): - print(f"{i}. {token_name}") - while True: - try: - choice = input("Enter the number of the token to switch to (or 'q' to quit): ") - if choice.lower() == "q": - return None - index = int(choice) - 1 - if 0 <= index < len(token_names): - return token_names[index] - else: - print("Invalid selection. Please try again.") - except ValueError: - print("Invalid input. Please enter a number or 'q' to quit.") - - def _select_token_name_tui(self, token_names: List[str]) -> Optional[str]: - choices = [Choice(token_name, name=token_name) for token_name in token_names] - try: - return inquirer.select( - message="Select a token to switch to:", - choices=choices, - default=None, - ).execute() - except KeyboardInterrupt: - logger.info("Token selection cancelled.") - return None - - -class AuthList(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - auth_list() - - -class AuthWhoami(BaseAuthCommand): - def run(self): - token = get_token() - if token is None: - print("Not logged in") - exit() - try: - info = self._api.whoami(token) - print(ANSI.bold("user: "), info["name"]) - orgs = [org["name"] for org in info["orgs"]] - if orgs: - print(ANSI.bold("orgs: "), ",".join(orgs)) - - if ENDPOINT != "https://huggingface.co": - print(f"Authenticated through private endpoint: {ENDPOINT}") - except HTTPError as e: - print(e) - print(ANSI.red(e.response.text)) - exit(1) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/cache.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/cache.py deleted file mode 100644 index cc36ef5efd2508bcc5e32b1fbe222bb55358777c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/cache.py +++ /dev/null @@ -1,403 +0,0 @@ -# coding=utf-8 -# Copyright 2025-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains the 'hf cache' command group with 'scan' and 'delete' subcommands.""" - -import os -import time -from argparse import Namespace, _SubParsersAction -from functools import wraps -from tempfile import mkstemp -from typing import Any, Callable, Iterable, List, Literal, Optional, Union - -from ..utils import CachedRepoInfo, CachedRevisionInfo, CacheNotFound, HFCacheInfo, scan_cache_dir -from . import BaseHuggingfaceCLICommand -from ._cli_utils import ANSI, tabulate - - -# --- DELETE helpers (from delete_cache.py) --- -try: - from InquirerPy import inquirer - from InquirerPy.base.control import Choice - from InquirerPy.separator import Separator - - _inquirer_py_available = True -except ImportError: - _inquirer_py_available = False - -SortingOption_T = Literal["alphabetical", "lastUpdated", "lastUsed", "size"] -_CANCEL_DELETION_STR = "CANCEL_DELETION" - - -def require_inquirer_py(fn: Callable) -> Callable: - @wraps(fn) - def _inner(*args, **kwargs): - if not _inquirer_py_available: - raise ImportError( - "The 'cache delete' command requires extra dependencies for the TUI.\n" - "Please run 'pip install \"huggingface_hub[cli]\"' to install them.\n" - "Otherwise, disable TUI using the '--disable-tui' flag." - ) - return fn(*args, **kwargs) - - return _inner - - -class CacheCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - cache_parser = parser.add_parser("cache", help="Manage local cache directory.") - cache_subparsers = cache_parser.add_subparsers(dest="cache_command", help="Cache subcommands") - - # Show help if no subcommand is provided - cache_parser.set_defaults(func=lambda args: cache_parser.print_help()) - - # Scan subcommand - scan_parser = cache_subparsers.add_parser("scan", help="Scan cache directory.") - scan_parser.add_argument( - "--dir", - type=str, - default=None, - help="cache directory to scan (optional). Default to the default HuggingFace cache.", - ) - scan_parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="show a more verbose output", - ) - scan_parser.set_defaults(func=CacheCommand, cache_command="scan") - # Delete subcommand - delete_parser = cache_subparsers.add_parser("delete", help="Delete revisions from the cache directory.") - delete_parser.add_argument( - "--dir", - type=str, - default=None, - help="cache directory (optional). Default to the default HuggingFace cache.", - ) - delete_parser.add_argument( - "--disable-tui", - action="store_true", - help=( - "Disable Terminal User Interface (TUI) mode. Useful if your platform/terminal doesn't support the multiselect menu." - ), - ) - delete_parser.add_argument( - "--sort", - nargs="?", - choices=["alphabetical", "lastUpdated", "lastUsed", "size"], - help=( - "Sort repositories by the specified criteria. Options: " - "'alphabetical' (A-Z), " - "'lastUpdated' (newest first), " - "'lastUsed' (most recent first), " - "'size' (largest first)." - ), - ) - delete_parser.set_defaults(func=CacheCommand, cache_command="delete") - - def __init__(self, args: Namespace) -> None: - self.args = args - self.verbosity: int = getattr(args, "verbose", 0) - self.cache_dir: Optional[str] = getattr(args, "dir", None) - self.disable_tui: bool = getattr(args, "disable_tui", False) - self.sort_by: Optional[SortingOption_T] = getattr(args, "sort", None) - self.cache_command: Optional[str] = getattr(args, "cache_command", None) - - def run(self): - if self.cache_command == "scan": - self._run_scan() - elif self.cache_command == "delete": - self._run_delete() - else: - print("Please specify a cache subcommand (scan or delete). Use -h for help.") - - def _run_scan(self): - try: - t0 = time.time() - hf_cache_info = scan_cache_dir(self.cache_dir) - t1 = time.time() - except CacheNotFound as exc: - cache_dir = exc.cache_dir - print(f"Cache directory not found: {cache_dir}") - return - print(get_table(hf_cache_info, verbosity=self.verbosity)) - print( - f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)" - f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}." - ) - if len(hf_cache_info.warnings) > 0: - message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning." - if self.verbosity >= 3: - print(ANSI.gray(message)) - for warning in hf_cache_info.warnings: - print(ANSI.gray(str(warning))) - else: - print(ANSI.gray(message + " Use -vvv to print details.")) - - def _run_delete(self): - hf_cache_info = scan_cache_dir(self.cache_dir) - if self.disable_tui: - selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) - else: - selected_hashes = _manual_review_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) - if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes: - confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?" - if self.disable_tui: - confirmed = _ask_for_confirmation_no_tui(confirm_message) - else: - confirmed = _ask_for_confirmation_tui(confirm_message) - if confirmed: - strategy = hf_cache_info.delete_revisions(*selected_hashes) - print("Start deletion.") - strategy.execute() - print( - f"Done. Deleted {len(strategy.repos)} repo(s) and" - f" {len(strategy.snapshots)} revision(s) for a total of" - f" {strategy.expected_freed_size_str}." - ) - return - print("Deletion is cancelled. Do nothing.") - - -def get_table(hf_cache_info: HFCacheInfo, *, verbosity: int = 0) -> str: - if verbosity == 0: - return tabulate( - rows=[ - [ - repo.repo_id, - repo.repo_type, - "{:>12}".format(repo.size_on_disk_str), - repo.nb_files, - repo.last_accessed_str, - repo.last_modified_str, - ", ".join(sorted(repo.refs)), - str(repo.repo_path), - ] - for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path) - ], - headers=[ - "REPO ID", - "REPO TYPE", - "SIZE ON DISK", - "NB FILES", - "LAST_ACCESSED", - "LAST_MODIFIED", - "REFS", - "LOCAL PATH", - ], - ) - else: - return tabulate( - rows=[ - [ - repo.repo_id, - repo.repo_type, - revision.commit_hash, - "{:>12}".format(revision.size_on_disk_str), - revision.nb_files, - revision.last_modified_str, - ", ".join(sorted(revision.refs)), - str(revision.snapshot_path), - ] - for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path) - for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash) - ], - headers=[ - "REPO ID", - "REPO TYPE", - "REVISION", - "SIZE ON DISK", - "NB FILES", - "LAST_MODIFIED", - "REFS", - "LOCAL PATH", - ], - ) - - -def _get_repo_sorting_key(repo: CachedRepoInfo, sort_by: Optional[SortingOption_T] = None): - if sort_by == "alphabetical": - return (repo.repo_type, repo.repo_id.lower()) - elif sort_by == "lastUpdated": - return -max(rev.last_modified for rev in repo.revisions) - elif sort_by == "lastUsed": - return -repo.last_accessed - elif sort_by == "size": - return -repo.size_on_disk - else: - return (repo.repo_type, repo.repo_id) - - -@require_inquirer_py -def _manual_review_tui( - hf_cache_info: HFCacheInfo, preselected: List[str], sort_by: Optional[SortingOption_T] = None -) -> List[str]: - choices = _get_tui_choices_from_scan(repos=hf_cache_info.repos, preselected=preselected, sort_by=sort_by) - checkbox = inquirer.checkbox( - message="Select revisions to delete:", - choices=choices, - cycle=False, - height=100, - instruction=_get_expectations_str( - hf_cache_info, selected_hashes=[c.value for c in choices if isinstance(c, Choice) and c.enabled] - ), - long_instruction="Press to select, to validate and to quit without modification.", - transformer=lambda result: f"{len(result)} revision(s) selected.", - ) - - def _update_expectations(_): - checkbox._instruction = _get_expectations_str( - hf_cache_info, - selected_hashes=[choice["value"] for choice in checkbox.content_control.choices if choice["enabled"]], - ) - - checkbox.kb_func_lookup["toggle"].append({"func": _update_expectations}) - try: - return checkbox.execute() - except KeyboardInterrupt: - return [] - - -@require_inquirer_py -def _ask_for_confirmation_tui(message: str, default: bool = True) -> bool: - return inquirer.confirm(message, default=default).execute() - - -def _get_tui_choices_from_scan( - repos: Iterable[CachedRepoInfo], preselected: List[str], sort_by: Optional[SortingOption_T] = None -) -> List: - choices: List[Union["Choice", "Separator"]] = [] - choices.append( - Choice( - _CANCEL_DELETION_STR, name="None of the following (if selected, nothing will be deleted).", enabled=False - ) - ) - sorted_repos = sorted(repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by)) - for repo in sorted_repos: - choices.append( - Separator( - f"\n{repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}, used {repo.last_accessed_str})" - ) - ) - for revision in sorted(repo.revisions, key=_revision_sorting_order): - choices.append( - Choice( - revision.commit_hash, - name=( - f"{revision.commit_hash[:8]}: {', '.join(sorted(revision.refs)) or '(detached)'} # modified {revision.last_modified_str}" - ), - enabled=revision.commit_hash in preselected, - ) - ) - return choices - - -def _manual_review_no_tui( - hf_cache_info: HFCacheInfo, preselected: List[str], sort_by: Optional[SortingOption_T] = None -) -> List[str]: - fd, tmp_path = mkstemp(suffix=".txt") - os.close(fd) - lines = [] - sorted_repos = sorted(hf_cache_info.repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by)) - for repo in sorted_repos: - lines.append( - f"\n# {repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}, used {repo.last_accessed_str})" - ) - for revision in sorted(repo.revisions, key=_revision_sorting_order): - lines.append( - f"{'' if revision.commit_hash in preselected else '#'} {revision.commit_hash} # Refs: {', '.join(sorted(revision.refs)) or '(detached)'} # modified {revision.last_modified_str}" - ) - with open(tmp_path, "w") as f: - f.write(_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS) - f.write("\n".join(lines)) - instructions = f""" - TUI is disabled. In order to select which revisions you want to delete, please edit - the following file using the text editor of your choice. Instructions for manual - editing are located at the beginning of the file. Edit the file, save it and confirm - to continue. - File to edit: {ANSI.bold(tmp_path)} - """ - print("\n".join(line.strip() for line in instructions.strip().split("\n"))) - while True: - selected_hashes = _read_manual_review_tmp_file(tmp_path) - if _ask_for_confirmation_no_tui( - _get_expectations_str(hf_cache_info, selected_hashes) + " Continue ?", default=False - ): - break - os.remove(tmp_path) - return sorted(selected_hashes) - - -def _ask_for_confirmation_no_tui(message: str, default: bool = True) -> bool: - YES = ("y", "yes", "1") - NO = ("n", "no", "0") - DEFAULT = "" - ALL = YES + NO + (DEFAULT,) - full_message = message + (" (Y/n) " if default else " (y/N) ") - while True: - answer = input(full_message).lower() - if answer == DEFAULT: - return default - if answer in YES: - return True - if answer in NO: - return False - print(f"Invalid input. Must be one of {ALL}") - - -def _get_expectations_str(hf_cache_info: HFCacheInfo, selected_hashes: List[str]) -> str: - if _CANCEL_DELETION_STR in selected_hashes: - return "Nothing will be deleted." - strategy = hf_cache_info.delete_revisions(*selected_hashes) - return f"{len(selected_hashes)} revisions selected counting for {strategy.expected_freed_size_str}." - - -def _read_manual_review_tmp_file(tmp_path: str) -> List[str]: - with open(tmp_path) as f: - content = f.read() - lines = [line.strip() for line in content.split("\n")] - selected_lines = [line for line in lines if not line.startswith("#")] - selected_hashes = [line.split("#")[0].strip() for line in selected_lines] - return [hash for hash in selected_hashes if len(hash) > 0] - - -_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS = f""" -# INSTRUCTIONS -# ------------ -# This is a temporary file created by running `hf cache delete --disable-tui`. It contains a set of revisions that can be deleted from your local cache directory. -# -# Please manually review the revisions you want to delete: -# - Revision hashes can be commented out with '#'. -# - Only non-commented revisions in this file will be deleted. -# - Revision hashes that are removed from this file are ignored as well. -# - If `{_CANCEL_DELETION_STR}` line is uncommented, the all cache deletion is cancelled and no changes will be applied. -# -# Once you've manually reviewed this file, please confirm deletion in the terminal. This file will be automatically removed once done. -# ------------ - -# KILL SWITCH -# ------------ -# Un-comment following line to completely cancel the deletion process -# {_CANCEL_DELETION_STR} -# ------------ - -# REVISIONS -# ------------ -""".strip() - - -def _revision_sorting_order(revision: CachedRevisionInfo) -> Any: - return revision.last_modified diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/download.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/download.py deleted file mode 100644 index 2660644e62955952f010701a823d7a8bdce1803b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/download.py +++ /dev/null @@ -1,181 +0,0 @@ -# coding=utf-8 -# Copyright 202-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to download files from the Hub with the CLI. - -Usage: - hf download --help - - # Download file - hf download gpt2 config.json - - # Download entire repo - hf download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78 - - # Download repo with filters - hf download gpt2 --include="*.safetensors" - - # Download with token - hf download Wauplin/private-model --token=hf_*** - - # Download quietly (no progress bar, no warnings, only the returned path) - hf download gpt2 config.json --quiet - - # Download to local dir - hf download gpt2 --local-dir=./models/gpt2 -""" - -import warnings -from argparse import Namespace, _SubParsersAction -from typing import List, Optional - -from huggingface_hub import logging -from huggingface_hub._snapshot_download import snapshot_download -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.file_download import hf_hub_download -from huggingface_hub.utils import disable_progress_bars, enable_progress_bars - - -logger = logging.get_logger(__name__) - - -class DownloadCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - download_parser = parser.add_parser("download", help="Download files from the Hub") - download_parser.add_argument( - "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)." - ) - download_parser.add_argument( - "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)." - ) - download_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Type of repo to download from (defaults to 'model').", - ) - download_parser.add_argument( - "--revision", - type=str, - help="An optional Git revision id which can be a branch name, a tag, or a commit hash.", - ) - download_parser.add_argument( - "--include", nargs="*", type=str, help="Glob patterns to match files to download." - ) - download_parser.add_argument( - "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download." - ) - download_parser.add_argument( - "--cache-dir", type=str, help="Path to the directory where to save the downloaded files." - ) - download_parser.add_argument( - "--local-dir", - type=str, - help=( - "If set, the downloaded file will be placed under this directory. Check out" - " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more" - " details." - ), - ) - download_parser.add_argument( - "--force-download", - action="store_true", - help="If True, the files will be downloaded even if they are already cached.", - ) - download_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - download_parser.add_argument( - "--quiet", - action="store_true", - help="If True, progress bars are disabled and only the path to the download files is printed.", - ) - download_parser.add_argument( - "--max-workers", - type=int, - default=8, - help="Maximum number of workers to use for downloading files. Default is 8.", - ) - download_parser.set_defaults(func=DownloadCommand) - - def __init__(self, args: Namespace) -> None: - self.token = args.token - self.repo_id: str = args.repo_id - self.filenames: List[str] = args.filenames - self.repo_type: str = args.repo_type - self.revision: Optional[str] = args.revision - self.include: Optional[List[str]] = args.include - self.exclude: Optional[List[str]] = args.exclude - self.cache_dir: Optional[str] = args.cache_dir - self.local_dir: Optional[str] = args.local_dir - self.force_download: bool = args.force_download - self.quiet: bool = args.quiet - self.max_workers: int = args.max_workers - - def run(self) -> None: - if self.quiet: - disable_progress_bars() - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - print(self._download()) # Print path to downloaded files - enable_progress_bars() - else: - logging.set_verbosity_info() - print(self._download()) # Print path to downloaded files - logging.set_verbosity_warning() - - def _download(self) -> str: - # Warn user if patterns are ignored - if len(self.filenames) > 0: - if self.include is not None and len(self.include) > 0: - warnings.warn("Ignoring `--include` since filenames have being explicitly set.") - if self.exclude is not None and len(self.exclude) > 0: - warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.") - - # Single file to download: use `hf_hub_download` - if len(self.filenames) == 1: - return hf_hub_download( - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - filename=self.filenames[0], - cache_dir=self.cache_dir, - force_download=self.force_download, - token=self.token, - local_dir=self.local_dir, - library_name="huggingface-cli", - ) - - # Otherwise: use `snapshot_download` to ensure all files comes from same revision - elif len(self.filenames) == 0: - allow_patterns = self.include - ignore_patterns = self.exclude - else: - allow_patterns = self.filenames - ignore_patterns = None - - return snapshot_download( - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - force_download=self.force_download, - cache_dir=self.cache_dir, - token=self.token, - local_dir=self.local_dir, - library_name="huggingface-cli", - max_workers=self.max_workers, - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/hf.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/hf.py deleted file mode 100644 index 2587918b294b427fb8f3e0f990884826b66514a8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/hf.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from argparse import ArgumentParser - -from huggingface_hub.cli.auth import AuthCommands -from huggingface_hub.cli.cache import CacheCommand -from huggingface_hub.cli.download import DownloadCommand -from huggingface_hub.cli.jobs import JobsCommands -from huggingface_hub.cli.lfs import LfsCommands -from huggingface_hub.cli.repo import RepoCommands -from huggingface_hub.cli.repo_files import RepoFilesCommand -from huggingface_hub.cli.system import EnvironmentCommand, VersionCommand -from huggingface_hub.cli.upload import UploadCommand -from huggingface_hub.cli.upload_large_folder import UploadLargeFolderCommand - - -def main(): - parser = ArgumentParser("hf", usage="hf []") - commands_parser = parser.add_subparsers(help="hf command helpers") - - # Register commands - AuthCommands.register_subcommand(commands_parser) - CacheCommand.register_subcommand(commands_parser) - DownloadCommand.register_subcommand(commands_parser) - JobsCommands.register_subcommand(commands_parser) - RepoCommands.register_subcommand(commands_parser) - RepoFilesCommand.register_subcommand(commands_parser) - UploadCommand.register_subcommand(commands_parser) - UploadLargeFolderCommand.register_subcommand(commands_parser) - - # System commands - EnvironmentCommand.register_subcommand(commands_parser) - VersionCommand.register_subcommand(commands_parser) - - # LFS commands (hidden in --help) - LfsCommands.register_subcommand(commands_parser) - - # Let's go - args = parser.parse_args() - if not hasattr(args, "func"): - parser.print_help() - exit(1) - - # Run - service = args.func(args) - if service is not None: - service.run() - - -if __name__ == "__main__": - main() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/jobs.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/jobs.py deleted file mode 100644 index 3a661c7df7d65813dbb1b2a8f449ca8410e320e0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/jobs.py +++ /dev/null @@ -1,1100 +0,0 @@ -# Copyright 2025 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains commands to interact with jobs on the Hugging Face Hub. - -Usage: - # run a job - hf jobs run - - # List running or completed jobs - hf jobs ps [-a] [-f key=value] [--format TEMPLATE] - - # Stream logs from a job - hf jobs logs - - # Inspect detailed information about a job - hf jobs inspect - - # Cancel a running job - hf jobs cancel -""" - -import json -import os -import re -from argparse import Namespace, _SubParsersAction -from dataclasses import asdict -from pathlib import Path -from typing import Dict, List, Optional, Union - -import requests - -from huggingface_hub import HfApi, SpaceHardware, get_token -from huggingface_hub.utils import logging -from huggingface_hub.utils._dotenv import load_dotenv - -from . import BaseHuggingfaceCLICommand - - -logger = logging.get_logger(__name__) - -SUGGESTED_FLAVORS = [item.value for item in SpaceHardware if item.value != "zero-a10g"] - - -class JobsCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - jobs_parser = parser.add_parser("jobs", help="Run and manage Jobs on the Hub.") - jobs_subparsers = jobs_parser.add_subparsers(help="huggingface.co jobs related commands") - - # Show help if no subcommand is provided - jobs_parser.set_defaults(func=lambda args: jobs_parser.print_help()) - - # Register commands - InspectCommand.register_subcommand(jobs_subparsers) - LogsCommand.register_subcommand(jobs_subparsers) - PsCommand.register_subcommand(jobs_subparsers) - RunCommand.register_subcommand(jobs_subparsers) - CancelCommand.register_subcommand(jobs_subparsers) - UvCommand.register_subcommand(jobs_subparsers) - ScheduledJobsCommands.register_subcommand(jobs_subparsers) - - -class RunCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction) -> None: - run_parser = parser.add_parser("run", help="Run a Job") - run_parser.add_argument("image", type=str, help="The Docker image to use.") - run_parser.add_argument("-e", "--env", action="append", help="Set environment variables. E.g. --env ENV=value") - run_parser.add_argument( - "-s", - "--secrets", - action="append", - help=( - "Set secret environment variables. E.g. --secrets SECRET=value " - "or `--secrets HF_TOKEN` to pass your Hugging Face token." - ), - ) - run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.") - run_parser.add_argument("--secrets-file", type=str, help="Read in a file of secret environment variables.") - run_parser.add_argument( - "--flavor", - type=str, - help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.", - ) - run_parser.add_argument( - "--timeout", - type=str, - help="Max duration: int/float with s (seconds, default), m (minutes), h (hours) or d (days).", - ) - run_parser.add_argument( - "-d", - "--detach", - action="store_true", - help="Run the Job in the background and print the Job ID.", - ) - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace where the Job will be created. Defaults to the current user's namespace.", - ) - run_parser.add_argument( - "--token", - type=str, - help="A User Access Token generated from https://huggingface.co/settings/tokens", - ) - run_parser.add_argument("command", nargs="...", help="The command to run.") - run_parser.set_defaults(func=RunCommand) - - def __init__(self, args: Namespace) -> None: - self.image: str = args.image - self.command: List[str] = args.command - self.env: dict[str, Optional[str]] = {} - if args.env_file: - self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy())) - for env_value in args.env or []: - self.env.update(load_dotenv(env_value, environ=os.environ.copy())) - self.secrets: dict[str, Optional[str]] = {} - extended_environ = _get_extended_environ() - if args.secrets_file: - self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ)) - for secret in args.secrets or []: - self.secrets.update(load_dotenv(secret, environ=extended_environ)) - self.flavor: Optional[SpaceHardware] = args.flavor - self.timeout: Optional[str] = args.timeout - self.detach: bool = args.detach - self.namespace: Optional[str] = args.namespace - self.token: Optional[str] = args.token - - def run(self) -> None: - api = HfApi(token=self.token) - job = api.run_job( - image=self.image, - command=self.command, - env=self.env, - secrets=self.secrets, - flavor=self.flavor, - timeout=self.timeout, - namespace=self.namespace, - ) - # Always print the job ID to the user - print(f"Job started with ID: {job.id}") - print(f"View at: {job.url}") - - if self.detach: - return - - # Now let's stream the logs - for log in api.fetch_job_logs(job_id=job.id): - print(log) - - -class LogsCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction) -> None: - run_parser = parser.add_parser("logs", help="Fetch the logs of a Job") - run_parser.add_argument("job_id", type=str, help="Job ID") - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace where the job is running. Defaults to the current user's namespace.", - ) - run_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - run_parser.set_defaults(func=LogsCommand) - - def __init__(self, args: Namespace) -> None: - self.job_id: str = args.job_id - self.namespace: Optional[str] = args.namespace - self.token: Optional[str] = args.token - - def run(self) -> None: - api = HfApi(token=self.token) - for log in api.fetch_job_logs(job_id=self.job_id, namespace=self.namespace): - print(log) - - -def _tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str: - """ - Inspired by: - - - stackoverflow.com/a/8356620/593036 - - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data - """ - col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] - terminal_width = max(os.get_terminal_size().columns, len(headers) * 12) - while len(headers) + sum(col_widths) > terminal_width: - col_to_minimize = col_widths.index(max(col_widths)) - col_widths[col_to_minimize] //= 2 - if len(headers) + sum(col_widths) <= terminal_width: - col_widths[col_to_minimize] = terminal_width - sum(col_widths) - len(headers) + col_widths[col_to_minimize] - row_format = ("{{:{}}} " * len(headers)).format(*col_widths) - lines = [] - lines.append(row_format.format(*headers)) - lines.append(row_format.format(*["-" * w for w in col_widths])) - for row in rows: - row_format_args = [ - str(x)[: col_width - 3] + "..." if len(str(x)) > col_width else str(x) - for x, col_width in zip(row, col_widths) - ] - lines.append(row_format.format(*row_format_args)) - return "\n".join(lines) - - -class PsCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction) -> None: - run_parser = parser.add_parser("ps", help="List Jobs") - run_parser.add_argument( - "-a", - "--all", - action="store_true", - help="Show all Jobs (default shows just running)", - ) - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace from where it lists the jobs. Defaults to the current user's namespace.", - ) - run_parser.add_argument( - "--token", - type=str, - help="A User Access Token generated from https://huggingface.co/settings/tokens", - ) - # Add Docker-style filtering argument - run_parser.add_argument( - "-f", - "--filter", - action="append", - default=[], - help="Filter output based on conditions provided (format: key=value)", - ) - # Add option to format output - run_parser.add_argument( - "--format", - type=str, - help="Format output using a custom template", - ) - run_parser.set_defaults(func=PsCommand) - - def __init__(self, args: Namespace) -> None: - self.all: bool = args.all - self.namespace: Optional[str] = args.namespace - self.token: Optional[str] = args.token - self.format: Optional[str] = args.format - self.filters: Dict[str, str] = {} - - # Parse filter arguments (key=value pairs) - for f in args.filter: - if "=" in f: - key, value = f.split("=", 1) - self.filters[key.lower()] = value - else: - print(f"Warning: Ignoring invalid filter format '{f}'. Use key=value format.") - - def run(self) -> None: - """ - Fetch and display job information for the current user. - Uses Docker-style filtering with -f/--filter flag and key=value pairs. - """ - try: - api = HfApi(token=self.token) - - # Fetch jobs data - jobs = api.list_jobs(namespace=self.namespace) - - # Define table headers - table_headers = ["JOB ID", "IMAGE/SPACE", "COMMAND", "CREATED", "STATUS"] - - # Process jobs data - rows = [] - - for job in jobs: - # Extract job data for filtering - status = job.status.stage if job.status else "UNKNOWN" - - # Skip job if not all jobs should be shown and status doesn't match criteria - if not self.all and status not in ("RUNNING", "UPDATING"): - continue - - # Extract job ID - job_id = job.id - - # Extract image or space information - image_or_space = job.docker_image or "N/A" - - # Extract and format command - command = job.command or [] - command_str = " ".join(command) if command else "N/A" - - # Extract creation time - created_at = job.created_at.strftime("%Y-%m-%d %H:%M:%S") if job.created_at else "N/A" - - # Create a dict with all job properties for filtering - job_properties = { - "id": job_id, - "image": image_or_space, - "status": status.lower(), - "command": command_str, - } - - # Check if job matches all filters - if not self._matches_filters(job_properties): - continue - - # Create row - rows.append([job_id, image_or_space, command_str, created_at, status]) - - # Handle empty results - if not rows: - filters_msg = "" - if self.filters: - filters_msg = f" matching filters: {', '.join([f'{k}={v}' for k, v in self.filters.items()])}" - - print(f"No jobs found{filters_msg}") - return - - # Apply custom format if provided or use default tabular format - self._print_output(rows, table_headers) - - except requests.RequestException as e: - print(f"Error fetching jobs data: {e}") - except (KeyError, ValueError, TypeError) as e: - print(f"Error processing jobs data: {e}") - except Exception as e: - print(f"Unexpected error - {type(e).__name__}: {e}") - - def _matches_filters(self, job_properties: Dict[str, str]) -> bool: - """Check if job matches all specified filters.""" - for key, pattern in self.filters.items(): - # Check if property exists - if key not in job_properties: - return False - - # Support pattern matching with wildcards - if "*" in pattern or "?" in pattern: - # Convert glob pattern to regex - regex_pattern = pattern.replace("*", ".*").replace("?", ".") - if not re.search(f"^{regex_pattern}$", job_properties[key], re.IGNORECASE): - return False - # Simple substring matching - elif pattern.lower() not in job_properties[key].lower(): - return False - - return True - - def _print_output(self, rows, headers): - """Print output according to the chosen format.""" - if self.format: - # Custom template formatting (simplified) - template = self.format - for row in rows: - line = template - for i, field in enumerate(["id", "image", "command", "created", "status"]): - placeholder = f"{{{{.{field}}}}}" - if placeholder in line: - line = line.replace(placeholder, str(row[i])) - print(line) - else: - # Default tabular format - print( - _tabulate( - rows, - headers=headers, - ) - ) - - -class InspectCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction) -> None: - run_parser = parser.add_parser("inspect", help="Display detailed information on one or more Jobs") - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace where the job is running. Defaults to the current user's namespace.", - ) - run_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - run_parser.add_argument("job_ids", nargs="...", help="The jobs to inspect") - run_parser.set_defaults(func=InspectCommand) - - def __init__(self, args: Namespace) -> None: - self.namespace: Optional[str] = args.namespace - self.token: Optional[str] = args.token - self.job_ids: List[str] = args.job_ids - - def run(self) -> None: - api = HfApi(token=self.token) - jobs = [api.inspect_job(job_id=job_id, namespace=self.namespace) for job_id in self.job_ids] - print(json.dumps([asdict(job) for job in jobs], indent=4, default=str)) - - -class CancelCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction) -> None: - run_parser = parser.add_parser("cancel", help="Cancel a Job") - run_parser.add_argument("job_id", type=str, help="Job ID") - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace where the job is running. Defaults to the current user's namespace.", - ) - run_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - run_parser.set_defaults(func=CancelCommand) - - def __init__(self, args: Namespace) -> None: - self.job_id: str = args.job_id - self.namespace = args.namespace - self.token: Optional[str] = args.token - - def run(self) -> None: - api = HfApi(token=self.token) - api.cancel_job(job_id=self.job_id, namespace=self.namespace) - - -class UvCommand(BaseHuggingfaceCLICommand): - """Run UV scripts on Hugging Face infrastructure.""" - - @staticmethod - def register_subcommand(parser): - """Register UV run subcommand.""" - uv_parser = parser.add_parser( - "uv", - help="Run UV scripts (Python with inline dependencies) on HF infrastructure", - ) - - subparsers = uv_parser.add_subparsers(dest="uv_command", help="UV commands", required=True) - - # Run command only - run_parser = subparsers.add_parser( - "run", - help="Run a UV script (local file or URL) on HF infrastructure", - ) - run_parser.add_argument("script", help="UV script to run (local file or URL)") - run_parser.add_argument("script_args", nargs="...", help="Arguments for the script", default=[]) - run_parser.add_argument("--image", type=str, help="Use a custom Docker image with `uv` installed.") - run_parser.add_argument( - "--repo", - help="Repository name for the script (creates ephemeral if not specified)", - ) - run_parser.add_argument( - "--flavor", - type=str, - help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.", - ) - run_parser.add_argument("-e", "--env", action="append", help="Environment variables") - run_parser.add_argument( - "-s", - "--secrets", - action="append", - help=( - "Set secret environment variables. E.g. --secrets SECRET=value " - "or `--secrets HF_TOKEN` to pass your Hugging Face token." - ), - ) - run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.") - run_parser.add_argument( - "--secrets-file", - type=str, - help="Read in a file of secret environment variables.", - ) - run_parser.add_argument("--timeout", type=str, help="Max duration (e.g., 30s, 5m, 1h)") - run_parser.add_argument("-d", "--detach", action="store_true", help="Run in background") - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace where the Job will be created. Defaults to the current user's namespace.", - ) - run_parser.add_argument("--token", type=str, help="HF token") - # UV options - run_parser.add_argument("--with", action="append", help="Run with the given packages installed", dest="with_") - run_parser.add_argument( - "-p", "--python", type=str, help="The Python interpreter to use for the run environment" - ) - run_parser.set_defaults(func=UvCommand) - - def __init__(self, args: Namespace) -> None: - """Initialize the command with parsed arguments.""" - self.script = args.script - self.script_args = args.script_args - self.dependencies = args.with_ - self.python = args.python - self.image = args.image - self.env: dict[str, Optional[str]] = {} - if args.env_file: - self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy())) - for env_value in args.env or []: - self.env.update(load_dotenv(env_value, environ=os.environ.copy())) - self.secrets: dict[str, Optional[str]] = {} - extended_environ = _get_extended_environ() - if args.secrets_file: - self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ)) - for secret in args.secrets or []: - self.secrets.update(load_dotenv(secret, environ=extended_environ)) - self.flavor: Optional[SpaceHardware] = args.flavor - self.timeout: Optional[str] = args.timeout - self.detach: bool = args.detach - self.namespace: Optional[str] = args.namespace - self.token: Optional[str] = args.token - self._repo = args.repo - - def run(self) -> None: - """Execute UV command.""" - logging.set_verbosity(logging.INFO) - api = HfApi(token=self.token) - job = api.run_uv_job( - script=self.script, - script_args=self.script_args, - dependencies=self.dependencies, - python=self.python, - image=self.image, - env=self.env, - secrets=self.secrets, - flavor=self.flavor, - timeout=self.timeout, - namespace=self.namespace, - _repo=self._repo, - ) - - # Always print the job ID to the user - print(f"Job started with ID: {job.id}") - print(f"View at: {job.url}") - - if self.detach: - return - - # Now let's stream the logs - for log in api.fetch_job_logs(job_id=job.id): - print(log) - - -def _get_extended_environ() -> Dict[str, str]: - extended_environ = os.environ.copy() - if (token := get_token()) is not None: - extended_environ["HF_TOKEN"] = token - return extended_environ - - -class ScheduledJobsCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - scheduled_jobs_parser = parser.add_parser("scheduled", help="Create and manage scheduled Jobs on the Hub.") - scheduled_jobs_subparsers = scheduled_jobs_parser.add_subparsers( - help="huggingface.co scheduled jobs related commands" - ) - - # Show help if no subcommand is provided - scheduled_jobs_parser.set_defaults(func=lambda args: scheduled_jobs_subparsers.print_help()) - - # Register commands - ScheduledRunCommand.register_subcommand(scheduled_jobs_subparsers) - ScheduledPsCommand.register_subcommand(scheduled_jobs_subparsers) - ScheduledInspectCommand.register_subcommand(scheduled_jobs_subparsers) - ScheduledDeleteCommand.register_subcommand(scheduled_jobs_subparsers) - ScheduledSuspendCommand.register_subcommand(scheduled_jobs_subparsers) - ScheduledResumeCommand.register_subcommand(scheduled_jobs_subparsers) - ScheduledUvCommand.register_subcommand(scheduled_jobs_subparsers) - - -class ScheduledRunCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction) -> None: - run_parser = parser.add_parser("run", help="Schedule a Job") - run_parser.add_argument( - "schedule", - type=str, - help="One of annually, yearly, monthly, weekly, daily, hourly, or a CRON schedule expression.", - ) - run_parser.add_argument("image", type=str, help="The Docker image to use.") - run_parser.add_argument( - "--suspend", - action="store_true", - help="Suspend (pause) the scheduled Job", - default=None, - ) - run_parser.add_argument( - "--concurrency", - action="store_true", - help="Allow multiple instances of this Job to run concurrently", - default=None, - ) - run_parser.add_argument("-e", "--env", action="append", help="Set environment variables. E.g. --env ENV=value") - run_parser.add_argument( - "-s", - "--secrets", - action="append", - help=( - "Set secret environment variables. E.g. --secrets SECRET=value " - "or `--secrets HF_TOKEN` to pass your Hugging Face token." - ), - ) - run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.") - run_parser.add_argument("--secrets-file", type=str, help="Read in a file of secret environment variables.") - run_parser.add_argument( - "--flavor", - type=str, - help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.", - ) - run_parser.add_argument( - "--timeout", - type=str, - help="Max duration: int/float with s (seconds, default), m (minutes), h (hours) or d (days).", - ) - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace where the scheduled Job will be created. Defaults to the current user's namespace.", - ) - run_parser.add_argument( - "--token", - type=str, - help="A User Access Token generated from https://huggingface.co/settings/tokens", - ) - run_parser.add_argument("command", nargs="...", help="The command to run.") - run_parser.set_defaults(func=ScheduledRunCommand) - - def __init__(self, args: Namespace) -> None: - self.schedule: str = args.schedule - self.image: str = args.image - self.command: List[str] = args.command - self.suspend: Optional[bool] = args.suspend - self.concurrency: Optional[bool] = args.concurrency - self.env: dict[str, Optional[str]] = {} - if args.env_file: - self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy())) - for env_value in args.env or []: - self.env.update(load_dotenv(env_value, environ=os.environ.copy())) - self.secrets: dict[str, Optional[str]] = {} - extended_environ = _get_extended_environ() - if args.secrets_file: - self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ)) - for secret in args.secrets or []: - self.secrets.update(load_dotenv(secret, environ=extended_environ)) - self.flavor: Optional[SpaceHardware] = args.flavor - self.timeout: Optional[str] = args.timeout - self.namespace: Optional[str] = args.namespace - self.token: Optional[str] = args.token - - def run(self) -> None: - api = HfApi(token=self.token) - scheduled_job = api.create_scheduled_job( - image=self.image, - command=self.command, - schedule=self.schedule, - suspend=self.suspend, - concurrency=self.concurrency, - env=self.env, - secrets=self.secrets, - flavor=self.flavor, - timeout=self.timeout, - namespace=self.namespace, - ) - # Always print the scheduled job ID to the user - print(f"Scheduled Job created with ID: {scheduled_job.id}") - - -class ScheduledPsCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction) -> None: - run_parser = parser.add_parser("ps", help="List scheduled Jobs") - run_parser.add_argument( - "-a", - "--all", - action="store_true", - help="Show all scheduled Jobs (default hides suspended)", - ) - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace from where it lists the jobs. Defaults to the current user's namespace.", - ) - run_parser.add_argument( - "--token", - type=str, - help="A User Access Token generated from https://huggingface.co/settings/tokens", - ) - # Add Docker-style filtering argument - run_parser.add_argument( - "-f", - "--filter", - action="append", - default=[], - help="Filter output based on conditions provided (format: key=value)", - ) - # Add option to format output - run_parser.add_argument( - "--format", - type=str, - help="Format output using a custom template", - ) - run_parser.set_defaults(func=ScheduledPsCommand) - - def __init__(self, args: Namespace) -> None: - self.all: bool = args.all - self.namespace: Optional[str] = args.namespace - self.token: Optional[str] = args.token - self.format: Optional[str] = args.format - self.filters: Dict[str, str] = {} - - # Parse filter arguments (key=value pairs) - for f in args.filter: - if "=" in f: - key, value = f.split("=", 1) - self.filters[key.lower()] = value - else: - print(f"Warning: Ignoring invalid filter format '{f}'. Use key=value format.") - - def run(self) -> None: - """ - Fetch and display scheduked job information for the current user. - Uses Docker-style filtering with -f/--filter flag and key=value pairs. - """ - try: - api = HfApi(token=self.token) - - # Fetch jobs data - scheduled_jobs = api.list_scheduled_jobs(namespace=self.namespace) - - # Define table headers - table_headers = [ - "ID", - "SCHEDULE", - "IMAGE/SPACE", - "COMMAND", - "LAST RUN", - "NEXT RUN", - "SUSPEND", - ] - - # Process jobs data - rows = [] - - for scheduled_job in scheduled_jobs: - # Extract job data for filtering - suspend = scheduled_job.suspend - - # Skip job if not all jobs should be shown and status doesn't match criteria - if not self.all and suspend: - continue - - # Extract job ID - scheduled_job_id = scheduled_job.id - - # Extract schedule - schedule = scheduled_job.schedule - - # Extract image or space information - image_or_space = scheduled_job.job_spec.docker_image or "N/A" - - # Extract and format command - command = scheduled_job.job_spec.command or [] - command_str = " ".join(command) if command else "N/A" - - # Extract status - last_job_at = ( - scheduled_job.status.last_job.at.strftime("%Y-%m-%d %H:%M:%S") - if scheduled_job.status.last_job - else "N/A" - ) - next_job_run_at = ( - scheduled_job.status.next_job_run_at.strftime("%Y-%m-%d %H:%M:%S") - if scheduled_job.status.next_job_run_at - else "N/A" - ) - - # Create a dict with all job properties for filtering - job_properties = { - "id": scheduled_job_id, - "image": image_or_space, - "suspend": str(suspend), - "command": command_str, - } - - # Check if job matches all filters - if not self._matches_filters(job_properties): - continue - - # Create row - rows.append( - [ - scheduled_job_id, - schedule, - image_or_space, - command_str, - last_job_at, - next_job_run_at, - suspend, - ] - ) - - # Handle empty results - if not rows: - filters_msg = "" - if self.filters: - filters_msg = f" matching filters: {', '.join([f'{k}={v}' for k, v in self.filters.items()])}" - - print(f"No scheduled jobs found{filters_msg}") - return - - # Apply custom format if provided or use default tabular format - self._print_output(rows, table_headers) - - except requests.RequestException as e: - print(f"Error fetching scheduled jobs data: {e}") - except (KeyError, ValueError, TypeError) as e: - print(f"Error processing scheduled jobs data: {e}") - except Exception as e: - print(f"Unexpected error - {type(e).__name__}: {e}") - - def _matches_filters(self, job_properties: Dict[str, str]) -> bool: - """Check if scheduled job matches all specified filters.""" - for key, pattern in self.filters.items(): - # Check if property exists - if key not in job_properties: - return False - - # Support pattern matching with wildcards - if "*" in pattern or "?" in pattern: - # Convert glob pattern to regex - regex_pattern = pattern.replace("*", ".*").replace("?", ".") - if not re.search(f"^{regex_pattern}$", job_properties[key], re.IGNORECASE): - return False - # Simple substring matching - elif pattern.lower() not in job_properties[key].lower(): - return False - - return True - - def _print_output(self, rows, headers): - """Print output according to the chosen format.""" - if self.format: - # Custom template formatting (simplified) - template = self.format - for row in rows: - line = template - for i, field in enumerate( - ["id", "schedule", "image", "command", "last_job_at", "next_job_run_at", "suspend"] - ): - placeholder = f"{{{{.{field}}}}}" - if placeholder in line: - line = line.replace(placeholder, str(row[i])) - print(line) - else: - # Default tabular format - print( - _tabulate( - rows, - headers=headers, - ) - ) - - -class ScheduledInspectCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction) -> None: - run_parser = parser.add_parser("inspect", help="Display detailed information on one or more scheduled Jobs") - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace where the scheduled job is. Defaults to the current user's namespace.", - ) - run_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - run_parser.add_argument("scheduled_job_ids", nargs="...", help="The scheduled jobs to inspect") - run_parser.set_defaults(func=ScheduledInspectCommand) - - def __init__(self, args: Namespace) -> None: - self.namespace: Optional[str] = args.namespace - self.token: Optional[str] = args.token - self.scheduled_job_ids: List[str] = args.scheduled_job_ids - - def run(self) -> None: - api = HfApi(token=self.token) - scheduled_jobs = [ - api.inspect_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=self.namespace) - for scheduled_job_id in self.scheduled_job_ids - ] - print(json.dumps([asdict(scheduled_job) for scheduled_job in scheduled_jobs], indent=4, default=str)) - - -class ScheduledDeleteCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction) -> None: - run_parser = parser.add_parser("delete", help="Delete a scheduled Job") - run_parser.add_argument("scheduled_job_id", type=str, help="Scheduled Job ID") - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace where the scheduled job is. Defaults to the current user's namespace.", - ) - run_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - run_parser.set_defaults(func=ScheduledDeleteCommand) - - def __init__(self, args: Namespace) -> None: - self.scheduled_job_id: str = args.scheduled_job_id - self.namespace = args.namespace - self.token: Optional[str] = args.token - - def run(self) -> None: - api = HfApi(token=self.token) - api.delete_scheduled_job(scheduled_job_id=self.scheduled_job_id, namespace=self.namespace) - - -class ScheduledSuspendCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction) -> None: - run_parser = parser.add_parser("suspend", help="Suspend (pause) a scheduled Job") - run_parser.add_argument("scheduled_job_id", type=str, help="Scheduled Job ID") - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace where the scheduled job is. Defaults to the current user's namespace.", - ) - run_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - run_parser.set_defaults(func=ScheduledSuspendCommand) - - def __init__(self, args: Namespace) -> None: - self.scheduled_job_id: str = args.scheduled_job_id - self.namespace = args.namespace - self.token: Optional[str] = args.token - - def run(self) -> None: - api = HfApi(token=self.token) - api.suspend_scheduled_job(scheduled_job_id=self.scheduled_job_id, namespace=self.namespace) - - -class ScheduledResumeCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction) -> None: - run_parser = parser.add_parser("resume", help="Resume (unpause) a scheduled Job") - run_parser.add_argument("scheduled_job_id", type=str, help="Scheduled Job ID") - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace where the scheduled job is. Defaults to the current user's namespace.", - ) - run_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - run_parser.set_defaults(func=ScheduledResumeCommand) - - def __init__(self, args: Namespace) -> None: - self.scheduled_job_id: str = args.scheduled_job_id - self.namespace = args.namespace - self.token: Optional[str] = args.token - - def run(self) -> None: - api = HfApi(token=self.token) - api.resume_scheduled_job(scheduled_job_id=self.scheduled_job_id, namespace=self.namespace) - - -class ScheduledUvCommand(BaseHuggingfaceCLICommand): - """Schedule UV scripts on Hugging Face infrastructure.""" - - @staticmethod - def register_subcommand(parser): - """Register UV run subcommand.""" - uv_parser = parser.add_parser( - "uv", - help="Schedule UV scripts (Python with inline dependencies) on HF infrastructure", - ) - - subparsers = uv_parser.add_subparsers(dest="uv_command", help="UV commands", required=True) - - # Run command only - run_parser = subparsers.add_parser( - "run", - help="Run a UV script (local file or URL) on HF infrastructure", - ) - run_parser.add_argument( - "schedule", - type=str, - help="One of annually, yearly, monthly, weekly, daily, hourly, or a CRON schedule expression.", - ) - run_parser.add_argument("script", help="UV script to run (local file or URL)") - run_parser.add_argument("script_args", nargs="...", help="Arguments for the script", default=[]) - run_parser.add_argument( - "--suspend", - action="store_true", - help="Suspend (pause) the scheduled Job", - default=None, - ) - run_parser.add_argument( - "--concurrency", - action="store_true", - help="Allow multiple instances of this Job to run concurrently", - default=None, - ) - run_parser.add_argument("--image", type=str, help="Use a custom Docker image with `uv` installed.") - run_parser.add_argument( - "--repo", - help="Repository name for the script (creates ephemeral if not specified)", - ) - run_parser.add_argument( - "--flavor", - type=str, - help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.", - ) - run_parser.add_argument("-e", "--env", action="append", help="Environment variables") - run_parser.add_argument( - "-s", - "--secrets", - action="append", - help=( - "Set secret environment variables. E.g. --secrets SECRET=value " - "or `--secrets HF_TOKEN` to pass your Hugging Face token." - ), - ) - run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.") - run_parser.add_argument( - "--secrets-file", - type=str, - help="Read in a file of secret environment variables.", - ) - run_parser.add_argument("--timeout", type=str, help="Max duration (e.g., 30s, 5m, 1h)") - run_parser.add_argument("-d", "--detach", action="store_true", help="Run in background") - run_parser.add_argument( - "--namespace", - type=str, - help="The namespace where the Job will be created. Defaults to the current user's namespace.", - ) - run_parser.add_argument("--token", type=str, help="HF token") - # UV options - run_parser.add_argument("--with", action="append", help="Run with the given packages installed", dest="with_") - run_parser.add_argument( - "-p", "--python", type=str, help="The Python interpreter to use for the run environment" - ) - run_parser.set_defaults(func=ScheduledUvCommand) - - def __init__(self, args: Namespace) -> None: - """Initialize the command with parsed arguments.""" - self.schedule: str = args.schedule - self.script = args.script - self.script_args = args.script_args - self.suspend: Optional[bool] = args.suspend - self.concurrency: Optional[bool] = args.concurrency - self.dependencies = args.with_ - self.python = args.python - self.image = args.image - self.env: dict[str, Optional[str]] = {} - if args.env_file: - self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy())) - for env_value in args.env or []: - self.env.update(load_dotenv(env_value, environ=os.environ.copy())) - self.secrets: dict[str, Optional[str]] = {} - extended_environ = _get_extended_environ() - if args.secrets_file: - self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ)) - for secret in args.secrets or []: - self.secrets.update(load_dotenv(secret, environ=extended_environ)) - self.flavor: Optional[SpaceHardware] = args.flavor - self.timeout: Optional[str] = args.timeout - self.detach: bool = args.detach - self.namespace: Optional[str] = args.namespace - self.token: Optional[str] = args.token - self._repo = args.repo - - def run(self) -> None: - """Schedule UV command.""" - logging.set_verbosity(logging.INFO) - api = HfApi(token=self.token) - job = api.create_scheduled_uv_job( - script=self.script, - script_args=self.script_args, - schedule=self.schedule, - suspend=self.suspend, - concurrency=self.concurrency, - dependencies=self.dependencies, - python=self.python, - image=self.image, - env=self.env, - secrets=self.secrets, - flavor=self.flavor, - timeout=self.timeout, - namespace=self.namespace, - _repo=self._repo, - ) - - # Always print the job ID to the user - print(f"Scheduled Job created with ID: {job.id}") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/lfs.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/lfs.py deleted file mode 100644 index e4c5b900c816494c260f6c440843a2d83703fab5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/lfs.py +++ /dev/null @@ -1,198 +0,0 @@ -""" -Implementation of a custom transfer agent for the transfer type "multipart" for -git-lfs. - -Inspired by: -github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py - -Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md - - -To launch debugger while developing: - -``` [lfs "customtransfer.multipart"] -path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678 ---wait-for-client -/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py -lfs-multipart-upload ```""" - -import json -import os -import subprocess -import sys -from argparse import _SubParsersAction -from typing import Dict, List, Optional - -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND - -from ..utils import get_session, hf_raise_for_status, logging -from ..utils._lfs import SliceFileObj - - -logger = logging.get_logger(__name__) - - -class LfsCommands(BaseHuggingfaceCLICommand): - """ - Implementation of a custom transfer agent for the transfer type "multipart" - for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom - transfer agent is: - https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md - - This introduces two commands to the CLI: - - 1. $ hf lfs-enable-largefiles - - This should be executed once for each model repo that contains a model file - >5GB. It's documented in the error message you get if you just try to git - push a 5GB file without having enabled it before. - - 2. $ hf lfs-multipart-upload - - This command is called by lfs directly and is not meant to be called by the - user. - """ - - @staticmethod - def register_subcommand(parser: _SubParsersAction): - enable_parser = parser.add_parser("lfs-enable-largefiles", add_help=False) - enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.") - enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args)) - - # Command will get called by git-lfs, do not call it directly. - upload_parser = parser.add_parser(LFS_MULTIPART_UPLOAD_COMMAND, add_help=False) - upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args)) - - -class LfsEnableCommand: - def __init__(self, args): - self.args = args - - def run(self): - local_path = os.path.abspath(self.args.path) - if not os.path.isdir(local_path): - print("This does not look like a valid git repo.") - exit(1) - subprocess.run( - "git config lfs.customtransfer.multipart.path hf".split(), - check=True, - cwd=local_path, - ) - subprocess.run( - f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(), - check=True, - cwd=local_path, - ) - print("Local repo set up for largefiles") - - -def write_msg(msg: Dict): - """Write out the message in Line delimited JSON.""" - msg_str = json.dumps(msg) + "\n" - sys.stdout.write(msg_str) - sys.stdout.flush() - - -def read_msg() -> Optional[Dict]: - """Read Line delimited JSON from stdin.""" - msg = json.loads(sys.stdin.readline().strip()) - - if "terminate" in (msg.get("type"), msg.get("event")): - # terminate message received - return None - - if msg.get("event") not in ("download", "upload"): - logger.critical("Received unexpected message") - sys.exit(1) - - return msg - - -class LfsUploadCommand: - def __init__(self, args) -> None: - self.args = args - - def run(self) -> None: - # Immediately after invoking a custom transfer process, git-lfs - # sends initiation data to the process over stdin. - # This tells the process useful information about the configuration. - init_msg = json.loads(sys.stdin.readline().strip()) - if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"): - write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}}) - sys.exit(1) - - # The transfer process should use the information it needs from the - # initiation structure, and also perform any one-off setup tasks it - # needs to do. It should then respond on stdout with a simple empty - # confirmation structure, as follows: - write_msg({}) - - # After the initiation exchange, git-lfs will send any number of - # transfer requests to the stdin of the transfer process, in a serial sequence. - while True: - msg = read_msg() - if msg is None: - # When all transfers have been processed, git-lfs will send - # a terminate event to the stdin of the transfer process. - # On receiving this message the transfer process should - # clean up and terminate. No response is expected. - sys.exit(0) - - oid = msg["oid"] - filepath = msg["path"] - completion_url = msg["action"]["href"] - header = msg["action"]["header"] - chunk_size = int(header.pop("chunk_size")) - presigned_urls: List[str] = list(header.values()) - - # Send a "started" progress event to allow other workers to start. - # Otherwise they're delayed until first "progress" event is reported, - # i.e. after the first 5GB by default (!) - write_msg( - { - "event": "progress", - "oid": oid, - "bytesSoFar": 1, - "bytesSinceLast": 0, - } - ) - - parts = [] - with open(filepath, "rb") as file: - for i, presigned_url in enumerate(presigned_urls): - with SliceFileObj( - file, - seek_from=i * chunk_size, - read_limit=chunk_size, - ) as data: - r = get_session().put(presigned_url, data=data) - hf_raise_for_status(r) - parts.append( - { - "etag": r.headers.get("etag"), - "partNumber": i + 1, - } - ) - # In order to support progress reporting while data is uploading / downloading, - # the transfer process should post messages to stdout - write_msg( - { - "event": "progress", - "oid": oid, - "bytesSoFar": (i + 1) * chunk_size, - "bytesSinceLast": chunk_size, - } - ) - # Not precise but that's ok. - - r = get_session().post( - completion_url, - json={ - "oid": oid, - "parts": parts, - }, - ) - hf_raise_for_status(r) - - write_msg({"event": "complete", "oid": oid}) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/repo.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/repo.py deleted file mode 100644 index ef0e3313580e3753a2617745e15762933229b15f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/repo.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright 2025 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains commands to interact with repositories on the Hugging Face Hub. - -Usage: - # create a new dataset repo on the Hub - hf repo create my-cool-dataset --repo-type=dataset - - # create a private model repo on the Hub - hf repo create my-cool-model --private -""" - -import argparse -from argparse import _SubParsersAction -from typing import Optional - -from requests.exceptions import HTTPError - -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.commands._cli_utils import ANSI -from huggingface_hub.constants import REPO_TYPES, SPACES_SDK_TYPES -from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError -from huggingface_hub.hf_api import HfApi -from huggingface_hub.utils import logging - - -logger = logging.get_logger(__name__) - - -class RepoCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - repo_parser = parser.add_parser("repo", help="Manage repos on the Hub.") - repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands") - - # Show help if no subcommand is provided - repo_parser.set_defaults(func=lambda args: repo_parser.print_help()) - - # CREATE - repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co") - repo_create_parser.add_argument( - "repo_id", - type=str, - help="The ID of the repo to create to (e.g. `username/repo-name`). The username is optional and will be set to your username if not provided.", - ) - repo_create_parser.add_argument( - "--repo-type", - type=str, - help='Optional: set to "dataset" or "space" if creating a dataset or space, default is model.', - ) - repo_create_parser.add_argument( - "--space_sdk", - type=str, - help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".', - choices=SPACES_SDK_TYPES, - ) - repo_create_parser.add_argument( - "--private", - action="store_true", - help="Whether to create a private repository. Defaults to public unless the organization's default is private.", - ) - repo_create_parser.add_argument( - "--token", - type=str, - help="Hugging Face token. Will default to the locally saved token if not provided.", - ) - repo_create_parser.add_argument( - "--exist-ok", - action="store_true", - help="Do not raise an error if repo already exists.", - ) - repo_create_parser.add_argument( - "--resource-group-id", - type=str, - help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.", - ) - repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args)) - - # TAG SUBCOMMANDS - repo_tag_parser = repo_subparsers.add_parser("tag", help="Manage tags for a repo on the Hub.") - tag_subparsers = repo_tag_parser.add_subparsers(help="Tag actions", dest="tag_action", required=True) - - # tag create - tag_create_parser = tag_subparsers.add_parser("create", help="Create a tag for a repo.") - tag_create_parser.add_argument( - "repo_id", type=str, help="The ID of the repo to tag (e.g. `username/repo-name`)." - ) - tag_create_parser.add_argument("tag", type=str, help="The name of the tag to create.") - tag_create_parser.add_argument("-m", "--message", type=str, help="The description of the tag to create.") - tag_create_parser.add_argument("--revision", type=str, help="The git revision to tag.") - tag_create_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens." - ) - tag_create_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Set the type of repository (model, dataset, or space).", - ) - tag_create_parser.set_defaults(func=lambda args: RepoTagCreateCommand(args)) - - # tag list - tag_list_parser = tag_subparsers.add_parser("list", help="List tags for a repo.") - tag_list_parser.add_argument( - "repo_id", type=str, help="The ID of the repo to list tags for (e.g. `username/repo-name`)." - ) - tag_list_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens." - ) - tag_list_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Set the type of repository (model, dataset, or space).", - ) - tag_list_parser.set_defaults(func=lambda args: RepoTagListCommand(args)) - - # tag delete - tag_delete_parser = tag_subparsers.add_parser("delete", help="Delete a tag from a repo.") - tag_delete_parser.add_argument( - "repo_id", type=str, help="The ID of the repo to delete the tag from (e.g. `username/repo-name`)." - ) - tag_delete_parser.add_argument("tag", type=str, help="The name of the tag to delete.") - tag_delete_parser.add_argument("-y", "--yes", action="store_true", help="Answer Yes to prompts automatically.") - tag_delete_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens." - ) - tag_delete_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Set the type of repository (model, dataset, or space).", - ) - tag_delete_parser.set_defaults(func=lambda args: RepoTagDeleteCommand(args)) - - -class RepoCreateCommand: - def __init__(self, args: argparse.Namespace): - self.repo_id: str = args.repo_id - self.repo_type: Optional[str] = args.repo_type - self.space_sdk: Optional[str] = args.space_sdk - self.private: bool = args.private - self.token: Optional[str] = args.token - self.exist_ok: bool = args.exist_ok - self.resource_group_id: Optional[str] = args.resource_group_id - self._api = HfApi() - - def run(self): - repo_url = self._api.create_repo( - repo_id=self.repo_id, - repo_type=self.repo_type, - private=self.private, - token=self.token, - exist_ok=self.exist_ok, - resource_group_id=self.resource_group_id, - space_sdk=self.space_sdk, - ) - print(f"Successfully created {ANSI.bold(repo_url.repo_id)} on the Hub.") - print(f"Your repo is now available at {ANSI.bold(repo_url)}") - - -class RepoTagCommand: - def __init__(self, args): - self.args = args - self.api = HfApi(token=getattr(args, "token", None)) - self.repo_id = args.repo_id - self.repo_type = getattr(args, "repo_type", "model") - if self.repo_type not in REPO_TYPES: - print("Invalid repo --repo-type") - exit(1) - - -class RepoTagCreateCommand(RepoTagCommand): - def run(self): - print( - f"You are about to create tag {ANSI.bold(str(self.args.tag))} on {self.repo_type} {ANSI.bold(self.repo_id)}" - ) - try: - self.api.create_tag( - repo_id=self.repo_id, - tag=self.args.tag, - tag_message=getattr(self.args, "message", None), - revision=getattr(self.args, "revision", None), - repo_type=self.repo_type, - ) - except RepositoryNotFoundError: - print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") - exit(1) - except RevisionNotFoundError: - print(f"Revision {ANSI.bold(str(getattr(self.args, 'revision', None)))} not found.") - exit(1) - except HfHubHTTPError as e: - if e.response.status_code == 409: - print(f"Tag {ANSI.bold(str(self.args.tag))} already exists on {ANSI.bold(self.repo_id)}") - exit(1) - raise e - print(f"Tag {ANSI.bold(str(self.args.tag))} created on {ANSI.bold(self.repo_id)}") - - -class RepoTagListCommand(RepoTagCommand): - def run(self): - try: - refs = self.api.list_repo_refs( - repo_id=self.repo_id, - repo_type=self.repo_type, - ) - except RepositoryNotFoundError: - print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") - exit(1) - except HTTPError as e: - print(e) - print(ANSI.red(e.response.text)) - exit(1) - if len(refs.tags) == 0: - print("No tags found") - exit(0) - print(f"Tags for {self.repo_type} {ANSI.bold(self.repo_id)}:") - for tag in refs.tags: - print(tag.name) - - -class RepoTagDeleteCommand(RepoTagCommand): - def run(self): - print(f"You are about to delete tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}") - if not getattr(self.args, "yes", False): - choice = input("Proceed? [Y/n] ").lower() - if choice not in ("", "y", "yes"): - print("Abort") - exit() - try: - self.api.delete_tag(repo_id=self.repo_id, tag=self.args.tag, repo_type=self.repo_type) - except RepositoryNotFoundError: - print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") - exit(1) - except RevisionNotFoundError: - print(f"Tag {ANSI.bold(self.args.tag)} not found on {ANSI.bold(self.repo_id)}") - exit(1) - print(f"Tag {ANSI.bold(self.args.tag)} deleted on {ANSI.bold(self.repo_id)}") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/repo_files.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/repo_files.py deleted file mode 100644 index 403d3126e234c7561cde5fbf8f1d49d7e3271da8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/repo_files.py +++ /dev/null @@ -1,128 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to update or delete files in a repository using the CLI. - -Usage: - # delete all - hf repo-files delete "*" - - # delete single file - hf repo-files delete file.txt - - # delete single folder - hf repo-files delete folder/ - - # delete multiple - hf repo-files delete file.txt folder/ file2.txt - - # delete multiple patterns - hf repo-files delete file.txt "*.json" "folder/*.parquet" - - # delete from different revision / repo-type - hf repo-files delete file.txt --revision=refs/pr/1 --repo-type=dataset -""" - -from argparse import _SubParsersAction -from typing import List, Optional - -from huggingface_hub import logging -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.hf_api import HfApi - - -logger = logging.get_logger(__name__) - - -class DeleteFilesSubCommand: - def __init__(self, args) -> None: - self.args = args - self.repo_id: str = args.repo_id - self.repo_type: Optional[str] = args.repo_type - self.revision: Optional[str] = args.revision - self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") - self.patterns: List[str] = args.patterns - self.commit_message: Optional[str] = args.commit_message - self.commit_description: Optional[str] = args.commit_description - self.create_pr: bool = args.create_pr - self.token: Optional[str] = args.token - - def run(self) -> None: - logging.set_verbosity_info() - url = self.api.delete_files( - delete_patterns=self.patterns, - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - commit_message=self.commit_message, - commit_description=self.commit_description, - create_pr=self.create_pr, - ) - print(f"Files correctly deleted from repo. Commit: {url}.") - logging.set_verbosity_warning() - - -class RepoFilesCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - repo_files_parser = parser.add_parser("repo-files", help="Manage files in a repo on the Hub.") - repo_files_subparsers = repo_files_parser.add_subparsers( - help="Action to execute against the files.", - required=True, - ) - delete_subparser = repo_files_subparsers.add_parser( - "delete", - help="Delete files from a repo on the Hub", - ) - delete_subparser.set_defaults(func=lambda args: DeleteFilesSubCommand(args)) - delete_subparser.add_argument( - "repo_id", type=str, help="The ID of the repo to manage (e.g. `username/repo-name`)." - ) - delete_subparser.add_argument( - "patterns", - nargs="+", - type=str, - help="Glob patterns to match files to delete.", - ) - delete_subparser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Type of the repo to upload to (e.g. `dataset`).", - ) - delete_subparser.add_argument( - "--revision", - type=str, - help=( - "An optional Git revision to push to. It can be a branch name " - "or a PR reference. If revision does not" - " exist and `--create-pr` is not set, a branch will be automatically created." - ), - ) - delete_subparser.add_argument( - "--commit-message", type=str, help="The summary / title / first line of the generated commit." - ) - delete_subparser.add_argument( - "--commit-description", type=str, help="The description of the generated commit." - ) - delete_subparser.add_argument( - "--create-pr", action="store_true", help="Whether to create a new Pull Request for these changes." - ) - delete_subparser.add_argument( - "--token", - type=str, - help="A User Access Token generated from https://huggingface.co/settings/tokens", - ) - - repo_files_parser.set_defaults(func=RepoFilesCommand) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/system.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/system.py deleted file mode 100644 index 03650175e9b71e329755de5c86e5bbf50569d4b7..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/system.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2022 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains commands to print information about the environment and version. - -Usage: - hf env - hf version -""" - -from argparse import _SubParsersAction - -from huggingface_hub import __version__ - -from ..utils import dump_environment_info -from . import BaseHuggingfaceCLICommand - - -class EnvironmentCommand(BaseHuggingfaceCLICommand): - def __init__(self, args): - self.args = args - - @staticmethod - def register_subcommand(parser: _SubParsersAction): - env_parser = parser.add_parser("env", help="Print information about the environment.") - env_parser.set_defaults(func=EnvironmentCommand) - - def run(self) -> None: - dump_environment_info() - - -class VersionCommand(BaseHuggingfaceCLICommand): - def __init__(self, args): - self.args = args - - @staticmethod - def register_subcommand(parser: _SubParsersAction): - version_parser = parser.add_parser("version", help="Print information about the hf version.") - version_parser.set_defaults(func=VersionCommand) - - def run(self) -> None: - print(f"huggingface_hub version: {__version__}") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/upload.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/upload.py deleted file mode 100644 index 0306bf9f5715fdc180dc4fa9819852388fca8b99..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/upload.py +++ /dev/null @@ -1,316 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to upload a repo or file with the CLI. - -Usage: - # Upload file (implicit) - hf upload my-cool-model ./my-cool-model.safetensors - - # Upload file (explicit) - hf upload my-cool-model ./my-cool-model.safetensors model.safetensors - - # Upload directory (implicit). If `my-cool-model/` is a directory it will be uploaded, otherwise an exception is raised. - hf upload my-cool-model - - # Upload directory (explicit) - hf upload my-cool-model ./models/my-cool-model . - - # Upload filtered directory (example: tensorboard logs except for the last run) - hf upload my-cool-model ./model/training /logs --include "*.tfevents.*" --exclude "*20230905*" - - # Upload with wildcard - hf upload my-cool-model "./model/training/*.safetensors" - - # Upload private dataset - hf upload Wauplin/my-cool-dataset ./data . --repo-type=dataset --private - - # Upload with token - hf upload Wauplin/my-cool-model --token=hf_**** - - # Sync local Space with Hub (upload new files, delete removed files) - hf upload Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub" - - # Schedule commits every 30 minutes - hf upload Wauplin/my-cool-model --every=30 -""" - -import os -import time -import warnings -from argparse import Namespace, _SubParsersAction -from typing import List, Optional - -from huggingface_hub import logging -from huggingface_hub._commit_scheduler import CommitScheduler -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.constants import HF_HUB_ENABLE_HF_TRANSFER -from huggingface_hub.errors import RevisionNotFoundError -from huggingface_hub.hf_api import HfApi -from huggingface_hub.utils import disable_progress_bars, enable_progress_bars -from huggingface_hub.utils._runtime import is_xet_available - - -logger = logging.get_logger(__name__) - - -class UploadCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - upload_parser = parser.add_parser( - "upload", help="Upload a file or a folder to the Hub. Recommended for single-commit uploads." - ) - upload_parser.add_argument( - "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)." - ) - upload_parser.add_argument( - "local_path", - nargs="?", - help="Local path to the file or folder to upload. Wildcard patterns are supported. Defaults to current directory.", - ) - upload_parser.add_argument( - "path_in_repo", - nargs="?", - help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.", - ) - upload_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Type of the repo to upload to (e.g. `dataset`).", - ) - upload_parser.add_argument( - "--revision", - type=str, - help=( - "An optional Git revision to push to. It can be a branch name or a PR reference. If revision does not" - " exist and `--create-pr` is not set, a branch will be automatically created." - ), - ) - upload_parser.add_argument( - "--private", - action="store_true", - help=( - "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already" - " exists." - ), - ) - upload_parser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.") - upload_parser.add_argument( - "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload." - ) - upload_parser.add_argument( - "--delete", - nargs="*", - type=str, - help="Glob patterns for file to be deleted from the repo while committing.", - ) - upload_parser.add_argument( - "--commit-message", type=str, help="The summary / title / first line of the generated commit." - ) - upload_parser.add_argument("--commit-description", type=str, help="The description of the generated commit.") - upload_parser.add_argument( - "--create-pr", action="store_true", help="Whether to upload content as a new Pull Request." - ) - upload_parser.add_argument( - "--every", - type=float, - help="If set, a background job is scheduled to create commits every `every` minutes.", - ) - upload_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - upload_parser.add_argument( - "--quiet", - action="store_true", - help="If True, progress bars are disabled and only the path to the uploaded files is printed.", - ) - upload_parser.set_defaults(func=UploadCommand) - - def __init__(self, args: Namespace) -> None: - self.repo_id: str = args.repo_id - self.repo_type: Optional[str] = args.repo_type - self.revision: Optional[str] = args.revision - self.private: bool = args.private - - self.include: Optional[List[str]] = args.include - self.exclude: Optional[List[str]] = args.exclude - self.delete: Optional[List[str]] = args.delete - - self.commit_message: Optional[str] = args.commit_message - self.commit_description: Optional[str] = args.commit_description - self.create_pr: bool = args.create_pr - self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") - self.quiet: bool = args.quiet # disable warnings and progress bars - - # Check `--every` is valid - if args.every is not None and args.every <= 0: - raise ValueError(f"`every` must be a positive value (got '{args.every}')") - self.every: Optional[float] = args.every - - # Resolve `local_path` and `path_in_repo` - repo_name: str = args.repo_id.split("/")[-1] # e.g. "Wauplin/my-cool-model" => "my-cool-model" - self.local_path: str - self.path_in_repo: str - - if args.local_path is not None and any(c in args.local_path for c in ["*", "?", "["]): - if args.include is not None: - raise ValueError("Cannot set `--include` when passing a `local_path` containing a wildcard.") - if args.path_in_repo is not None and args.path_in_repo != ".": - raise ValueError("Cannot set `path_in_repo` when passing a `local_path` containing a wildcard.") - self.local_path = "." - self.include = args.local_path - self.path_in_repo = "." - elif args.local_path is None and os.path.isfile(repo_name): - # Implicit case 1: user provided only a repo_id which happen to be a local file as well => upload it with same name - self.local_path = repo_name - self.path_in_repo = repo_name - elif args.local_path is None and os.path.isdir(repo_name): - # Implicit case 2: user provided only a repo_id which happen to be a local folder as well => upload it at root - self.local_path = repo_name - self.path_in_repo = "." - elif args.local_path is None: - # Implicit case 3: user provided only a repo_id that does not match a local file or folder - # => the user must explicitly provide a local_path => raise exception - raise ValueError(f"'{repo_name}' is not a local file or folder. Please set `local_path` explicitly.") - elif args.path_in_repo is None and os.path.isfile(args.local_path): - # Explicit local path to file, no path in repo => upload it at root with same name - self.local_path = args.local_path - self.path_in_repo = os.path.basename(args.local_path) - elif args.path_in_repo is None: - # Explicit local path to folder, no path in repo => upload at root - self.local_path = args.local_path - self.path_in_repo = "." - else: - # Finally, if both paths are explicit - self.local_path = args.local_path - self.path_in_repo = args.path_in_repo - - def run(self) -> None: - if self.quiet: - disable_progress_bars() - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - print(self._upload()) - enable_progress_bars() - else: - logging.set_verbosity_info() - print(self._upload()) - logging.set_verbosity_warning() - - def _upload(self) -> str: - if os.path.isfile(self.local_path): - if self.include is not None and len(self.include) > 0: - warnings.warn("Ignoring `--include` since a single file is uploaded.") - if self.exclude is not None and len(self.exclude) > 0: - warnings.warn("Ignoring `--exclude` since a single file is uploaded.") - if self.delete is not None and len(self.delete) > 0: - warnings.warn("Ignoring `--delete` since a single file is uploaded.") - - if not is_xet_available() and not HF_HUB_ENABLE_HF_TRANSFER: - logger.info( - "Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See" - " https://huggingface.co/docs/huggingface_hub/hf_transfer for more details." - ) - - # Schedule commits if `every` is set - if self.every is not None: - if os.path.isfile(self.local_path): - # If file => watch entire folder + use allow_patterns - folder_path = os.path.dirname(self.local_path) - path_in_repo = ( - self.path_in_repo[: -len(self.local_path)] # remove filename from path_in_repo - if self.path_in_repo.endswith(self.local_path) - else self.path_in_repo - ) - allow_patterns = [self.local_path] - ignore_patterns = [] - else: - folder_path = self.local_path - path_in_repo = self.path_in_repo - allow_patterns = self.include or [] - ignore_patterns = self.exclude or [] - if self.delete is not None and len(self.delete) > 0: - warnings.warn("Ignoring `--delete` when uploading with scheduled commits.") - - scheduler = CommitScheduler( - folder_path=folder_path, - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - path_in_repo=path_in_repo, - private=self.private, - every=self.every, - hf_api=self.api, - ) - print(f"Scheduling commits every {self.every} minutes to {scheduler.repo_id}.") - try: # Block main thread until KeyboardInterrupt - while True: - time.sleep(100) - except KeyboardInterrupt: - scheduler.stop() - return "Stopped scheduled commits." - - # Otherwise, create repo and proceed with the upload - if not os.path.isfile(self.local_path) and not os.path.isdir(self.local_path): - raise FileNotFoundError(f"No such file or directory: '{self.local_path}'.") - repo_id = self.api.create_repo( - repo_id=self.repo_id, - repo_type=self.repo_type, - exist_ok=True, - private=self.private, - space_sdk="gradio" if self.repo_type == "space" else None, - # ^ We don't want it to fail when uploading to a Space => let's set Gradio by default. - # ^ I'd rather not add CLI args to set it explicitly as we already have `hf repo create` for that. - ).repo_id - - # Check if branch already exists and if not, create it - if self.revision is not None and not self.create_pr: - try: - self.api.repo_info(repo_id=repo_id, repo_type=self.repo_type, revision=self.revision) - except RevisionNotFoundError: - logger.info(f"Branch '{self.revision}' not found. Creating it...") - self.api.create_branch(repo_id=repo_id, repo_type=self.repo_type, branch=self.revision, exist_ok=True) - # ^ `exist_ok=True` to avoid race concurrency issues - - # File-based upload - if os.path.isfile(self.local_path): - return self.api.upload_file( - path_or_fileobj=self.local_path, - path_in_repo=self.path_in_repo, - repo_id=repo_id, - repo_type=self.repo_type, - revision=self.revision, - commit_message=self.commit_message, - commit_description=self.commit_description, - create_pr=self.create_pr, - ) - - # Folder-based upload - else: - return self.api.upload_folder( - folder_path=self.local_path, - path_in_repo=self.path_in_repo, - repo_id=repo_id, - repo_type=self.repo_type, - revision=self.revision, - commit_message=self.commit_message, - commit_description=self.commit_description, - create_pr=self.create_pr, - allow_patterns=self.include, - ignore_patterns=self.exclude, - delete_patterns=self.delete, - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/upload_large_folder.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/upload_large_folder.py deleted file mode 100644 index 675c9ffe3dcd70242a9acd7837c6c2f00d8836df..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/cli/upload_large_folder.py +++ /dev/null @@ -1,132 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to upload a large folder with the CLI.""" - -import os -from argparse import Namespace, _SubParsersAction -from typing import List, Optional - -from huggingface_hub import logging -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.hf_api import HfApi -from huggingface_hub.utils import disable_progress_bars - -from ._cli_utils import ANSI - - -logger = logging.get_logger(__name__) - - -class UploadLargeFolderCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - subparser = parser.add_parser( - "upload-large-folder", - help="Upload a large folder to the Hub. Recommended for resumable uploads.", - ) - subparser.add_argument( - "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)." - ) - subparser.add_argument("local_path", type=str, help="Local path to the file or folder to upload.") - subparser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - help="Type of the repo to upload to (e.g. `dataset`).", - ) - subparser.add_argument( - "--revision", - type=str, - help=("An optional Git revision to push to. It can be a branch name or a PR reference."), - ) - subparser.add_argument( - "--private", - action="store_true", - help=( - "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists." - ), - ) - subparser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.") - subparser.add_argument("--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload.") - subparser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - subparser.add_argument( - "--num-workers", type=int, help="Number of workers to use to hash, upload and commit files." - ) - subparser.add_argument("--no-report", action="store_true", help="Whether to disable regular status report.") - subparser.add_argument("--no-bars", action="store_true", help="Whether to disable progress bars.") - subparser.set_defaults(func=UploadLargeFolderCommand) - - def __init__(self, args: Namespace) -> None: - self.repo_id: str = args.repo_id - self.local_path: str = args.local_path - self.repo_type: str = args.repo_type - self.revision: Optional[str] = args.revision - self.private: bool = args.private - - self.include: Optional[List[str]] = args.include - self.exclude: Optional[List[str]] = args.exclude - - self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") - - self.num_workers: Optional[int] = args.num_workers - self.no_report: bool = args.no_report - self.no_bars: bool = args.no_bars - - if not os.path.isdir(self.local_path): - raise ValueError("Large upload is only supported for folders.") - - def run(self) -> None: - logging.set_verbosity_info() - - print( - ANSI.yellow( - "You are about to upload a large folder to the Hub using `hf upload-large-folder`. " - "This is a new feature so feedback is very welcome!\n" - "\n" - "A few things to keep in mind:\n" - " - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n" - " - Do not start several processes in parallel.\n" - " - You can interrupt and resume the process at any time. " - "The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n" - " - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n" - "\n" - f"Some temporary metadata will be stored under `{self.local_path}/.cache/huggingface`.\n" - " - You must not modify those files manually.\n" - " - You must not delete the `./.cache/huggingface/` folder while a process is running.\n" - " - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n" - "\n" - "If the process output is too verbose, you can disable the progress bars with `--no-bars`. " - "You can also entirely disable the status report with `--no-report`.\n" - "\n" - "For more details, run `hf upload-large-folder --help` or check the documentation at " - "https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder." - ) - ) - - if self.no_bars: - disable_progress_bars() - - self.api.upload_large_folder( - repo_id=self.repo_id, - folder_path=self.local_path, - repo_type=self.repo_type, - revision=self.revision, - private=self.private, - allow_patterns=self.include, - ignore_patterns=self.exclude, - num_workers=self.num_workers, - print_report=not self.no_report, - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/__init__.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/__init__.py deleted file mode 100644 index 49d088214505b9604964ab142e7f8a5b38ccd5ef..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from abc import ABC, abstractmethod -from argparse import _SubParsersAction - - -class BaseHuggingfaceCLICommand(ABC): - @staticmethod - @abstractmethod - def register_subcommand(parser: _SubParsersAction): - raise NotImplementedError() - - @abstractmethod - def run(self): - raise NotImplementedError() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/_cli_utils.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/_cli_utils.py deleted file mode 100644 index bf4a1c0373b4d4bb71a3f4e8ea39da5a01cc79a7..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/_cli_utils.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2022 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains a utility for good-looking prints.""" - -import os -from typing import List, Union - - -class ANSI: - """ - Helper for en.wikipedia.org/wiki/ANSI_escape_code - """ - - _bold = "\u001b[1m" - _gray = "\u001b[90m" - _red = "\u001b[31m" - _reset = "\u001b[0m" - _yellow = "\u001b[33m" - - @classmethod - def bold(cls, s: str) -> str: - return cls._format(s, cls._bold) - - @classmethod - def gray(cls, s: str) -> str: - return cls._format(s, cls._gray) - - @classmethod - def red(cls, s: str) -> str: - return cls._format(s, cls._bold + cls._red) - - @classmethod - def yellow(cls, s: str) -> str: - return cls._format(s, cls._yellow) - - @classmethod - def _format(cls, s: str, code: str) -> str: - if os.environ.get("NO_COLOR"): - # See https://no-color.org/ - return s - return f"{code}{s}{cls._reset}" - - -def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str: - """ - Inspired by: - - - stackoverflow.com/a/8356620/593036 - - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data - """ - col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] - row_format = ("{{:{}}} " * len(headers)).format(*col_widths) - lines = [] - lines.append(row_format.format(*headers)) - lines.append(row_format.format(*["-" * w for w in col_widths])) - for row in rows: - lines.append(row_format.format(*row)) - return "\n".join(lines) - - -def show_deprecation_warning(old_command: str, new_command: str): - """Show a yellow warning about deprecated CLI command.""" - print(ANSI.yellow(f"⚠️ Warning: '{old_command}' is deprecated. Use '{new_command}' instead.")) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/delete_cache.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/delete_cache.py deleted file mode 100644 index 78ea1179678371807b3686b8acf17b9f0997035f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/delete_cache.py +++ /dev/null @@ -1,476 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to delete some revisions from the HF cache directory. - -Usage: - huggingface-cli delete-cache - huggingface-cli delete-cache --disable-tui - huggingface-cli delete-cache --dir ~/.cache/huggingface/hub - huggingface-cli delete-cache --sort=size - -NOTE: - This command is based on `InquirerPy` to build the multiselect menu in the terminal. - This dependency has to be installed with `pip install "huggingface_hub[cli]"`. Since - we want to avoid as much as possible cross-platform issues, I chose a library that - is built on top of `python-prompt-toolkit` which seems to be a reference in terminal - GUI (actively maintained on both Unix and Windows, 7.9k stars). - - For the moment, the TUI feature is in beta. - - See: - - https://github.com/kazhala/InquirerPy - - https://inquirerpy.readthedocs.io/en/latest/ - - https://github.com/prompt-toolkit/python-prompt-toolkit - - Other solutions could have been: - - `simple_term_menu`: would be good as well for our use case but some issues suggest - that Windows is less supported. - See: https://github.com/IngoMeyer441/simple-term-menu - - `PyInquirer`: very similar to `InquirerPy` but older and not maintained anymore. - In particular, no support of Python3.10. - See: https://github.com/CITGuru/PyInquirer - - `pick` (or `pickpack`): easy to use and flexible but built on top of Python's - standard library `curses` that is specific to Unix (not implemented on Windows). - See https://github.com/wong2/pick and https://github.com/anafvana/pickpack. - - `inquirer`: lot of traction (700 stars) but explicitly states "experimental - support of Windows". Not built on top of `python-prompt-toolkit`. - See https://github.com/magmax/python-inquirer - -TODO: add support for `huggingface-cli delete-cache aaaaaa bbbbbb cccccc (...)` ? -TODO: add "--keep-last" arg to delete revisions that are not on `main` ref -TODO: add "--filter" arg to filter repositories by name ? -TODO: add "--limit" arg to limit to X repos ? -TODO: add "-y" arg for immediate deletion ? -See discussions in https://github.com/huggingface/huggingface_hub/issues/1025. -""" - -import os -from argparse import Namespace, _SubParsersAction -from functools import wraps -from tempfile import mkstemp -from typing import Any, Callable, Iterable, List, Literal, Optional, Union - -from ..utils import CachedRepoInfo, CachedRevisionInfo, HFCacheInfo, scan_cache_dir -from . import BaseHuggingfaceCLICommand -from ._cli_utils import ANSI, show_deprecation_warning - - -try: - from InquirerPy import inquirer - from InquirerPy.base.control import Choice - from InquirerPy.separator import Separator - - _inquirer_py_available = True -except ImportError: - _inquirer_py_available = False - -SortingOption_T = Literal["alphabetical", "lastUpdated", "lastUsed", "size"] - - -def require_inquirer_py(fn: Callable) -> Callable: - """Decorator to flag methods that require `InquirerPy`.""" - - # TODO: refactor this + imports in a unified pattern across codebase - @wraps(fn) - def _inner(*args, **kwargs): - if not _inquirer_py_available: - raise ImportError( - "The `delete-cache` command requires extra dependencies to work with" - ' the TUI.\nPlease run `pip install "huggingface_hub[cli]"` to install' - " them.\nOtherwise, disable TUI using the `--disable-tui` flag." - ) - - return fn(*args, **kwargs) - - return _inner - - -# Possibility for the user to cancel deletion -_CANCEL_DELETION_STR = "CANCEL_DELETION" - - -class DeleteCacheCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - delete_cache_parser = parser.add_parser("delete-cache", help="Delete revisions from the cache directory.") - - delete_cache_parser.add_argument( - "--dir", - type=str, - default=None, - help="cache directory (optional). Default to the default HuggingFace cache.", - ) - - delete_cache_parser.add_argument( - "--disable-tui", - action="store_true", - help=( - "Disable Terminal User Interface (TUI) mode. Useful if your" - " platform/terminal doesn't support the multiselect menu." - ), - ) - - delete_cache_parser.add_argument( - "--sort", - nargs="?", - choices=["alphabetical", "lastUpdated", "lastUsed", "size"], - help=( - "Sort repositories by the specified criteria. Options: " - "'alphabetical' (A-Z), " - "'lastUpdated' (newest first), " - "'lastUsed' (most recent first), " - "'size' (largest first)." - ), - ) - - delete_cache_parser.set_defaults(func=DeleteCacheCommand) - - def __init__(self, args: Namespace) -> None: - self.cache_dir: Optional[str] = args.dir - self.disable_tui: bool = args.disable_tui - self.sort_by: Optional[SortingOption_T] = args.sort - - def run(self): - """Run `delete-cache` command with or without TUI.""" - show_deprecation_warning("huggingface-cli delete-cache", "hf cache delete") - - # Scan cache directory - hf_cache_info = scan_cache_dir(self.cache_dir) - - # Manual review from the user - if self.disable_tui: - selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) - else: - selected_hashes = _manual_review_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) - - # If deletion is not cancelled - if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes: - confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?" - - # Confirm deletion - if self.disable_tui: - confirmed = _ask_for_confirmation_no_tui(confirm_message) - else: - confirmed = _ask_for_confirmation_tui(confirm_message) - - # Deletion is confirmed - if confirmed: - strategy = hf_cache_info.delete_revisions(*selected_hashes) - print("Start deletion.") - strategy.execute() - print( - f"Done. Deleted {len(strategy.repos)} repo(s) and" - f" {len(strategy.snapshots)} revision(s) for a total of" - f" {strategy.expected_freed_size_str}." - ) - return - - # Deletion is cancelled - print("Deletion is cancelled. Do nothing.") - - -def _get_repo_sorting_key(repo: CachedRepoInfo, sort_by: Optional[SortingOption_T] = None): - if sort_by == "alphabetical": - return (repo.repo_type, repo.repo_id.lower()) # by type then name - elif sort_by == "lastUpdated": - return -max(rev.last_modified for rev in repo.revisions) # newest first - elif sort_by == "lastUsed": - return -repo.last_accessed # most recently used first - elif sort_by == "size": - return -repo.size_on_disk # largest first - else: - return (repo.repo_type, repo.repo_id) # default stable order - - -@require_inquirer_py -def _manual_review_tui( - hf_cache_info: HFCacheInfo, - preselected: List[str], - sort_by: Optional[SortingOption_T] = None, -) -> List[str]: - """Ask the user for a manual review of the revisions to delete. - - Displays a multi-select menu in the terminal (TUI). - """ - # Define multiselect list - choices = _get_tui_choices_from_scan( - repos=hf_cache_info.repos, - preselected=preselected, - sort_by=sort_by, - ) - checkbox = inquirer.checkbox( - message="Select revisions to delete:", - choices=choices, # List of revisions with some pre-selection - cycle=False, # No loop between top and bottom - height=100, # Large list if possible - # We use the instruction to display to the user the expected effect of the - # deletion. - instruction=_get_expectations_str( - hf_cache_info, - selected_hashes=[c.value for c in choices if isinstance(c, Choice) and c.enabled], - ), - # We use the long instruction to should keybindings instructions to the user - long_instruction="Press to select, to validate and to quit without modification.", - # Message that is displayed once the user validates its selection. - transformer=lambda result: f"{len(result)} revision(s) selected.", - ) - - # Add a callback to update the information line when a revision is - # selected/unselected - def _update_expectations(_) -> None: - # Hacky way to dynamically set an instruction message to the checkbox when - # a revision hash is selected/unselected. - checkbox._instruction = _get_expectations_str( - hf_cache_info, - selected_hashes=[choice["value"] for choice in checkbox.content_control.choices if choice["enabled"]], - ) - - checkbox.kb_func_lookup["toggle"].append({"func": _update_expectations}) - - # Finally display the form to the user. - try: - return checkbox.execute() - except KeyboardInterrupt: - return [] # Quit without deletion - - -@require_inquirer_py -def _ask_for_confirmation_tui(message: str, default: bool = True) -> bool: - """Ask for confirmation using Inquirer.""" - return inquirer.confirm(message, default=default).execute() - - -def _get_tui_choices_from_scan( - repos: Iterable[CachedRepoInfo], - preselected: List[str], - sort_by: Optional[SortingOption_T] = None, -) -> List: - """Build a list of choices from the scanned repos. - - Args: - repos (*Iterable[`CachedRepoInfo`]*): - List of scanned repos on which we want to delete revisions. - preselected (*List[`str`]*): - List of revision hashes that will be preselected. - sort_by (*Optional[SortingOption_T]*): - Sorting direction. Choices: "alphabetical", "lastUpdated", "lastUsed", "size". - - Return: - The list of choices to pass to `inquirer.checkbox`. - """ - choices: List[Union[Choice, Separator]] = [] - - # First choice is to cancel the deletion - choices.append( - Choice( - _CANCEL_DELETION_STR, - name="None of the following (if selected, nothing will be deleted).", - enabled=False, - ) - ) - - # Sort repos based on specified criteria - sorted_repos = sorted(repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by)) - - for repo in sorted_repos: - # Repo as separator - choices.append( - Separator( - f"\n{repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}," - f" used {repo.last_accessed_str})" - ) - ) - for revision in sorted(repo.revisions, key=_revision_sorting_order): - # Revision as choice - choices.append( - Choice( - revision.commit_hash, - name=( - f"{revision.commit_hash[:8]}:" - f" {', '.join(sorted(revision.refs)) or '(detached)'} #" - f" modified {revision.last_modified_str}" - ), - enabled=revision.commit_hash in preselected, - ) - ) - - # Return choices - return choices - - -def _manual_review_no_tui( - hf_cache_info: HFCacheInfo, - preselected: List[str], - sort_by: Optional[SortingOption_T] = None, -) -> List[str]: - """Ask the user for a manual review of the revisions to delete. - - Used when TUI is disabled. Manual review happens in a separate tmp file that the - user can manually edit. - """ - # 1. Generate temporary file with delete commands. - fd, tmp_path = mkstemp(suffix=".txt") # suffix to make it easier to find by editors - os.close(fd) - - lines = [] - - sorted_repos = sorted(hf_cache_info.repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by)) - - for repo in sorted_repos: - lines.append( - f"\n# {repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}," - f" used {repo.last_accessed_str})" - ) - for revision in sorted(repo.revisions, key=_revision_sorting_order): - lines.append( - # Deselect by prepending a '#' - f"{'' if revision.commit_hash in preselected else '#'} " - f" {revision.commit_hash} # Refs:" - # Print `refs` as comment on same line - f" {', '.join(sorted(revision.refs)) or '(detached)'} # modified" - # Print `last_modified` as comment on same line - f" {revision.last_modified_str}" - ) - - with open(tmp_path, "w") as f: - f.write(_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS) - f.write("\n".join(lines)) - - # 2. Prompt instructions to user. - instructions = f""" - TUI is disabled. In order to select which revisions you want to delete, please edit - the following file using the text editor of your choice. Instructions for manual - editing are located at the beginning of the file. Edit the file, save it and confirm - to continue. - File to edit: {ANSI.bold(tmp_path)} - """ - print("\n".join(line.strip() for line in instructions.strip().split("\n"))) - - # 3. Wait for user confirmation. - while True: - selected_hashes = _read_manual_review_tmp_file(tmp_path) - if _ask_for_confirmation_no_tui( - _get_expectations_str(hf_cache_info, selected_hashes) + " Continue ?", - default=False, - ): - break - - # 4. Return selected_hashes sorted to maintain stable order - os.remove(tmp_path) - return sorted(selected_hashes) # Sort to maintain stable order - - -def _ask_for_confirmation_no_tui(message: str, default: bool = True) -> bool: - """Ask for confirmation using pure-python.""" - YES = ("y", "yes", "1") - NO = ("n", "no", "0") - DEFAULT = "" - ALL = YES + NO + (DEFAULT,) - full_message = message + (" (Y/n) " if default else " (y/N) ") - while True: - answer = input(full_message).lower() - if answer == DEFAULT: - return default - if answer in YES: - return True - if answer in NO: - return False - print(f"Invalid input. Must be one of {ALL}") - - -def _get_expectations_str(hf_cache_info: HFCacheInfo, selected_hashes: List[str]) -> str: - """Format a string to display to the user how much space would be saved. - - Example: - ``` - >>> _get_expectations_str(hf_cache_info, selected_hashes) - '7 revisions selected counting for 4.3G.' - ``` - """ - if _CANCEL_DELETION_STR in selected_hashes: - return "Nothing will be deleted." - strategy = hf_cache_info.delete_revisions(*selected_hashes) - return f"{len(selected_hashes)} revisions selected counting for {strategy.expected_freed_size_str}." - - -def _read_manual_review_tmp_file(tmp_path: str) -> List[str]: - """Read the manually reviewed instruction file and return a list of revision hash. - - Example: - ```txt - # This is the tmp file content - ### - - # Commented out line - 123456789 # revision hash - - # Something else - # a_newer_hash # 2 days ago - an_older_hash # 3 days ago - ``` - - ```py - >>> _read_manual_review_tmp_file(tmp_path) - ['123456789', 'an_older_hash'] - ``` - """ - with open(tmp_path) as f: - content = f.read() - - # Split lines - lines = [line.strip() for line in content.split("\n")] - - # Filter commented lines - selected_lines = [line for line in lines if not line.startswith("#")] - - # Select only before comment - selected_hashes = [line.split("#")[0].strip() for line in selected_lines] - - # Return revision hashes - return [hash for hash in selected_hashes if len(hash) > 0] - - -_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS = f""" -# INSTRUCTIONS -# ------------ -# This is a temporary file created by running `huggingface-cli delete-cache` with the -# `--disable-tui` option. It contains a set of revisions that can be deleted from your -# local cache directory. -# -# Please manually review the revisions you want to delete: -# - Revision hashes can be commented out with '#'. -# - Only non-commented revisions in this file will be deleted. -# - Revision hashes that are removed from this file are ignored as well. -# - If `{_CANCEL_DELETION_STR}` line is uncommented, the all cache deletion is cancelled and -# no changes will be applied. -# -# Once you've manually reviewed this file, please confirm deletion in the terminal. This -# file will be automatically removed once done. -# ------------ - -# KILL SWITCH -# ------------ -# Un-comment following line to completely cancel the deletion process -# {_CANCEL_DELETION_STR} -# ------------ - -# REVISIONS -# ------------ -""".strip() - - -def _revision_sorting_order(revision: CachedRevisionInfo) -> Any: - # Sort by last modified (oldest first) - return revision.last_modified diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/download.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/download.py deleted file mode 100644 index 0dd2c1070ead01f9ad6855de3929928d268279c2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/download.py +++ /dev/null @@ -1,204 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to download files from the Hub with the CLI. - -Usage: - huggingface-cli download --help - - # Download file - huggingface-cli download gpt2 config.json - - # Download entire repo - huggingface-cli download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78 - - # Download repo with filters - huggingface-cli download gpt2 --include="*.safetensors" - - # Download with token - huggingface-cli download Wauplin/private-model --token=hf_*** - - # Download quietly (no progress bar, no warnings, only the returned path) - huggingface-cli download gpt2 config.json --quiet - - # Download to local dir - huggingface-cli download gpt2 --local-dir=./models/gpt2 -""" - -import warnings -from argparse import Namespace, _SubParsersAction -from typing import List, Optional - -from huggingface_hub import logging -from huggingface_hub._snapshot_download import snapshot_download -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.file_download import hf_hub_download -from huggingface_hub.utils import disable_progress_bars, enable_progress_bars - -from ._cli_utils import show_deprecation_warning - - -logger = logging.get_logger(__name__) - - -class DownloadCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - download_parser = parser.add_parser("download", help="Download files from the Hub") - download_parser.add_argument( - "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)." - ) - download_parser.add_argument( - "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)." - ) - download_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Type of repo to download from (defaults to 'model').", - ) - download_parser.add_argument( - "--revision", - type=str, - help="An optional Git revision id which can be a branch name, a tag, or a commit hash.", - ) - download_parser.add_argument( - "--include", nargs="*", type=str, help="Glob patterns to match files to download." - ) - download_parser.add_argument( - "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download." - ) - download_parser.add_argument( - "--cache-dir", type=str, help="Path to the directory where to save the downloaded files." - ) - download_parser.add_argument( - "--local-dir", - type=str, - help=( - "If set, the downloaded file will be placed under this directory. Check out" - " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more" - " details." - ), - ) - download_parser.add_argument( - "--local-dir-use-symlinks", - choices=["auto", "True", "False"], - help=("Deprecated and ignored. Downloading to a local directory does not use symlinks anymore."), - ) - download_parser.add_argument( - "--force-download", - action="store_true", - help="If True, the files will be downloaded even if they are already cached.", - ) - download_parser.add_argument( - "--resume-download", - action="store_true", - help="Deprecated and ignored. Downloading a file to local dir always attempts to resume previously interrupted downloads (unless hf-transfer is enabled).", - ) - download_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - download_parser.add_argument( - "--quiet", - action="store_true", - help="If True, progress bars are disabled and only the path to the download files is printed.", - ) - download_parser.add_argument( - "--max-workers", - type=int, - default=8, - help="Maximum number of workers to use for downloading files. Default is 8.", - ) - download_parser.set_defaults(func=DownloadCommand) - - def __init__(self, args: Namespace) -> None: - self.token = args.token - self.repo_id: str = args.repo_id - self.filenames: List[str] = args.filenames - self.repo_type: str = args.repo_type - self.revision: Optional[str] = args.revision - self.include: Optional[List[str]] = args.include - self.exclude: Optional[List[str]] = args.exclude - self.cache_dir: Optional[str] = args.cache_dir - self.local_dir: Optional[str] = args.local_dir - self.force_download: bool = args.force_download - self.resume_download: Optional[bool] = args.resume_download or None - self.quiet: bool = args.quiet - self.max_workers: int = args.max_workers - - if args.local_dir_use_symlinks is not None: - warnings.warn( - "Ignoring --local-dir-use-symlinks. Downloading to a local directory does not use symlinks anymore.", - FutureWarning, - ) - - def run(self) -> None: - show_deprecation_warning("huggingface-cli download", "hf download") - - if self.quiet: - disable_progress_bars() - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - print(self._download()) # Print path to downloaded files - enable_progress_bars() - else: - logging.set_verbosity_info() - print(self._download()) # Print path to downloaded files - logging.set_verbosity_warning() - - def _download(self) -> str: - # Warn user if patterns are ignored - if len(self.filenames) > 0: - if self.include is not None and len(self.include) > 0: - warnings.warn("Ignoring `--include` since filenames have being explicitly set.") - if self.exclude is not None and len(self.exclude) > 0: - warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.") - - # Single file to download: use `hf_hub_download` - if len(self.filenames) == 1: - return hf_hub_download( - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - filename=self.filenames[0], - cache_dir=self.cache_dir, - resume_download=self.resume_download, - force_download=self.force_download, - token=self.token, - local_dir=self.local_dir, - library_name="huggingface-cli", - ) - - # Otherwise: use `snapshot_download` to ensure all files comes from same revision - elif len(self.filenames) == 0: - allow_patterns = self.include - ignore_patterns = self.exclude - else: - allow_patterns = self.filenames - ignore_patterns = None - - return snapshot_download( - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - resume_download=self.resume_download, - force_download=self.force_download, - cache_dir=self.cache_dir, - token=self.token, - local_dir=self.local_dir, - library_name="huggingface-cli", - max_workers=self.max_workers, - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/env.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/env.py deleted file mode 100644 index ad674738b2f137ec0b79c11ef35057a351de6d86..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/env.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2022 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to print information about the environment. - -Usage: - huggingface-cli env -""" - -from argparse import _SubParsersAction - -from ..utils import dump_environment_info -from . import BaseHuggingfaceCLICommand -from ._cli_utils import show_deprecation_warning - - -class EnvironmentCommand(BaseHuggingfaceCLICommand): - def __init__(self, args): - self.args = args - - @staticmethod - def register_subcommand(parser: _SubParsersAction): - env_parser = parser.add_parser("env", help="Print information about the environment.") - env_parser.set_defaults(func=EnvironmentCommand) - - def run(self) -> None: - show_deprecation_warning("huggingface-cli env", "hf env") - - dump_environment_info() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/huggingface_cli.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/huggingface_cli.py deleted file mode 100644 index 697c85d1e386d9c954be0f8112cb12e1bc84e7fe..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/huggingface_cli.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from argparse import ArgumentParser - -from huggingface_hub.commands._cli_utils import show_deprecation_warning -from huggingface_hub.commands.delete_cache import DeleteCacheCommand -from huggingface_hub.commands.download import DownloadCommand -from huggingface_hub.commands.env import EnvironmentCommand -from huggingface_hub.commands.lfs import LfsCommands -from huggingface_hub.commands.repo import RepoCommands -from huggingface_hub.commands.repo_files import RepoFilesCommand -from huggingface_hub.commands.scan_cache import ScanCacheCommand -from huggingface_hub.commands.tag import TagCommands -from huggingface_hub.commands.upload import UploadCommand -from huggingface_hub.commands.upload_large_folder import UploadLargeFolderCommand -from huggingface_hub.commands.user import UserCommands -from huggingface_hub.commands.version import VersionCommand - - -def main(): - parser = ArgumentParser("huggingface-cli", usage="huggingface-cli []") - commands_parser = parser.add_subparsers(help="huggingface-cli command helpers") - - # Register commands - DownloadCommand.register_subcommand(commands_parser) - UploadCommand.register_subcommand(commands_parser) - RepoFilesCommand.register_subcommand(commands_parser) - EnvironmentCommand.register_subcommand(commands_parser) - UserCommands.register_subcommand(commands_parser) - RepoCommands.register_subcommand(commands_parser) - LfsCommands.register_subcommand(commands_parser) - ScanCacheCommand.register_subcommand(commands_parser) - DeleteCacheCommand.register_subcommand(commands_parser) - TagCommands.register_subcommand(commands_parser) - VersionCommand.register_subcommand(commands_parser) - - # Experimental - UploadLargeFolderCommand.register_subcommand(commands_parser) - - # Let's go - args = parser.parse_args() - if not hasattr(args, "func"): - show_deprecation_warning("huggingface-cli", "hf") - parser.print_help() - exit(1) - - # Run - service = args.func(args) - service.run() - - -if __name__ == "__main__": - main() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/lfs.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/lfs.py deleted file mode 100644 index e510e345e6a4bf6da03f71b35cbfa2a4f0eb7325..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/lfs.py +++ /dev/null @@ -1,200 +0,0 @@ -""" -Implementation of a custom transfer agent for the transfer type "multipart" for -git-lfs. - -Inspired by: -github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py - -Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md - - -To launch debugger while developing: - -``` [lfs "customtransfer.multipart"] -path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678 ---wait-for-client -/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py -lfs-multipart-upload ```""" - -import json -import os -import subprocess -import sys -from argparse import _SubParsersAction -from typing import Dict, List, Optional - -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND - -from ..utils import get_session, hf_raise_for_status, logging -from ..utils._lfs import SliceFileObj - - -logger = logging.get_logger(__name__) - - -class LfsCommands(BaseHuggingfaceCLICommand): - """ - Implementation of a custom transfer agent for the transfer type "multipart" - for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom - transfer agent is: - https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md - - This introduces two commands to the CLI: - - 1. $ huggingface-cli lfs-enable-largefiles - - This should be executed once for each model repo that contains a model file - >5GB. It's documented in the error message you get if you just try to git - push a 5GB file without having enabled it before. - - 2. $ huggingface-cli lfs-multipart-upload - - This command is called by lfs directly and is not meant to be called by the - user. - """ - - @staticmethod - def register_subcommand(parser: _SubParsersAction): - enable_parser = parser.add_parser( - "lfs-enable-largefiles", help="Configure your repository to enable upload of files > 5GB." - ) - enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.") - enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args)) - - # Command will get called by git-lfs, do not call it directly. - upload_parser = parser.add_parser(LFS_MULTIPART_UPLOAD_COMMAND, add_help=False) - upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args)) - - -class LfsEnableCommand: - def __init__(self, args): - self.args = args - - def run(self): - local_path = os.path.abspath(self.args.path) - if not os.path.isdir(local_path): - print("This does not look like a valid git repo.") - exit(1) - subprocess.run( - "git config lfs.customtransfer.multipart.path huggingface-cli".split(), - check=True, - cwd=local_path, - ) - subprocess.run( - f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(), - check=True, - cwd=local_path, - ) - print("Local repo set up for largefiles") - - -def write_msg(msg: Dict): - """Write out the message in Line delimited JSON.""" - msg_str = json.dumps(msg) + "\n" - sys.stdout.write(msg_str) - sys.stdout.flush() - - -def read_msg() -> Optional[Dict]: - """Read Line delimited JSON from stdin.""" - msg = json.loads(sys.stdin.readline().strip()) - - if "terminate" in (msg.get("type"), msg.get("event")): - # terminate message received - return None - - if msg.get("event") not in ("download", "upload"): - logger.critical("Received unexpected message") - sys.exit(1) - - return msg - - -class LfsUploadCommand: - def __init__(self, args) -> None: - self.args = args - - def run(self) -> None: - # Immediately after invoking a custom transfer process, git-lfs - # sends initiation data to the process over stdin. - # This tells the process useful information about the configuration. - init_msg = json.loads(sys.stdin.readline().strip()) - if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"): - write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}}) - sys.exit(1) - - # The transfer process should use the information it needs from the - # initiation structure, and also perform any one-off setup tasks it - # needs to do. It should then respond on stdout with a simple empty - # confirmation structure, as follows: - write_msg({}) - - # After the initiation exchange, git-lfs will send any number of - # transfer requests to the stdin of the transfer process, in a serial sequence. - while True: - msg = read_msg() - if msg is None: - # When all transfers have been processed, git-lfs will send - # a terminate event to the stdin of the transfer process. - # On receiving this message the transfer process should - # clean up and terminate. No response is expected. - sys.exit(0) - - oid = msg["oid"] - filepath = msg["path"] - completion_url = msg["action"]["href"] - header = msg["action"]["header"] - chunk_size = int(header.pop("chunk_size")) - presigned_urls: List[str] = list(header.values()) - - # Send a "started" progress event to allow other workers to start. - # Otherwise they're delayed until first "progress" event is reported, - # i.e. after the first 5GB by default (!) - write_msg( - { - "event": "progress", - "oid": oid, - "bytesSoFar": 1, - "bytesSinceLast": 0, - } - ) - - parts = [] - with open(filepath, "rb") as file: - for i, presigned_url in enumerate(presigned_urls): - with SliceFileObj( - file, - seek_from=i * chunk_size, - read_limit=chunk_size, - ) as data: - r = get_session().put(presigned_url, data=data) - hf_raise_for_status(r) - parts.append( - { - "etag": r.headers.get("etag"), - "partNumber": i + 1, - } - ) - # In order to support progress reporting while data is uploading / downloading, - # the transfer process should post messages to stdout - write_msg( - { - "event": "progress", - "oid": oid, - "bytesSoFar": (i + 1) * chunk_size, - "bytesSinceLast": chunk_size, - } - ) - # Not precise but that's ok. - - r = get_session().post( - completion_url, - json={ - "oid": oid, - "parts": parts, - }, - ) - hf_raise_for_status(r) - - write_msg({"event": "complete", "oid": oid}) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/repo.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/repo.py deleted file mode 100644 index fe75349d67bdc0314afe737daa7224b2a090f810..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/repo.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2025 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains commands to interact with repositories on the Hugging Face Hub. - -Usage: - # create a new dataset repo on the Hub - huggingface-cli repo create my-cool-dataset --repo-type=dataset - - # create a private model repo on the Hub - huggingface-cli repo create my-cool-model --private -""" - -import argparse -from argparse import _SubParsersAction -from typing import Optional - -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.commands._cli_utils import ANSI -from huggingface_hub.constants import SPACES_SDK_TYPES -from huggingface_hub.hf_api import HfApi -from huggingface_hub.utils import logging - -from ._cli_utils import show_deprecation_warning - - -logger = logging.get_logger(__name__) - - -class RepoCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - repo_parser = parser.add_parser("repo", help="{create} Commands to interact with your huggingface.co repos.") - repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands") - repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co") - repo_create_parser.add_argument( - "repo_id", - type=str, - help="The ID of the repo to create to (e.g. `username/repo-name`). The username is optional and will be set to your username if not provided.", - ) - repo_create_parser.add_argument( - "--repo-type", - type=str, - help='Optional: set to "dataset" or "space" if creating a dataset or space, default is model.', - ) - repo_create_parser.add_argument( - "--space_sdk", - type=str, - help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".', - choices=SPACES_SDK_TYPES, - ) - repo_create_parser.add_argument( - "--private", - action="store_true", - help="Whether to create a private repository. Defaults to public unless the organization's default is private.", - ) - repo_create_parser.add_argument( - "--token", - type=str, - help="Hugging Face token. Will default to the locally saved token if not provided.", - ) - repo_create_parser.add_argument( - "--exist-ok", - action="store_true", - help="Do not raise an error if repo already exists.", - ) - repo_create_parser.add_argument( - "--resource-group-id", - type=str, - help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.", - ) - repo_create_parser.add_argument( - "--type", - type=str, - help="[Deprecated]: use --repo-type instead.", - ) - repo_create_parser.add_argument( - "-y", - "--yes", - action="store_true", - help="[Deprecated] no effect.", - ) - repo_create_parser.add_argument( - "--organization", type=str, help="[Deprecated] Pass the organization namespace directly in the repo_id." - ) - repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args)) - - -class RepoCreateCommand: - def __init__(self, args: argparse.Namespace): - self.repo_id: str = args.repo_id - self.repo_type: Optional[str] = args.repo_type or args.type - self.space_sdk: Optional[str] = args.space_sdk - self.organization: Optional[str] = args.organization - self.yes: bool = args.yes - self.private: bool = args.private - self.token: Optional[str] = args.token - self.exist_ok: bool = args.exist_ok - self.resource_group_id: Optional[str] = args.resource_group_id - - if args.type is not None: - print( - ANSI.yellow( - "The --type argument is deprecated and will be removed in a future version. Use --repo-type instead." - ) - ) - if self.organization is not None: - print( - ANSI.yellow( - "The --organization argument is deprecated and will be removed in a future version. Pass the organization namespace directly in the repo_id." - ) - ) - if self.yes: - print( - ANSI.yellow( - "The --yes argument is deprecated and will be removed in a future version. It does not have any effect." - ) - ) - - self._api = HfApi() - - def run(self): - show_deprecation_warning("huggingface-cli repo", "hf repo") - - if self.organization is not None: - if "/" in self.repo_id: - print(ANSI.red("You cannot pass both --organization and a repo_id with a namespace.")) - exit(1) - self.repo_id = f"{self.organization}/{self.repo_id}" - - repo_url = self._api.create_repo( - repo_id=self.repo_id, - repo_type=self.repo_type, - private=self.private, - token=self.token, - exist_ok=self.exist_ok, - resource_group_id=self.resource_group_id, - space_sdk=self.space_sdk, - ) - print(f"Successfully created {ANSI.bold(repo_url.repo_id)} on the Hub.") - print(f"Your repo is now available at {ANSI.bold(repo_url)}") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/repo_files.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/repo_files.py deleted file mode 100644 index da9685315ea67dc9d1e9921ecb2656244cae8783..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/repo_files.py +++ /dev/null @@ -1,132 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to update or delete files in a repository using the CLI. - -Usage: - # delete all - huggingface-cli repo-files delete "*" - - # delete single file - huggingface-cli repo-files delete file.txt - - # delete single folder - huggingface-cli repo-files delete folder/ - - # delete multiple - huggingface-cli repo-files delete file.txt folder/ file2.txt - - # delete multiple patterns - huggingface-cli repo-files delete file.txt "*.json" "folder/*.parquet" - - # delete from different revision / repo-type - huggingface-cli repo-files delete file.txt --revision=refs/pr/1 --repo-type=dataset -""" - -from argparse import _SubParsersAction -from typing import List, Optional - -from huggingface_hub import logging -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.hf_api import HfApi - -from ._cli_utils import show_deprecation_warning - - -logger = logging.get_logger(__name__) - - -class DeleteFilesSubCommand: - def __init__(self, args) -> None: - self.args = args - self.repo_id: str = args.repo_id - self.repo_type: Optional[str] = args.repo_type - self.revision: Optional[str] = args.revision - self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") - self.patterns: List[str] = args.patterns - self.commit_message: Optional[str] = args.commit_message - self.commit_description: Optional[str] = args.commit_description - self.create_pr: bool = args.create_pr - self.token: Optional[str] = args.token - - def run(self) -> None: - show_deprecation_warning("huggingface-cli repo-files", "hf repo-files") - - logging.set_verbosity_info() - url = self.api.delete_files( - delete_patterns=self.patterns, - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - commit_message=self.commit_message, - commit_description=self.commit_description, - create_pr=self.create_pr, - ) - print(f"Files correctly deleted from repo. Commit: {url}.") - logging.set_verbosity_warning() - - -class RepoFilesCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - repo_files_parser = parser.add_parser("repo-files", help="Manage files in a repo on the Hub") - repo_files_parser.add_argument( - "repo_id", type=str, help="The ID of the repo to manage (e.g. `username/repo-name`)." - ) - repo_files_subparsers = repo_files_parser.add_subparsers( - help="Action to execute against the files.", - required=True, - ) - delete_subparser = repo_files_subparsers.add_parser( - "delete", - help="Delete files from a repo on the Hub", - ) - delete_subparser.set_defaults(func=lambda args: DeleteFilesSubCommand(args)) - delete_subparser.add_argument( - "patterns", - nargs="+", - type=str, - help="Glob patterns to match files to delete.", - ) - delete_subparser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Type of the repo to upload to (e.g. `dataset`).", - ) - delete_subparser.add_argument( - "--revision", - type=str, - help=( - "An optional Git revision to push to. It can be a branch name " - "or a PR reference. If revision does not" - " exist and `--create-pr` is not set, a branch will be automatically created." - ), - ) - delete_subparser.add_argument( - "--commit-message", type=str, help="The summary / title / first line of the generated commit." - ) - delete_subparser.add_argument( - "--commit-description", type=str, help="The description of the generated commit." - ) - delete_subparser.add_argument( - "--create-pr", action="store_true", help="Whether to create a new Pull Request for these changes." - ) - repo_files_parser.add_argument( - "--token", - type=str, - help="A User Access Token generated from https://huggingface.co/settings/tokens", - ) - - repo_files_parser.set_defaults(func=RepoFilesCommand) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/scan_cache.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/scan_cache.py deleted file mode 100644 index 711a5d09cc2b64b9c7f22a298e26a198b4dc48f1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/scan_cache.py +++ /dev/null @@ -1,183 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to scan the HF cache directory. - -Usage: - huggingface-cli scan-cache - huggingface-cli scan-cache -v - huggingface-cli scan-cache -vvv - huggingface-cli scan-cache --dir ~/.cache/huggingface/hub -""" - -import time -from argparse import Namespace, _SubParsersAction -from typing import Optional - -from ..utils import CacheNotFound, HFCacheInfo, scan_cache_dir -from . import BaseHuggingfaceCLICommand -from ._cli_utils import ANSI, show_deprecation_warning, tabulate - - -class ScanCacheCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - scan_cache_parser = parser.add_parser("scan-cache", help="Scan cache directory.") - - scan_cache_parser.add_argument( - "--dir", - type=str, - default=None, - help="cache directory to scan (optional). Default to the default HuggingFace cache.", - ) - scan_cache_parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="show a more verbose output", - ) - scan_cache_parser.set_defaults(func=ScanCacheCommand) - - def __init__(self, args: Namespace) -> None: - self.verbosity: int = args.verbose - self.cache_dir: Optional[str] = args.dir - - def run(self): - show_deprecation_warning("huggingface-cli scan-cache", "hf cache scan") - - try: - t0 = time.time() - hf_cache_info = scan_cache_dir(self.cache_dir) - t1 = time.time() - except CacheNotFound as exc: - cache_dir = exc.cache_dir - print(f"Cache directory not found: {cache_dir}") - return - - self._print_hf_cache_info_as_table(hf_cache_info) - - print( - f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)" - f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}." - ) - if len(hf_cache_info.warnings) > 0: - message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning." - if self.verbosity >= 3: - print(ANSI.gray(message)) - for warning in hf_cache_info.warnings: - print(ANSI.gray(str(warning))) - else: - print(ANSI.gray(message + " Use -vvv to print details.")) - - def _print_hf_cache_info_as_table(self, hf_cache_info: HFCacheInfo) -> None: - print(get_table(hf_cache_info, verbosity=self.verbosity)) - - -def get_table(hf_cache_info: HFCacheInfo, *, verbosity: int = 0) -> str: - """Generate a table from the [`HFCacheInfo`] object. - - Pass `verbosity=0` to get a table with a single row per repo, with columns - "repo_id", "repo_type", "size_on_disk", "nb_files", "last_accessed", "last_modified", "refs", "local_path". - - Pass `verbosity=1` to get a table with a row per repo and revision (thus multiple rows can appear for a single repo), with columns - "repo_id", "repo_type", "revision", "size_on_disk", "nb_files", "last_modified", "refs", "local_path". - - Example: - ```py - >>> from huggingface_hub.utils import scan_cache_dir - >>> from huggingface_hub.commands.scan_cache import get_table - - >>> hf_cache_info = scan_cache_dir() - HFCacheInfo(...) - - >>> print(get_table(hf_cache_info, verbosity=0)) - REPO ID REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED LAST_MODIFIED REFS LOCAL PATH - --------------------------------------------------- --------- ------------ -------- ------------- ------------- ---- -------------------------------------------------------------------------------------------------- - roberta-base model 2.7M 5 1 day ago 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--roberta-base - suno/bark model 8.8K 1 1 week ago 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--suno--bark - t5-base model 893.8M 4 4 days ago 7 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-base - t5-large model 3.0G 4 5 weeks ago 5 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-large - - >>> print(get_table(hf_cache_info, verbosity=1)) - REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED REFS LOCAL PATH - --------------------------------------------------- --------- ---------------------------------------- ------------ -------- ------------- ---- ----------------------------------------------------------------------------------------------------------------------------------------------------- - roberta-base model e2da8e2f811d1448a5b465c236feacd80ffbac7b 2.7M 5 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--roberta-base\\snapshots\\e2da8e2f811d1448a5b465c236feacd80ffbac7b - suno/bark model 70a8a7d34168586dc5d028fa9666aceade177992 8.8K 1 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--suno--bark\\snapshots\\70a8a7d34168586dc5d028fa9666aceade177992 - t5-base model a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 893.8M 4 7 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-base\\snapshots\\a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 - t5-large model 150ebc2c4b72291e770f58e6057481c8d2ed331a 3.0G 4 5 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-large\\snapshots\\150ebc2c4b72291e770f58e6057481c8d2ed331a ``` - ``` - - Args: - hf_cache_info ([`HFCacheInfo`]): - The HFCacheInfo object to print. - verbosity (`int`, *optional*): - The verbosity level. Defaults to 0. - - Returns: - `str`: The table as a string. - """ - if verbosity == 0: - return tabulate( - rows=[ - [ - repo.repo_id, - repo.repo_type, - "{:>12}".format(repo.size_on_disk_str), - repo.nb_files, - repo.last_accessed_str, - repo.last_modified_str, - ", ".join(sorted(repo.refs)), - str(repo.repo_path), - ] - for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path) - ], - headers=[ - "REPO ID", - "REPO TYPE", - "SIZE ON DISK", - "NB FILES", - "LAST_ACCESSED", - "LAST_MODIFIED", - "REFS", - "LOCAL PATH", - ], - ) - else: - return tabulate( - rows=[ - [ - repo.repo_id, - repo.repo_type, - revision.commit_hash, - "{:>12}".format(revision.size_on_disk_str), - revision.nb_files, - revision.last_modified_str, - ", ".join(sorted(revision.refs)), - str(revision.snapshot_path), - ] - for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path) - for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash) - ], - headers=[ - "REPO ID", - "REPO TYPE", - "REVISION", - "SIZE ON DISK", - "NB FILES", - "LAST_MODIFIED", - "REFS", - "LOCAL PATH", - ], - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/tag.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/tag.py deleted file mode 100644 index 405d407f8135d940cf078f905a6e66acd4b1dacc..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/tag.py +++ /dev/null @@ -1,161 +0,0 @@ -# coding=utf-8 -# Copyright 2024-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Contains commands to perform tag management with the CLI. - -Usage Examples: - - Create a tag: - $ huggingface-cli tag user/my-model 1.0 --message "First release" - $ huggingface-cli tag user/my-model 1.0 -m "First release" --revision develop - $ huggingface-cli tag user/my-dataset 1.0 -m "First release" --repo-type dataset - $ huggingface-cli tag user/my-space 1.0 - - List all tags: - $ huggingface-cli tag -l user/my-model - $ huggingface-cli tag --list user/my-dataset --repo-type dataset - - Delete a tag: - $ huggingface-cli tag -d user/my-model 1.0 - $ huggingface-cli tag --delete user/my-dataset 1.0 --repo-type dataset - $ huggingface-cli tag -d user/my-space 1.0 -y -""" - -from argparse import Namespace, _SubParsersAction - -from requests.exceptions import HTTPError - -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.constants import ( - REPO_TYPES, -) -from huggingface_hub.hf_api import HfApi - -from ..errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError -from ._cli_utils import ANSI, show_deprecation_warning - - -class TagCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - tag_parser = parser.add_parser("tag", help="(create, list, delete) tags for a repo in the hub") - - tag_parser.add_argument("repo_id", type=str, help="The ID of the repo to tag (e.g. `username/repo-name`).") - tag_parser.add_argument("tag", nargs="?", type=str, help="The name of the tag for creation or deletion.") - tag_parser.add_argument("-m", "--message", type=str, help="The description of the tag to create.") - tag_parser.add_argument("--revision", type=str, help="The git revision to tag.") - tag_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens." - ) - tag_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Set the type of repository (model, dataset, or space).", - ) - tag_parser.add_argument("-y", "--yes", action="store_true", help="Answer Yes to prompts automatically.") - - tag_parser.add_argument("-l", "--list", action="store_true", help="List tags for a repository.") - tag_parser.add_argument("-d", "--delete", action="store_true", help="Delete a tag for a repository.") - - tag_parser.set_defaults(func=lambda args: handle_commands(args)) - - -def handle_commands(args: Namespace): - show_deprecation_warning("huggingface-cli tag", "hf repo tag") - - if args.list: - return TagListCommand(args) - elif args.delete: - return TagDeleteCommand(args) - else: - return TagCreateCommand(args) - - -class TagCommand: - def __init__(self, args: Namespace): - self.args = args - self.api = HfApi(token=self.args.token) - self.repo_id = self.args.repo_id - self.repo_type = self.args.repo_type - if self.repo_type not in REPO_TYPES: - print("Invalid repo --repo-type") - exit(1) - - -class TagCreateCommand(TagCommand): - def run(self): - print(f"You are about to create tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}") - - try: - self.api.create_tag( - repo_id=self.repo_id, - tag=self.args.tag, - tag_message=self.args.message, - revision=self.args.revision, - repo_type=self.repo_type, - ) - except RepositoryNotFoundError: - print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") - exit(1) - except RevisionNotFoundError: - print(f"Revision {ANSI.bold(self.args.revision)} not found.") - exit(1) - except HfHubHTTPError as e: - if e.response.status_code == 409: - print(f"Tag {ANSI.bold(self.args.tag)} already exists on {ANSI.bold(self.repo_id)}") - exit(1) - raise e - - print(f"Tag {ANSI.bold(self.args.tag)} created on {ANSI.bold(self.repo_id)}") - - -class TagListCommand(TagCommand): - def run(self): - try: - refs = self.api.list_repo_refs( - repo_id=self.repo_id, - repo_type=self.repo_type, - ) - except RepositoryNotFoundError: - print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") - exit(1) - except HTTPError as e: - print(e) - print(ANSI.red(e.response.text)) - exit(1) - if len(refs.tags) == 0: - print("No tags found") - exit(0) - print(f"Tags for {self.repo_type} {ANSI.bold(self.repo_id)}:") - for tag in refs.tags: - print(tag.name) - - -class TagDeleteCommand(TagCommand): - def run(self): - print(f"You are about to delete tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}") - - if not self.args.yes: - choice = input("Proceed? [Y/n] ").lower() - if choice not in ("", "y", "yes"): - print("Abort") - exit() - try: - self.api.delete_tag(repo_id=self.repo_id, tag=self.args.tag, repo_type=self.repo_type) - except RepositoryNotFoundError: - print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") - exit(1) - except RevisionNotFoundError: - print(f"Tag {ANSI.bold(self.args.tag)} not found on {ANSI.bold(self.repo_id)}") - exit(1) - print(f"Tag {ANSI.bold(self.args.tag)} deleted on {ANSI.bold(self.repo_id)}") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/upload.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/upload.py deleted file mode 100644 index c778555cda56eb17c905f0728fef6712acc75cb8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/upload.py +++ /dev/null @@ -1,318 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to upload a repo or file with the CLI. - -Usage: - # Upload file (implicit) - huggingface-cli upload my-cool-model ./my-cool-model.safetensors - - # Upload file (explicit) - huggingface-cli upload my-cool-model ./my-cool-model.safetensors model.safetensors - - # Upload directory (implicit). If `my-cool-model/` is a directory it will be uploaded, otherwise an exception is raised. - huggingface-cli upload my-cool-model - - # Upload directory (explicit) - huggingface-cli upload my-cool-model ./models/my-cool-model . - - # Upload filtered directory (example: tensorboard logs except for the last run) - huggingface-cli upload my-cool-model ./model/training /logs --include "*.tfevents.*" --exclude "*20230905*" - - # Upload with wildcard - huggingface-cli upload my-cool-model "./model/training/*.safetensors" - - # Upload private dataset - huggingface-cli upload Wauplin/my-cool-dataset ./data . --repo-type=dataset --private - - # Upload with token - huggingface-cli upload Wauplin/my-cool-model --token=hf_**** - - # Sync local Space with Hub (upload new files, delete removed files) - huggingface-cli upload Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub" - - # Schedule commits every 30 minutes - huggingface-cli upload Wauplin/my-cool-model --every=30 -""" - -import os -import time -import warnings -from argparse import Namespace, _SubParsersAction -from typing import List, Optional - -from huggingface_hub import logging -from huggingface_hub._commit_scheduler import CommitScheduler -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.constants import HF_HUB_ENABLE_HF_TRANSFER -from huggingface_hub.errors import RevisionNotFoundError -from huggingface_hub.hf_api import HfApi -from huggingface_hub.utils import disable_progress_bars, enable_progress_bars -from huggingface_hub.utils._runtime import is_xet_available - -from ._cli_utils import show_deprecation_warning - - -logger = logging.get_logger(__name__) - - -class UploadCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - upload_parser = parser.add_parser("upload", help="Upload a file or a folder to a repo on the Hub") - upload_parser.add_argument( - "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)." - ) - upload_parser.add_argument( - "local_path", - nargs="?", - help="Local path to the file or folder to upload. Wildcard patterns are supported. Defaults to current directory.", - ) - upload_parser.add_argument( - "path_in_repo", - nargs="?", - help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.", - ) - upload_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Type of the repo to upload to (e.g. `dataset`).", - ) - upload_parser.add_argument( - "--revision", - type=str, - help=( - "An optional Git revision to push to. It can be a branch name or a PR reference. If revision does not" - " exist and `--create-pr` is not set, a branch will be automatically created." - ), - ) - upload_parser.add_argument( - "--private", - action="store_true", - help=( - "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already" - " exists." - ), - ) - upload_parser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.") - upload_parser.add_argument( - "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload." - ) - upload_parser.add_argument( - "--delete", - nargs="*", - type=str, - help="Glob patterns for file to be deleted from the repo while committing.", - ) - upload_parser.add_argument( - "--commit-message", type=str, help="The summary / title / first line of the generated commit." - ) - upload_parser.add_argument("--commit-description", type=str, help="The description of the generated commit.") - upload_parser.add_argument( - "--create-pr", action="store_true", help="Whether to upload content as a new Pull Request." - ) - upload_parser.add_argument( - "--every", - type=float, - help="If set, a background job is scheduled to create commits every `every` minutes.", - ) - upload_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - upload_parser.add_argument( - "--quiet", - action="store_true", - help="If True, progress bars are disabled and only the path to the uploaded files is printed.", - ) - upload_parser.set_defaults(func=UploadCommand) - - def __init__(self, args: Namespace) -> None: - self.repo_id: str = args.repo_id - self.repo_type: Optional[str] = args.repo_type - self.revision: Optional[str] = args.revision - self.private: bool = args.private - - self.include: Optional[List[str]] = args.include - self.exclude: Optional[List[str]] = args.exclude - self.delete: Optional[List[str]] = args.delete - - self.commit_message: Optional[str] = args.commit_message - self.commit_description: Optional[str] = args.commit_description - self.create_pr: bool = args.create_pr - self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") - self.quiet: bool = args.quiet # disable warnings and progress bars - - # Check `--every` is valid - if args.every is not None and args.every <= 0: - raise ValueError(f"`every` must be a positive value (got '{args.every}')") - self.every: Optional[float] = args.every - - # Resolve `local_path` and `path_in_repo` - repo_name: str = args.repo_id.split("/")[-1] # e.g. "Wauplin/my-cool-model" => "my-cool-model" - self.local_path: str - self.path_in_repo: str - - if args.local_path is not None and any(c in args.local_path for c in ["*", "?", "["]): - if args.include is not None: - raise ValueError("Cannot set `--include` when passing a `local_path` containing a wildcard.") - if args.path_in_repo is not None and args.path_in_repo != ".": - raise ValueError("Cannot set `path_in_repo` when passing a `local_path` containing a wildcard.") - self.local_path = "." - self.include = args.local_path - self.path_in_repo = "." - elif args.local_path is None and os.path.isfile(repo_name): - # Implicit case 1: user provided only a repo_id which happen to be a local file as well => upload it with same name - self.local_path = repo_name - self.path_in_repo = repo_name - elif args.local_path is None and os.path.isdir(repo_name): - # Implicit case 2: user provided only a repo_id which happen to be a local folder as well => upload it at root - self.local_path = repo_name - self.path_in_repo = "." - elif args.local_path is None: - # Implicit case 3: user provided only a repo_id that does not match a local file or folder - # => the user must explicitly provide a local_path => raise exception - raise ValueError(f"'{repo_name}' is not a local file or folder. Please set `local_path` explicitly.") - elif args.path_in_repo is None and os.path.isfile(args.local_path): - # Explicit local path to file, no path in repo => upload it at root with same name - self.local_path = args.local_path - self.path_in_repo = os.path.basename(args.local_path) - elif args.path_in_repo is None: - # Explicit local path to folder, no path in repo => upload at root - self.local_path = args.local_path - self.path_in_repo = "." - else: - # Finally, if both paths are explicit - self.local_path = args.local_path - self.path_in_repo = args.path_in_repo - - def run(self) -> None: - show_deprecation_warning("huggingface-cli upload", "hf upload") - - if self.quiet: - disable_progress_bars() - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - print(self._upload()) - enable_progress_bars() - else: - logging.set_verbosity_info() - print(self._upload()) - logging.set_verbosity_warning() - - def _upload(self) -> str: - if os.path.isfile(self.local_path): - if self.include is not None and len(self.include) > 0: - warnings.warn("Ignoring `--include` since a single file is uploaded.") - if self.exclude is not None and len(self.exclude) > 0: - warnings.warn("Ignoring `--exclude` since a single file is uploaded.") - if self.delete is not None and len(self.delete) > 0: - warnings.warn("Ignoring `--delete` since a single file is uploaded.") - - if not is_xet_available() and not HF_HUB_ENABLE_HF_TRANSFER: - logger.info( - "Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See" - " https://huggingface.co/docs/huggingface_hub/hf_transfer for more details." - ) - - # Schedule commits if `every` is set - if self.every is not None: - if os.path.isfile(self.local_path): - # If file => watch entire folder + use allow_patterns - folder_path = os.path.dirname(self.local_path) - path_in_repo = ( - self.path_in_repo[: -len(self.local_path)] # remove filename from path_in_repo - if self.path_in_repo.endswith(self.local_path) - else self.path_in_repo - ) - allow_patterns = [self.local_path] - ignore_patterns = [] - else: - folder_path = self.local_path - path_in_repo = self.path_in_repo - allow_patterns = self.include or [] - ignore_patterns = self.exclude or [] - if self.delete is not None and len(self.delete) > 0: - warnings.warn("Ignoring `--delete` when uploading with scheduled commits.") - - scheduler = CommitScheduler( - folder_path=folder_path, - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - path_in_repo=path_in_repo, - private=self.private, - every=self.every, - hf_api=self.api, - ) - print(f"Scheduling commits every {self.every} minutes to {scheduler.repo_id}.") - try: # Block main thread until KeyboardInterrupt - while True: - time.sleep(100) - except KeyboardInterrupt: - scheduler.stop() - return "Stopped scheduled commits." - - # Otherwise, create repo and proceed with the upload - if not os.path.isfile(self.local_path) and not os.path.isdir(self.local_path): - raise FileNotFoundError(f"No such file or directory: '{self.local_path}'.") - repo_id = self.api.create_repo( - repo_id=self.repo_id, - repo_type=self.repo_type, - exist_ok=True, - private=self.private, - space_sdk="gradio" if self.repo_type == "space" else None, - # ^ We don't want it to fail when uploading to a Space => let's set Gradio by default. - # ^ I'd rather not add CLI args to set it explicitly as we already have `huggingface-cli repo create` for that. - ).repo_id - - # Check if branch already exists and if not, create it - if self.revision is not None and not self.create_pr: - try: - self.api.repo_info(repo_id=repo_id, repo_type=self.repo_type, revision=self.revision) - except RevisionNotFoundError: - logger.info(f"Branch '{self.revision}' not found. Creating it...") - self.api.create_branch(repo_id=repo_id, repo_type=self.repo_type, branch=self.revision, exist_ok=True) - # ^ `exist_ok=True` to avoid race concurrency issues - - # File-based upload - if os.path.isfile(self.local_path): - return self.api.upload_file( - path_or_fileobj=self.local_path, - path_in_repo=self.path_in_repo, - repo_id=repo_id, - repo_type=self.repo_type, - revision=self.revision, - commit_message=self.commit_message, - commit_description=self.commit_description, - create_pr=self.create_pr, - ) - - # Folder-based upload - else: - return self.api.upload_folder( - folder_path=self.local_path, - path_in_repo=self.path_in_repo, - repo_id=repo_id, - repo_type=self.repo_type, - revision=self.revision, - commit_message=self.commit_message, - commit_description=self.commit_description, - create_pr=self.create_pr, - allow_patterns=self.include, - ignore_patterns=self.exclude, - delete_patterns=self.delete, - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/upload_large_folder.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/upload_large_folder.py deleted file mode 100644 index 3105ba3f57f5644aa18e627aa5d1d18e61515ae7..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/upload_large_folder.py +++ /dev/null @@ -1,131 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to upload a large folder with the CLI.""" - -import os -from argparse import Namespace, _SubParsersAction -from typing import List, Optional - -from huggingface_hub import logging -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.hf_api import HfApi -from huggingface_hub.utils import disable_progress_bars - -from ._cli_utils import ANSI, show_deprecation_warning - - -logger = logging.get_logger(__name__) - - -class UploadLargeFolderCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - subparser = parser.add_parser("upload-large-folder", help="Upload a large folder to a repo on the Hub") - subparser.add_argument( - "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)." - ) - subparser.add_argument("local_path", type=str, help="Local path to the file or folder to upload.") - subparser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - help="Type of the repo to upload to (e.g. `dataset`).", - ) - subparser.add_argument( - "--revision", - type=str, - help=("An optional Git revision to push to. It can be a branch name or a PR reference."), - ) - subparser.add_argument( - "--private", - action="store_true", - help=( - "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists." - ), - ) - subparser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.") - subparser.add_argument("--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload.") - subparser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - subparser.add_argument( - "--num-workers", type=int, help="Number of workers to use to hash, upload and commit files." - ) - subparser.add_argument("--no-report", action="store_true", help="Whether to disable regular status report.") - subparser.add_argument("--no-bars", action="store_true", help="Whether to disable progress bars.") - subparser.set_defaults(func=UploadLargeFolderCommand) - - def __init__(self, args: Namespace) -> None: - self.repo_id: str = args.repo_id - self.local_path: str = args.local_path - self.repo_type: str = args.repo_type - self.revision: Optional[str] = args.revision - self.private: bool = args.private - - self.include: Optional[List[str]] = args.include - self.exclude: Optional[List[str]] = args.exclude - - self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") - - self.num_workers: Optional[int] = args.num_workers - self.no_report: bool = args.no_report - self.no_bars: bool = args.no_bars - - if not os.path.isdir(self.local_path): - raise ValueError("Large upload is only supported for folders.") - - def run(self) -> None: - show_deprecation_warning("huggingface-cli upload-large-folder", "hf upload-large-folder") - - logging.set_verbosity_info() - - print( - ANSI.yellow( - "You are about to upload a large folder to the Hub using `huggingface-cli upload-large-folder`. " - "This is a new feature so feedback is very welcome!\n" - "\n" - "A few things to keep in mind:\n" - " - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n" - " - Do not start several processes in parallel.\n" - " - You can interrupt and resume the process at any time. " - "The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n" - " - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n" - "\n" - f"Some temporary metadata will be stored under `{self.local_path}/.cache/huggingface`.\n" - " - You must not modify those files manually.\n" - " - You must not delete the `./.cache/huggingface/` folder while a process is running.\n" - " - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n" - "\n" - "If the process output is too verbose, you can disable the progress bars with `--no-bars`. " - "You can also entirely disable the status report with `--no-report`.\n" - "\n" - "For more details, run `huggingface-cli upload-large-folder --help` or check the documentation at " - "https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder." - ) - ) - - if self.no_bars: - disable_progress_bars() - - self.api.upload_large_folder( - repo_id=self.repo_id, - folder_path=self.local_path, - repo_type=self.repo_type, - revision=self.revision, - private=self.private, - allow_patterns=self.include, - ignore_patterns=self.exclude, - num_workers=self.num_workers, - print_report=not self.no_report, - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/user.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/user.py deleted file mode 100644 index 3f4da0f45d0dae5bc4458f844f776db9c3971208..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/user.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains commands to authenticate to the Hugging Face Hub and interact with your repositories. - -Usage: - # login and save token locally. - huggingface-cli login --token=hf_*** --add-to-git-credential - - # switch between tokens - huggingface-cli auth switch - - # list all tokens - huggingface-cli auth list - - # logout from a specific token, if no token-name is provided, all tokens will be deleted from your machine. - huggingface-cli logout --token-name=your_token_name - - # find out which huggingface.co account you are logged in as - huggingface-cli whoami -""" - -from argparse import _SubParsersAction -from typing import List, Optional - -from requests.exceptions import HTTPError - -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.constants import ENDPOINT -from huggingface_hub.hf_api import HfApi - -from .._login import auth_list, auth_switch, login, logout -from ..utils import get_stored_tokens, get_token, logging -from ._cli_utils import ANSI, show_deprecation_warning - - -logger = logging.get_logger(__name__) - -try: - from InquirerPy import inquirer - from InquirerPy.base.control import Choice - - _inquirer_py_available = True -except ImportError: - _inquirer_py_available = False - - -class UserCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - login_parser = parser.add_parser("login", help="Log in using a token from huggingface.co/settings/tokens") - login_parser.add_argument( - "--token", - type=str, - help="Token generated from https://huggingface.co/settings/tokens", - ) - login_parser.add_argument( - "--add-to-git-credential", - action="store_true", - help="Optional: Save token to git credential helper.", - ) - login_parser.set_defaults(func=lambda args: LoginCommand(args)) - whoami_parser = parser.add_parser("whoami", help="Find out which huggingface.co account you are logged in as.") - whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args)) - - logout_parser = parser.add_parser("logout", help="Log out") - logout_parser.add_argument( - "--token-name", - type=str, - help="Optional: Name of the access token to log out from.", - ) - logout_parser.set_defaults(func=lambda args: LogoutCommand(args)) - - auth_parser = parser.add_parser("auth", help="Other authentication related commands") - auth_subparsers = auth_parser.add_subparsers(help="Authentication subcommands") - auth_switch_parser = auth_subparsers.add_parser("switch", help="Switch between access tokens") - auth_switch_parser.add_argument( - "--token-name", - type=str, - help="Optional: Name of the access token to switch to.", - ) - auth_switch_parser.add_argument( - "--add-to-git-credential", - action="store_true", - help="Optional: Save token to git credential helper.", - ) - auth_switch_parser.set_defaults(func=lambda args: AuthSwitchCommand(args)) - auth_list_parser = auth_subparsers.add_parser("list", help="List all stored access tokens") - auth_list_parser.set_defaults(func=lambda args: AuthListCommand(args)) - - -class BaseUserCommand: - def __init__(self, args): - self.args = args - self._api = HfApi() - - -class LoginCommand(BaseUserCommand): - def run(self): - show_deprecation_warning("huggingface-cli login", "hf auth login") - - logging.set_verbosity_info() - login( - token=self.args.token, - add_to_git_credential=self.args.add_to_git_credential, - ) - - -class LogoutCommand(BaseUserCommand): - def run(self): - show_deprecation_warning("huggingface-cli logout", "hf auth logout") - - logging.set_verbosity_info() - logout(token_name=self.args.token_name) - - -class AuthSwitchCommand(BaseUserCommand): - def run(self): - show_deprecation_warning("huggingface-cli auth switch", "hf auth switch") - - logging.set_verbosity_info() - token_name = self.args.token_name - if token_name is None: - token_name = self._select_token_name() - - if token_name is None: - print("No token name provided. Aborting.") - exit() - auth_switch(token_name, add_to_git_credential=self.args.add_to_git_credential) - - def _select_token_name(self) -> Optional[str]: - token_names = list(get_stored_tokens().keys()) - - if not token_names: - logger.error("No stored tokens found. Please login first.") - return None - - if _inquirer_py_available: - return self._select_token_name_tui(token_names) - # if inquirer is not available, use a simpler terminal UI - print("Available stored tokens:") - for i, token_name in enumerate(token_names, 1): - print(f"{i}. {token_name}") - while True: - try: - choice = input("Enter the number of the token to switch to (or 'q' to quit): ") - if choice.lower() == "q": - return None - index = int(choice) - 1 - if 0 <= index < len(token_names): - return token_names[index] - else: - print("Invalid selection. Please try again.") - except ValueError: - print("Invalid input. Please enter a number or 'q' to quit.") - - def _select_token_name_tui(self, token_names: List[str]) -> Optional[str]: - choices = [Choice(token_name, name=token_name) for token_name in token_names] - try: - return inquirer.select( - message="Select a token to switch to:", - choices=choices, - default=None, - ).execute() - except KeyboardInterrupt: - logger.info("Token selection cancelled.") - return None - - -class AuthListCommand(BaseUserCommand): - def run(self): - show_deprecation_warning("huggingface-cli auth list", "hf auth list") - - logging.set_verbosity_info() - auth_list() - - -class WhoamiCommand(BaseUserCommand): - def run(self): - show_deprecation_warning("huggingface-cli whoami", "hf auth whoami") - - token = get_token() - if token is None: - print("Not logged in") - exit() - try: - info = self._api.whoami(token) - print(info["name"]) - orgs = [org["name"] for org in info["orgs"]] - if orgs: - print(ANSI.bold("orgs: "), ",".join(orgs)) - - if ENDPOINT != "https://huggingface.co": - print(f"Authenticated through private endpoint: {ENDPOINT}") - except HTTPError as e: - print(e) - print(ANSI.red(e.response.text)) - exit(1) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/version.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/version.py deleted file mode 100644 index 10d341bcdb93e0616fcf80370ac8dde63b15ce9c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/commands/version.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2022 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to print information about the version. - -Usage: - huggingface-cli version -""" - -from argparse import _SubParsersAction - -from huggingface_hub import __version__ - -from . import BaseHuggingfaceCLICommand -from ._cli_utils import show_deprecation_warning - - -class VersionCommand(BaseHuggingfaceCLICommand): - def __init__(self, args): - self.args = args - - @staticmethod - def register_subcommand(parser: _SubParsersAction): - version_parser = parser.add_parser("version", help="Print information about the huggingface-cli version.") - version_parser.set_defaults(func=VersionCommand) - - def run(self) -> None: - show_deprecation_warning("huggingface-cli version", "hf version") - - print(f"huggingface_hub version: {__version__}") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/community.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/community.py deleted file mode 100644 index ffaab355174689b1dfb5b1c95f06fc088859d4cf..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/community.py +++ /dev/null @@ -1,363 +0,0 @@ -""" -Data structures to interact with Discussions and Pull Requests on the Hub. - -See [the Discussions and Pull Requests guide](https://huggingface.co/docs/hub/repositories-pull-requests-discussions) -for more information on Pull Requests, Discussions, and the community tab. -""" - -from dataclasses import dataclass -from datetime import datetime -from typing import List, Literal, Optional, TypedDict, Union - -from . import constants -from .utils import parse_datetime - - -DiscussionStatus = Literal["open", "closed", "merged", "draft"] - - -@dataclass -class Discussion: - """ - A Discussion or Pull Request on the Hub. - - This dataclass is not intended to be instantiated directly. - - Attributes: - title (`str`): - The title of the Discussion / Pull Request - status (`str`): - The status of the Discussion / Pull Request. - It must be one of: - * `"open"` - * `"closed"` - * `"merged"` (only for Pull Requests ) - * `"draft"` (only for Pull Requests ) - num (`int`): - The number of the Discussion / Pull Request. - repo_id (`str`): - The id (`"{namespace}/{repo_name}"`) of the repo on which - the Discussion / Pull Request was open. - repo_type (`str`): - The type of the repo on which the Discussion / Pull Request was open. - Possible values are: `"model"`, `"dataset"`, `"space"`. - author (`str`): - The username of the Discussion / Pull Request author. - Can be `"deleted"` if the user has been deleted since. - is_pull_request (`bool`): - Whether or not this is a Pull Request. - created_at (`datetime`): - The `datetime` of creation of the Discussion / Pull Request. - endpoint (`str`): - Endpoint of the Hub. Default is https://huggingface.co. - git_reference (`str`, *optional*): - (property) Git reference to which changes can be pushed if this is a Pull Request, `None` otherwise. - url (`str`): - (property) URL of the discussion on the Hub. - """ - - title: str - status: DiscussionStatus - num: int - repo_id: str - repo_type: str - author: str - is_pull_request: bool - created_at: datetime - endpoint: str - - @property - def git_reference(self) -> Optional[str]: - """ - If this is a Pull Request , returns the git reference to which changes can be pushed. - Returns `None` otherwise. - """ - if self.is_pull_request: - return f"refs/pr/{self.num}" - return None - - @property - def url(self) -> str: - """Returns the URL of the discussion on the Hub.""" - if self.repo_type is None or self.repo_type == constants.REPO_TYPE_MODEL: - return f"{self.endpoint}/{self.repo_id}/discussions/{self.num}" - return f"{self.endpoint}/{self.repo_type}s/{self.repo_id}/discussions/{self.num}" - - -@dataclass -class DiscussionWithDetails(Discussion): - """ - Subclass of [`Discussion`]. - - Attributes: - title (`str`): - The title of the Discussion / Pull Request - status (`str`): - The status of the Discussion / Pull Request. - It can be one of: - * `"open"` - * `"closed"` - * `"merged"` (only for Pull Requests ) - * `"draft"` (only for Pull Requests ) - num (`int`): - The number of the Discussion / Pull Request. - repo_id (`str`): - The id (`"{namespace}/{repo_name}"`) of the repo on which - the Discussion / Pull Request was open. - repo_type (`str`): - The type of the repo on which the Discussion / Pull Request was open. - Possible values are: `"model"`, `"dataset"`, `"space"`. - author (`str`): - The username of the Discussion / Pull Request author. - Can be `"deleted"` if the user has been deleted since. - is_pull_request (`bool`): - Whether or not this is a Pull Request. - created_at (`datetime`): - The `datetime` of creation of the Discussion / Pull Request. - events (`list` of [`DiscussionEvent`]) - The list of [`DiscussionEvents`] in this Discussion or Pull Request. - conflicting_files (`Union[List[str], bool, None]`, *optional*): - A list of conflicting files if this is a Pull Request. - `None` if `self.is_pull_request` is `False`. - `True` if there are conflicting files but the list can't be retrieved. - target_branch (`str`, *optional*): - The branch into which changes are to be merged if this is a - Pull Request . `None` if `self.is_pull_request` is `False`. - merge_commit_oid (`str`, *optional*): - If this is a merged Pull Request , this is set to the OID / SHA of - the merge commit, `None` otherwise. - diff (`str`, *optional*): - The git diff if this is a Pull Request , `None` otherwise. - endpoint (`str`): - Endpoint of the Hub. Default is https://huggingface.co. - git_reference (`str`, *optional*): - (property) Git reference to which changes can be pushed if this is a Pull Request, `None` otherwise. - url (`str`): - (property) URL of the discussion on the Hub. - """ - - events: List["DiscussionEvent"] - conflicting_files: Union[List[str], bool, None] - target_branch: Optional[str] - merge_commit_oid: Optional[str] - diff: Optional[str] - - -class DiscussionEventArgs(TypedDict): - id: str - type: str - created_at: datetime - author: str - _event: dict - - -@dataclass -class DiscussionEvent: - """ - An event in a Discussion or Pull Request. - - Use concrete classes: - * [`DiscussionComment`] - * [`DiscussionStatusChange`] - * [`DiscussionCommit`] - * [`DiscussionTitleChange`] - - Attributes: - id (`str`): - The ID of the event. An hexadecimal string. - type (`str`): - The type of the event. - created_at (`datetime`): - A [`datetime`](https://docs.python.org/3/library/datetime.html?highlight=datetime#datetime.datetime) - object holding the creation timestamp for the event. - author (`str`): - The username of the Discussion / Pull Request author. - Can be `"deleted"` if the user has been deleted since. - """ - - id: str - type: str - created_at: datetime - author: str - - _event: dict - """Stores the original event data, in case we need to access it later.""" - - -@dataclass -class DiscussionComment(DiscussionEvent): - """A comment in a Discussion / Pull Request. - - Subclass of [`DiscussionEvent`]. - - - Attributes: - id (`str`): - The ID of the event. An hexadecimal string. - type (`str`): - The type of the event. - created_at (`datetime`): - A [`datetime`](https://docs.python.org/3/library/datetime.html?highlight=datetime#datetime.datetime) - object holding the creation timestamp for the event. - author (`str`): - The username of the Discussion / Pull Request author. - Can be `"deleted"` if the user has been deleted since. - content (`str`): - The raw markdown content of the comment. Mentions, links and images are not rendered. - edited (`bool`): - Whether or not this comment has been edited. - hidden (`bool`): - Whether or not this comment has been hidden. - """ - - content: str - edited: bool - hidden: bool - - @property - def rendered(self) -> str: - """The rendered comment, as a HTML string""" - return self._event["data"]["latest"]["html"] - - @property - def last_edited_at(self) -> datetime: - """The last edit time, as a `datetime` object.""" - return parse_datetime(self._event["data"]["latest"]["updatedAt"]) - - @property - def last_edited_by(self) -> str: - """The last edit time, as a `datetime` object.""" - return self._event["data"]["latest"].get("author", {}).get("name", "deleted") - - @property - def edit_history(self) -> List[dict]: - """The edit history of the comment""" - return self._event["data"]["history"] - - @property - def number_of_edits(self) -> int: - return len(self.edit_history) - - -@dataclass -class DiscussionStatusChange(DiscussionEvent): - """A change of status in a Discussion / Pull Request. - - Subclass of [`DiscussionEvent`]. - - Attributes: - id (`str`): - The ID of the event. An hexadecimal string. - type (`str`): - The type of the event. - created_at (`datetime`): - A [`datetime`](https://docs.python.org/3/library/datetime.html?highlight=datetime#datetime.datetime) - object holding the creation timestamp for the event. - author (`str`): - The username of the Discussion / Pull Request author. - Can be `"deleted"` if the user has been deleted since. - new_status (`str`): - The status of the Discussion / Pull Request after the change. - It can be one of: - * `"open"` - * `"closed"` - * `"merged"` (only for Pull Requests ) - """ - - new_status: str - - -@dataclass -class DiscussionCommit(DiscussionEvent): - """A commit in a Pull Request. - - Subclass of [`DiscussionEvent`]. - - Attributes: - id (`str`): - The ID of the event. An hexadecimal string. - type (`str`): - The type of the event. - created_at (`datetime`): - A [`datetime`](https://docs.python.org/3/library/datetime.html?highlight=datetime#datetime.datetime) - object holding the creation timestamp for the event. - author (`str`): - The username of the Discussion / Pull Request author. - Can be `"deleted"` if the user has been deleted since. - summary (`str`): - The summary of the commit. - oid (`str`): - The OID / SHA of the commit, as a hexadecimal string. - """ - - summary: str - oid: str - - -@dataclass -class DiscussionTitleChange(DiscussionEvent): - """A rename event in a Discussion / Pull Request. - - Subclass of [`DiscussionEvent`]. - - Attributes: - id (`str`): - The ID of the event. An hexadecimal string. - type (`str`): - The type of the event. - created_at (`datetime`): - A [`datetime`](https://docs.python.org/3/library/datetime.html?highlight=datetime#datetime.datetime) - object holding the creation timestamp for the event. - author (`str`): - The username of the Discussion / Pull Request author. - Can be `"deleted"` if the user has been deleted since. - old_title (`str`): - The previous title for the Discussion / Pull Request. - new_title (`str`): - The new title. - """ - - old_title: str - new_title: str - - -def deserialize_event(event: dict) -> DiscussionEvent: - """Instantiates a [`DiscussionEvent`] from a dict""" - event_id: str = event["id"] - event_type: str = event["type"] - created_at = parse_datetime(event["createdAt"]) - - common_args: DiscussionEventArgs = { - "id": event_id, - "type": event_type, - "created_at": created_at, - "author": event.get("author", {}).get("name", "deleted"), - "_event": event, - } - - if event_type == "comment": - return DiscussionComment( - **common_args, - edited=event["data"]["edited"], - hidden=event["data"]["hidden"], - content=event["data"]["latest"]["raw"], - ) - if event_type == "status-change": - return DiscussionStatusChange( - **common_args, - new_status=event["data"]["status"], - ) - if event_type == "commit": - return DiscussionCommit( - **common_args, - summary=event["data"]["subject"], - oid=event["data"]["oid"], - ) - if event_type == "title-change": - return DiscussionTitleChange( - **common_args, - old_title=event["data"]["from"], - new_title=event["data"]["to"], - ) - - return DiscussionEvent(**common_args) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/constants.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/constants.py deleted file mode 100644 index b30b2c01d99c5ee5428875f3711227024f5d0829..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/constants.py +++ /dev/null @@ -1,294 +0,0 @@ -import os -import re -import typing -from typing import Literal, Optional, Tuple - - -# Possible values for env variables - - -ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"} -ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"}) - - -def _is_true(value: Optional[str]) -> bool: - if value is None: - return False - return value.upper() in ENV_VARS_TRUE_VALUES - - -def _as_int(value: Optional[str]) -> Optional[int]: - if value is None: - return None - return int(value) - - -# Constants for file downloads - -PYTORCH_WEIGHTS_NAME = "pytorch_model.bin" -TF2_WEIGHTS_NAME = "tf_model.h5" -TF_WEIGHTS_NAME = "model.ckpt" -FLAX_WEIGHTS_NAME = "flax_model.msgpack" -CONFIG_NAME = "config.json" -REPOCARD_NAME = "README.md" -DEFAULT_ETAG_TIMEOUT = 10 -DEFAULT_DOWNLOAD_TIMEOUT = 10 -DEFAULT_REQUEST_TIMEOUT = 10 -DOWNLOAD_CHUNK_SIZE = 10 * 1024 * 1024 -HF_TRANSFER_CONCURRENCY = 100 -MAX_HTTP_DOWNLOAD_SIZE = 50 * 1000 * 1000 * 1000 # 50 GB - -# Constants for serialization - -PYTORCH_WEIGHTS_FILE_PATTERN = "pytorch_model{suffix}.bin" # Unsafe pickle: use safetensors instead -SAFETENSORS_WEIGHTS_FILE_PATTERN = "model{suffix}.safetensors" -TF2_WEIGHTS_FILE_PATTERN = "tf_model{suffix}.h5" - -# Constants for safetensors repos - -SAFETENSORS_SINGLE_FILE = "model.safetensors" -SAFETENSORS_INDEX_FILE = "model.safetensors.index.json" -SAFETENSORS_MAX_HEADER_LENGTH = 25_000_000 - -# Timeout of aquiring file lock and logging the attempt -FILELOCK_LOG_EVERY_SECONDS = 10 - -# Git-related constants - -DEFAULT_REVISION = "main" -REGEX_COMMIT_OID = re.compile(r"[A-Fa-f0-9]{5,40}") - -HUGGINGFACE_CO_URL_HOME = "https://huggingface.co/" - -_staging_mode = _is_true(os.environ.get("HUGGINGFACE_CO_STAGING")) - -_HF_DEFAULT_ENDPOINT = "https://huggingface.co" -_HF_DEFAULT_STAGING_ENDPOINT = "https://hub-ci.huggingface.co" -ENDPOINT = os.getenv("HF_ENDPOINT", _HF_DEFAULT_ENDPOINT).rstrip("/") -HUGGINGFACE_CO_URL_TEMPLATE = ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}" - -if _staging_mode: - ENDPOINT = _HF_DEFAULT_STAGING_ENDPOINT - HUGGINGFACE_CO_URL_TEMPLATE = _HF_DEFAULT_STAGING_ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}" - -HUGGINGFACE_HEADER_X_REPO_COMMIT = "X-Repo-Commit" -HUGGINGFACE_HEADER_X_LINKED_ETAG = "X-Linked-Etag" -HUGGINGFACE_HEADER_X_LINKED_SIZE = "X-Linked-Size" -HUGGINGFACE_HEADER_X_BILL_TO = "X-HF-Bill-To" - -INFERENCE_ENDPOINT = os.environ.get("HF_INFERENCE_ENDPOINT", "https://api-inference.huggingface.co") - -# See https://huggingface.co/docs/inference-endpoints/index -INFERENCE_ENDPOINTS_ENDPOINT = "https://api.endpoints.huggingface.cloud/v2" -INFERENCE_CATALOG_ENDPOINT = "https://endpoints.huggingface.co/api/catalog" - -# See https://api.endpoints.huggingface.cloud/#post-/v2/endpoint/-namespace- -INFERENCE_ENDPOINT_IMAGE_KEYS = [ - "custom", - "huggingface", - "huggingfaceNeuron", - "llamacpp", - "tei", - "tgi", - "tgiNeuron", -] - -# Proxy for third-party providers -INFERENCE_PROXY_TEMPLATE = "https://router.huggingface.co/{provider}" - -REPO_ID_SEPARATOR = "--" -# ^ this substring is not allowed in repo_ids on hf.co -# and is the canonical one we use for serialization of repo ids elsewhere. - - -REPO_TYPE_DATASET = "dataset" -REPO_TYPE_SPACE = "space" -REPO_TYPE_MODEL = "model" -REPO_TYPES = [None, REPO_TYPE_MODEL, REPO_TYPE_DATASET, REPO_TYPE_SPACE] -SPACES_SDK_TYPES = ["gradio", "streamlit", "docker", "static"] - -REPO_TYPES_URL_PREFIXES = { - REPO_TYPE_DATASET: "datasets/", - REPO_TYPE_SPACE: "spaces/", -} -REPO_TYPES_MAPPING = { - "datasets": REPO_TYPE_DATASET, - "spaces": REPO_TYPE_SPACE, - "models": REPO_TYPE_MODEL, -} - -DiscussionTypeFilter = Literal["all", "discussion", "pull_request"] -DISCUSSION_TYPES: Tuple[DiscussionTypeFilter, ...] = typing.get_args(DiscussionTypeFilter) -DiscussionStatusFilter = Literal["all", "open", "closed"] -DISCUSSION_STATUS: Tuple[DiscussionTypeFilter, ...] = typing.get_args(DiscussionStatusFilter) - -# Webhook subscription types -WEBHOOK_DOMAIN_T = Literal["repo", "discussions"] - -# default cache -default_home = os.path.join(os.path.expanduser("~"), ".cache") -HF_HOME = os.path.expandvars( - os.path.expanduser( - os.getenv( - "HF_HOME", - os.path.join(os.getenv("XDG_CACHE_HOME", default_home), "huggingface"), - ) - ) -) -hf_cache_home = HF_HOME # for backward compatibility. TODO: remove this in 1.0.0 - -default_cache_path = os.path.join(HF_HOME, "hub") -default_assets_cache_path = os.path.join(HF_HOME, "assets") - -# Legacy env variables -HUGGINGFACE_HUB_CACHE = os.getenv("HUGGINGFACE_HUB_CACHE", default_cache_path) -HUGGINGFACE_ASSETS_CACHE = os.getenv("HUGGINGFACE_ASSETS_CACHE", default_assets_cache_path) - -# New env variables -HF_HUB_CACHE = os.path.expandvars( - os.path.expanduser( - os.getenv( - "HF_HUB_CACHE", - HUGGINGFACE_HUB_CACHE, - ) - ) -) -HF_ASSETS_CACHE = os.path.expandvars( - os.path.expanduser( - os.getenv( - "HF_ASSETS_CACHE", - HUGGINGFACE_ASSETS_CACHE, - ) - ) -) - -HF_HUB_OFFLINE = _is_true(os.environ.get("HF_HUB_OFFLINE") or os.environ.get("TRANSFORMERS_OFFLINE")) - -# If set, log level will be set to DEBUG and all requests made to the Hub will be logged -# as curl commands for reproducibility. -HF_DEBUG = _is_true(os.environ.get("HF_DEBUG")) - -# Opt-out from telemetry requests -HF_HUB_DISABLE_TELEMETRY = ( - _is_true(os.environ.get("HF_HUB_DISABLE_TELEMETRY")) # HF-specific env variable - or _is_true(os.environ.get("DISABLE_TELEMETRY")) - or _is_true(os.environ.get("DO_NOT_TRACK")) # https://consoledonottrack.com/ -) - -HF_TOKEN_PATH = os.path.expandvars( - os.path.expanduser( - os.getenv( - "HF_TOKEN_PATH", - os.path.join(HF_HOME, "token"), - ) - ) -) -HF_STORED_TOKENS_PATH = os.path.join(os.path.dirname(HF_TOKEN_PATH), "stored_tokens") - -if _staging_mode: - # In staging mode, we use a different cache to ensure we don't mix up production and staging data or tokens - # In practice in `huggingface_hub` tests, we monkeypatch these values with temporary directories. The following - # lines are only used in third-party libraries tests (e.g. `transformers`, `diffusers`, etc.). - _staging_home = os.path.join(os.path.expanduser("~"), ".cache", "huggingface_staging") - HUGGINGFACE_HUB_CACHE = os.path.join(_staging_home, "hub") - HF_TOKEN_PATH = os.path.join(_staging_home, "token") - -# Here, `True` will disable progress bars globally without possibility of enabling it -# programmatically. `False` will enable them without possibility of disabling them. -# If environment variable is not set (None), then the user is free to enable/disable -# them programmatically. -# TL;DR: env variable has priority over code -__HF_HUB_DISABLE_PROGRESS_BARS = os.environ.get("HF_HUB_DISABLE_PROGRESS_BARS") -HF_HUB_DISABLE_PROGRESS_BARS: Optional[bool] = ( - _is_true(__HF_HUB_DISABLE_PROGRESS_BARS) if __HF_HUB_DISABLE_PROGRESS_BARS is not None else None -) - -# Disable warning on machines that do not support symlinks (e.g. Windows non-developer) -HF_HUB_DISABLE_SYMLINKS_WARNING: bool = _is_true(os.environ.get("HF_HUB_DISABLE_SYMLINKS_WARNING")) - -# Disable warning when using experimental features -HF_HUB_DISABLE_EXPERIMENTAL_WARNING: bool = _is_true(os.environ.get("HF_HUB_DISABLE_EXPERIMENTAL_WARNING")) - -# Disable sending the cached token by default is all HTTP requests to the Hub -HF_HUB_DISABLE_IMPLICIT_TOKEN: bool = _is_true(os.environ.get("HF_HUB_DISABLE_IMPLICIT_TOKEN")) - -# Enable fast-download using external dependency "hf_transfer" -# See: -# - https://pypi.org/project/hf-transfer/ -# - https://github.com/huggingface/hf_transfer (private) -HF_HUB_ENABLE_HF_TRANSFER: bool = _is_true(os.environ.get("HF_HUB_ENABLE_HF_TRANSFER")) - - -# UNUSED -# We don't use symlinks in local dir anymore. -HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD: int = ( - _as_int(os.environ.get("HF_HUB_LOCAL_DIR_AUTO_SYMLINK_THRESHOLD")) or 5 * 1024 * 1024 -) - -# Used to override the etag timeout on a system level -HF_HUB_ETAG_TIMEOUT: int = _as_int(os.environ.get("HF_HUB_ETAG_TIMEOUT")) or DEFAULT_ETAG_TIMEOUT - -# Used to override the get request timeout on a system level -HF_HUB_DOWNLOAD_TIMEOUT: int = _as_int(os.environ.get("HF_HUB_DOWNLOAD_TIMEOUT")) or DEFAULT_DOWNLOAD_TIMEOUT - -# Allows to add information about the requester in the user-agent (eg. partner name) -HF_HUB_USER_AGENT_ORIGIN: Optional[str] = os.environ.get("HF_HUB_USER_AGENT_ORIGIN") - -# List frameworks that are handled by the InferenceAPI service. Useful to scan endpoints and check which models are -# deployed and running. Since 95% of the models are using the top 4 frameworks listed below, we scan only those by -# default. We still keep the full list of supported frameworks in case we want to scan all of them. -MAIN_INFERENCE_API_FRAMEWORKS = [ - "diffusers", - "sentence-transformers", - "text-generation-inference", - "transformers", -] - -ALL_INFERENCE_API_FRAMEWORKS = MAIN_INFERENCE_API_FRAMEWORKS + [ - "adapter-transformers", - "allennlp", - "asteroid", - "bertopic", - "doctr", - "espnet", - "fairseq", - "fastai", - "fasttext", - "flair", - "k2", - "keras", - "mindspore", - "nemo", - "open_clip", - "paddlenlp", - "peft", - "pyannote-audio", - "sklearn", - "spacy", - "span-marker", - "speechbrain", - "stanza", - "timm", -] - -# If OAuth didn't work after 2 redirects, there's likely a third-party cookie issue in the Space iframe view. -# In this case, we redirect the user to the non-iframe view. -OAUTH_MAX_REDIRECTS = 2 - -# OAuth-related environment variables injected by the Space -OAUTH_CLIENT_ID = os.environ.get("OAUTH_CLIENT_ID") -OAUTH_CLIENT_SECRET = os.environ.get("OAUTH_CLIENT_SECRET") -OAUTH_SCOPES = os.environ.get("OAUTH_SCOPES") -OPENID_PROVIDER_URL = os.environ.get("OPENID_PROVIDER_URL") - -# Xet constants -HUGGINGFACE_HEADER_X_XET_ENDPOINT = "X-Xet-Cas-Url" -HUGGINGFACE_HEADER_X_XET_ACCESS_TOKEN = "X-Xet-Access-Token" -HUGGINGFACE_HEADER_X_XET_EXPIRATION = "X-Xet-Token-Expiration" -HUGGINGFACE_HEADER_X_XET_HASH = "X-Xet-Hash" -HUGGINGFACE_HEADER_X_XET_REFRESH_ROUTE = "X-Xet-Refresh-Route" -HUGGINGFACE_HEADER_LINK_XET_AUTH_KEY = "xet-auth" - -default_xet_cache_path = os.path.join(HF_HOME, "xet") -HF_XET_CACHE = os.getenv("HF_XET_CACHE", default_xet_cache_path) -HF_HUB_DISABLE_XET: bool = _is_true(os.environ.get("HF_HUB_DISABLE_XET")) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/dataclasses.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/dataclasses.py deleted file mode 100644 index 636a0ac64b327448e6f8f56b10add54528071f29..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/dataclasses.py +++ /dev/null @@ -1,484 +0,0 @@ -import inspect -from dataclasses import _MISSING_TYPE, MISSING, Field, field, fields -from functools import wraps -from typing import ( - Any, - Callable, - Dict, - ForwardRef, - List, - Literal, - Optional, - Tuple, - Type, - TypeVar, - Union, - get_args, - get_origin, - overload, -) - -from .errors import ( - StrictDataclassClassValidationError, - StrictDataclassDefinitionError, - StrictDataclassFieldValidationError, -) - - -Validator_T = Callable[[Any], None] -T = TypeVar("T") - - -# The overload decorator helps type checkers understand the different return types -@overload -def strict(cls: Type[T]) -> Type[T]: ... - - -@overload -def strict(*, accept_kwargs: bool = False) -> Callable[[Type[T]], Type[T]]: ... - - -def strict( - cls: Optional[Type[T]] = None, *, accept_kwargs: bool = False -) -> Union[Type[T], Callable[[Type[T]], Type[T]]]: - """ - Decorator to add strict validation to a dataclass. - - This decorator must be used on top of `@dataclass` to ensure IDEs and static typing tools - recognize the class as a dataclass. - - Can be used with or without arguments: - - `@strict` - - `@strict(accept_kwargs=True)` - - Args: - cls: - The class to convert to a strict dataclass. - accept_kwargs (`bool`, *optional*): - If True, allows arbitrary keyword arguments in `__init__`. Defaults to False. - - Returns: - The enhanced dataclass with strict validation on field assignment. - - Example: - ```py - >>> from dataclasses import dataclass - >>> from huggingface_hub.dataclasses import as_validated_field, strict, validated_field - - >>> @as_validated_field - >>> def positive_int(value: int): - ... if not value >= 0: - ... raise ValueError(f"Value must be positive, got {value}") - - >>> @strict(accept_kwargs=True) - ... @dataclass - ... class User: - ... name: str - ... age: int = positive_int(default=10) - - # Initialize - >>> User(name="John") - User(name='John', age=10) - - # Extra kwargs are accepted - >>> User(name="John", age=30, lastname="Doe") - User(name='John', age=30, *lastname='Doe') - - # Invalid type => raises - >>> User(name="John", age="30") - huggingface_hub.errors.StrictDataclassFieldValidationError: Validation error for field 'age': - TypeError: Field 'age' expected int, got str (value: '30') - - # Invalid value => raises - >>> User(name="John", age=-1) - huggingface_hub.errors.StrictDataclassFieldValidationError: Validation error for field 'age': - ValueError: Value must be positive, got -1 - ``` - """ - - def wrap(cls: Type[T]) -> Type[T]: - if not hasattr(cls, "__dataclass_fields__"): - raise StrictDataclassDefinitionError( - f"Class '{cls.__name__}' must be a dataclass before applying @strict." - ) - - # List and store validators - field_validators: Dict[str, List[Validator_T]] = {} - for f in fields(cls): # type: ignore [arg-type] - validators = [] - validators.append(_create_type_validator(f)) - custom_validator = f.metadata.get("validator") - if custom_validator is not None: - if not isinstance(custom_validator, list): - custom_validator = [custom_validator] - for validator in custom_validator: - if not _is_validator(validator): - raise StrictDataclassDefinitionError( - f"Invalid validator for field '{f.name}': {validator}. Must be a callable taking a single argument." - ) - validators.extend(custom_validator) - field_validators[f.name] = validators - cls.__validators__ = field_validators # type: ignore - - # Override __setattr__ to validate fields on assignment - original_setattr = cls.__setattr__ - - def __strict_setattr__(self: Any, name: str, value: Any) -> None: - """Custom __setattr__ method for strict dataclasses.""" - # Run all validators - for validator in self.__validators__.get(name, []): - try: - validator(value) - except (ValueError, TypeError) as e: - raise StrictDataclassFieldValidationError(field=name, cause=e) from e - - # If validation passed, set the attribute - original_setattr(self, name, value) - - cls.__setattr__ = __strict_setattr__ # type: ignore[method-assign] - - if accept_kwargs: - # (optional) Override __init__ to accept arbitrary keyword arguments - original_init = cls.__init__ - - @wraps(original_init) - def __init__(self, **kwargs: Any) -> None: - # Extract only the fields that are part of the dataclass - dataclass_fields = {f.name for f in fields(cls)} # type: ignore [arg-type] - standard_kwargs = {k: v for k, v in kwargs.items() if k in dataclass_fields} - - # Call the original __init__ with standard fields - original_init(self, **standard_kwargs) - - # Add any additional kwargs as attributes - for name, value in kwargs.items(): - if name not in dataclass_fields: - self.__setattr__(name, value) - - cls.__init__ = __init__ # type: ignore[method-assign] - - # (optional) Override __repr__ to include additional kwargs - original_repr = cls.__repr__ - - @wraps(original_repr) - def __repr__(self) -> str: - # Call the original __repr__ to get the standard fields - standard_repr = original_repr(self) - - # Get additional kwargs - additional_kwargs = [ - # add a '*' in front of additional kwargs to let the user know they are not part of the dataclass - f"*{k}={v!r}" - for k, v in self.__dict__.items() - if k not in cls.__dataclass_fields__ # type: ignore [attr-defined] - ] - additional_repr = ", ".join(additional_kwargs) - - # Combine both representations - return f"{standard_repr[:-1]}, {additional_repr})" if additional_kwargs else standard_repr - - cls.__repr__ = __repr__ # type: ignore [method-assign] - - # List all public methods starting with `validate_` => class validators. - class_validators = [] - - for name in dir(cls): - if not name.startswith("validate_"): - continue - method = getattr(cls, name) - if not callable(method): - continue - if len(inspect.signature(method).parameters) != 1: - raise StrictDataclassDefinitionError( - f"Class '{cls.__name__}' has a class validator '{name}' that takes more than one argument." - " Class validators must take only 'self' as an argument. Methods starting with 'validate_'" - " are considered to be class validators." - ) - class_validators.append(method) - - cls.__class_validators__ = class_validators # type: ignore [attr-defined] - - # Add `validate` method to the class, but first check if it already exists - def validate(self: T) -> None: - """Run class validators on the instance.""" - for validator in cls.__class_validators__: # type: ignore [attr-defined] - try: - validator(self) - except (ValueError, TypeError) as e: - raise StrictDataclassClassValidationError(validator=validator.__name__, cause=e) from e - - # Hack to be able to raise if `.validate()` already exists except if it was created by this decorator on a parent class - # (in which case we just override it) - validate.__is_defined_by_strict_decorator__ = True # type: ignore [attr-defined] - - if hasattr(cls, "validate"): - if not getattr(cls.validate, "__is_defined_by_strict_decorator__", False): # type: ignore [attr-defined] - raise StrictDataclassDefinitionError( - f"Class '{cls.__name__}' already implements a method called 'validate'." - " This method name is reserved when using the @strict decorator on a dataclass." - " If you want to keep your own method, please rename it." - ) - - cls.validate = validate # type: ignore - - # Run class validators after initialization - initial_init = cls.__init__ - - @wraps(initial_init) - def init_with_validate(self, *args, **kwargs) -> None: - """Run class validators after initialization.""" - initial_init(self, *args, **kwargs) # type: ignore [call-arg] - cls.validate(self) # type: ignore [attr-defined] - - setattr(cls, "__init__", init_with_validate) - - return cls - - # Return wrapped class or the decorator itself - return wrap(cls) if cls is not None else wrap - - -def validated_field( - validator: Union[List[Validator_T], Validator_T], - default: Union[Any, _MISSING_TYPE] = MISSING, - default_factory: Union[Callable[[], Any], _MISSING_TYPE] = MISSING, - init: bool = True, - repr: bool = True, - hash: Optional[bool] = None, - compare: bool = True, - metadata: Optional[Dict] = None, - **kwargs: Any, -) -> Any: - """ - Create a dataclass field with a custom validator. - - Useful to apply several checks to a field. If only applying one rule, check out the [`as_validated_field`] decorator. - - Args: - validator (`Callable` or `List[Callable]`): - A method that takes a value as input and raises ValueError/TypeError if the value is invalid. - Can be a list of validators to apply multiple checks. - **kwargs: - Additional arguments to pass to `dataclasses.field()`. - - Returns: - A field with the validator attached in metadata - """ - if not isinstance(validator, list): - validator = [validator] - if metadata is None: - metadata = {} - metadata["validator"] = validator - return field( # type: ignore - default=default, # type: ignore [arg-type] - default_factory=default_factory, # type: ignore [arg-type] - init=init, - repr=repr, - hash=hash, - compare=compare, - metadata=metadata, - **kwargs, - ) - - -def as_validated_field(validator: Validator_T): - """ - Decorates a validator function as a [`validated_field`] (i.e. a dataclass field with a custom validator). - - Args: - validator (`Callable`): - A method that takes a value as input and raises ValueError/TypeError if the value is invalid. - """ - - def _inner( - default: Union[Any, _MISSING_TYPE] = MISSING, - default_factory: Union[Callable[[], Any], _MISSING_TYPE] = MISSING, - init: bool = True, - repr: bool = True, - hash: Optional[bool] = None, - compare: bool = True, - metadata: Optional[Dict] = None, - **kwargs: Any, - ): - return validated_field( - validator, - default=default, - default_factory=default_factory, - init=init, - repr=repr, - hash=hash, - compare=compare, - metadata=metadata, - **kwargs, - ) - - return _inner - - -def type_validator(name: str, value: Any, expected_type: Any) -> None: - """Validate that 'value' matches 'expected_type'.""" - origin = get_origin(expected_type) - args = get_args(expected_type) - - if expected_type is Any: - return - elif validator := _BASIC_TYPE_VALIDATORS.get(origin): - validator(name, value, args) - elif isinstance(expected_type, type): # simple types - _validate_simple_type(name, value, expected_type) - elif isinstance(expected_type, ForwardRef) or isinstance(expected_type, str): - return - else: - raise TypeError(f"Unsupported type for field '{name}': {expected_type}") - - -def _validate_union(name: str, value: Any, args: Tuple[Any, ...]) -> None: - """Validate that value matches one of the types in a Union.""" - errors = [] - for t in args: - try: - type_validator(name, value, t) - return # Valid if any type matches - except TypeError as e: - errors.append(str(e)) - - raise TypeError( - f"Field '{name}' with value {repr(value)} doesn't match any type in {args}. Errors: {'; '.join(errors)}" - ) - - -def _validate_literal(name: str, value: Any, args: Tuple[Any, ...]) -> None: - """Validate Literal type.""" - if value not in args: - raise TypeError(f"Field '{name}' expected one of {args}, got {value}") - - -def _validate_list(name: str, value: Any, args: Tuple[Any, ...]) -> None: - """Validate List[T] type.""" - if not isinstance(value, list): - raise TypeError(f"Field '{name}' expected a list, got {type(value).__name__}") - - # Validate each item in the list - item_type = args[0] - for i, item in enumerate(value): - try: - type_validator(f"{name}[{i}]", item, item_type) - except TypeError as e: - raise TypeError(f"Invalid item at index {i} in list '{name}'") from e - - -def _validate_dict(name: str, value: Any, args: Tuple[Any, ...]) -> None: - """Validate Dict[K, V] type.""" - if not isinstance(value, dict): - raise TypeError(f"Field '{name}' expected a dict, got {type(value).__name__}") - - # Validate keys and values - key_type, value_type = args - for k, v in value.items(): - try: - type_validator(f"{name}.key", k, key_type) - type_validator(f"{name}[{k!r}]", v, value_type) - except TypeError as e: - raise TypeError(f"Invalid key or value in dict '{name}'") from e - - -def _validate_tuple(name: str, value: Any, args: Tuple[Any, ...]) -> None: - """Validate Tuple type.""" - if not isinstance(value, tuple): - raise TypeError(f"Field '{name}' expected a tuple, got {type(value).__name__}") - - # Handle variable-length tuples: Tuple[T, ...] - if len(args) == 2 and args[1] is Ellipsis: - for i, item in enumerate(value): - try: - type_validator(f"{name}[{i}]", item, args[0]) - except TypeError as e: - raise TypeError(f"Invalid item at index {i} in tuple '{name}'") from e - # Handle fixed-length tuples: Tuple[T1, T2, ...] - elif len(args) != len(value): - raise TypeError(f"Field '{name}' expected a tuple of length {len(args)}, got {len(value)}") - else: - for i, (item, expected) in enumerate(zip(value, args)): - try: - type_validator(f"{name}[{i}]", item, expected) - except TypeError as e: - raise TypeError(f"Invalid item at index {i} in tuple '{name}'") from e - - -def _validate_set(name: str, value: Any, args: Tuple[Any, ...]) -> None: - """Validate Set[T] type.""" - if not isinstance(value, set): - raise TypeError(f"Field '{name}' expected a set, got {type(value).__name__}") - - # Validate each item in the set - item_type = args[0] - for i, item in enumerate(value): - try: - type_validator(f"{name} item", item, item_type) - except TypeError as e: - raise TypeError(f"Invalid item in set '{name}'") from e - - -def _validate_simple_type(name: str, value: Any, expected_type: type) -> None: - """Validate simple type (int, str, etc.).""" - if not isinstance(value, expected_type): - raise TypeError( - f"Field '{name}' expected {expected_type.__name__}, got {type(value).__name__} (value: {repr(value)})" - ) - - -def _create_type_validator(field: Field) -> Validator_T: - """Create a type validator function for a field.""" - # Hacky: we cannot use a lambda here because of reference issues - - def validator(value: Any) -> None: - type_validator(field.name, value, field.type) - - return validator - - -def _is_validator(validator: Any) -> bool: - """Check if a function is a validator. - - A validator is a Callable that can be called with a single positional argument. - The validator can have more arguments with default values. - - Basically, returns True if `validator(value)` is possible. - """ - if not callable(validator): - return False - - signature = inspect.signature(validator) - parameters = list(signature.parameters.values()) - if len(parameters) == 0: - return False - if parameters[0].kind not in ( - inspect.Parameter.POSITIONAL_OR_KEYWORD, - inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.VAR_POSITIONAL, - ): - return False - for parameter in parameters[1:]: - if parameter.default == inspect.Parameter.empty: - return False - return True - - -_BASIC_TYPE_VALIDATORS = { - Union: _validate_union, - Literal: _validate_literal, - list: _validate_list, - dict: _validate_dict, - tuple: _validate_tuple, - set: _validate_set, -} - - -__all__ = [ - "strict", - "validated_field", - "Validator_T", - "StrictDataclassClassValidationError", - "StrictDataclassDefinitionError", - "StrictDataclassFieldValidationError", -] diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/errors.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/errors.py deleted file mode 100644 index e7cc5647ef02bade7f4eb81c93ca31825437af0e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/errors.py +++ /dev/null @@ -1,379 +0,0 @@ -"""Contains all custom errors.""" - -from pathlib import Path -from typing import Optional, Union - -from requests import HTTPError, Response - - -# CACHE ERRORS - - -class CacheNotFound(Exception): - """Exception thrown when the Huggingface cache is not found.""" - - cache_dir: Union[str, Path] - - def __init__(self, msg: str, cache_dir: Union[str, Path], *args, **kwargs): - super().__init__(msg, *args, **kwargs) - self.cache_dir = cache_dir - - -class CorruptedCacheException(Exception): - """Exception for any unexpected structure in the Huggingface cache-system.""" - - -# HEADERS ERRORS - - -class LocalTokenNotFoundError(EnvironmentError): - """Raised if local token is required but not found.""" - - -# HTTP ERRORS - - -class OfflineModeIsEnabled(ConnectionError): - """Raised when a request is made but `HF_HUB_OFFLINE=1` is set as environment variable.""" - - -class HfHubHTTPError(HTTPError): - """ - HTTPError to inherit from for any custom HTTP Error raised in HF Hub. - - Any HTTPError is converted at least into a `HfHubHTTPError`. If some information is - sent back by the server, it will be added to the error message. - - Added details: - - Request ID sourced from headers in order of precedence: "X-Request-Id", "X-Amzn-Trace-Id", "X-Amz-Cf-Id". - - Server error message from the header "X-Error-Message". - - Server error message if we can found one in the response body. - - Example: - ```py - import requests - from huggingface_hub.utils import get_session, hf_raise_for_status, HfHubHTTPError - - response = get_session().post(...) - try: - hf_raise_for_status(response) - except HfHubHTTPError as e: - print(str(e)) # formatted message - e.request_id, e.server_message # details returned by server - - # Complete the error message with additional information once it's raised - e.append_to_message("\n`create_commit` expects the repository to exist.") - raise - ``` - """ - - def __init__(self, message: str, response: Optional[Response] = None, *, server_message: Optional[str] = None): - self.request_id = ( - response.headers.get("x-request-id") - or response.headers.get("X-Amzn-Trace-Id") - or response.headers.get("x-amz-cf-id") - if response is not None - else None - ) - self.server_message = server_message - - super().__init__( - message, - response=response, # type: ignore [arg-type] - request=response.request if response is not None else None, # type: ignore [arg-type] - ) - - def append_to_message(self, additional_message: str) -> None: - """Append additional information to the `HfHubHTTPError` initial message.""" - self.args = (self.args[0] + additional_message,) + self.args[1:] - - -# INFERENCE CLIENT ERRORS - - -class InferenceTimeoutError(HTTPError, TimeoutError): - """Error raised when a model is unavailable or the request times out.""" - - -# INFERENCE ENDPOINT ERRORS - - -class InferenceEndpointError(Exception): - """Generic exception when dealing with Inference Endpoints.""" - - -class InferenceEndpointTimeoutError(InferenceEndpointError, TimeoutError): - """Exception for timeouts while waiting for Inference Endpoint.""" - - -# SAFETENSORS ERRORS - - -class SafetensorsParsingError(Exception): - """Raised when failing to parse a safetensors file metadata. - - This can be the case if the file is not a safetensors file or does not respect the specification. - """ - - -class NotASafetensorsRepoError(Exception): - """Raised when a repo is not a Safetensors repo i.e. doesn't have either a `model.safetensors` or a - `model.safetensors.index.json` file. - """ - - -# TEXT GENERATION ERRORS - - -class TextGenerationError(HTTPError): - """Generic error raised if text-generation went wrong.""" - - -# Text Generation Inference Errors -class ValidationError(TextGenerationError): - """Server-side validation error.""" - - -class GenerationError(TextGenerationError): - pass - - -class OverloadedError(TextGenerationError): - pass - - -class IncompleteGenerationError(TextGenerationError): - pass - - -class UnknownError(TextGenerationError): - pass - - -# VALIDATION ERRORS - - -class HFValidationError(ValueError): - """Generic exception thrown by `huggingface_hub` validators. - - Inherits from [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError). - """ - - -# FILE METADATA ERRORS - - -class FileMetadataError(OSError): - """Error triggered when the metadata of a file on the Hub cannot be retrieved (missing ETag or commit_hash). - - Inherits from `OSError` for backward compatibility. - """ - - -# REPOSITORY ERRORS - - -class RepositoryNotFoundError(HfHubHTTPError): - """ - Raised when trying to access a hf.co URL with an invalid repository name, or - with a private repo name the user does not have access to. - - Example: - - ```py - >>> from huggingface_hub import model_info - >>> model_info("") - (...) - huggingface_hub.utils._errors.RepositoryNotFoundError: 401 Client Error. (Request ID: PvMw_VjBMjVdMz53WKIzP) - - Repository Not Found for url: https://huggingface.co/api/models/%3Cnon_existent_repository%3E. - Please make sure you specified the correct `repo_id` and `repo_type`. - If the repo is private, make sure you are authenticated. - Invalid username or password. - ``` - """ - - -class GatedRepoError(RepositoryNotFoundError): - """ - Raised when trying to access a gated repository for which the user is not on the - authorized list. - - Note: derives from `RepositoryNotFoundError` to ensure backward compatibility. - - Example: - - ```py - >>> from huggingface_hub import model_info - >>> model_info("") - (...) - huggingface_hub.utils._errors.GatedRepoError: 403 Client Error. (Request ID: ViT1Bf7O_026LGSQuVqfa) - - Cannot access gated repo for url https://huggingface.co/api/models/ardent-figment/gated-model. - Access to model ardent-figment/gated-model is restricted and you are not in the authorized list. - Visit https://huggingface.co/ardent-figment/gated-model to ask for access. - ``` - """ - - -class DisabledRepoError(HfHubHTTPError): - """ - Raised when trying to access a repository that has been disabled by its author. - - Example: - - ```py - >>> from huggingface_hub import dataset_info - >>> dataset_info("laion/laion-art") - (...) - huggingface_hub.utils._errors.DisabledRepoError: 403 Client Error. (Request ID: Root=1-659fc3fa-3031673e0f92c71a2260dbe2;bc6f4dfb-b30a-4862-af0a-5cfe827610d8) - - Cannot access repository for url https://huggingface.co/api/datasets/laion/laion-art. - Access to this resource is disabled. - ``` - """ - - -# REVISION ERROR - - -class RevisionNotFoundError(HfHubHTTPError): - """ - Raised when trying to access a hf.co URL with a valid repository but an invalid - revision. - - Example: - - ```py - >>> from huggingface_hub import hf_hub_download - >>> hf_hub_download('bert-base-cased', 'config.json', revision='') - (...) - huggingface_hub.utils._errors.RevisionNotFoundError: 404 Client Error. (Request ID: Mwhe_c3Kt650GcdKEFomX) - - Revision Not Found for url: https://huggingface.co/bert-base-cased/resolve/%3Cnon-existent-revision%3E/config.json. - ``` - """ - - -# ENTRY ERRORS -class EntryNotFoundError(HfHubHTTPError): - """ - Raised when trying to access a hf.co URL with a valid repository and revision - but an invalid filename. - - Example: - - ```py - >>> from huggingface_hub import hf_hub_download - >>> hf_hub_download('bert-base-cased', '') - (...) - huggingface_hub.utils._errors.EntryNotFoundError: 404 Client Error. (Request ID: 53pNl6M0MxsnG5Sw8JA6x) - - Entry Not Found for url: https://huggingface.co/bert-base-cased/resolve/main/%3Cnon-existent-file%3E. - ``` - """ - - -class LocalEntryNotFoundError(EntryNotFoundError, FileNotFoundError, ValueError): - """ - Raised when trying to access a file or snapshot that is not on the disk when network is - disabled or unavailable (connection issue). The entry may exist on the Hub. - - Note: `ValueError` type is to ensure backward compatibility. - Note: `LocalEntryNotFoundError` derives from `HTTPError` because of `EntryNotFoundError` - even when it is not a network issue. - - Example: - - ```py - >>> from huggingface_hub import hf_hub_download - >>> hf_hub_download('bert-base-cased', '', local_files_only=True) - (...) - huggingface_hub.utils._errors.LocalEntryNotFoundError: Cannot find the requested files in the disk cache and outgoing traffic has been disabled. To enable hf.co look-ups and downloads online, set 'local_files_only' to False. - ``` - """ - - def __init__(self, message: str): - super().__init__(message, response=None) - - -# REQUEST ERROR -class BadRequestError(HfHubHTTPError, ValueError): - """ - Raised by `hf_raise_for_status` when the server returns a HTTP 400 error. - - Example: - - ```py - >>> resp = requests.post("hf.co/api/check", ...) - >>> hf_raise_for_status(resp, endpoint_name="check") - huggingface_hub.utils._errors.BadRequestError: Bad request for check endpoint: {details} (Request ID: XXX) - ``` - """ - - -# DDUF file format ERROR - - -class DDUFError(Exception): - """Base exception for errors related to the DDUF format.""" - - -class DDUFCorruptedFileError(DDUFError): - """Exception thrown when the DDUF file is corrupted.""" - - -class DDUFExportError(DDUFError): - """Base exception for errors during DDUF export.""" - - -class DDUFInvalidEntryNameError(DDUFExportError): - """Exception thrown when the entry name is invalid.""" - - -# STRICT DATACLASSES ERRORS - - -class StrictDataclassError(Exception): - """Base exception for strict dataclasses.""" - - -class StrictDataclassDefinitionError(StrictDataclassError): - """Exception thrown when a strict dataclass is defined incorrectly.""" - - -class StrictDataclassFieldValidationError(StrictDataclassError): - """Exception thrown when a strict dataclass fails validation for a given field.""" - - def __init__(self, field: str, cause: Exception): - error_message = f"Validation error for field '{field}':" - error_message += f"\n {cause.__class__.__name__}: {cause}" - super().__init__(error_message) - - -class StrictDataclassClassValidationError(StrictDataclassError): - """Exception thrown when a strict dataclass fails validation on a class validator.""" - - def __init__(self, validator: str, cause: Exception): - error_message = f"Class validation error for validator '{validator}':" - error_message += f"\n {cause.__class__.__name__}: {cause}" - super().__init__(error_message) - - -# XET ERRORS - - -class XetError(Exception): - """Base exception for errors related to Xet Storage.""" - - -class XetAuthorizationError(XetError): - """Exception thrown when the user does not have the right authorization to use Xet Storage.""" - - -class XetRefreshTokenError(XetError): - """Exception thrown when the refresh token is invalid.""" - - -class XetDownloadError(Exception): - """Exception thrown when the download from Xet Storage fails.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/fastai_utils.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/fastai_utils.py deleted file mode 100644 index fc3b42323a251140aac813da24493918be267472..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/fastai_utils.py +++ /dev/null @@ -1,415 +0,0 @@ -import json -import os -from pathlib import Path -from pickle import DEFAULT_PROTOCOL, PicklingError -from typing import Any, Dict, List, Optional, Union - -from packaging import version - -from huggingface_hub import constants, snapshot_download -from huggingface_hub.hf_api import HfApi -from huggingface_hub.utils import ( - SoftTemporaryDirectory, - get_fastai_version, - get_fastcore_version, - get_python_version, -) - -from .utils import logging, validate_hf_hub_args -from .utils._runtime import _PY_VERSION # noqa: F401 # for backward compatibility... - - -logger = logging.get_logger(__name__) - - -def _check_fastai_fastcore_versions( - fastai_min_version: str = "2.4", - fastcore_min_version: str = "1.3.27", -): - """ - Checks that the installed fastai and fastcore versions are compatible for pickle serialization. - - Args: - fastai_min_version (`str`, *optional*): - The minimum fastai version supported. - fastcore_min_version (`str`, *optional*): - The minimum fastcore version supported. - - > [!TIP] - > Raises the following error: - > - > - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) - > if the fastai or fastcore libraries are not available or are of an invalid version. - """ - - if (get_fastcore_version() or get_fastai_version()) == "N/A": - raise ImportError( - f"fastai>={fastai_min_version} and fastcore>={fastcore_min_version} are" - f" required. Currently using fastai=={get_fastai_version()} and" - f" fastcore=={get_fastcore_version()}." - ) - - current_fastai_version = version.Version(get_fastai_version()) - current_fastcore_version = version.Version(get_fastcore_version()) - - if current_fastai_version < version.Version(fastai_min_version): - raise ImportError( - "`push_to_hub_fastai` and `from_pretrained_fastai` require a" - f" fastai>={fastai_min_version} version, but you are using fastai version" - f" {get_fastai_version()} which is incompatible. Upgrade with `pip install" - " fastai==2.5.6`." - ) - - if current_fastcore_version < version.Version(fastcore_min_version): - raise ImportError( - "`push_to_hub_fastai` and `from_pretrained_fastai` require a" - f" fastcore>={fastcore_min_version} version, but you are using fastcore" - f" version {get_fastcore_version()} which is incompatible. Upgrade with" - " `pip install fastcore==1.3.27`." - ) - - -def _check_fastai_fastcore_pyproject_versions( - storage_folder: str, - fastai_min_version: str = "2.4", - fastcore_min_version: str = "1.3.27", -): - """ - Checks that the `pyproject.toml` file in the directory `storage_folder` has fastai and fastcore versions - that are compatible with `from_pretrained_fastai` and `push_to_hub_fastai`. If `pyproject.toml` does not exist - or does not contain versions for fastai and fastcore, then it logs a warning. - - Args: - storage_folder (`str`): - Folder to look for the `pyproject.toml` file. - fastai_min_version (`str`, *optional*): - The minimum fastai version supported. - fastcore_min_version (`str`, *optional*): - The minimum fastcore version supported. - - > [!TIP] - > Raises the following errors: - > - > - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) - > if the `toml` module is not installed. - > - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) - > if the `pyproject.toml` indicates a lower than minimum supported version of fastai or fastcore. - """ - - try: - import toml - except ModuleNotFoundError: - raise ImportError( - "`push_to_hub_fastai` and `from_pretrained_fastai` require the toml module." - " Install it with `pip install toml`." - ) - - # Checks that a `pyproject.toml`, with `build-system` and `requires` sections, exists in the repository. If so, get a list of required packages. - if not os.path.isfile(f"{storage_folder}/pyproject.toml"): - logger.warning( - "There is no `pyproject.toml` in the repository that contains the fastai" - " `Learner`. The `pyproject.toml` would allow us to verify that your fastai" - " and fastcore versions are compatible with those of the model you want to" - " load." - ) - return - pyproject_toml = toml.load(f"{storage_folder}/pyproject.toml") - - if "build-system" not in pyproject_toml.keys(): - logger.warning( - "There is no `build-system` section in the pyproject.toml of the repository" - " that contains the fastai `Learner`. The `build-system` would allow us to" - " verify that your fastai and fastcore versions are compatible with those" - " of the model you want to load." - ) - return - build_system_toml = pyproject_toml["build-system"] - - if "requires" not in build_system_toml.keys(): - logger.warning( - "There is no `requires` section in the pyproject.toml of the repository" - " that contains the fastai `Learner`. The `requires` would allow us to" - " verify that your fastai and fastcore versions are compatible with those" - " of the model you want to load." - ) - return - package_versions = build_system_toml["requires"] - - # Extracts contains fastai and fastcore versions from `pyproject.toml` if available. - # If the package is specified but not the version (e.g. "fastai" instead of "fastai=2.4"), the default versions are the highest. - fastai_packages = [pck for pck in package_versions if pck.startswith("fastai")] - if len(fastai_packages) == 0: - logger.warning("The repository does not have a fastai version specified in the `pyproject.toml`.") - # fastai_version is an empty string if not specified - else: - fastai_version = str(fastai_packages[0]).partition("=")[2] - if fastai_version != "" and version.Version(fastai_version) < version.Version(fastai_min_version): - raise ImportError( - "`from_pretrained_fastai` requires" - f" fastai>={fastai_min_version} version but the model to load uses" - f" {fastai_version} which is incompatible." - ) - - fastcore_packages = [pck for pck in package_versions if pck.startswith("fastcore")] - if len(fastcore_packages) == 0: - logger.warning("The repository does not have a fastcore version specified in the `pyproject.toml`.") - # fastcore_version is an empty string if not specified - else: - fastcore_version = str(fastcore_packages[0]).partition("=")[2] - if fastcore_version != "" and version.Version(fastcore_version) < version.Version(fastcore_min_version): - raise ImportError( - "`from_pretrained_fastai` requires" - f" fastcore>={fastcore_min_version} version, but you are using fastcore" - f" version {fastcore_version} which is incompatible." - ) - - -README_TEMPLATE = """--- -tags: -- fastai ---- - -# Amazing! - -🥳 Congratulations on hosting your fastai model on the Hugging Face Hub! - -# Some next steps -1. Fill out this model card with more information (see the template below and the [documentation here](https://huggingface.co/docs/hub/model-repos))! - -2. Create a demo in Gradio or Streamlit using 🤗 Spaces ([documentation here](https://huggingface.co/docs/hub/spaces)). - -3. Join the fastai community on the [Fastai Discord](https://discord.com/invite/YKrxeNn)! - -Greetings fellow fastlearner 🤝! Don't forget to delete this content from your model card. - - ---- - - -# Model card - -## Model description -More information needed - -## Intended uses & limitations -More information needed - -## Training and evaluation data -More information needed -""" - -PYPROJECT_TEMPLATE = f"""[build-system] -requires = ["setuptools>=40.8.0", "wheel", "python={get_python_version()}", "fastai={get_fastai_version()}", "fastcore={get_fastcore_version()}"] -build-backend = "setuptools.build_meta:__legacy__" -""" - - -def _create_model_card(repo_dir: Path): - """ - Creates a model card for the repository. - - Args: - repo_dir (`Path`): - Directory where model card is created. - """ - readme_path = repo_dir / "README.md" - - if not readme_path.exists(): - with readme_path.open("w", encoding="utf-8") as f: - f.write(README_TEMPLATE) - - -def _create_model_pyproject(repo_dir: Path): - """ - Creates a `pyproject.toml` for the repository. - - Args: - repo_dir (`Path`): - Directory where `pyproject.toml` is created. - """ - pyproject_path = repo_dir / "pyproject.toml" - - if not pyproject_path.exists(): - with pyproject_path.open("w", encoding="utf-8") as f: - f.write(PYPROJECT_TEMPLATE) - - -def _save_pretrained_fastai( - learner, - save_directory: Union[str, Path], - config: Optional[Dict[str, Any]] = None, -): - """ - Saves a fastai learner to `save_directory` in pickle format using the default pickle protocol for the version of python used. - - Args: - learner (`Learner`): - The `fastai.Learner` you'd like to save. - save_directory (`str` or `Path`): - Specific directory in which you want to save the fastai learner. - config (`dict`, *optional*): - Configuration object. Will be uploaded as a .json file. Example: 'https://huggingface.co/espejelomar/fastai-pet-breeds-classification/blob/main/config.json'. - - > [!TIP] - > Raises the following error: - > - > - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError) - > if the config file provided is not a dictionary. - """ - _check_fastai_fastcore_versions() - - os.makedirs(save_directory, exist_ok=True) - - # if the user provides config then we update it with the fastai and fastcore versions in CONFIG_TEMPLATE. - if config is not None: - if not isinstance(config, dict): - raise RuntimeError(f"Provided config should be a dict. Got: '{type(config)}'") - path = os.path.join(save_directory, constants.CONFIG_NAME) - with open(path, "w") as f: - json.dump(config, f) - - _create_model_card(Path(save_directory)) - _create_model_pyproject(Path(save_directory)) - - # learner.export saves the model in `self.path`. - learner.path = Path(save_directory) - os.makedirs(save_directory, exist_ok=True) - try: - learner.export( - fname="model.pkl", - pickle_protocol=DEFAULT_PROTOCOL, - ) - except PicklingError: - raise PicklingError( - "You are using a lambda function, i.e., an anonymous function. `pickle`" - " cannot pickle function objects and requires that all functions have" - " names. One possible solution is to name the function." - ) - - -@validate_hf_hub_args -def from_pretrained_fastai( - repo_id: str, - revision: Optional[str] = None, -): - """ - Load pretrained fastai model from the Hub or from a local directory. - - Args: - repo_id (`str`): - The location where the pickled fastai.Learner is. It can be either of the two: - - Hosted on the Hugging Face Hub. E.g.: 'espejelomar/fatai-pet-breeds-classification' or 'distilgpt2'. - You can add a `revision` by appending `@` at the end of `repo_id`. E.g.: `dbmdz/bert-base-german-cased@main`. - Revision is the specific model version to use. Since we use a git-based system for storing models and other - artifacts on the Hugging Face Hub, it can be a branch name, a tag name, or a commit id. - - Hosted locally. `repo_id` would be a directory containing the pickle and a pyproject.toml - indicating the fastai and fastcore versions used to build the `fastai.Learner`. E.g.: `./my_model_directory/`. - revision (`str`, *optional*): - Revision at which the repo's files are downloaded. See documentation of `snapshot_download`. - - Returns: - The `fastai.Learner` model in the `repo_id` repo. - """ - _check_fastai_fastcore_versions() - - # Load the `repo_id` repo. - # `snapshot_download` returns the folder where the model was stored. - # `cache_dir` will be the default '/root/.cache/huggingface/hub' - if not os.path.isdir(repo_id): - storage_folder = snapshot_download( - repo_id=repo_id, - revision=revision, - library_name="fastai", - library_version=get_fastai_version(), - ) - else: - storage_folder = repo_id - - _check_fastai_fastcore_pyproject_versions(storage_folder) - - from fastai.learner import load_learner # type: ignore - - return load_learner(os.path.join(storage_folder, "model.pkl")) - - -@validate_hf_hub_args -def push_to_hub_fastai( - learner, - *, - repo_id: str, - commit_message: str = "Push FastAI model using huggingface_hub.", - private: Optional[bool] = None, - token: Optional[str] = None, - config: Optional[dict] = None, - branch: Optional[str] = None, - create_pr: Optional[bool] = None, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - delete_patterns: Optional[Union[List[str], str]] = None, - api_endpoint: Optional[str] = None, -): - """ - Upload learner checkpoint files to the Hub. - - Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be pushed to the hub. Use - `delete_patterns` to delete existing remote files in the same commit. See [`upload_folder`] reference for more - details. - - Args: - learner (`Learner`): - The `fastai.Learner' you'd like to push to the Hub. - repo_id (`str`): - The repository id for your model in Hub in the format of "namespace/repo_name". The namespace can be your individual account or an organization to which you have write access (for example, 'stanfordnlp/stanza-de'). - commit_message (`str`, *optional*): - Message to commit while pushing. Will default to :obj:`"add model"`. - private (`bool`, *optional*): - Whether or not the repository created should be private. - If `None` (default), will default to been public except if the organization's default is private. - token (`str`, *optional*): - The Hugging Face account token to use as HTTP bearer authorization for remote files. If :obj:`None`, the token will be asked by a prompt. - config (`dict`, *optional*): - Configuration object to be saved alongside the model weights. - branch (`str`, *optional*): - The git branch on which to push the model. This defaults to - the default branch as specified in your repository, which - defaults to `"main"`. - create_pr (`boolean`, *optional*): - Whether or not to create a Pull Request from `branch` with that commit. - Defaults to `False`. - api_endpoint (`str`, *optional*): - The API endpoint to use when pushing the model to the hub. - allow_patterns (`List[str]` or `str`, *optional*): - If provided, only files matching at least one pattern are pushed. - ignore_patterns (`List[str]` or `str`, *optional*): - If provided, files matching any of the patterns are not pushed. - delete_patterns (`List[str]` or `str`, *optional*): - If provided, remote files matching any of the patterns will be deleted from the repo. - - Returns: - The url of the commit of your model in the given repository. - - > [!TIP] - > Raises the following error: - > - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if the user is not log on to the Hugging Face Hub. - """ - _check_fastai_fastcore_versions() - api = HfApi(endpoint=api_endpoint) - repo_id = api.create_repo(repo_id=repo_id, token=token, private=private, exist_ok=True).repo_id - - # Push the files to the repo in a single commit - with SoftTemporaryDirectory() as tmp: - saved_path = Path(tmp) / repo_id - _save_pretrained_fastai(learner, saved_path, config=config) - return api.upload_folder( - repo_id=repo_id, - token=token, - folder_path=saved_path, - commit_message=commit_message, - revision=branch, - create_pr=create_pr, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - delete_patterns=delete_patterns, - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/file_download.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/file_download.py deleted file mode 100644 index aff7236b4da41271e77a3cd58e84352904362ee9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/file_download.py +++ /dev/null @@ -1,1820 +0,0 @@ -import copy -import errno -import inspect -import os -import re -import shutil -import stat -import time -import uuid -import warnings -from dataclasses import dataclass -from pathlib import Path -from typing import Any, BinaryIO, Dict, Literal, NoReturn, Optional, Tuple, Union -from urllib.parse import quote, urlparse - -import requests - -from . import ( - __version__, # noqa: F401 # for backward compatibility - constants, -) -from ._local_folder import get_local_download_paths, read_download_metadata, write_download_metadata -from .constants import ( - HUGGINGFACE_CO_URL_TEMPLATE, # noqa: F401 # for backward compatibility - HUGGINGFACE_HUB_CACHE, # noqa: F401 # for backward compatibility -) -from .errors import ( - EntryNotFoundError, - FileMetadataError, - GatedRepoError, - HfHubHTTPError, - LocalEntryNotFoundError, - RepositoryNotFoundError, - RevisionNotFoundError, -) -from .utils import ( - OfflineModeIsEnabled, - SoftTemporaryDirectory, - WeakFileLock, - XetFileData, - build_hf_headers, - get_fastai_version, # noqa: F401 # for backward compatibility - get_fastcore_version, # noqa: F401 # for backward compatibility - get_graphviz_version, # noqa: F401 # for backward compatibility - get_jinja_version, # noqa: F401 # for backward compatibility - get_pydot_version, # noqa: F401 # for backward compatibility - get_tf_version, # noqa: F401 # for backward compatibility - get_torch_version, # noqa: F401 # for backward compatibility - hf_raise_for_status, - is_fastai_available, # noqa: F401 # for backward compatibility - is_fastcore_available, # noqa: F401 # for backward compatibility - is_graphviz_available, # noqa: F401 # for backward compatibility - is_jinja_available, # noqa: F401 # for backward compatibility - is_pydot_available, # noqa: F401 # for backward compatibility - is_tf_available, # noqa: F401 # for backward compatibility - is_torch_available, # noqa: F401 # for backward compatibility - logging, - parse_xet_file_data_from_response, - refresh_xet_connection_info, - reset_sessions, - tqdm, - validate_hf_hub_args, -) -from .utils._http import _adjust_range_header, http_backoff -from .utils._runtime import _PY_VERSION, is_xet_available # noqa: F401 # for backward compatibility -from .utils._typing import HTTP_METHOD_T -from .utils.sha import sha_fileobj -from .utils.tqdm import _get_progress_bar_context - - -logger = logging.get_logger(__name__) - -# Return value when trying to load a file from cache but the file does not exist in the distant repo. -_CACHED_NO_EXIST = object() -_CACHED_NO_EXIST_T = Any - -# Regex to get filename from a "Content-Disposition" header for CDN-served files -HEADER_FILENAME_PATTERN = re.compile(r'filename="(?P.*?)";') - -# Regex to check if the revision IS directly a commit_hash -REGEX_COMMIT_HASH = re.compile(r"^[0-9a-f]{40}$") - -# Regex to check if the file etag IS a valid sha256 -REGEX_SHA256 = re.compile(r"^[0-9a-f]{64}$") - -_are_symlinks_supported_in_dir: Dict[str, bool] = {} - - -def are_symlinks_supported(cache_dir: Union[str, Path, None] = None) -> bool: - """Return whether the symlinks are supported on the machine. - - Since symlinks support can change depending on the mounted disk, we need to check - on the precise cache folder. By default, the default HF cache directory is checked. - - Args: - cache_dir (`str`, `Path`, *optional*): - Path to the folder where cached files are stored. - - Returns: [bool] Whether symlinks are supported in the directory. - """ - # Defaults to HF cache - if cache_dir is None: - cache_dir = constants.HF_HUB_CACHE - cache_dir = str(Path(cache_dir).expanduser().resolve()) # make it unique - - # Check symlink compatibility only once (per cache directory) at first time use - if cache_dir not in _are_symlinks_supported_in_dir: - _are_symlinks_supported_in_dir[cache_dir] = True - - os.makedirs(cache_dir, exist_ok=True) - with SoftTemporaryDirectory(dir=cache_dir) as tmpdir: - src_path = Path(tmpdir) / "dummy_file_src" - src_path.touch() - dst_path = Path(tmpdir) / "dummy_file_dst" - - # Relative source path as in `_create_symlink`` - relative_src = os.path.relpath(src_path, start=os.path.dirname(dst_path)) - try: - os.symlink(relative_src, dst_path) - except OSError: - # Likely running on Windows - _are_symlinks_supported_in_dir[cache_dir] = False - - if not constants.HF_HUB_DISABLE_SYMLINKS_WARNING: - message = ( - "`huggingface_hub` cache-system uses symlinks by default to" - " efficiently store duplicated files but your machine does not" - f" support them in {cache_dir}. Caching files will still work" - " but in a degraded version that might require more space on" - " your disk. This warning can be disabled by setting the" - " `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For" - " more details, see" - " https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations." - ) - if os.name == "nt": - message += ( - "\nTo support symlinks on Windows, you either need to" - " activate Developer Mode or to run Python as an" - " administrator. In order to activate developer mode," - " see this article:" - " https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development" - ) - warnings.warn(message) - - return _are_symlinks_supported_in_dir[cache_dir] - - -@dataclass(frozen=True) -class HfFileMetadata: - """Data structure containing information about a file versioned on the Hub. - - Returned by [`get_hf_file_metadata`] based on a URL. - - Args: - commit_hash (`str`, *optional*): - The commit_hash related to the file. - etag (`str`, *optional*): - Etag of the file on the server. - location (`str`): - Location where to download the file. Can be a Hub url or not (CDN). - size (`size`): - Size of the file. In case of an LFS file, contains the size of the actual - LFS file, not the pointer. - xet_file_data (`XetFileData`, *optional*): - Xet information for the file. This is only set if the file is stored using Xet storage. - """ - - commit_hash: Optional[str] - etag: Optional[str] - location: str - size: Optional[int] - xet_file_data: Optional[XetFileData] - - -@validate_hf_hub_args -def hf_hub_url( - repo_id: str, - filename: str, - *, - subfolder: Optional[str] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - endpoint: Optional[str] = None, -) -> str: - """Construct the URL of a file from the given information. - - The resolved address can either be a huggingface.co-hosted url, or a link to - Cloudfront (a Content Delivery Network, or CDN) for large files which are - more than a few MBs. - - Args: - repo_id (`str`): - A namespace (user or an organization) name and a repo name separated - by a `/`. - filename (`str`): - The name of the file in the repo. - subfolder (`str`, *optional*): - An optional value corresponding to a folder inside the repo. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if downloading from a dataset or space, - `None` or `"model"` if downloading from a model. Default is `None`. - revision (`str`, *optional*): - An optional Git revision id which can be a branch name, a tag, or a - commit hash. - - Example: - - ```python - >>> from huggingface_hub import hf_hub_url - - >>> hf_hub_url( - ... repo_id="julien-c/EsperBERTo-small", filename="pytorch_model.bin" - ... ) - 'https://huggingface.co/julien-c/EsperBERTo-small/resolve/main/pytorch_model.bin' - ``` - - > [!TIP] - > Notes: - > - > Cloudfront is replicated over the globe so downloads are way faster for - > the end user (and it also lowers our bandwidth costs). - > - > Cloudfront aggressively caches files by default (default TTL is 24 - > hours), however this is not an issue here because we implement a - > git-based versioning system on huggingface.co, which means that we store - > the files on S3/Cloudfront in a content-addressable way (i.e., the file - > name is its hash). Using content-addressable filenames means cache can't - > ever be stale. - > - > In terms of client-side caching from this library, we base our caching - > on the objects' entity tag (`ETag`), which is an identifier of a - > specific version of a resource [1]_. An object's ETag is: its git-sha1 - > if stored in git, or its sha256 if stored in git-lfs. - - References: - - - [1] https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag - """ - if subfolder == "": - subfolder = None - if subfolder is not None: - filename = f"{subfolder}/{filename}" - - if repo_type not in constants.REPO_TYPES: - raise ValueError("Invalid repo type") - - if repo_type in constants.REPO_TYPES_URL_PREFIXES: - repo_id = constants.REPO_TYPES_URL_PREFIXES[repo_type] + repo_id - - if revision is None: - revision = constants.DEFAULT_REVISION - url = HUGGINGFACE_CO_URL_TEMPLATE.format( - repo_id=repo_id, revision=quote(revision, safe=""), filename=quote(filename) - ) - # Update endpoint if provided - if endpoint is not None and url.startswith(constants.ENDPOINT): - url = endpoint + url[len(constants.ENDPOINT) :] - return url - - -def _request_wrapper( - method: HTTP_METHOD_T, url: str, *, follow_relative_redirects: bool = False, **params -) -> requests.Response: - """Wrapper around requests methods to follow relative redirects if `follow_relative_redirects=True` even when - `allow_redirection=False`. - - A backoff mechanism retries the HTTP call on 5xx errors and network errors. - - Args: - method (`str`): - HTTP method, such as 'GET' or 'HEAD'. - url (`str`): - The URL of the resource to fetch. - follow_relative_redirects (`bool`, *optional*, defaults to `False`) - If True, relative redirection (redirection to the same site) will be resolved even when `allow_redirection` - kwarg is set to False. Useful when we want to follow a redirection to a renamed repository without - following redirection to a CDN. - **params (`dict`, *optional*): - Params to pass to `requests.request`. - """ - # Recursively follow relative redirects - if follow_relative_redirects: - response = _request_wrapper( - method=method, - url=url, - follow_relative_redirects=False, - **params, - ) - - # If redirection, we redirect only relative paths. - # This is useful in case of a renamed repository. - if 300 <= response.status_code <= 399: - parsed_target = urlparse(response.headers["Location"]) - if parsed_target.netloc == "": - # This means it is a relative 'location' headers, as allowed by RFC 7231. - # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') - # We want to follow this relative redirect ! - # - # Highly inspired by `resolve_redirects` from requests library. - # See https://github.com/psf/requests/blob/main/requests/sessions.py#L159 - next_url = urlparse(url)._replace(path=parsed_target.path).geturl() - return _request_wrapper(method=method, url=next_url, follow_relative_redirects=True, **params) - return response - - # Perform request and return if status_code is not in the retry list. - response = http_backoff(method=method, url=url, **params) - hf_raise_for_status(response) - return response - - -def _get_file_length_from_http_response(response: requests.Response) -> Optional[int]: - """ - Get the length of the file from the HTTP response headers. - - This function extracts the file size from the HTTP response headers, either from the - `Content-Range` or `Content-Length` header, if available (in that order). - - Args: - response (`requests.Response`): - The HTTP response object. - - Returns: - `int` or `None`: The length of the file in bytes, or None if not available. - """ - - # If HTTP response contains compressed body (e.g. gzip), the `Content-Length` header will - # contain the length of the compressed body, not the uncompressed file size. - # And at the start of transmission there's no way to know the uncompressed file size for gzip, - # thus we return None in that case. - content_encoding = response.headers.get("Content-Encoding", "identity").lower() - if content_encoding != "identity": - # gzip/br/deflate/zstd etc - return None - - content_range = response.headers.get("Content-Range") - if content_range is not None: - return int(content_range.rsplit("/")[-1]) - - content_length = response.headers.get("Content-Length") - if content_length is not None: - return int(content_length) - - return None - - -def http_get( - url: str, - temp_file: BinaryIO, - *, - proxies: Optional[Dict] = None, - resume_size: int = 0, - headers: Optional[Dict[str, Any]] = None, - expected_size: Optional[int] = None, - displayed_filename: Optional[str] = None, - _nb_retries: int = 5, - _tqdm_bar: Optional[tqdm] = None, -) -> None: - """ - Download a remote file. Do not gobble up errors, and will return errors tailored to the Hugging Face Hub. - - If ConnectionError (SSLError) or ReadTimeout happen while streaming data from the server, it is most likely a - transient error (network outage?). We log a warning message and try to resume the download a few times before - giving up. The method gives up after 5 attempts if no new data has being received from the server. - - Args: - url (`str`): - The URL of the file to download. - temp_file (`BinaryIO`): - The file-like object where to save the file. - proxies (`dict`, *optional*): - Dictionary mapping protocol to the URL of the proxy passed to `requests.request`. - resume_size (`int`, *optional*): - The number of bytes already downloaded. If set to 0 (default), the whole file is download. If set to a - positive number, the download will resume at the given position. - headers (`dict`, *optional*): - Dictionary of HTTP Headers to send with the request. - expected_size (`int`, *optional*): - The expected size of the file to download. If set, the download will raise an error if the size of the - received content is different from the expected one. - displayed_filename (`str`, *optional*): - The filename of the file that is being downloaded. Value is used only to display a nice progress bar. If - not set, the filename is guessed from the URL or the `Content-Disposition` header. - """ - if expected_size is not None and resume_size == expected_size: - # If the file is already fully downloaded, we don't need to download it again. - return - - has_custom_range_header = headers is not None and any(h.lower() == "range" for h in headers) - hf_transfer = None - if constants.HF_HUB_ENABLE_HF_TRANSFER: - if resume_size != 0: - warnings.warn("'hf_transfer' does not support `resume_size`: falling back to regular download method") - elif proxies is not None: - warnings.warn("'hf_transfer' does not support `proxies`: falling back to regular download method") - elif has_custom_range_header: - warnings.warn("'hf_transfer' ignores custom 'Range' headers; falling back to regular download method") - else: - try: - import hf_transfer # type: ignore[no-redef] - except ImportError: - raise ValueError( - "Fast download using 'hf_transfer' is enabled" - " (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is not" - " available in your environment. Try `pip install hf_transfer`." - ) - - initial_headers = headers - headers = copy.deepcopy(headers) or {} - if resume_size > 0: - headers["Range"] = _adjust_range_header(headers.get("Range"), resume_size) - elif expected_size and expected_size > constants.MAX_HTTP_DOWNLOAD_SIZE: - # Any files over 50GB will not be available through basic http request. - # Setting the range header to 0-0 will force the server to return the file size in the Content-Range header. - # Since hf_transfer splits the download into chunks, the process will succeed afterwards. - if hf_transfer: - headers["Range"] = "bytes=0-0" - else: - raise ValueError( - "The file is too large to be downloaded using the regular download method. Use `hf_transfer` or `hf_xet` instead." - " Try `pip install hf_transfer` or `pip install hf_xet`." - ) - - r = _request_wrapper( - method="GET", url=url, stream=True, proxies=proxies, headers=headers, timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT - ) - hf_raise_for_status(r) - - # If we requested a Range but got 200 back, the server ignored our Range header - # (e.g. CloudFront with Accept-Encoding: gzip). Reset file to avoid corruption. - if resume_size > 0 and r.status_code == 200: - temp_file.seek(0) - temp_file.truncate() - resume_size = 0 - - total: Optional[int] = _get_file_length_from_http_response(r) - - if displayed_filename is None: - displayed_filename = url - content_disposition = r.headers.get("Content-Disposition") - if content_disposition is not None: - match = HEADER_FILENAME_PATTERN.search(content_disposition) - if match is not None: - # Means file is on CDN - displayed_filename = match.groupdict()["filename"] - - # Truncate filename if too long to display - if len(displayed_filename) > 40: - displayed_filename = f"(…){displayed_filename[-40:]}" - - consistency_error_message = ( - f"Consistency check failed: file should be of size {expected_size} but has size" - f" {{actual_size}} ({displayed_filename}).\nThis is usually due to network issues while downloading the file." - " Please retry with `force_download=True`." - ) - progress_cm = _get_progress_bar_context( - desc=displayed_filename, - log_level=logger.getEffectiveLevel(), - total=total, - initial=resume_size, - name="huggingface_hub.http_get", - _tqdm_bar=_tqdm_bar, - ) - - with progress_cm as progress: - if hf_transfer and total is not None and total > 5 * constants.DOWNLOAD_CHUNK_SIZE: - supports_callback = "callback" in inspect.signature(hf_transfer.download).parameters - if not supports_callback: - warnings.warn( - "You are using an outdated version of `hf_transfer`. " - "Consider upgrading to latest version to enable progress bars " - "using `pip install -U hf_transfer`." - ) - try: - hf_transfer.download( - url=url, - filename=temp_file.name, - max_files=constants.HF_TRANSFER_CONCURRENCY, - chunk_size=constants.DOWNLOAD_CHUNK_SIZE, - headers=initial_headers, - parallel_failures=3, - max_retries=5, - **({"callback": progress.update} if supports_callback else {}), - ) - except Exception as e: - raise RuntimeError( - "An error occurred while downloading using `hf_transfer`. Consider" - " disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling." - ) from e - if not supports_callback: - progress.update(total) - if expected_size is not None and expected_size != os.path.getsize(temp_file.name): - raise EnvironmentError( - consistency_error_message.format( - actual_size=os.path.getsize(temp_file.name), - ) - ) - return - new_resume_size = resume_size - try: - for chunk in r.iter_content(chunk_size=constants.DOWNLOAD_CHUNK_SIZE): - if chunk: # filter out keep-alive new chunks - progress.update(len(chunk)) - temp_file.write(chunk) - new_resume_size += len(chunk) - # Some data has been downloaded from the server so we reset the number of retries. - _nb_retries = 5 - except (requests.ConnectionError, requests.ReadTimeout) as e: - # If ConnectionError (SSLError) or ReadTimeout happen while streaming data from the server, it is most likely - # a transient error (network outage?). We log a warning message and try to resume the download a few times - # before giving up. Tre retry mechanism is basic but should be enough in most cases. - if _nb_retries <= 0: - logger.warning("Error while downloading from %s: %s\nMax retries exceeded.", url, str(e)) - raise - logger.warning("Error while downloading from %s: %s\nTrying to resume download...", url, str(e)) - time.sleep(1) - reset_sessions() # In case of SSLError it's best to reset the shared requests.Session objects - return http_get( - url=url, - temp_file=temp_file, - proxies=proxies, - resume_size=new_resume_size, - headers=initial_headers, - expected_size=expected_size, - _nb_retries=_nb_retries - 1, - _tqdm_bar=_tqdm_bar, - ) - - if expected_size is not None and expected_size != temp_file.tell(): - raise EnvironmentError( - consistency_error_message.format( - actual_size=temp_file.tell(), - ) - ) - - -def xet_get( - *, - incomplete_path: Path, - xet_file_data: XetFileData, - headers: Dict[str, str], - expected_size: Optional[int] = None, - displayed_filename: Optional[str] = None, - _tqdm_bar: Optional[tqdm] = None, -) -> None: - """ - Download a file using Xet storage service. - - Args: - incomplete_path (`Path`): - The path to the file to download. - xet_file_data (`XetFileData`): - The file metadata needed to make the request to the xet storage service. - headers (`Dict[str, str]`): - The headers to send to the xet storage service. - expected_size (`int`, *optional*): - The expected size of the file to download. If set, the download will raise an error if the size of the - received content is different from the expected one. - displayed_filename (`str`, *optional*): - The filename of the file that is being downloaded. Value is used only to display a nice progress bar. If - not set, the filename is guessed from the URL or the `Content-Disposition` header. - - **How it works:** - The file download system uses Xet storage, which is a content-addressable storage system that breaks files into chunks - for efficient storage and transfer. - - `hf_xet.download_files` manages downloading files by: - - Taking a list of files to download (each with its unique content hash) - - Connecting to a storage server (CAS server) that knows how files are chunked - - Using authentication to ensure secure access - - Providing progress updates during download - - Authentication works by regularly refreshing access tokens through `refresh_xet_connection_info` to maintain a valid - connection to the storage server. - - The download process works like this: - 1. Create a local cache folder at `~/.cache/huggingface/xet/chunk-cache` to store reusable file chunks - 2. Download files in parallel: - 2.1. Prepare to write the file to disk - 2.2. Ask the server "how is this file split into chunks?" using the file's unique hash - The server responds with: - - Which chunks make up the complete file - - Where each chunk can be downloaded from - 2.3. For each needed chunk: - - Checks if we already have it in our local cache - - If not, download it from cloud storage (S3) - - Save it to cache for future use - - Assemble the chunks in order to recreate the original file - - """ - try: - from hf_xet import PyXetDownloadInfo, download_files # type: ignore[no-redef] - except ImportError: - raise ValueError( - "To use optimized download using Xet storage, you need to install the hf_xet package. " - 'Try `pip install "huggingface_hub[hf_xet]"` or `pip install hf_xet`.' - ) - - connection_info = refresh_xet_connection_info(file_data=xet_file_data, headers=headers) - - def token_refresher() -> Tuple[str, int]: - connection_info = refresh_xet_connection_info(file_data=xet_file_data, headers=headers) - if connection_info is None: - raise ValueError("Failed to refresh token using xet metadata.") - return connection_info.access_token, connection_info.expiration_unix_epoch - - xet_download_info = [ - PyXetDownloadInfo( - destination_path=str(incomplete_path.absolute()), hash=xet_file_data.file_hash, file_size=expected_size - ) - ] - - if not displayed_filename: - displayed_filename = incomplete_path.name - - # Truncate filename if too long to display - if len(displayed_filename) > 40: - displayed_filename = f"{displayed_filename[:40]}(…)" - - progress_cm = _get_progress_bar_context( - desc=displayed_filename, - log_level=logger.getEffectiveLevel(), - total=expected_size, - initial=0, - name="huggingface_hub.xet_get", - _tqdm_bar=_tqdm_bar, - ) - - with progress_cm as progress: - - def progress_updater(progress_bytes: float): - progress.update(progress_bytes) - - download_files( - xet_download_info, - endpoint=connection_info.endpoint, - token_info=(connection_info.access_token, connection_info.expiration_unix_epoch), - token_refresher=token_refresher, - progress_updater=[progress_updater], - ) - - -def _normalize_etag(etag: Optional[str]) -> Optional[str]: - """Normalize ETag HTTP header, so it can be used to create nice filepaths. - - The HTTP spec allows two forms of ETag: - ETag: W/"" - ETag: "" - - For now, we only expect the second form from the server, but we want to be future-proof so we support both. For - more context, see `TestNormalizeEtag` tests and https://github.com/huggingface/huggingface_hub/pull/1428. - - Args: - etag (`str`, *optional*): HTTP header - - Returns: - `str` or `None`: string that can be used as a nice directory name. - Returns `None` if input is None. - """ - if etag is None: - return None - return etag.lstrip("W/").strip('"') - - -def _create_relative_symlink(src: str, dst: str, new_blob: bool = False) -> None: - """Alias method used in `transformers` conversion script.""" - return _create_symlink(src=src, dst=dst, new_blob=new_blob) - - -def _create_symlink(src: str, dst: str, new_blob: bool = False) -> None: - """Create a symbolic link named dst pointing to src. - - By default, it will try to create a symlink using a relative path. Relative paths have 2 advantages: - - If the cache_folder is moved (example: back-up on a shared drive), relative paths within the cache folder will - not break. - - Relative paths seems to be better handled on Windows. Issue was reported 3 times in less than a week when - changing from relative to absolute paths. See https://github.com/huggingface/huggingface_hub/issues/1398, - https://github.com/huggingface/diffusers/issues/2729 and https://github.com/huggingface/transformers/pull/22228. - NOTE: The issue with absolute paths doesn't happen on admin mode. - When creating a symlink from the cache to a local folder, it is possible that a relative path cannot be created. - This happens when paths are not on the same volume. In that case, we use absolute paths. - - - The result layout looks something like - └── [ 128] snapshots - ├── [ 128] 2439f60ef33a0d46d85da5001d52aeda5b00ce9f - │ ├── [ 52] README.md -> ../../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812 - │ └── [ 76] pytorch_model.bin -> ../../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd - - If symlinks cannot be created on this platform (most likely to be Windows), the workaround is to avoid symlinks by - having the actual file in `dst`. If it is a new file (`new_blob=True`), we move it to `dst`. If it is not a new file - (`new_blob=False`), we don't know if the blob file is already referenced elsewhere. To avoid breaking existing - cache, the file is duplicated on the disk. - - In case symlinks are not supported, a warning message is displayed to the user once when loading `huggingface_hub`. - The warning message can be disabled with the `DISABLE_SYMLINKS_WARNING` environment variable. - """ - try: - os.remove(dst) - except OSError: - pass - - abs_src = os.path.abspath(os.path.expanduser(src)) - abs_dst = os.path.abspath(os.path.expanduser(dst)) - abs_dst_folder = os.path.dirname(abs_dst) - - # Use relative_dst in priority - try: - relative_src = os.path.relpath(abs_src, abs_dst_folder) - except ValueError: - # Raised on Windows if src and dst are not on the same volume. This is the case when creating a symlink to a - # local_dir instead of within the cache directory. - # See https://docs.python.org/3/library/os.path.html#os.path.relpath - relative_src = None - - try: - commonpath = os.path.commonpath([abs_src, abs_dst]) - _support_symlinks = are_symlinks_supported(commonpath) - except ValueError: - # Raised if src and dst are not on the same volume. Symlinks will still work on Linux/Macos. - # See https://docs.python.org/3/library/os.path.html#os.path.commonpath - _support_symlinks = os.name != "nt" - except PermissionError: - # Permission error means src and dst are not in the same volume (e.g. destination path has been provided - # by the user via `local_dir`. Let's test symlink support there) - _support_symlinks = are_symlinks_supported(abs_dst_folder) - except OSError as e: - # OS error (errno=30) means that the commonpath is readonly on Linux/MacOS. - if e.errno == errno.EROFS: - _support_symlinks = are_symlinks_supported(abs_dst_folder) - else: - raise - - # Symlinks are supported => let's create a symlink. - if _support_symlinks: - src_rel_or_abs = relative_src or abs_src - logger.debug(f"Creating pointer from {src_rel_or_abs} to {abs_dst}") - try: - os.symlink(src_rel_or_abs, abs_dst) - return - except FileExistsError: - if os.path.islink(abs_dst) and os.path.realpath(abs_dst) == os.path.realpath(abs_src): - # `abs_dst` already exists and is a symlink to the `abs_src` blob. It is most likely that the file has - # been cached twice concurrently (exactly between `os.remove` and `os.symlink`). Do nothing. - return - else: - # Very unlikely to happen. Means a file `dst` has been created exactly between `os.remove` and - # `os.symlink` and is not a symlink to the `abs_src` blob file. Raise exception. - raise - except PermissionError: - # Permission error means src and dst are not in the same volume (e.g. download to local dir) and symlink - # is supported on both volumes but not between them. Let's just make a hard copy in that case. - pass - - # Symlinks are not supported => let's move or copy the file. - if new_blob: - logger.info(f"Symlink not supported. Moving file from {abs_src} to {abs_dst}") - shutil.move(abs_src, abs_dst, copy_function=_copy_no_matter_what) - else: - logger.info(f"Symlink not supported. Copying file from {abs_src} to {abs_dst}") - shutil.copyfile(abs_src, abs_dst) - - -def _cache_commit_hash_for_specific_revision(storage_folder: str, revision: str, commit_hash: str) -> None: - """Cache reference between a revision (tag, branch or truncated commit hash) and the corresponding commit hash. - - Does nothing if `revision` is already a proper `commit_hash` or reference is already cached. - """ - if revision != commit_hash: - ref_path = Path(storage_folder) / "refs" / revision - ref_path.parent.mkdir(parents=True, exist_ok=True) - if not ref_path.exists() or commit_hash != ref_path.read_text(): - # Update ref only if has been updated. Could cause useless error in case - # repo is already cached and user doesn't have write access to cache folder. - # See https://github.com/huggingface/huggingface_hub/issues/1216. - ref_path.write_text(commit_hash) - - -@validate_hf_hub_args -def repo_folder_name(*, repo_id: str, repo_type: str) -> str: - """Return a serialized version of a hf.co repo name and type, safe for disk storage - as a single non-nested folder. - - Example: models--julien-c--EsperBERTo-small - """ - # remove all `/` occurrences to correctly convert repo to directory name - parts = [f"{repo_type}s", *repo_id.split("/")] - return constants.REPO_ID_SEPARATOR.join(parts) - - -def _check_disk_space(expected_size: int, target_dir: Union[str, Path]) -> None: - """Check disk usage and log a warning if there is not enough disk space to download the file. - - Args: - expected_size (`int`): - The expected size of the file in bytes. - target_dir (`str`): - The directory where the file will be stored after downloading. - """ - - target_dir = Path(target_dir) # format as `Path` - for path in [target_dir] + list(target_dir.parents): # first check target_dir, then each parents one by one - try: - target_dir_free = shutil.disk_usage(path).free - if target_dir_free < expected_size: - warnings.warn( - "Not enough free disk space to download the file. " - f"The expected file size is: {expected_size / 1e6:.2f} MB. " - f"The target location {target_dir} only has {target_dir_free / 1e6:.2f} MB free disk space." - ) - return - except OSError: # raise on anything: file does not exist or space disk cannot be checked - pass - - -@validate_hf_hub_args -def hf_hub_download( - repo_id: str, - filename: str, - *, - subfolder: Optional[str] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - library_name: Optional[str] = None, - library_version: Optional[str] = None, - cache_dir: Union[str, Path, None] = None, - local_dir: Union[str, Path, None] = None, - user_agent: Union[Dict, str, None] = None, - force_download: bool = False, - proxies: Optional[Dict] = None, - etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, - token: Union[bool, str, None] = None, - local_files_only: bool = False, - headers: Optional[Dict[str, str]] = None, - endpoint: Optional[str] = None, - resume_download: Optional[bool] = None, - force_filename: Optional[str] = None, - local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto", -) -> str: - """Download a given file if it's not already present in the local cache. - - The new cache file layout looks like this: - - The cache directory contains one subfolder per repo_id (namespaced by repo type) - - inside each repo folder: - - refs is a list of the latest known revision => commit_hash pairs - - blobs contains the actual file blobs (identified by their git-sha or sha256, depending on - whether they're LFS files or not) - - snapshots contains one subfolder per commit, each "commit" contains the subset of the files - that have been resolved at that particular commit. Each filename is a symlink to the blob - at that particular commit. - - ``` - [ 96] . - └── [ 160] models--julien-c--EsperBERTo-small - ├── [ 160] blobs - │ ├── [321M] 403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd - │ ├── [ 398] 7cb18dc9bafbfcf74629a4b760af1b160957a83e - │ └── [1.4K] d7edf6bd2a681fb0175f7735299831ee1b22b812 - ├── [ 96] refs - │ └── [ 40] main - └── [ 128] snapshots - ├── [ 128] 2439f60ef33a0d46d85da5001d52aeda5b00ce9f - │ ├── [ 52] README.md -> ../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812 - │ └── [ 76] pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd - └── [ 128] bbc77c8132af1cc5cf678da3f1ddf2de43606d48 - ├── [ 52] README.md -> ../../blobs/7cb18dc9bafbfcf74629a4b760af1b160957a83e - └── [ 76] pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd - ``` - - If `local_dir` is provided, the file structure from the repo will be replicated in this location. When using this - option, the `cache_dir` will not be used and a `.cache/huggingface/` folder will be created at the root of `local_dir` - to store some metadata related to the downloaded files. While this mechanism is not as robust as the main - cache-system, it's optimized for regularly pulling the latest version of a repository. - - Args: - repo_id (`str`): - A user or an organization name and a repo name separated by a `/`. - filename (`str`): - The name of the file in the repo. - subfolder (`str`, *optional*): - An optional value corresponding to a folder inside the model repo. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if downloading from a dataset or space, - `None` or `"model"` if downloading from a model. Default is `None`. - revision (`str`, *optional*): - An optional Git revision id which can be a branch name, a tag, or a - commit hash. - library_name (`str`, *optional*): - The name of the library to which the object corresponds. - library_version (`str`, *optional*): - The version of the library. - cache_dir (`str`, `Path`, *optional*): - Path to the folder where cached files are stored. - local_dir (`str` or `Path`, *optional*): - If provided, the downloaded file will be placed under this directory. - user_agent (`dict`, `str`, *optional*): - The user-agent info in the form of a dictionary or a string. - force_download (`bool`, *optional*, defaults to `False`): - Whether the file should be downloaded even if it already exists in - the local cache. - proxies (`dict`, *optional*): - Dictionary mapping protocol to the URL of the proxy passed to - `requests.request`. - etag_timeout (`float`, *optional*, defaults to `10`): - When fetching ETag, how many seconds to wait for the server to send - data before giving up which is passed to `requests.request`. - token (`str`, `bool`, *optional*): - A token to be used for the download. - - If `True`, the token is read from the HuggingFace config - folder. - - If a string, it's used as the authentication token. - local_files_only (`bool`, *optional*, defaults to `False`): - If `True`, avoid downloading the file and return the path to the - local cached file if it exists. - headers (`dict`, *optional*): - Additional headers to be sent with the request. - - Returns: - `str`: Local path of file or if networking is off, last version of file cached on disk. - - Raises: - [`~utils.RepositoryNotFoundError`] - If the repository to download from cannot be found. This may be because it doesn't exist, - or because it is set to `private` and you do not have access. - [`~utils.RevisionNotFoundError`] - If the revision to download from cannot be found. - [`~utils.EntryNotFoundError`] - If the file to download cannot be found. - [`~utils.LocalEntryNotFoundError`] - If network is disabled or unavailable and file is not found in cache. - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) - If `token=True` but the token cannot be found. - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) - If ETag cannot be determined. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If some parameter value is invalid. - - """ - if constants.HF_HUB_ETAG_TIMEOUT != constants.DEFAULT_ETAG_TIMEOUT: - # Respect environment variable above user value - etag_timeout = constants.HF_HUB_ETAG_TIMEOUT - - if force_filename is not None: - warnings.warn( - "The `force_filename` parameter is deprecated as a new caching system, " - "which keeps the filenames as they are on the Hub, is now in place.", - FutureWarning, - ) - if resume_download is not None: - warnings.warn( - "`resume_download` is deprecated and will be removed in version 1.0.0. " - "Downloads always resume when possible. " - "If you want to force a new download, use `force_download=True`.", - FutureWarning, - ) - - if cache_dir is None: - cache_dir = constants.HF_HUB_CACHE - if revision is None: - revision = constants.DEFAULT_REVISION - if isinstance(cache_dir, Path): - cache_dir = str(cache_dir) - if isinstance(local_dir, Path): - local_dir = str(local_dir) - - if subfolder == "": - subfolder = None - if subfolder is not None: - # This is used to create a URL, and not a local path, hence the forward slash. - filename = f"{subfolder}/{filename}" - - if repo_type is None: - repo_type = "model" - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES)}") - - hf_headers = build_hf_headers( - token=token, - library_name=library_name, - library_version=library_version, - user_agent=user_agent, - headers=headers, - ) - - if local_dir is not None: - if local_dir_use_symlinks != "auto": - warnings.warn( - "`local_dir_use_symlinks` parameter is deprecated and will be ignored. " - "The process to download files to a local folder has been updated and do " - "not rely on symlinks anymore. You only need to pass a destination folder " - "as`local_dir`.\n" - "For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder." - ) - - return _hf_hub_download_to_local_dir( - # Destination - local_dir=local_dir, - # File info - repo_id=repo_id, - repo_type=repo_type, - filename=filename, - revision=revision, - # HTTP info - endpoint=endpoint, - etag_timeout=etag_timeout, - headers=hf_headers, - proxies=proxies, - token=token, - # Additional options - cache_dir=cache_dir, - force_download=force_download, - local_files_only=local_files_only, - ) - else: - return _hf_hub_download_to_cache_dir( - # Destination - cache_dir=cache_dir, - # File info - repo_id=repo_id, - filename=filename, - repo_type=repo_type, - revision=revision, - # HTTP info - endpoint=endpoint, - etag_timeout=etag_timeout, - headers=hf_headers, - proxies=proxies, - token=token, - # Additional options - local_files_only=local_files_only, - force_download=force_download, - ) - - -def _hf_hub_download_to_cache_dir( - *, - # Destination - cache_dir: str, - # File info - repo_id: str, - filename: str, - repo_type: str, - revision: str, - # HTTP info - endpoint: Optional[str], - etag_timeout: float, - headers: Dict[str, str], - proxies: Optional[Dict], - token: Optional[Union[bool, str]], - # Additional options - local_files_only: bool, - force_download: bool, -) -> str: - """Download a given file to a cache folder, if not already present. - - Method should not be called directly. Please use `hf_hub_download` instead. - """ - locks_dir = os.path.join(cache_dir, ".locks") - storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type)) - - # cross platform transcription of filename, to be used as a local file path. - relative_filename = os.path.join(*filename.split("/")) - if os.name == "nt": - if relative_filename.startswith("..\\") or "\\..\\" in relative_filename: - raise ValueError( - f"Invalid filename: cannot handle filename '{relative_filename}' on Windows. Please ask the repository" - " owner to rename this file." - ) - - # if user provides a commit_hash and they already have the file on disk, shortcut everything. - if REGEX_COMMIT_HASH.match(revision): - pointer_path = _get_pointer_path(storage_folder, revision, relative_filename) - if os.path.exists(pointer_path) and not force_download: - return pointer_path - - # Try to get metadata (etag, commit_hash, url, size) from the server. - # If we can't, a HEAD request error is returned. - (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_call_error) = _get_metadata_or_catch_error( - repo_id=repo_id, - filename=filename, - repo_type=repo_type, - revision=revision, - endpoint=endpoint, - proxies=proxies, - etag_timeout=etag_timeout, - headers=headers, - token=token, - local_files_only=local_files_only, - storage_folder=storage_folder, - relative_filename=relative_filename, - ) - - # etag can be None for several reasons: - # 1. we passed local_files_only. - # 2. we don't have a connection - # 3. Hub is down (HTTP 500, 503, 504) - # 4. repo is not found -for example private or gated- and invalid/missing token sent - # 5. Hub is blocked by a firewall or proxy is not set correctly. - # => Try to get the last downloaded one from the specified revision. - # - # If the specified revision is a commit hash, look inside "snapshots". - # If the specified revision is a branch or tag, look inside "refs". - if head_call_error is not None: - # Couldn't make a HEAD call => let's try to find a local file - if not force_download: - commit_hash = None - if REGEX_COMMIT_HASH.match(revision): - commit_hash = revision - else: - ref_path = os.path.join(storage_folder, "refs", revision) - if os.path.isfile(ref_path): - with open(ref_path) as f: - commit_hash = f.read() - - # Return pointer file if exists - if commit_hash is not None: - pointer_path = _get_pointer_path(storage_folder, commit_hash, relative_filename) - if os.path.exists(pointer_path) and not force_download: - return pointer_path - - # Otherwise, raise appropriate error - _raise_on_head_call_error(head_call_error, force_download, local_files_only) - - # From now on, etag, commit_hash, url and size are not None. - assert etag is not None, "etag must have been retrieved from server" - assert commit_hash is not None, "commit_hash must have been retrieved from server" - assert url_to_download is not None, "file location must have been retrieved from server" - assert expected_size is not None, "expected_size must have been retrieved from server" - blob_path = os.path.join(storage_folder, "blobs", etag) - pointer_path = _get_pointer_path(storage_folder, commit_hash, relative_filename) - - os.makedirs(os.path.dirname(blob_path), exist_ok=True) - os.makedirs(os.path.dirname(pointer_path), exist_ok=True) - - # if passed revision is not identical to commit_hash - # then revision has to be a branch name or tag name. - # In that case store a ref. - _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash) - - # Prevent parallel downloads of the same file with a lock. - # etag could be duplicated across repos, - lock_path = os.path.join(locks_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type), f"{etag}.lock") - - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix. - if ( - os.name == "nt" - and len(os.path.abspath(lock_path)) > 255 - and not os.path.abspath(lock_path).startswith("\\\\?\\") - ): - lock_path = "\\\\?\\" + os.path.abspath(lock_path) - - if ( - os.name == "nt" - and len(os.path.abspath(blob_path)) > 255 - and not os.path.abspath(blob_path).startswith("\\\\?\\") - ): - blob_path = "\\\\?\\" + os.path.abspath(blob_path) - - Path(lock_path).parent.mkdir(parents=True, exist_ok=True) - - # pointer already exists -> immediate return - if not force_download and os.path.exists(pointer_path): - return pointer_path - - # Blob exists but pointer must be (safely) created -> take the lock - if not force_download and os.path.exists(blob_path): - with WeakFileLock(lock_path): - if not os.path.exists(pointer_path): - _create_symlink(blob_path, pointer_path, new_blob=False) - return pointer_path - - # Local file doesn't exist or etag isn't a match => retrieve file from remote (or cache) - - with WeakFileLock(lock_path): - _download_to_tmp_and_move( - incomplete_path=Path(blob_path + ".incomplete"), - destination_path=Path(blob_path), - url_to_download=url_to_download, - proxies=proxies, - headers=headers, - expected_size=expected_size, - filename=filename, - force_download=force_download, - etag=etag, - xet_file_data=xet_file_data, - ) - if not os.path.exists(pointer_path): - _create_symlink(blob_path, pointer_path, new_blob=True) - - return pointer_path - - -def _hf_hub_download_to_local_dir( - *, - # Destination - local_dir: Union[str, Path], - # File info - repo_id: str, - repo_type: str, - filename: str, - revision: str, - # HTTP info - endpoint: Optional[str], - etag_timeout: float, - headers: Dict[str, str], - proxies: Optional[Dict], - token: Union[bool, str, None], - # Additional options - cache_dir: str, - force_download: bool, - local_files_only: bool, -) -> str: - """Download a given file to a local folder, if not already present. - - Method should not be called directly. Please use `hf_hub_download` instead. - """ - # Some Windows versions do not allow for paths longer than 255 characters. - # In this case, we must specify it as an extended path by using the "\\?\" prefix. - if os.name == "nt" and len(os.path.abspath(local_dir)) > 255: - local_dir = "\\\\?\\" + os.path.abspath(local_dir) - local_dir = Path(local_dir) - paths = get_local_download_paths(local_dir=local_dir, filename=filename) - local_metadata = read_download_metadata(local_dir=local_dir, filename=filename) - - # Local file exists + metadata exists + commit_hash matches => return file - if ( - not force_download - and REGEX_COMMIT_HASH.match(revision) - and paths.file_path.is_file() - and local_metadata is not None - and local_metadata.commit_hash == revision - ): - return str(paths.file_path) - - # Local file doesn't exist or commit_hash doesn't match => we need the etag - (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_call_error) = _get_metadata_or_catch_error( - repo_id=repo_id, - filename=filename, - repo_type=repo_type, - revision=revision, - endpoint=endpoint, - proxies=proxies, - etag_timeout=etag_timeout, - headers=headers, - token=token, - local_files_only=local_files_only, - ) - - if head_call_error is not None: - # No HEAD call but local file exists => default to local file - if not force_download and paths.file_path.is_file(): - logger.warning( - f"Couldn't access the Hub to check for update but local file already exists. Defaulting to existing file. (error: {head_call_error})" - ) - return str(paths.file_path) - # Otherwise => raise - _raise_on_head_call_error(head_call_error, force_download, local_files_only) - - # From now on, etag, commit_hash, url and size are not None. - assert etag is not None, "etag must have been retrieved from server" - assert commit_hash is not None, "commit_hash must have been retrieved from server" - assert url_to_download is not None, "file location must have been retrieved from server" - assert expected_size is not None, "expected_size must have been retrieved from server" - - # Local file exists => check if it's up-to-date - if not force_download and paths.file_path.is_file(): - # etag matches => update metadata and return file - if local_metadata is not None and local_metadata.etag == etag: - write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag) - return str(paths.file_path) - - # metadata is outdated + etag is a sha256 - # => means it's an LFS file (large) - # => let's compute local hash and compare - # => if match, update metadata and return file - if local_metadata is None and REGEX_SHA256.match(etag) is not None: - with open(paths.file_path, "rb") as f: - file_hash = sha_fileobj(f).hex() - if file_hash == etag: - write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag) - return str(paths.file_path) - - # Local file doesn't exist or etag isn't a match => retrieve file from remote (or cache) - - # If we are lucky enough, the file is already in the cache => copy it - if not force_download: - cached_path = try_to_load_from_cache( - repo_id=repo_id, - filename=filename, - cache_dir=cache_dir, - revision=commit_hash, - repo_type=repo_type, - ) - if isinstance(cached_path, str): - with WeakFileLock(paths.lock_path): - paths.file_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(cached_path, paths.file_path) - write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag) - return str(paths.file_path) - - # Otherwise, let's download the file! - with WeakFileLock(paths.lock_path): - paths.file_path.unlink(missing_ok=True) # delete outdated file first - _download_to_tmp_and_move( - incomplete_path=paths.incomplete_path(etag), - destination_path=paths.file_path, - url_to_download=url_to_download, - proxies=proxies, - headers=headers, - expected_size=expected_size, - filename=filename, - force_download=force_download, - etag=etag, - xet_file_data=xet_file_data, - ) - - write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag) - return str(paths.file_path) - - -@validate_hf_hub_args -def try_to_load_from_cache( - repo_id: str, - filename: str, - cache_dir: Union[str, Path, None] = None, - revision: Optional[str] = None, - repo_type: Optional[str] = None, -) -> Union[str, _CACHED_NO_EXIST_T, None]: - """ - Explores the cache to return the latest cached file for a given revision if found. - - This function will not raise any exception if the file in not cached. - - Args: - cache_dir (`str` or `os.PathLike`): - The folder where the cached files lie. - repo_id (`str`): - The ID of the repo on huggingface.co. - filename (`str`): - The filename to look for inside `repo_id`. - revision (`str`, *optional*): - The specific model version to use. Will default to `"main"` if it's not provided and no `commit_hash` is - provided either. - repo_type (`str`, *optional*): - The type of the repository. Will default to `"model"`. - - Returns: - `Optional[str]` or `_CACHED_NO_EXIST`: - Will return `None` if the file was not cached. Otherwise: - - The exact path to the cached file if it's found in the cache - - A special value `_CACHED_NO_EXIST` if the file does not exist at the given commit hash and this fact was - cached. - - Example: - - ```python - from huggingface_hub import try_to_load_from_cache, _CACHED_NO_EXIST - - filepath = try_to_load_from_cache() - if isinstance(filepath, str): - # file exists and is cached - ... - elif filepath is _CACHED_NO_EXIST: - # non-existence of file is cached - ... - else: - # file is not cached - ... - ``` - """ - if revision is None: - revision = "main" - if repo_type is None: - repo_type = "model" - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES)}") - if cache_dir is None: - cache_dir = constants.HF_HUB_CACHE - - object_id = repo_id.replace("/", "--") - repo_cache = os.path.join(cache_dir, f"{repo_type}s--{object_id}") - if not os.path.isdir(repo_cache): - # No cache for this model - return None - - refs_dir = os.path.join(repo_cache, "refs") - snapshots_dir = os.path.join(repo_cache, "snapshots") - no_exist_dir = os.path.join(repo_cache, ".no_exist") - - # Resolve refs (for instance to convert main to the associated commit sha) - if os.path.isdir(refs_dir): - revision_file = os.path.join(refs_dir, revision) - if os.path.isfile(revision_file): - with open(revision_file) as f: - revision = f.read() - - # Check if file is cached as "no_exist" - if os.path.isfile(os.path.join(no_exist_dir, revision, filename)): - return _CACHED_NO_EXIST - - # Check if revision folder exists - if not os.path.exists(snapshots_dir): - return None - cached_shas = os.listdir(snapshots_dir) - if revision not in cached_shas: - # No cache for this revision and we won't try to return a random revision - return None - - # Check if file exists in cache - cached_file = os.path.join(snapshots_dir, revision, filename) - return cached_file if os.path.isfile(cached_file) else None - - -@validate_hf_hub_args -def get_hf_file_metadata( - url: str, - token: Union[bool, str, None] = None, - proxies: Optional[Dict] = None, - timeout: Optional[float] = constants.DEFAULT_REQUEST_TIMEOUT, - library_name: Optional[str] = None, - library_version: Optional[str] = None, - user_agent: Union[Dict, str, None] = None, - headers: Optional[Dict[str, str]] = None, - endpoint: Optional[str] = None, -) -> HfFileMetadata: - """Fetch metadata of a file versioned on the Hub for a given url. - - Args: - url (`str`): - File url, for example returned by [`hf_hub_url`]. - token (`str` or `bool`, *optional*): - A token to be used for the download. - - If `True`, the token is read from the HuggingFace config - folder. - - If `False` or `None`, no token is provided. - - If a string, it's used as the authentication token. - proxies (`dict`, *optional*): - Dictionary mapping protocol to the URL of the proxy passed to - `requests.request`. - timeout (`float`, *optional*, defaults to 10): - How many seconds to wait for the server to send metadata before giving up. - library_name (`str`, *optional*): - The name of the library to which the object corresponds. - library_version (`str`, *optional*): - The version of the library. - user_agent (`dict`, `str`, *optional*): - The user-agent info in the form of a dictionary or a string. - headers (`dict`, *optional*): - Additional headers to be sent with the request. - endpoint (`str`, *optional*): - Endpoint of the Hub. Defaults to . - - Returns: - A [`HfFileMetadata`] object containing metadata such as location, etag, size and - commit_hash. - """ - hf_headers = build_hf_headers( - token=token, - library_name=library_name, - library_version=library_version, - user_agent=user_agent, - headers=headers, - ) - hf_headers["Accept-Encoding"] = "identity" # prevent any compression => we want to know the real size of the file - - # Retrieve metadata - r = _request_wrapper( - method="HEAD", - url=url, - headers=hf_headers, - allow_redirects=False, - follow_relative_redirects=True, - proxies=proxies, - timeout=timeout, - ) - hf_raise_for_status(r) - - # Return - return HfFileMetadata( - commit_hash=r.headers.get(constants.HUGGINGFACE_HEADER_X_REPO_COMMIT), - # We favor a custom header indicating the etag of the linked resource, and - # we fallback to the regular etag header. - etag=_normalize_etag(r.headers.get(constants.HUGGINGFACE_HEADER_X_LINKED_ETAG) or r.headers.get("ETag")), - # Either from response headers (if redirected) or defaults to request url - # Do not use directly `url`, as `_request_wrapper` might have followed relative - # redirects. - location=r.headers.get("Location") or r.request.url, # type: ignore - size=_int_or_none( - r.headers.get(constants.HUGGINGFACE_HEADER_X_LINKED_SIZE) or r.headers.get("Content-Length") - ), - xet_file_data=parse_xet_file_data_from_response(r, endpoint=endpoint), # type: ignore - ) - - -def _get_metadata_or_catch_error( - *, - repo_id: str, - filename: str, - repo_type: str, - revision: str, - endpoint: Optional[str], - proxies: Optional[Dict], - etag_timeout: Optional[float], - headers: Dict[str, str], # mutated inplace! - token: Union[bool, str, None], - local_files_only: bool, - relative_filename: Optional[str] = None, # only used to store `.no_exists` in cache - storage_folder: Optional[str] = None, # only used to store `.no_exists` in cache -) -> Union[ - # Either an exception is caught and returned - Tuple[None, None, None, None, None, Exception], - # Or the metadata is returned as - # `(url_to_download, etag, commit_hash, expected_size, xet_file_data, None)` - Tuple[str, str, str, int, Optional[XetFileData], None], -]: - """Get metadata for a file on the Hub, safely handling network issues. - - Returns either the etag, commit_hash and expected size of the file, or the error - raised while fetching the metadata. - - NOTE: This function mutates `headers` inplace! It removes the `authorization` header - if the file is a LFS blob and the domain of the url is different from the - domain of the location (typically an S3 bucket). - """ - if local_files_only: - return ( - None, - None, - None, - None, - None, - OfflineModeIsEnabled( - f"Cannot access file since 'local_files_only=True' as been set. (repo_id: {repo_id}, repo_type: {repo_type}, revision: {revision}, filename: {filename})" - ), - ) - - url = hf_hub_url(repo_id, filename, repo_type=repo_type, revision=revision, endpoint=endpoint) - url_to_download: str = url - etag: Optional[str] = None - commit_hash: Optional[str] = None - expected_size: Optional[int] = None - head_error_call: Optional[Exception] = None - xet_file_data: Optional[XetFileData] = None - - # Try to get metadata from the server. - # Do not raise yet if the file is not found or not accessible. - if not local_files_only: - try: - try: - metadata = get_hf_file_metadata( - url=url, proxies=proxies, timeout=etag_timeout, headers=headers, token=token, endpoint=endpoint - ) - except EntryNotFoundError as http_error: - if storage_folder is not None and relative_filename is not None: - # Cache the non-existence of the file - commit_hash = http_error.response.headers.get(constants.HUGGINGFACE_HEADER_X_REPO_COMMIT) - if commit_hash is not None: - no_exist_file_path = Path(storage_folder) / ".no_exist" / commit_hash / relative_filename - try: - no_exist_file_path.parent.mkdir(parents=True, exist_ok=True) - no_exist_file_path.touch() - except OSError as e: - logger.error( - f"Could not cache non-existence of file. Will ignore error and continue. Error: {e}" - ) - _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash) - raise - - # Commit hash must exist - commit_hash = metadata.commit_hash - if commit_hash is None: - raise FileMetadataError( - "Distant resource does not seem to be on huggingface.co. It is possible that a configuration issue" - " prevents you from downloading resources from https://huggingface.co. Please check your firewall" - " and proxy settings and make sure your SSL certificates are updated." - ) - - # Etag must exist - # If we don't have any of those, raise an error. - etag = metadata.etag - if etag is None: - raise FileMetadataError( - "Distant resource does not have an ETag, we won't be able to reliably ensure reproducibility." - ) - - # Size must exist - expected_size = metadata.size - if expected_size is None: - raise FileMetadataError("Distant resource does not have a Content-Length.") - - xet_file_data = metadata.xet_file_data - - # In case of a redirect, save an extra redirect on the request.get call, - # and ensure we download the exact atomic version even if it changed - # between the HEAD and the GET (unlikely, but hey). - # - # If url domain is different => we are downloading from a CDN => url is signed => don't send auth - # If url domain is the same => redirect due to repo rename AND downloading a regular file => keep auth - if xet_file_data is None and url != metadata.location: - url_to_download = metadata.location - if urlparse(url).netloc != urlparse(metadata.location).netloc: - # Remove authorization header when downloading a LFS blob - headers.pop("authorization", None) - except (requests.exceptions.SSLError, requests.exceptions.ProxyError): - # Actually raise for those subclasses of ConnectionError - raise - except ( - requests.exceptions.ConnectionError, - requests.exceptions.Timeout, - OfflineModeIsEnabled, - ) as error: - # Otherwise, our Internet connection is down. - # etag is None - head_error_call = error - except (RevisionNotFoundError, EntryNotFoundError): - # The repo was found but the revision or entry doesn't exist on the Hub (never existed or got deleted) - raise - except requests.HTTPError as error: - # Multiple reasons for an http error: - # - Repository is private and invalid/missing token sent - # - Repository is gated and invalid/missing token sent - # - Hub is down (error 500 or 504) - # => let's switch to 'local_files_only=True' to check if the files are already cached. - # (if it's not the case, the error will be re-raised) - head_error_call = error - except FileMetadataError as error: - # Multiple reasons for a FileMetadataError: - # - Wrong network configuration (proxy, firewall, SSL certificates) - # - Inconsistency on the Hub - # => let's switch to 'local_files_only=True' to check if the files are already cached. - # (if it's not the case, the error will be re-raised) - head_error_call = error - - if not (local_files_only or etag is not None or head_error_call is not None): - raise RuntimeError("etag is empty due to uncovered problems") - - return (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_error_call) # type: ignore [return-value] - - -def _raise_on_head_call_error(head_call_error: Exception, force_download: bool, local_files_only: bool) -> NoReturn: - """Raise an appropriate error when the HEAD call failed and we cannot locate a local file.""" - # No head call => we cannot force download. - if force_download: - if local_files_only: - raise ValueError("Cannot pass 'force_download=True' and 'local_files_only=True' at the same time.") - elif isinstance(head_call_error, OfflineModeIsEnabled): - raise ValueError("Cannot pass 'force_download=True' when offline mode is enabled.") from head_call_error - else: - raise ValueError("Force download failed due to the above error.") from head_call_error - - # No head call + couldn't find an appropriate file on disk => raise an error. - if local_files_only: - raise LocalEntryNotFoundError( - "Cannot find the requested files in the disk cache and outgoing traffic has been disabled. To enable" - " hf.co look-ups and downloads online, set 'local_files_only' to False." - ) - elif isinstance(head_call_error, (RepositoryNotFoundError, GatedRepoError)) or ( - isinstance(head_call_error, HfHubHTTPError) and head_call_error.response.status_code == 401 - ): - # Repo not found or gated => let's raise the actual error - # Unauthorized => likely a token issue => let's raise the actual error - raise head_call_error - else: - # Otherwise: most likely a connection issue or Hub downtime => let's warn the user - raise LocalEntryNotFoundError( - "An error happened while trying to locate the file on the Hub and we cannot find the requested files" - " in the local cache. Please check your connection and try again or make sure your Internet connection" - " is on." - ) from head_call_error - - -def _download_to_tmp_and_move( - incomplete_path: Path, - destination_path: Path, - url_to_download: str, - proxies: Optional[Dict], - headers: Dict[str, str], - expected_size: Optional[int], - filename: str, - force_download: bool, - etag: Optional[str], - xet_file_data: Optional[XetFileData], -) -> None: - """Download content from a URL to a destination path. - - Internal logic: - - return early if file is already downloaded - - resume download if possible (from incomplete file) - - do not resume download if `force_download=True` or `HF_HUB_ENABLE_HF_TRANSFER=True` - - check disk space before downloading - - download content to a temporary file - - set correct permissions on temporary file - - move the temporary file to the destination path - - Both `incomplete_path` and `destination_path` must be on the same volume to avoid a local copy. - """ - if destination_path.exists() and not force_download: - # Do nothing if already exists (except if force_download=True) - return - - if incomplete_path.exists() and (force_download or (constants.HF_HUB_ENABLE_HF_TRANSFER and not proxies)): - # By default, we will try to resume the download if possible. - # However, if the user has set `force_download=True` or if `hf_transfer` is enabled, then we should - # not resume the download => delete the incomplete file. - message = f"Removing incomplete file '{incomplete_path}'" - if force_download: - message += " (force_download=True)" - elif constants.HF_HUB_ENABLE_HF_TRANSFER and not proxies: - message += " (hf_transfer=True)" - logger.info(message) - incomplete_path.unlink(missing_ok=True) - - with incomplete_path.open("ab") as f: - resume_size = f.tell() - message = f"Downloading '{filename}' to '{incomplete_path}'" - if resume_size > 0 and expected_size is not None: - message += f" (resume from {resume_size}/{expected_size})" - logger.info(message) - - if expected_size is not None: # might be None if HTTP header not set correctly - # Check disk space in both tmp and destination path - _check_disk_space(expected_size, incomplete_path.parent) - _check_disk_space(expected_size, destination_path.parent) - - if xet_file_data is not None and is_xet_available(): - logger.debug("Xet Storage is enabled for this repo. Downloading file from Xet Storage..") - xet_get( - incomplete_path=incomplete_path, - xet_file_data=xet_file_data, - headers=headers, - expected_size=expected_size, - displayed_filename=filename, - ) - else: - if xet_file_data is not None and not constants.HF_HUB_DISABLE_XET: - logger.warning( - "Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed. " - "Falling back to regular HTTP download. " - "For better performance, install the package with: `pip install huggingface_hub[hf_xet]` or `pip install hf_xet`" - ) - - http_get( - url_to_download, - f, - proxies=proxies, - resume_size=resume_size, - headers=headers, - expected_size=expected_size, - ) - - logger.info(f"Download complete. Moving file to {destination_path}") - _chmod_and_move(incomplete_path, destination_path) - - -def _int_or_none(value: Optional[str]) -> Optional[int]: - try: - return int(value) # type: ignore - except (TypeError, ValueError): - return None - - -def _chmod_and_move(src: Path, dst: Path) -> None: - """Set correct permission before moving a blob from tmp directory to cache dir. - - Do not take into account the `umask` from the process as there is no convenient way - to get it that is thread-safe. - - See: - - About umask: https://docs.python.org/3/library/os.html#os.umask - - Thread-safety: https://stackoverflow.com/a/70343066 - - About solution: https://github.com/huggingface/huggingface_hub/pull/1220#issuecomment-1326211591 - - Fix issue: https://github.com/huggingface/huggingface_hub/issues/1141 - - Fix issue: https://github.com/huggingface/huggingface_hub/issues/1215 - """ - # Get umask by creating a temporary file in the cached repo folder. - tmp_file = dst.parent.parent / f"tmp_{uuid.uuid4()}" - try: - tmp_file.touch() - cache_dir_mode = Path(tmp_file).stat().st_mode - os.chmod(str(src), stat.S_IMODE(cache_dir_mode)) - except OSError as e: - logger.warning( - f"Could not set the permissions on the file '{src}'. Error: {e}.\nContinuing without setting permissions." - ) - finally: - try: - tmp_file.unlink() - except OSError: - # fails if `tmp_file.touch()` failed => do nothing - # See https://github.com/huggingface/huggingface_hub/issues/2359 - pass - - shutil.move(str(src), str(dst), copy_function=_copy_no_matter_what) - - -def _copy_no_matter_what(src: str, dst: str) -> None: - """Copy file from src to dst. - - If `shutil.copy2` fails, fallback to `shutil.copyfile`. - """ - try: - # Copy file with metadata and permission - # Can fail e.g. if dst is an S3 mount - shutil.copy2(src, dst) - except OSError: - # Copy only file content - shutil.copyfile(src, dst) - - -def _get_pointer_path(storage_folder: str, revision: str, relative_filename: str) -> str: - # Using `os.path.abspath` instead of `Path.resolve()` to avoid resolving symlinks - snapshot_path = os.path.join(storage_folder, "snapshots") - pointer_path = os.path.join(snapshot_path, revision, relative_filename) - if Path(os.path.abspath(snapshot_path)) not in Path(os.path.abspath(pointer_path)).parents: - raise ValueError( - "Invalid pointer path: cannot create pointer path in snapshot folder if" - f" `storage_folder='{storage_folder}'`, `revision='{revision}'` and" - f" `relative_filename='{relative_filename}'`." - ) - return pointer_path diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/hf_api.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/hf_api.py deleted file mode 100644 index 8977e202da9631530dee3009427eae6ad26ec17f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/hf_api.py +++ /dev/null @@ -1,11036 +0,0 @@ -# coding=utf-8 -# Copyright 2019-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import annotations - -import inspect -import json -import re -import struct -import time -import warnings -from collections import defaultdict -from concurrent.futures import Future, ThreadPoolExecutor -from dataclasses import asdict, dataclass, field -from datetime import datetime -from functools import wraps -from itertools import islice -from pathlib import Path -from textwrap import dedent -from typing import ( - TYPE_CHECKING, - Any, - BinaryIO, - Callable, - Dict, - Iterable, - Iterator, - List, - Literal, - Optional, - Tuple, - Type, - TypeVar, - Union, - overload, -) -from urllib.parse import quote - -import requests -from requests.exceptions import HTTPError -from tqdm.auto import tqdm as base_tqdm -from tqdm.contrib.concurrent import thread_map - -from . import constants -from ._commit_api import ( - CommitOperation, - CommitOperationAdd, - CommitOperationCopy, - CommitOperationDelete, - _fetch_files_to_copy, - _fetch_upload_modes, - _prepare_commit_payload, - _upload_files, - _warn_on_overwriting_operations, -) -from ._inference_endpoints import InferenceEndpoint, InferenceEndpointType -from ._jobs_api import JobInfo, JobSpec, ScheduledJobInfo, _create_job_spec -from ._space_api import SpaceHardware, SpaceRuntime, SpaceStorage, SpaceVariable -from ._upload_large_folder import upload_large_folder_internal -from .community import ( - Discussion, - DiscussionComment, - DiscussionStatusChange, - DiscussionTitleChange, - DiscussionWithDetails, - deserialize_event, -) -from .constants import ( - DEFAULT_ETAG_TIMEOUT, # noqa: F401 # kept for backward compatibility - DEFAULT_REQUEST_TIMEOUT, # noqa: F401 # kept for backward compatibility - DEFAULT_REVISION, # noqa: F401 # kept for backward compatibility - DISCUSSION_STATUS, # noqa: F401 # kept for backward compatibility - DISCUSSION_TYPES, # noqa: F401 # kept for backward compatibility - ENDPOINT, # noqa: F401 # kept for backward compatibility - INFERENCE_ENDPOINTS_ENDPOINT, # noqa: F401 # kept for backward compatibility - REGEX_COMMIT_OID, # noqa: F401 # kept for backward compatibility - REPO_TYPE_MODEL, # noqa: F401 # kept for backward compatibility - REPO_TYPES, # noqa: F401 # kept for backward compatibility - REPO_TYPES_MAPPING, # noqa: F401 # kept for backward compatibility - REPO_TYPES_URL_PREFIXES, # noqa: F401 # kept for backward compatibility - SAFETENSORS_INDEX_FILE, # noqa: F401 # kept for backward compatibility - SAFETENSORS_MAX_HEADER_LENGTH, # noqa: F401 # kept for backward compatibility - SAFETENSORS_SINGLE_FILE, # noqa: F401 # kept for backward compatibility - SPACES_SDK_TYPES, # noqa: F401 # kept for backward compatibility - WEBHOOK_DOMAIN_T, # noqa: F401 # kept for backward compatibility - DiscussionStatusFilter, # noqa: F401 # kept for backward compatibility - DiscussionTypeFilter, # noqa: F401 # kept for backward compatibility -) -from .errors import ( - BadRequestError, - EntryNotFoundError, - GatedRepoError, - HfHubHTTPError, - RepositoryNotFoundError, - RevisionNotFoundError, -) -from .file_download import HfFileMetadata, get_hf_file_metadata, hf_hub_url -from .repocard_data import DatasetCardData, ModelCardData, SpaceCardData -from .utils import ( - DEFAULT_IGNORE_PATTERNS, - HfFolder, # noqa: F401 # kept for backward compatibility - LocalTokenNotFoundError, - NotASafetensorsRepoError, - SafetensorsFileMetadata, - SafetensorsParsingError, - SafetensorsRepoMetadata, - TensorInfo, - build_hf_headers, - chunk_iterable, - experimental, - filter_repo_objects, - fix_hf_endpoint_in_url, - get_session, - get_token, - hf_raise_for_status, - logging, - paginate, - parse_datetime, - validate_hf_hub_args, -) -from .utils import tqdm as hf_tqdm -from .utils._auth import _get_token_from_environment, _get_token_from_file, _get_token_from_google_colab -from .utils._deprecation import _deprecate_arguments, _deprecate_method -from .utils._typing import CallableT -from .utils.endpoint_helpers import _is_emission_within_threshold - - -if TYPE_CHECKING: - from .inference._providers import PROVIDER_T - -R = TypeVar("R") # Return type -CollectionItemType_T = Literal["model", "dataset", "space", "paper", "collection"] - -ExpandModelProperty_T = Literal[ - "author", - "baseModels", - "cardData", - "childrenModelCount", - "config", - "createdAt", - "disabled", - "downloads", - "downloadsAllTime", - "gated", - "gguf", - "inference", - "inferenceProviderMapping", - "lastModified", - "library_name", - "likes", - "mask_token", - "model-index", - "pipeline_tag", - "private", - "resourceGroup", - "safetensors", - "sha", - "siblings", - "spaces", - "tags", - "transformersInfo", - "trendingScore", - "usedStorage", - "widgetData", - "xetEnabled", -] - -ExpandDatasetProperty_T = Literal[ - "author", - "cardData", - "citation", - "createdAt", - "description", - "disabled", - "downloads", - "downloadsAllTime", - "gated", - "lastModified", - "likes", - "paperswithcode_id", - "private", - "resourceGroup", - "sha", - "siblings", - "tags", - "trendingScore", - "usedStorage", - "xetEnabled", -] - -ExpandSpaceProperty_T = Literal[ - "author", - "cardData", - "createdAt", - "datasets", - "disabled", - "lastModified", - "likes", - "models", - "private", - "resourceGroup", - "runtime", - "sdk", - "sha", - "siblings", - "subdomain", - "tags", - "trendingScore", - "usedStorage", - "xetEnabled", -] - -USERNAME_PLACEHOLDER = "hf_user" -_REGEX_DISCUSSION_URL = re.compile(r".*/discussions/(\d+)$") - -_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE = ( - "\nNote: Creating a commit assumes that the repo already exists on the" - " Huggingface Hub. Please use `create_repo` if it's not the case." -) -_AUTH_CHECK_NO_REPO_ERROR_MESSAGE = ( - "\nNote: The repository either does not exist or you do not have access rights." - " Please check the repository ID and your access permissions." - " If this is a private repository, ensure that your token is correct." -) -logger = logging.get_logger(__name__) - - -def repo_type_and_id_from_hf_id(hf_id: str, hub_url: Optional[str] = None) -> Tuple[Optional[str], Optional[str], str]: - """ - Returns the repo type and ID from a huggingface.co URL linking to a - repository - - Args: - hf_id (`str`): - An URL or ID of a repository on the HF hub. Accepted values are: - - - https://huggingface.co/// - - https://huggingface.co// - - hf://// - - hf:/// - - // - - / - - - hub_url (`str`, *optional*): - The URL of the HuggingFace Hub, defaults to https://huggingface.co - - Returns: - A tuple with three items: repo_type (`str` or `None`), namespace (`str` or - `None`) and repo_id (`str`). - - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If URL cannot be parsed. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If `repo_type` is unknown. - """ - input_hf_id = hf_id - - hub_url = re.sub(r"https?://", "", hub_url if hub_url is not None else constants.ENDPOINT) - is_hf_url = hub_url in hf_id and "@" not in hf_id - - HFFS_PREFIX = "hf://" - if hf_id.startswith(HFFS_PREFIX): # Remove "hf://" prefix if exists - hf_id = hf_id[len(HFFS_PREFIX) :] - - url_segments = hf_id.split("/") - is_hf_id = len(url_segments) <= 3 - - namespace: Optional[str] - if is_hf_url: - namespace, repo_id = url_segments[-2:] - if namespace == hub_url: - namespace = None - if len(url_segments) > 2 and hub_url not in url_segments[-3]: - repo_type = url_segments[-3] - elif namespace in constants.REPO_TYPES_MAPPING: - # Mean canonical dataset or model - repo_type = constants.REPO_TYPES_MAPPING[namespace] - namespace = None - else: - repo_type = None - elif is_hf_id: - if len(url_segments) == 3: - # Passed // or // - repo_type, namespace, repo_id = url_segments[-3:] - elif len(url_segments) == 2: - if url_segments[0] in constants.REPO_TYPES_MAPPING: - # Passed '' or 'datasets/' for a canonical model or dataset - repo_type = constants.REPO_TYPES_MAPPING[url_segments[0]] - namespace = None - repo_id = hf_id.split("/")[-1] - else: - # Passed / or / - namespace, repo_id = hf_id.split("/")[-2:] - repo_type = None - else: - # Passed - repo_id = url_segments[0] - namespace, repo_type = None, None - else: - raise ValueError(f"Unable to retrieve user and repo ID from the passed HF ID: {hf_id}") - - # Check if repo type is known (mapping "spaces" => "space" + empty value => `None`) - if repo_type in constants.REPO_TYPES_MAPPING: - repo_type = constants.REPO_TYPES_MAPPING[repo_type] - if repo_type == "": - repo_type = None - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Unknown `repo_type`: '{repo_type}' ('{input_hf_id}')") - - return repo_type, namespace, repo_id - - -@dataclass -class LastCommitInfo(dict): - oid: str - title: str - date: datetime - - def __post_init__(self): # hack to make LastCommitInfo backward compatible - self.update(asdict(self)) - - -@dataclass -class BlobLfsInfo(dict): - size: int - sha256: str - pointer_size: int - - def __post_init__(self): # hack to make BlobLfsInfo backward compatible - self.update(asdict(self)) - - -@dataclass -class BlobSecurityInfo(dict): - safe: bool # duplicate information with "status" field, keeping it for backward compatibility - status: str - av_scan: Optional[Dict] - pickle_import_scan: Optional[Dict] - - def __post_init__(self): # hack to make BlogSecurityInfo backward compatible - self.update(asdict(self)) - - -@dataclass -class TransformersInfo(dict): - auto_model: str - custom_class: Optional[str] = None - # possible `pipeline_tag` values: https://github.com/huggingface/huggingface.js/blob/3ee32554b8620644a6287e786b2a83bf5caf559c/packages/tasks/src/pipelines.ts#L72 - pipeline_tag: Optional[str] = None - processor: Optional[str] = None - - def __post_init__(self): # hack to make TransformersInfo backward compatible - self.update(asdict(self)) - - -@dataclass -class SafeTensorsInfo(dict): - parameters: Dict[str, int] - total: int - - def __post_init__(self): # hack to make SafeTensorsInfo backward compatible - self.update(asdict(self)) - - -@dataclass -class CommitInfo(str): - """Data structure containing information about a newly created commit. - - Returned by any method that creates a commit on the Hub: [`create_commit`], [`upload_file`], [`upload_folder`], - [`delete_file`], [`delete_folder`]. It inherits from `str` for backward compatibility but using methods specific - to `str` is deprecated. - - Attributes: - commit_url (`str`): - Url where to find the commit. - - commit_message (`str`): - The summary (first line) of the commit that has been created. - - commit_description (`str`): - Description of the commit that has been created. Can be empty. - - oid (`str`): - Commit hash id. Example: `"91c54ad1727ee830252e457677f467be0bfd8a57"`. - - pr_url (`str`, *optional*): - Url to the PR that has been created, if any. Populated when `create_pr=True` - is passed. - - pr_revision (`str`, *optional*): - Revision of the PR that has been created, if any. Populated when - `create_pr=True` is passed. Example: `"refs/pr/1"`. - - pr_num (`int`, *optional*): - Number of the PR discussion that has been created, if any. Populated when - `create_pr=True` is passed. Can be passed as `discussion_num` in - [`get_discussion_details`]. Example: `1`. - - repo_url (`RepoUrl`): - Repo URL of the commit containing info like repo_id, repo_type, etc. - - _url (`str`, *optional*): - Legacy url for `str` compatibility. Can be the url to the uploaded file on the Hub (if returned by - [`upload_file`]), to the uploaded folder on the Hub (if returned by [`upload_folder`]) or to the commit on - the Hub (if returned by [`create_commit`]). Defaults to `commit_url`. It is deprecated to use this - attribute. Please use `commit_url` instead. - """ - - commit_url: str - commit_message: str - commit_description: str - oid: str - pr_url: Optional[str] = None - - # Computed from `commit_url` in `__post_init__` - repo_url: RepoUrl = field(init=False) - - # Computed from `pr_url` in `__post_init__` - pr_revision: Optional[str] = field(init=False) - pr_num: Optional[str] = field(init=False) - - # legacy url for `str` compatibility (ex: url to uploaded file, url to uploaded folder, url to PR, etc.) - _url: str = field(repr=False, default=None) # type: ignore # defaults to `commit_url` - - def __new__(cls, *args, commit_url: str, _url: Optional[str] = None, **kwargs): - return str.__new__(cls, _url or commit_url) - - def __post_init__(self): - """Populate pr-related fields after initialization. - - See https://docs.python.org/3.10/library/dataclasses.html#post-init-processing. - """ - # Repo info - self.repo_url = RepoUrl(self.commit_url.split("/commit/")[0]) - - # PR info - if self.pr_url is not None: - self.pr_revision = _parse_revision_from_pr_url(self.pr_url) - self.pr_num = int(self.pr_revision.split("/")[-1]) - else: - self.pr_revision = None - self.pr_num = None - - -@dataclass -class AccessRequest: - """Data structure containing information about a user access request. - - Attributes: - username (`str`): - Username of the user who requested access. - fullname (`str`): - Fullname of the user who requested access. - email (`Optional[str]`): - Email of the user who requested access. - Can only be `None` in the /accepted list if the user was granted access manually. - timestamp (`datetime`): - Timestamp of the request. - status (`Literal["pending", "accepted", "rejected"]`): - Status of the request. Can be one of `["pending", "accepted", "rejected"]`. - fields (`Dict[str, Any]`, *optional*): - Additional fields filled by the user in the gate form. - """ - - username: str - fullname: str - email: Optional[str] - timestamp: datetime - status: Literal["pending", "accepted", "rejected"] - - # Additional fields filled by the user in the gate form - fields: Optional[Dict[str, Any]] = None - - -@dataclass -class WebhookWatchedItem: - """Data structure containing information about the items watched by a webhook. - - Attributes: - type (`Literal["dataset", "model", "org", "space", "user"]`): - Type of the item to be watched. Can be one of `["dataset", "model", "org", "space", "user"]`. - name (`str`): - Name of the item to be watched. Can be the username, organization name, model name, dataset name or space name. - """ - - type: Literal["dataset", "model", "org", "space", "user"] - name: str - - -@dataclass -class WebhookInfo: - """Data structure containing information about a webhook. - - One of `url` or `job` is specified, but not both. - - Attributes: - id (`str`): - ID of the webhook. - url (`str`, *optional*): - URL of the webhook. - job (`JobSpec`, *optional*): - Specifications of the Job to trigger. - watched (`List[WebhookWatchedItem]`): - List of items watched by the webhook, see [`WebhookWatchedItem`]. - domains (`List[WEBHOOK_DOMAIN_T]`): - List of domains the webhook is watching. Can be one of `["repo", "discussions"]`. - secret (`str`, *optional*): - Secret of the webhook. - disabled (`bool`): - Whether the webhook is disabled or not. - """ - - id: str - url: Optional[str] - job: Optional[JobSpec] - watched: List[WebhookWatchedItem] - domains: List[constants.WEBHOOK_DOMAIN_T] - secret: Optional[str] - disabled: bool - - -class RepoUrl(str): - """Subclass of `str` describing a repo URL on the Hub. - - `RepoUrl` is returned by `HfApi.create_repo`. It inherits from `str` for backward - compatibility. At initialization, the URL is parsed to populate properties: - - endpoint (`str`) - - namespace (`Optional[str]`) - - repo_name (`str`) - - repo_id (`str`) - - repo_type (`Literal["model", "dataset", "space"]`) - - url (`str`) - - Args: - url (`Any`): - String value of the repo url. - endpoint (`str`, *optional*): - Endpoint of the Hub. Defaults to . - - Example: - ```py - >>> RepoUrl('https://huggingface.co/gpt2') - RepoUrl('https://huggingface.co/gpt2', endpoint='https://huggingface.co', repo_type='model', repo_id='gpt2') - - >>> RepoUrl('https://hub-ci.huggingface.co/datasets/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co') - RepoUrl('https://hub-ci.huggingface.co/datasets/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co', repo_type='dataset', repo_id='dummy_user/dummy_dataset') - - >>> RepoUrl('hf://datasets/my-user/my-dataset') - RepoUrl('hf://datasets/my-user/my-dataset', endpoint='https://huggingface.co', repo_type='dataset', repo_id='user/dataset') - - >>> HfApi.create_repo("dummy_model") - RepoUrl('https://huggingface.co/Wauplin/dummy_model', endpoint='https://huggingface.co', repo_type='model', repo_id='Wauplin/dummy_model') - ``` - - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If URL cannot be parsed. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If `repo_type` is unknown. - """ - - def __new__(cls, url: Any, endpoint: Optional[str] = None): - url = fix_hf_endpoint_in_url(url, endpoint=endpoint) - return super(RepoUrl, cls).__new__(cls, url) - - def __init__(self, url: Any, endpoint: Optional[str] = None) -> None: - super().__init__() - # Parse URL - self.endpoint = endpoint or constants.ENDPOINT - repo_type, namespace, repo_name = repo_type_and_id_from_hf_id(self, hub_url=self.endpoint) - - # Populate fields - self.namespace = namespace - self.repo_name = repo_name - self.repo_id = repo_name if namespace is None else f"{namespace}/{repo_name}" - self.repo_type = repo_type or constants.REPO_TYPE_MODEL - self.url = str(self) # just in case it's needed - - def __repr__(self) -> str: - return f"RepoUrl('{self}', endpoint='{self.endpoint}', repo_type='{self.repo_type}', repo_id='{self.repo_id}')" - - -@dataclass -class RepoSibling: - """ - Contains basic information about a repo file inside a repo on the Hub. - - > [!TIP] - > All attributes of this class are optional except `rfilename`. This is because only the file names are returned when - > listing repositories on the Hub (with [`list_models`], [`list_datasets`] or [`list_spaces`]). If you need more - > information like file size, blob id or lfs details, you must request them specifically from one repo at a time - > (using [`model_info`], [`dataset_info`] or [`space_info`]) as it adds more constraints on the backend server to - > retrieve these. - - Attributes: - rfilename (str): - file name, relative to the repo root. - size (`int`, *optional*): - The file's size, in bytes. This attribute is defined when `files_metadata` argument of [`repo_info`] is set - to `True`. It's `None` otherwise. - blob_id (`str`, *optional*): - The file's git OID. This attribute is defined when `files_metadata` argument of [`repo_info`] is set to - `True`. It's `None` otherwise. - lfs (`BlobLfsInfo`, *optional*): - The file's LFS metadata. This attribute is defined when`files_metadata` argument of [`repo_info`] is set to - `True` and the file is stored with Git LFS. It's `None` otherwise. - """ - - rfilename: str - size: Optional[int] = None - blob_id: Optional[str] = None - lfs: Optional[BlobLfsInfo] = None - - -@dataclass -class RepoFile: - """ - Contains information about a file on the Hub. - - Attributes: - path (str): - file path relative to the repo root. - size (`int`): - The file's size, in bytes. - blob_id (`str`): - The file's git OID. - lfs (`BlobLfsInfo`): - The file's LFS metadata. - last_commit (`LastCommitInfo`, *optional*): - The file's last commit metadata. Only defined if [`list_repo_tree`] and [`get_paths_info`] - are called with `expand=True`. - security (`BlobSecurityInfo`, *optional*): - The file's security scan metadata. Only defined if [`list_repo_tree`] and [`get_paths_info`] - are called with `expand=True`. - """ - - path: str - size: int - blob_id: str - lfs: Optional[BlobLfsInfo] = None - last_commit: Optional[LastCommitInfo] = None - security: Optional[BlobSecurityInfo] = None - - def __init__(self, **kwargs): - self.path = kwargs.pop("path") - self.size = kwargs.pop("size") - self.blob_id = kwargs.pop("oid") - lfs = kwargs.pop("lfs", None) - if lfs is not None: - lfs = BlobLfsInfo(size=lfs["size"], sha256=lfs["oid"], pointer_size=lfs["pointerSize"]) - self.lfs = lfs - last_commit = kwargs.pop("lastCommit", None) or kwargs.pop("last_commit", None) - if last_commit is not None: - last_commit = LastCommitInfo( - oid=last_commit["id"], title=last_commit["title"], date=parse_datetime(last_commit["date"]) - ) - self.last_commit = last_commit - security = kwargs.pop("securityFileStatus", None) - if security is not None: - safe = security["status"] == "safe" - security = BlobSecurityInfo( - safe=safe, - status=security["status"], - av_scan=security["avScan"], - pickle_import_scan=security["pickleImportScan"], - ) - self.security = security - - # backwards compatibility - self.rfilename = self.path - self.lastCommit = self.last_commit - - -@dataclass -class RepoFolder: - """ - Contains information about a folder on the Hub. - - Attributes: - path (str): - folder path relative to the repo root. - tree_id (`str`): - The folder's git OID. - last_commit (`LastCommitInfo`, *optional*): - The folder's last commit metadata. Only defined if [`list_repo_tree`] and [`get_paths_info`] - are called with `expand=True`. - """ - - path: str - tree_id: str - last_commit: Optional[LastCommitInfo] = None - - def __init__(self, **kwargs): - self.path = kwargs.pop("path") - self.tree_id = kwargs.pop("oid") - last_commit = kwargs.pop("lastCommit", None) or kwargs.pop("last_commit", None) - if last_commit is not None: - last_commit = LastCommitInfo( - oid=last_commit["id"], title=last_commit["title"], date=parse_datetime(last_commit["date"]) - ) - self.last_commit = last_commit - - -@dataclass -class InferenceProviderMapping: - provider: "PROVIDER_T" # Provider name - hf_model_id: str # ID of the model on the Hugging Face Hub - provider_id: str # ID of the model on the provider's side - status: Literal["error", "live", "staging"] - task: str - - adapter: Optional[str] = None - adapter_weights_path: Optional[str] = None - type: Optional[Literal["single-model", "tag-filter"]] = None - - def __init__(self, **kwargs): - self.provider = kwargs.pop("provider") - self.hf_model_id = kwargs.pop("hf_model_id") - self.provider_id = kwargs.pop("providerId") - self.status = kwargs.pop("status") - self.task = kwargs.pop("task") - - self.adapter = kwargs.pop("adapter", None) - self.adapter_weights_path = kwargs.pop("adapterWeightsPath", None) - self.type = kwargs.pop("type", None) - self.__dict__.update(**kwargs) - - -@dataclass -class ModelInfo: - """ - Contains information about a model on the Hub. This object is returned by [`model_info`] and [`list_models`]. - - > [!TIP] - > Most attributes of this class are optional. This is because the data returned by the Hub depends on the query made. - > In general, the more specific the query, the more information is returned. On the contrary, when listing models - > using [`list_models`] only a subset of the attributes are returned. - - Attributes: - id (`str`): - ID of model. - author (`str`, *optional*): - Author of the model. - sha (`str`, *optional*): - Repo SHA at this particular revision. - created_at (`datetime`, *optional*): - Date of creation of the repo on the Hub. Note that the lowest value is `2022-03-02T23:29:04.000Z`, - corresponding to the date when we began to store creation dates. - last_modified (`datetime`, *optional*): - Date of last commit to the repo. - private (`bool`): - Is the repo private. - disabled (`bool`, *optional*): - Is the repo disabled. - downloads (`int`): - Number of downloads of the model over the last 30 days. - downloads_all_time (`int`): - Cumulated number of downloads of the model since its creation. - gated (`Literal["auto", "manual", False]`, *optional*): - Is the repo gated. - If so, whether there is manual or automatic approval. - gguf (`Dict`, *optional*): - GGUF information of the model. - inference (`Literal["warm"]`, *optional*): - Status of the model on Inference Providers. Warm if the model is served by at least one provider. - inference_provider_mapping (`List[InferenceProviderMapping]`, *optional*): - A list of [`InferenceProviderMapping`] ordered after the user's provider order. - likes (`int`): - Number of likes of the model. - library_name (`str`, *optional*): - Library associated with the model. - tags (`List[str]`): - List of tags of the model. Compared to `card_data.tags`, contains extra tags computed by the Hub - (e.g. supported libraries, model's arXiv). - pipeline_tag (`str`, *optional*): - Pipeline tag associated with the model. - mask_token (`str`, *optional*): - Mask token used by the model. - widget_data (`Any`, *optional*): - Widget data associated with the model. - model_index (`Dict`, *optional*): - Model index for evaluation. - config (`Dict`, *optional*): - Model configuration. - transformers_info (`TransformersInfo`, *optional*): - Transformers-specific info (auto class, processor, etc.) associated with the model. - trending_score (`int`, *optional*): - Trending score of the model. - card_data (`ModelCardData`, *optional*): - Model Card Metadata as a [`huggingface_hub.repocard_data.ModelCardData`] object. - siblings (`List[RepoSibling]`): - List of [`huggingface_hub.hf_api.RepoSibling`] objects that constitute the model. - spaces (`List[str]`, *optional*): - List of spaces using the model. - safetensors (`SafeTensorsInfo`, *optional*): - Model's safetensors information. - security_repo_status (`Dict`, *optional*): - Model's security scan status. - """ - - id: str - author: Optional[str] - sha: Optional[str] - created_at: Optional[datetime] - last_modified: Optional[datetime] - private: Optional[bool] - disabled: Optional[bool] - downloads: Optional[int] - downloads_all_time: Optional[int] - gated: Optional[Literal["auto", "manual", False]] - gguf: Optional[Dict] - inference: Optional[Literal["warm"]] - inference_provider_mapping: Optional[List[InferenceProviderMapping]] - likes: Optional[int] - library_name: Optional[str] - tags: Optional[List[str]] - pipeline_tag: Optional[str] - mask_token: Optional[str] - card_data: Optional[ModelCardData] - widget_data: Optional[Any] - model_index: Optional[Dict] - config: Optional[Dict] - transformers_info: Optional[TransformersInfo] - trending_score: Optional[int] - siblings: Optional[List[RepoSibling]] - spaces: Optional[List[str]] - safetensors: Optional[SafeTensorsInfo] - security_repo_status: Optional[Dict] - xet_enabled: Optional[bool] - - def __init__(self, **kwargs): - self.id = kwargs.pop("id") - self.author = kwargs.pop("author", None) - self.sha = kwargs.pop("sha", None) - last_modified = kwargs.pop("lastModified", None) or kwargs.pop("last_modified", None) - self.last_modified = parse_datetime(last_modified) if last_modified else None - created_at = kwargs.pop("createdAt", None) or kwargs.pop("created_at", None) - self.created_at = parse_datetime(created_at) if created_at else None - self.private = kwargs.pop("private", None) - self.gated = kwargs.pop("gated", None) - self.disabled = kwargs.pop("disabled", None) - self.downloads = kwargs.pop("downloads", None) - self.downloads_all_time = kwargs.pop("downloadsAllTime", None) - self.likes = kwargs.pop("likes", None) - self.library_name = kwargs.pop("library_name", None) - self.gguf = kwargs.pop("gguf", None) - - self.inference = kwargs.pop("inference", None) - - # little hack to simplify Inference Providers logic and make it backward and forward compatible - # right now, API returns a dict on model_info and a list on list_models. Let's harmonize to list. - mapping = kwargs.pop("inferenceProviderMapping", None) - if isinstance(mapping, list): - self.inference_provider_mapping = [ - InferenceProviderMapping(**{**value, "hf_model_id": self.id}) for value in mapping - ] - elif isinstance(mapping, dict): - self.inference_provider_mapping = [ - InferenceProviderMapping(**{**value, "hf_model_id": self.id, "provider": provider}) - for provider, value in mapping.items() - ] - elif mapping is None: - self.inference_provider_mapping = None - else: - raise ValueError( - f"Unexpected type for `inferenceProviderMapping`. Expecting `dict` or `list`. Got {mapping}." - ) - - self.tags = kwargs.pop("tags", None) - self.pipeline_tag = kwargs.pop("pipeline_tag", None) - self.mask_token = kwargs.pop("mask_token", None) - self.trending_score = kwargs.pop("trendingScore", None) - - card_data = kwargs.pop("cardData", None) or kwargs.pop("card_data", None) - self.card_data = ( - ModelCardData(**card_data, ignore_metadata_errors=True) if isinstance(card_data, dict) else card_data - ) - - self.widget_data = kwargs.pop("widgetData", None) - self.model_index = kwargs.pop("model-index", None) or kwargs.pop("model_index", None) - self.config = kwargs.pop("config", None) - transformers_info = kwargs.pop("transformersInfo", None) or kwargs.pop("transformers_info", None) - self.transformers_info = TransformersInfo(**transformers_info) if transformers_info else None - siblings = kwargs.pop("siblings", None) - self.siblings = ( - [ - RepoSibling( - rfilename=sibling["rfilename"], - size=sibling.get("size"), - blob_id=sibling.get("blobId"), - lfs=( - BlobLfsInfo( - size=sibling["lfs"]["size"], - sha256=sibling["lfs"]["sha256"], - pointer_size=sibling["lfs"]["pointerSize"], - ) - if sibling.get("lfs") - else None - ), - ) - for sibling in siblings - ] - if siblings is not None - else None - ) - self.spaces = kwargs.pop("spaces", None) - safetensors = kwargs.pop("safetensors", None) - self.safetensors = ( - SafeTensorsInfo( - parameters=safetensors["parameters"], - total=safetensors["total"], - ) - if safetensors - else None - ) - self.security_repo_status = kwargs.pop("securityRepoStatus", None) - self.xet_enabled = kwargs.pop("xetEnabled", None) - # backwards compatibility - self.lastModified = self.last_modified - self.cardData = self.card_data - self.transformersInfo = self.transformers_info - self.__dict__.update(**kwargs) - - -@dataclass -class DatasetInfo: - """ - Contains information about a dataset on the Hub. This object is returned by [`dataset_info`] and [`list_datasets`]. - - > [!TIP] - > Most attributes of this class are optional. This is because the data returned by the Hub depends on the query made. - > In general, the more specific the query, the more information is returned. On the contrary, when listing datasets - > using [`list_datasets`] only a subset of the attributes are returned. - - Attributes: - id (`str`): - ID of dataset. - author (`str`): - Author of the dataset. - sha (`str`): - Repo SHA at this particular revision. - created_at (`datetime`, *optional*): - Date of creation of the repo on the Hub. Note that the lowest value is `2022-03-02T23:29:04.000Z`, - corresponding to the date when we began to store creation dates. - last_modified (`datetime`, *optional*): - Date of last commit to the repo. - private (`bool`): - Is the repo private. - disabled (`bool`, *optional*): - Is the repo disabled. - gated (`Literal["auto", "manual", False]`, *optional*): - Is the repo gated. - If so, whether there is manual or automatic approval. - downloads (`int`): - Number of downloads of the dataset over the last 30 days. - downloads_all_time (`int`): - Cumulated number of downloads of the model since its creation. - likes (`int`): - Number of likes of the dataset. - tags (`List[str]`): - List of tags of the dataset. - card_data (`DatasetCardData`, *optional*): - Model Card Metadata as a [`huggingface_hub.repocard_data.DatasetCardData`] object. - siblings (`List[RepoSibling]`): - List of [`huggingface_hub.hf_api.RepoSibling`] objects that constitute the dataset. - paperswithcode_id (`str`, *optional*): - Papers with code ID of the dataset. - trending_score (`int`, *optional*): - Trending score of the dataset. - """ - - id: str - author: Optional[str] - sha: Optional[str] - created_at: Optional[datetime] - last_modified: Optional[datetime] - private: Optional[bool] - gated: Optional[Literal["auto", "manual", False]] - disabled: Optional[bool] - downloads: Optional[int] - downloads_all_time: Optional[int] - likes: Optional[int] - paperswithcode_id: Optional[str] - tags: Optional[List[str]] - trending_score: Optional[int] - card_data: Optional[DatasetCardData] - siblings: Optional[List[RepoSibling]] - xet_enabled: Optional[bool] - - def __init__(self, **kwargs): - self.id = kwargs.pop("id") - self.author = kwargs.pop("author", None) - self.sha = kwargs.pop("sha", None) - created_at = kwargs.pop("createdAt", None) or kwargs.pop("created_at", None) - self.created_at = parse_datetime(created_at) if created_at else None - last_modified = kwargs.pop("lastModified", None) or kwargs.pop("last_modified", None) - self.last_modified = parse_datetime(last_modified) if last_modified else None - self.private = kwargs.pop("private", None) - self.gated = kwargs.pop("gated", None) - self.disabled = kwargs.pop("disabled", None) - self.downloads = kwargs.pop("downloads", None) - self.downloads_all_time = kwargs.pop("downloadsAllTime", None) - self.likes = kwargs.pop("likes", None) - self.paperswithcode_id = kwargs.pop("paperswithcode_id", None) - self.tags = kwargs.pop("tags", None) - self.trending_score = kwargs.pop("trendingScore", None) - - card_data = kwargs.pop("cardData", None) or kwargs.pop("card_data", None) - self.card_data = ( - DatasetCardData(**card_data, ignore_metadata_errors=True) if isinstance(card_data, dict) else card_data - ) - siblings = kwargs.pop("siblings", None) - self.siblings = ( - [ - RepoSibling( - rfilename=sibling["rfilename"], - size=sibling.get("size"), - blob_id=sibling.get("blobId"), - lfs=( - BlobLfsInfo( - size=sibling["lfs"]["size"], - sha256=sibling["lfs"]["sha256"], - pointer_size=sibling["lfs"]["pointerSize"], - ) - if sibling.get("lfs") - else None - ), - ) - for sibling in siblings - ] - if siblings is not None - else None - ) - self.xet_enabled = kwargs.pop("xetEnabled", None) - # backwards compatibility - self.lastModified = self.last_modified - self.cardData = self.card_data - self.__dict__.update(**kwargs) - - -@dataclass -class SpaceInfo: - """ - Contains information about a Space on the Hub. This object is returned by [`space_info`] and [`list_spaces`]. - - > [!TIP] - > Most attributes of this class are optional. This is because the data returned by the Hub depends on the query made. - > In general, the more specific the query, the more information is returned. On the contrary, when listing spaces - > using [`list_spaces`] only a subset of the attributes are returned. - - Attributes: - id (`str`): - ID of the Space. - author (`str`, *optional*): - Author of the Space. - sha (`str`, *optional*): - Repo SHA at this particular revision. - created_at (`datetime`, *optional*): - Date of creation of the repo on the Hub. Note that the lowest value is `2022-03-02T23:29:04.000Z`, - corresponding to the date when we began to store creation dates. - last_modified (`datetime`, *optional*): - Date of last commit to the repo. - private (`bool`): - Is the repo private. - gated (`Literal["auto", "manual", False]`, *optional*): - Is the repo gated. - If so, whether there is manual or automatic approval. - disabled (`bool`, *optional*): - Is the Space disabled. - host (`str`, *optional*): - Host URL of the Space. - subdomain (`str`, *optional*): - Subdomain of the Space. - likes (`int`): - Number of likes of the Space. - tags (`List[str]`): - List of tags of the Space. - siblings (`List[RepoSibling]`): - List of [`huggingface_hub.hf_api.RepoSibling`] objects that constitute the Space. - card_data (`SpaceCardData`, *optional*): - Space Card Metadata as a [`huggingface_hub.repocard_data.SpaceCardData`] object. - runtime (`SpaceRuntime`, *optional*): - Space runtime information as a [`huggingface_hub.hf_api.SpaceRuntime`] object. - sdk (`str`, *optional*): - SDK used by the Space. - models (`List[str]`, *optional*): - List of models used by the Space. - datasets (`List[str]`, *optional*): - List of datasets used by the Space. - trending_score (`int`, *optional*): - Trending score of the Space. - """ - - id: str - author: Optional[str] - sha: Optional[str] - created_at: Optional[datetime] - last_modified: Optional[datetime] - private: Optional[bool] - gated: Optional[Literal["auto", "manual", False]] - disabled: Optional[bool] - host: Optional[str] - subdomain: Optional[str] - likes: Optional[int] - sdk: Optional[str] - tags: Optional[List[str]] - siblings: Optional[List[RepoSibling]] - trending_score: Optional[int] - card_data: Optional[SpaceCardData] - runtime: Optional[SpaceRuntime] - models: Optional[List[str]] - datasets: Optional[List[str]] - xet_enabled: Optional[bool] - - def __init__(self, **kwargs): - self.id = kwargs.pop("id") - self.author = kwargs.pop("author", None) - self.sha = kwargs.pop("sha", None) - created_at = kwargs.pop("createdAt", None) or kwargs.pop("created_at", None) - self.created_at = parse_datetime(created_at) if created_at else None - last_modified = kwargs.pop("lastModified", None) or kwargs.pop("last_modified", None) - self.last_modified = parse_datetime(last_modified) if last_modified else None - self.private = kwargs.pop("private", None) - self.gated = kwargs.pop("gated", None) - self.disabled = kwargs.pop("disabled", None) - self.host = kwargs.pop("host", None) - self.subdomain = kwargs.pop("subdomain", None) - self.likes = kwargs.pop("likes", None) - self.sdk = kwargs.pop("sdk", None) - self.tags = kwargs.pop("tags", None) - self.trending_score = kwargs.pop("trendingScore", None) - card_data = kwargs.pop("cardData", None) or kwargs.pop("card_data", None) - self.card_data = ( - SpaceCardData(**card_data, ignore_metadata_errors=True) if isinstance(card_data, dict) else card_data - ) - siblings = kwargs.pop("siblings", None) - self.siblings = ( - [ - RepoSibling( - rfilename=sibling["rfilename"], - size=sibling.get("size"), - blob_id=sibling.get("blobId"), - lfs=( - BlobLfsInfo( - size=sibling["lfs"]["size"], - sha256=sibling["lfs"]["sha256"], - pointer_size=sibling["lfs"]["pointerSize"], - ) - if sibling.get("lfs") - else None - ), - ) - for sibling in siblings - ] - if siblings is not None - else None - ) - runtime = kwargs.pop("runtime", None) - self.runtime = SpaceRuntime(runtime) if runtime else None - self.models = kwargs.pop("models", None) - self.datasets = kwargs.pop("datasets", None) - self.xet_enabled = kwargs.pop("xetEnabled", None) - # backwards compatibility - self.lastModified = self.last_modified - self.cardData = self.card_data - self.__dict__.update(**kwargs) - - -@dataclass -class CollectionItem: - """ - Contains information about an item of a Collection (model, dataset, Space, paper or collection). - - Attributes: - item_object_id (`str`): - Unique ID of the item in the collection. - item_id (`str`): - ID of the underlying object on the Hub. Can be either a repo_id, a paper id or a collection slug. - e.g. `"jbilcke-hf/ai-comic-factory"`, `"2307.09288"`, `"celinah/cerebras-function-calling-682607169c35fbfa98b30b9a"`. - item_type (`str`): - Type of the underlying object. Can be one of `"model"`, `"dataset"`, `"space"`, `"paper"` or `"collection"`. - position (`int`): - Position of the item in the collection. - note (`str`, *optional*): - Note associated with the item, as plain text. - """ - - item_object_id: str # id in database - item_id: str # repo_id or paper id - item_type: str - position: int - note: Optional[str] = None - - def __init__( - self, - _id: str, - id: str, - type: CollectionItemType_T, - position: int, - note: Optional[Dict] = None, - **kwargs, - ) -> None: - self.item_object_id: str = _id # id in database - self.item_id: str = id # repo_id or paper id - # if the item is a collection, override item_id with the slug - slug = kwargs.get("slug") - if slug is not None: - self.item_id = slug # collection slug - self.item_type: CollectionItemType_T = type - self.position: int = position - self.note: str = note["text"] if note is not None else None - - -@dataclass -class Collection: - """ - Contains information about a Collection on the Hub. - - Attributes: - slug (`str`): - Slug of the collection. E.g. `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. - title (`str`): - Title of the collection. E.g. `"Recent models"`. - owner (`str`): - Owner of the collection. E.g. `"TheBloke"`. - items (`List[CollectionItem]`): - List of items in the collection. - last_updated (`datetime`): - Date of the last update of the collection. - position (`int`): - Position of the collection in the list of collections of the owner. - private (`bool`): - Whether the collection is private or not. - theme (`str`): - Theme of the collection. E.g. `"green"`. - upvotes (`int`): - Number of upvotes of the collection. - description (`str`, *optional*): - Description of the collection, as plain text. - url (`str`): - (property) URL of the collection on the Hub. - """ - - slug: str - title: str - owner: str - items: List[CollectionItem] - last_updated: datetime - position: int - private: bool - theme: str - upvotes: int - description: Optional[str] = None - - def __init__(self, **kwargs) -> None: - self.slug = kwargs.pop("slug") - self.title = kwargs.pop("title") - self.owner = kwargs.pop("owner") - self.items = [CollectionItem(**item) for item in kwargs.pop("items")] - self.last_updated = parse_datetime(kwargs.pop("lastUpdated")) - self.position = kwargs.pop("position") - self.private = kwargs.pop("private") - self.theme = kwargs.pop("theme") - self.upvotes = kwargs.pop("upvotes") - self.description = kwargs.pop("description", None) - endpoint = kwargs.pop("endpoint", None) - if endpoint is None: - endpoint = constants.ENDPOINT - self._url = f"{endpoint}/collections/{self.slug}" - - @property - def url(self) -> str: - """Returns the URL of the collection on the Hub.""" - return self._url - - -@dataclass -class GitRefInfo: - """ - Contains information about a git reference for a repo on the Hub. - - Attributes: - name (`str`): - Name of the reference (e.g. tag name or branch name). - ref (`str`): - Full git ref on the Hub (e.g. `"refs/heads/main"` or `"refs/tags/v1.0"`). - target_commit (`str`): - OID of the target commit for the ref (e.g. `"e7da7f221d5bf496a48136c0cd264e630fe9fcc8"`) - """ - - name: str - ref: str - target_commit: str - - -@dataclass -class GitRefs: - """ - Contains information about all git references for a repo on the Hub. - - Object is returned by [`list_repo_refs`]. - - Attributes: - branches (`List[GitRefInfo]`): - A list of [`GitRefInfo`] containing information about branches on the repo. - converts (`List[GitRefInfo]`): - A list of [`GitRefInfo`] containing information about "convert" refs on the repo. - Converts are refs used (internally) to push preprocessed data in Dataset repos. - tags (`List[GitRefInfo]`): - A list of [`GitRefInfo`] containing information about tags on the repo. - pull_requests (`List[GitRefInfo]`, *optional*): - A list of [`GitRefInfo`] containing information about pull requests on the repo. - Only returned if `include_prs=True` is set. - """ - - branches: List[GitRefInfo] - converts: List[GitRefInfo] - tags: List[GitRefInfo] - pull_requests: Optional[List[GitRefInfo]] = None - - -@dataclass -class GitCommitInfo: - """ - Contains information about a git commit for a repo on the Hub. Check out [`list_repo_commits`] for more details. - - Attributes: - commit_id (`str`): - OID of the commit (e.g. `"e7da7f221d5bf496a48136c0cd264e630fe9fcc8"`) - authors (`List[str]`): - List of authors of the commit. - created_at (`datetime`): - Datetime when the commit was created. - title (`str`): - Title of the commit. This is a free-text value entered by the authors. - message (`str`): - Description of the commit. This is a free-text value entered by the authors. - formatted_title (`str`): - Title of the commit formatted as HTML. Only returned if `formatted=True` is set. - formatted_message (`str`): - Description of the commit formatted as HTML. Only returned if `formatted=True` is set. - """ - - commit_id: str - - authors: List[str] - created_at: datetime - title: str - message: str - - formatted_title: Optional[str] - formatted_message: Optional[str] - - -@dataclass -class UserLikes: - """ - Contains information about a user likes on the Hub. - - Attributes: - user (`str`): - Name of the user for which we fetched the likes. - total (`int`): - Total number of likes. - datasets (`List[str]`): - List of datasets liked by the user (as repo_ids). - models (`List[str]`): - List of models liked by the user (as repo_ids). - spaces (`List[str]`): - List of spaces liked by the user (as repo_ids). - """ - - # Metadata - user: str - total: int - - # User likes - datasets: List[str] - models: List[str] - spaces: List[str] - - -@dataclass -class Organization: - """ - Contains information about an organization on the Hub. - - Attributes: - avatar_url (`str`): - URL of the organization's avatar. - name (`str`): - Name of the organization on the Hub (unique). - fullname (`str`): - Organization's full name. - details (`str`, *optional*): - Organization's description. - is_verified (`bool`, *optional*): - Whether the organization is verified. - is_following (`bool`, *optional*): - Whether the authenticated user follows this organization. - num_users (`int`, *optional*): - Number of members in the organization. - num_models (`int`, *optional*): - Number of models owned by the organization. - num_spaces (`int`, *optional*): - Number of Spaces owned by the organization. - num_datasets (`int`, *optional*): - Number of datasets owned by the organization. - num_followers (`int`, *optional*): - Number of followers of the organization. - """ - - avatar_url: str - name: str - fullname: str - details: Optional[str] = None - is_verified: Optional[bool] = None - is_following: Optional[bool] = None - num_users: Optional[int] = None - num_models: Optional[int] = None - num_spaces: Optional[int] = None - num_datasets: Optional[int] = None - num_followers: Optional[int] = None - - def __init__(self, **kwargs) -> None: - self.avatar_url = kwargs.pop("avatarUrl", "") - self.name = kwargs.pop("name", "") - self.fullname = kwargs.pop("fullname", "") - self.details = kwargs.pop("details", None) - self.is_verified = kwargs.pop("isVerified", None) - self.is_following = kwargs.pop("isFollowing", None) - self.num_users = kwargs.pop("numUsers", None) - self.num_models = kwargs.pop("numModels", None) - self.num_spaces = kwargs.pop("numSpaces", None) - self.num_datasets = kwargs.pop("numDatasets", None) - self.num_followers = kwargs.pop("numFollowers", None) - - # forward compatibility - self.__dict__.update(**kwargs) - - -@dataclass -class User: - """ - Contains information about a user on the Hub. - - Attributes: - username (`str`): - Name of the user on the Hub (unique). - fullname (`str`): - User's full name. - avatar_url (`str`): - URL of the user's avatar. - details (`str`, *optional*): - User's details. - is_following (`bool`, *optional*): - Whether the authenticated user is following this user. - is_pro (`bool`, *optional*): - Whether the user is a pro user. - num_models (`int`, *optional*): - Number of models created by the user. - num_datasets (`int`, *optional*): - Number of datasets created by the user. - num_spaces (`int`, *optional*): - Number of spaces created by the user. - num_discussions (`int`, *optional*): - Number of discussions initiated by the user. - num_papers (`int`, *optional*): - Number of papers authored by the user. - num_upvotes (`int`, *optional*): - Number of upvotes received by the user. - num_likes (`int`, *optional*): - Number of likes given by the user. - num_following (`int`, *optional*): - Number of users this user is following. - num_followers (`int`, *optional*): - Number of users following this user. - orgs (list of [`Organization`]): - List of organizations the user is part of. - """ - - # Metadata - username: str - fullname: str - avatar_url: str - details: Optional[str] = None - is_following: Optional[bool] = None - is_pro: Optional[bool] = None - num_models: Optional[int] = None - num_datasets: Optional[int] = None - num_spaces: Optional[int] = None - num_discussions: Optional[int] = None - num_papers: Optional[int] = None - num_upvotes: Optional[int] = None - num_likes: Optional[int] = None - num_following: Optional[int] = None - num_followers: Optional[int] = None - orgs: List[Organization] = field(default_factory=list) - - def __init__(self, **kwargs) -> None: - self.username = kwargs.pop("user", "") - self.fullname = kwargs.pop("fullname", "") - self.avatar_url = kwargs.pop("avatarUrl", "") - self.is_following = kwargs.pop("isFollowing", None) - self.is_pro = kwargs.pop("isPro", None) - self.details = kwargs.pop("details", None) - self.num_models = kwargs.pop("numModels", None) - self.num_datasets = kwargs.pop("numDatasets", None) - self.num_spaces = kwargs.pop("numSpaces", None) - self.num_discussions = kwargs.pop("numDiscussions", None) - self.num_papers = kwargs.pop("numPapers", None) - self.num_upvotes = kwargs.pop("numUpvotes", None) - self.num_likes = kwargs.pop("numLikes", None) - self.num_following = kwargs.pop("numFollowing", None) - self.num_followers = kwargs.pop("numFollowers", None) - self.user_type = kwargs.pop("type", None) - self.orgs = [Organization(**org) for org in kwargs.pop("orgs", [])] - - # forward compatibility - self.__dict__.update(**kwargs) - - -@dataclass -class PaperInfo: - """ - Contains information about a paper on the Hub. - - Attributes: - id (`str`): - arXiv paper ID. - authors (`List[str]`, **optional**): - Names of paper authors - published_at (`datetime`, **optional**): - Date paper published. - title (`str`, **optional**): - Title of the paper. - summary (`str`, **optional**): - Summary of the paper. - upvotes (`int`, **optional**): - Number of upvotes for the paper on the Hub. - discussion_id (`str`, **optional**): - Discussion ID for the paper on the Hub. - source (`str`, **optional**): - Source of the paper. - comments (`int`, **optional**): - Number of comments for the paper on the Hub. - submitted_at (`datetime`, **optional**): - Date paper appeared in daily papers on the Hub. - submitted_by (`User`, **optional**): - Information about who submitted the daily paper. - """ - - id: str - authors: Optional[List[str]] - published_at: Optional[datetime] - title: Optional[str] - summary: Optional[str] - upvotes: Optional[int] - discussion_id: Optional[str] - source: Optional[str] - comments: Optional[int] - submitted_at: Optional[datetime] - submitted_by: Optional[User] - - def __init__(self, **kwargs) -> None: - paper = kwargs.pop("paper", {}) - self.id = kwargs.pop("id", None) or paper.pop("id", None) - authors = paper.pop("authors", None) or kwargs.pop("authors", None) - self.authors = [author.pop("name", None) for author in authors] if authors else None - published_at = paper.pop("publishedAt", None) or kwargs.pop("publishedAt", None) - self.published_at = parse_datetime(published_at) if published_at else None - self.title = kwargs.pop("title", None) - self.source = kwargs.pop("source", None) - self.summary = paper.pop("summary", None) or kwargs.pop("summary", None) - self.upvotes = paper.pop("upvotes", None) or kwargs.pop("upvotes", None) - self.discussion_id = paper.pop("discussionId", None) or kwargs.pop("discussionId", None) - self.comments = kwargs.pop("numComments", 0) - submitted_at = kwargs.pop("publishedAt", None) or kwargs.pop("submittedOnDailyAt", None) - self.submitted_at = parse_datetime(submitted_at) if submitted_at else None - submitted_by = kwargs.pop("submittedBy", None) or kwargs.pop("submittedOnDailyBy", None) - self.submitted_by = User(**submitted_by) if submitted_by else None - - # forward compatibility - self.__dict__.update(**kwargs) - - -@dataclass -class LFSFileInfo: - """ - Contains information about a file stored as LFS on a repo on the Hub. - - Used in the context of listing and permanently deleting LFS files from a repo to free-up space. - See [`list_lfs_files`] and [`permanently_delete_lfs_files`] for more details. - - Git LFS files are tracked using SHA-256 object IDs, rather than file paths, to optimize performance - This approach is necessary because a single object can be referenced by multiple paths across different commits, - making it impractical to search and resolve these connections. Check out [our documentation](https://huggingface.co/docs/hub/storage-limits#advanced-track-lfs-file-references) - to learn how to know which filename(s) is(are) associated with each SHA. - - Attributes: - file_oid (`str`): - SHA-256 object ID of the file. This is the identifier to pass when permanently deleting the file. - filename (`str`): - Possible filename for the LFS object. See the note above for more information. - oid (`str`): - OID of the LFS object. - pushed_at (`datetime`): - Date the LFS object was pushed to the repo. - ref (`str`, *optional*): - Ref where the LFS object has been pushed (if any). - size (`int`): - Size of the LFS object. - - Example: - ```py - >>> from huggingface_hub import HfApi - >>> api = HfApi() - >>> lfs_files = api.list_lfs_files("username/my-cool-repo") - - # Filter files files to delete based on a combination of `filename`, `pushed_at`, `ref` or `size`. - # e.g. select only LFS files in the "checkpoints" folder - >>> lfs_files_to_delete = (lfs_file for lfs_file in lfs_files if lfs_file.filename.startswith("checkpoints/")) - - # Permanently delete LFS files - >>> api.permanently_delete_lfs_files("username/my-cool-repo", lfs_files_to_delete) - ``` - """ - - file_oid: str - filename: str - oid: str - pushed_at: datetime - ref: Optional[str] - size: int - - def __init__(self, **kwargs) -> None: - self.file_oid = kwargs.pop("fileOid") - self.filename = kwargs.pop("filename") - self.oid = kwargs.pop("oid") - self.pushed_at = parse_datetime(kwargs.pop("pushedAt")) - self.ref = kwargs.pop("ref", None) - self.size = kwargs.pop("size") - - # forward compatibility - self.__dict__.update(**kwargs) - - -def future_compatible(fn: CallableT) -> CallableT: - """Wrap a method of `HfApi` to handle `run_as_future=True`. - - A method flagged as "future_compatible" will be called in a thread if `run_as_future=True` and return a - `concurrent.futures.Future` instance. Otherwise, it will be called normally and return the result. - """ - sig = inspect.signature(fn) - args_params = list(sig.parameters)[1:] # remove "self" from list - - @wraps(fn) - def _inner(self, *args, **kwargs): - # Get `run_as_future` value if provided (default to False) - if "run_as_future" in kwargs: - run_as_future = kwargs["run_as_future"] - kwargs["run_as_future"] = False # avoid recursion error - else: - run_as_future = False - for param, value in zip(args_params, args): - if param == "run_as_future": - run_as_future = value - break - - # Call the function in a thread if `run_as_future=True` - if run_as_future: - return self.run_as_future(fn, self, *args, **kwargs) - - # Otherwise, call the function normally - return fn(self, *args, **kwargs) - - _inner.is_future_compatible = True # type: ignore - return _inner # type: ignore - - -class HfApi: - """ - Client to interact with the Hugging Face Hub via HTTP. - - The client is initialized with some high-level settings used in all requests - made to the Hub (HF endpoint, authentication, user agents...). Using the `HfApi` - client is preferred but not mandatory as all of its public methods are exposed - directly at the root of `huggingface_hub`. - - Args: - endpoint (`str`, *optional*): - Endpoint of the Hub. Defaults to . - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - library_name (`str`, *optional*): - The name of the library that is making the HTTP request. Will be added to - the user-agent header. Example: `"transformers"`. - library_version (`str`, *optional*): - The version of the library that is making the HTTP request. Will be added - to the user-agent header. Example: `"4.24.0"`. - user_agent (`str`, `dict`, *optional*): - The user agent info in the form of a dictionary or a single string. It will - be completed with information about the installed packages. - headers (`dict`, *optional*): - Additional headers to be sent with each request. Example: `{"X-My-Header": "value"}`. - Headers passed here are taking precedence over the default headers. - """ - - def __init__( - self, - endpoint: Optional[str] = None, - token: Union[str, bool, None] = None, - library_name: Optional[str] = None, - library_version: Optional[str] = None, - user_agent: Union[Dict, str, None] = None, - headers: Optional[Dict[str, str]] = None, - ) -> None: - self.endpoint = endpoint if endpoint is not None else constants.ENDPOINT - self.token = token - self.library_name = library_name - self.library_version = library_version - self.user_agent = user_agent - self.headers = headers - self._thread_pool: Optional[ThreadPoolExecutor] = None - - def run_as_future(self, fn: Callable[..., R], *args, **kwargs) -> Future[R]: - """ - Run a method in the background and return a Future instance. - - The main goal is to run methods without blocking the main thread (e.g. to push data during a training). - Background jobs are queued to preserve order but are not ran in parallel. If you need to speed-up your scripts - by parallelizing lots of call to the API, you must setup and use your own [ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor). - - Note: Most-used methods like [`upload_file`], [`upload_folder`] and [`create_commit`] have a `run_as_future: bool` - argument to directly call them in the background. This is equivalent to calling `api.run_as_future(...)` on them - but less verbose. - - Args: - fn (`Callable`): - The method to run in the background. - *args, **kwargs: - Arguments with which the method will be called. - - Return: - `Future`: a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) instance to - get the result of the task. - - Example: - ```py - >>> from huggingface_hub import HfApi - >>> api = HfApi() - >>> future = api.run_as_future(api.whoami) # instant - >>> future.done() - False - >>> future.result() # wait until complete and return result - (...) - >>> future.done() - True - ``` - """ - if self._thread_pool is None: - self._thread_pool = ThreadPoolExecutor(max_workers=1) - self._thread_pool - return self._thread_pool.submit(fn, *args, **kwargs) - - @validate_hf_hub_args - def whoami(self, token: Union[bool, str, None] = None) -> Dict: - """ - Call HF API to know "whoami". - - Args: - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - """ - # Get the effective token using the helper function get_token - effective_token = token or self.token or get_token() or True - r = get_session().get( - f"{self.endpoint}/api/whoami-v2", - headers=self._build_hf_headers(token=effective_token), - ) - try: - hf_raise_for_status(r) - except HTTPError as e: - if e.response.status_code == 401: - error_message = "Invalid user token." - # Check which token is the effective one and generate the error message accordingly - if effective_token == _get_token_from_google_colab(): - error_message += " The token from Google Colab vault is invalid. Please update it from the UI." - elif effective_token == _get_token_from_environment(): - error_message += ( - " The token from HF_TOKEN environment variable is invalid. " - "Note that HF_TOKEN takes precedence over `hf auth login`." - ) - elif effective_token == _get_token_from_file(): - error_message += " The token stored is invalid. Please run `hf auth login` to update it." - raise HTTPError(error_message, request=e.request, response=e.response) from e - raise - return r.json() - - @_deprecate_method( - version="1.0", - message=( - "Permissions are more complex than when `get_token_permission` was first introduced. " - "OAuth and fine-grain tokens allows for more detailed permissions. " - "If you need to know the permissions associated with a token, please use `whoami` and check the `'auth'` key." - ), - ) - def get_token_permission( - self, token: Union[bool, str, None] = None - ) -> Literal["read", "write", "fineGrained", None]: - """ - Check if a given `token` is valid and return its permissions. - - > [!WARNING] - > This method is deprecated and will be removed in version 1.0. Permissions are more complex than when - > `get_token_permission` was first introduced. OAuth and fine-grain tokens allows for more detailed permissions. - > If you need to know the permissions associated with a token, please use `whoami` and check the `'auth'` key. - - For more details about tokens, please refer to https://huggingface.co/docs/hub/security-tokens#what-are-user-access-tokens. - - Args: - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Literal["read", "write", "fineGrained", None]`: Permission granted by the token ("read" or "write"). Returns `None` if no - token passed, if token is invalid or if role is not returned by the server. This typically happens when the token is an OAuth token. - """ - try: - return self.whoami(token=token)["auth"]["accessToken"]["role"] - except (LocalTokenNotFoundError, HTTPError, KeyError): - return None - - def get_model_tags(self) -> Dict: - """ - List all valid model tags as a nested namespace object - """ - path = f"{self.endpoint}/api/models-tags-by-type" - r = get_session().get(path) - hf_raise_for_status(r) - return r.json() - - def get_dataset_tags(self) -> Dict: - """ - List all valid dataset tags as a nested namespace object. - """ - path = f"{self.endpoint}/api/datasets-tags-by-type" - r = get_session().get(path) - hf_raise_for_status(r) - return r.json() - - @_deprecate_arguments( - version="1.0", deprecated_args=["language", "library", "task", "tags"], custom_message="Use `filter` instead." - ) - @validate_hf_hub_args - def list_models( - self, - *, - # Search-query parameter - filter: Union[str, Iterable[str], None] = None, - author: Optional[str] = None, - apps: Optional[Union[str, List[str]]] = None, - gated: Optional[bool] = None, - inference: Optional[Literal["warm"]] = None, - inference_provider: Optional[Union[Literal["all"], "PROVIDER_T", List["PROVIDER_T"]]] = None, - model_name: Optional[str] = None, - trained_dataset: Optional[Union[str, List[str]]] = None, - search: Optional[str] = None, - pipeline_tag: Optional[str] = None, - emissions_thresholds: Optional[Tuple[float, float]] = None, - # Sorting and pagination parameters - sort: Union[Literal["last_modified"], str, None] = None, - direction: Optional[Literal[-1]] = None, - limit: Optional[int] = None, - # Additional data to fetch - expand: Optional[List[ExpandModelProperty_T]] = None, - full: Optional[bool] = None, - cardData: bool = False, - fetch_config: bool = False, - token: Union[bool, str, None] = None, - # Deprecated arguments - use `filter` instead - language: Optional[Union[str, List[str]]] = None, - library: Optional[Union[str, List[str]]] = None, - tags: Optional[Union[str, List[str]]] = None, - task: Optional[Union[str, List[str]]] = None, - ) -> Iterable[ModelInfo]: - """ - List models hosted on the Huggingface Hub, given some filters. - - Args: - filter (`str` or `Iterable[str]`, *optional*): - A string or list of string to filter models on the Hub. - Models can be filtered by library, language, task, tags, and more. - author (`str`, *optional*): - A string which identify the author (user or organization) of the - returned models. - apps (`str` or `List`, *optional*): - A string or list of strings to filter models on the Hub that - support the specified apps. Example values include `"ollama"` or `["ollama", "vllm"]`. - gated (`bool`, *optional*): - A boolean to filter models on the Hub that are gated or not. By default, all models are returned. - If `gated=True` is passed, only gated models are returned. - If `gated=False` is passed, only non-gated models are returned. - inference (`Literal["warm"]`, *optional*): - If "warm", filter models on the Hub currently served by at least one provider. - inference_provider (`Literal["all"]` or `str`, *optional*): - A string to filter models on the Hub that are served by a specific provider. - Pass `"all"` to get all models served by at least one provider. - library (`str` or `List`, *optional*): - Deprecated. Pass a library name in `filter` to filter models by library. - language (`str` or `List`, *optional*): - Deprecated. Pass a language in `filter` to filter models by language. - model_name (`str`, *optional*): - A string that contain complete or partial names for models on the - Hub, such as "bert" or "bert-base-cased" - task (`str` or `List`, *optional*): - Deprecated. Pass a task in `filter` to filter models by task. - trained_dataset (`str` or `List`, *optional*): - A string tag or a list of string tags of the trained dataset for a - model on the Hub. - tags (`str` or `List`, *optional*): - Deprecated. Pass tags in `filter` to filter models by tags. - search (`str`, *optional*): - A string that will be contained in the returned model ids. - pipeline_tag (`str`, *optional*): - A string pipeline tag to filter models on the Hub by, such as `summarization`. - emissions_thresholds (`Tuple`, *optional*): - A tuple of two ints or floats representing a minimum and maximum - carbon footprint to filter the resulting models with in grams. - sort (`Literal["last_modified"]` or `str`, *optional*): - The key with which to sort the resulting models. Possible values are "last_modified", "trending_score", - "created_at", "downloads" and "likes". - direction (`Literal[-1]` or `int`, *optional*): - Direction in which to sort. The value `-1` sorts by descending - order while all other values sort by ascending order. - limit (`int`, *optional*): - The limit on the number of models fetched. Leaving this option - to `None` fetches all models. - expand (`List[ExpandModelProperty_T]`, *optional*): - List properties to return in the response. When used, only the properties in the list will be returned. - This parameter cannot be used if `full`, `cardData` or `fetch_config` are passed. - Possible values are `"author"`, `"cardData"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"gguf"`, `"inference"`, `"inferenceProviderMapping"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"`, `"trendingScore"`, `"widgetData"`, `"resourceGroup"` and `"xetEnabled"`. - full (`bool`, *optional*): - Whether to fetch all model data, including the `last_modified`, - the `sha`, the files and the `tags`. This is set to `True` by - default when using a filter. - cardData (`bool`, *optional*): - Whether to grab the metadata for the model as well. Can contain - useful information such as carbon emissions, metrics, and - datasets trained on. - fetch_config (`bool`, *optional*): - Whether to fetch the model configs as well. This is not included - in `full` due to its size. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - - Returns: - `Iterable[ModelInfo]`: an iterable of [`huggingface_hub.hf_api.ModelInfo`] objects. - - Example: - - ```python - >>> from huggingface_hub import HfApi - - >>> api = HfApi() - - # List all models - >>> api.list_models() - - # List text classification models - >>> api.list_models(filter="text-classification") - - # List models from the KerasHub library - >>> api.list_models(filter="keras-hub") - - # List models served by Cohere - >>> api.list_models(inference_provider="cohere") - - # List models with "bert" in their name - >>> api.list_models(search="bert") - - # List models with "bert" in their name and pushed by google - >>> api.list_models(search="bert", author="google") - ``` - """ - if expand and (full or cardData or fetch_config): - raise ValueError("`expand` cannot be used if `full`, `cardData` or `fetch_config` are passed.") - - if emissions_thresholds is not None and not cardData: - raise ValueError("`emissions_thresholds` were passed without setting `cardData=True`.") - - path = f"{self.endpoint}/api/models" - headers = self._build_hf_headers(token=token) - params: Dict[str, Any] = {} - - # Build the filter list - filter_list: List[str] = [] - if filter: - filter_list.extend([filter] if isinstance(filter, str) else filter) - if library: - filter_list.extend([library] if isinstance(library, str) else library) - if task: - filter_list.extend([task] if isinstance(task, str) else task) - if trained_dataset: - if isinstance(trained_dataset, str): - trained_dataset = [trained_dataset] - for dataset in trained_dataset: - if not dataset.startswith("dataset:"): - dataset = f"dataset:{dataset}" - filter_list.append(dataset) - if language: - filter_list.extend([language] if isinstance(language, str) else language) - if tags: - filter_list.extend([tags] if isinstance(tags, str) else tags) - if len(filter_list) > 0: - params["filter"] = filter_list - - # Handle other query params - if author: - params["author"] = author - if apps: - if isinstance(apps, str): - apps = [apps] - params["apps"] = apps - if gated is not None: - params["gated"] = gated - if inference is not None: - params["inference"] = inference - if inference_provider is not None: - params["inference_provider"] = inference_provider - if pipeline_tag: - params["pipeline_tag"] = pipeline_tag - search_list = [] - if model_name: - search_list.append(model_name) - if search: - search_list.append(search) - if len(search_list) > 0: - params["search"] = search_list - if sort is not None: - params["sort"] = ( - "lastModified" - if sort == "last_modified" - else "trendingScore" - if sort == "trending_score" - else "createdAt" - if sort == "created_at" - else sort - ) - if direction is not None: - params["direction"] = direction - if limit is not None: - params["limit"] = limit - - # Request additional data - if full: - params["full"] = True - if fetch_config: - params["config"] = True - if cardData: - params["cardData"] = True - if expand: - params["expand"] = expand - - # `items` is a generator - items = paginate(path, params=params, headers=headers) - if limit is not None: - items = islice(items, limit) # Do not iterate over all pages - for item in items: - if "siblings" not in item: - item["siblings"] = None - model_info = ModelInfo(**item) - if emissions_thresholds is None or _is_emission_within_threshold(model_info, *emissions_thresholds): - yield model_info - - @_deprecate_arguments(version="1.0", deprecated_args=["tags"], custom_message="Use `filter` instead.") - @validate_hf_hub_args - def list_datasets( - self, - *, - # Search-query parameter - filter: Union[str, Iterable[str], None] = None, - author: Optional[str] = None, - benchmark: Optional[Union[str, List[str]]] = None, - dataset_name: Optional[str] = None, - gated: Optional[bool] = None, - language_creators: Optional[Union[str, List[str]]] = None, - language: Optional[Union[str, List[str]]] = None, - multilinguality: Optional[Union[str, List[str]]] = None, - size_categories: Optional[Union[str, List[str]]] = None, - task_categories: Optional[Union[str, List[str]]] = None, - task_ids: Optional[Union[str, List[str]]] = None, - search: Optional[str] = None, - # Sorting and pagination parameters - sort: Optional[Union[Literal["last_modified"], str]] = None, - direction: Optional[Literal[-1]] = None, - limit: Optional[int] = None, - # Additional data to fetch - expand: Optional[List[ExpandDatasetProperty_T]] = None, - full: Optional[bool] = None, - token: Union[bool, str, None] = None, - # Deprecated arguments - use `filter` instead - tags: Optional[Union[str, List[str]]] = None, - ) -> Iterable[DatasetInfo]: - """ - List datasets hosted on the Huggingface Hub, given some filters. - - Args: - filter (`str` or `Iterable[str]`, *optional*): - A string or list of string to filter datasets on the hub. - author (`str`, *optional*): - A string which identify the author of the returned datasets. - benchmark (`str` or `List`, *optional*): - A string or list of strings that can be used to identify datasets on - the Hub by their official benchmark. - dataset_name (`str`, *optional*): - A string or list of strings that can be used to identify datasets on - the Hub by its name, such as `SQAC` or `wikineural` - gated (`bool`, *optional*): - A boolean to filter datasets on the Hub that are gated or not. By default, all datasets are returned. - If `gated=True` is passed, only gated datasets are returned. - If `gated=False` is passed, only non-gated datasets are returned. - language_creators (`str` or `List`, *optional*): - A string or list of strings that can be used to identify datasets on - the Hub with how the data was curated, such as `crowdsourced` or - `machine_generated`. - language (`str` or `List`, *optional*): - A string or list of strings representing a two-character language to - filter datasets by on the Hub. - multilinguality (`str` or `List`, *optional*): - A string or list of strings representing a filter for datasets that - contain multiple languages. - size_categories (`str` or `List`, *optional*): - A string or list of strings that can be used to identify datasets on - the Hub by the size of the dataset such as `100K>> from huggingface_hub import HfApi - - >>> api = HfApi() - - # List all datasets - >>> api.list_datasets() - - - # List only the text classification datasets - >>> api.list_datasets(filter="task_categories:text-classification") - - - # List only the datasets in russian for language modeling - >>> api.list_datasets( - ... filter=("language:ru", "task_ids:language-modeling") - ... ) - - # List FiftyOne datasets (identified by the tag "fiftyone" in dataset card) - >>> api.list_datasets(tags="fiftyone") - ``` - - Example usage with the `search` argument: - - ```python - >>> from huggingface_hub import HfApi - - >>> api = HfApi() - - # List all datasets with "text" in their name - >>> api.list_datasets(search="text") - - # List all datasets with "text" in their name made by google - >>> api.list_datasets(search="text", author="google") - ``` - """ - if expand and full: - raise ValueError("`expand` cannot be used if `full` is passed.") - - path = f"{self.endpoint}/api/datasets" - headers = self._build_hf_headers(token=token) - params: Dict[str, Any] = {} - - # Build `filter` list - filter_list = [] - if filter is not None: - if isinstance(filter, str): - filter_list.append(filter) - else: - filter_list.extend(filter) - for key, value in ( - ("benchmark", benchmark), - ("language_creators", language_creators), - ("language", language), - ("multilinguality", multilinguality), - ("size_categories", size_categories), - ("task_categories", task_categories), - ("task_ids", task_ids), - ): - if value: - if isinstance(value, str): - value = [value] - for value_item in value: - if not value_item.startswith(f"{key}:"): - data = f"{key}:{value_item}" - filter_list.append(data) - if tags is not None: - filter_list.extend([tags] if isinstance(tags, str) else tags) - if len(filter_list) > 0: - params["filter"] = filter_list - - # Handle other query params - if author: - params["author"] = author - if gated is not None: - params["gated"] = gated - search_list = [] - if dataset_name: - search_list.append(dataset_name) - if search: - search_list.append(search) - if len(search_list) > 0: - params["search"] = search_list - if sort is not None: - params["sort"] = ( - "lastModified" - if sort == "last_modified" - else "trendingScore" - if sort == "trending_score" - else "createdAt" - if sort == "created_at" - else sort - ) - if direction is not None: - params["direction"] = direction - if limit is not None: - params["limit"] = limit - - # Request additional data - if expand: - params["expand"] = expand - if full: - params["full"] = True - - items = paginate(path, params=params, headers=headers) - if limit is not None: - items = islice(items, limit) # Do not iterate over all pages - for item in items: - if "siblings" not in item: - item["siblings"] = None - yield DatasetInfo(**item) - - @validate_hf_hub_args - def list_spaces( - self, - *, - # Search-query parameter - filter: Union[str, Iterable[str], None] = None, - author: Optional[str] = None, - search: Optional[str] = None, - datasets: Union[str, Iterable[str], None] = None, - models: Union[str, Iterable[str], None] = None, - linked: bool = False, - # Sorting and pagination parameters - sort: Union[Literal["last_modified"], str, None] = None, - direction: Optional[Literal[-1]] = None, - limit: Optional[int] = None, - # Additional data to fetch - expand: Optional[List[ExpandSpaceProperty_T]] = None, - full: Optional[bool] = None, - token: Union[bool, str, None] = None, - ) -> Iterable[SpaceInfo]: - """ - List spaces hosted on the Huggingface Hub, given some filters. - - Args: - filter (`str` or `Iterable`, *optional*): - A string tag or list of tags that can be used to identify Spaces on the Hub. - author (`str`, *optional*): - A string which identify the author of the returned Spaces. - search (`str`, *optional*): - A string that will be contained in the returned Spaces. - datasets (`str` or `Iterable`, *optional*): - Whether to return Spaces that make use of a dataset. - The name of a specific dataset can be passed as a string. - models (`str` or `Iterable`, *optional*): - Whether to return Spaces that make use of a model. - The name of a specific model can be passed as a string. - linked (`bool`, *optional*): - Whether to return Spaces that make use of either a model or a dataset. - sort (`Literal["last_modified"]` or `str`, *optional*): - The key with which to sort the resulting models. Possible values are "last_modified", "trending_score", - "created_at" and "likes". - direction (`Literal[-1]` or `int`, *optional*): - Direction in which to sort. The value `-1` sorts by descending - order while all other values sort by ascending order. - limit (`int`, *optional*): - The limit on the number of Spaces fetched. Leaving this option - to `None` fetches all Spaces. - expand (`List[ExpandSpaceProperty_T]`, *optional*): - List properties to return in the response. When used, only the properties in the list will be returned. - This parameter cannot be used if `full` is passed. - Possible values are `"author"`, `"cardData"`, `"datasets"`, `"disabled"`, `"lastModified"`, `"createdAt"`, `"likes"`, `"models"`, `"private"`, `"runtime"`, `"sdk"`, `"siblings"`, `"sha"`, `"subdomain"`, `"tags"`, `"trendingScore"`, `"usedStorage"`, `"resourceGroup"` and `"xetEnabled"`. - full (`bool`, *optional*): - Whether to fetch all Spaces data, including the `last_modified`, `siblings` - and `card_data` fields. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Iterable[SpaceInfo]`: an iterable of [`huggingface_hub.hf_api.SpaceInfo`] objects. - """ - if expand and full: - raise ValueError("`expand` cannot be used if `full` is passed.") - - path = f"{self.endpoint}/api/spaces" - headers = self._build_hf_headers(token=token) - params: Dict[str, Any] = {} - if filter is not None: - params["filter"] = filter - if author is not None: - params["author"] = author - if search is not None: - params["search"] = search - if sort is not None: - params["sort"] = ( - "lastModified" - if sort == "last_modified" - else "trendingScore" - if sort == "trending_score" - else "createdAt" - if sort == "created_at" - else sort - ) - if direction is not None: - params["direction"] = direction - if limit is not None: - params["limit"] = limit - if linked: - params["linked"] = True - if datasets is not None: - params["datasets"] = datasets - if models is not None: - params["models"] = models - - # Request additional data - if expand: - params["expand"] = expand - if full: - params["full"] = True - - items = paginate(path, params=params, headers=headers) - if limit is not None: - items = islice(items, limit) # Do not iterate over all pages - for item in items: - if "siblings" not in item: - item["siblings"] = None - yield SpaceInfo(**item) - - @validate_hf_hub_args - def unlike( - self, - repo_id: str, - *, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - ) -> None: - """ - Unlike a given repo on the Hub (e.g. remove from favorite list). - - To prevent spam usage, it is not possible to `like` a repository from a script. - - See also [`list_liked_repos`]. - - Args: - repo_id (`str`): - The repository to unlike. Example: `"user/my-cool-model"`. - - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if unliking a dataset or space, `None` or - `"model"` if unliking a model. Default is `None`. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If repository is not found (error 404): wrong repo_id/repo_type, private - but not authenticated or repo does not exist. - - Example: - ```python - >>> from huggingface_hub import list_liked_repos, unlike - >>> "gpt2" in list_liked_repos().models # we assume you have already liked gpt2 - True - >>> unlike("gpt2") - >>> "gpt2" in list_liked_repos().models - False - ``` - """ - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - response = get_session().delete( - url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/like", headers=self._build_hf_headers(token=token) - ) - hf_raise_for_status(response) - - @validate_hf_hub_args - def list_liked_repos( - self, - user: Optional[str] = None, - *, - token: Union[bool, str, None] = None, - ) -> UserLikes: - """ - List all public repos liked by a user on huggingface.co. - - This list is public so token is optional. If `user` is not passed, it defaults to - the logged in user. - - See also [`unlike`]. - - Args: - user (`str`, *optional*): - Name of the user for which you want to fetch the likes. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`UserLikes`]: object containing the user name and 3 lists of repo ids (1 for - models, 1 for datasets and 1 for Spaces). - - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If `user` is not passed and no token found (either from argument or from machine). - - Example: - ```python - >>> from huggingface_hub import list_liked_repos - - >>> likes = list_liked_repos("julien-c") - - >>> likes.user - "julien-c" - - >>> likes.models - ["osanseviero/streamlit_1.15", "Xhaheen/ChatGPT_HF", ...] - ``` - """ - # User is either provided explicitly or retrieved from current token. - if user is None: - me = self.whoami(token=token) - if me["type"] == "user": - user = me["name"] - else: - raise ValueError( - "Cannot list liked repos. You must provide a 'user' as input or be logged in as a user." - ) - - path = f"{self.endpoint}/api/users/{user}/likes" - headers = self._build_hf_headers(token=token) - - likes = list(paginate(path, params={}, headers=headers)) - # Looping over a list of items similar to: - # { - # 'createdAt': '2021-09-09T21:53:27.000Z', - # 'repo': { - # 'name': 'PaddlePaddle/PaddleOCR', - # 'type': 'space' - # } - # } - # Let's loop 3 times over the received list. Less efficient but more straightforward to read. - return UserLikes( - user=user, - total=len(likes), - models=[like["repo"]["name"] for like in likes if like["repo"]["type"] == "model"], - datasets=[like["repo"]["name"] for like in likes if like["repo"]["type"] == "dataset"], - spaces=[like["repo"]["name"] for like in likes if like["repo"]["type"] == "space"], - ) - - @validate_hf_hub_args - def list_repo_likers( - self, - repo_id: str, - *, - repo_type: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> Iterable[User]: - """ - List all users who liked a given repo on the hugging Face Hub. - - See also [`list_liked_repos`]. - - Args: - repo_id (`str`): - The repository to retrieve . Example: `"user/my-cool-model"`. - - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - - Returns: - `Iterable[User]`: an iterable of [`huggingface_hub.hf_api.User`] objects. - """ - - # Construct the API endpoint - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/likers" - for liker in paginate(path, params={}, headers=self._build_hf_headers(token=token)): - yield User(username=liker["user"], fullname=liker["fullname"], avatar_url=liker["avatarUrl"]) - - @validate_hf_hub_args - def model_info( - self, - repo_id: str, - *, - revision: Optional[str] = None, - timeout: Optional[float] = None, - securityStatus: Optional[bool] = None, - files_metadata: bool = False, - expand: Optional[List[ExpandModelProperty_T]] = None, - token: Union[bool, str, None] = None, - ) -> ModelInfo: - """ - Get info on one specific model on huggingface.co - - Model can be private if you pass an acceptable token or are logged in. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - revision (`str`, *optional*): - The revision of the model repository from which to get the - information. - timeout (`float`, *optional*): - Whether to set a timeout for the request to the Hub. - securityStatus (`bool`, *optional*): - Whether to retrieve the security status from the model - repository as well. The security status will be returned in the `security_repo_status` field. - files_metadata (`bool`, *optional*): - Whether or not to retrieve metadata for files in the repository - (size, LFS metadata, etc). Defaults to `False`. - expand (`List[ExpandModelProperty_T]`, *optional*): - List properties to return in the response. When used, only the properties in the list will be returned. - This parameter cannot be used if `securityStatus` or `files_metadata` are passed. - Possible values are `"author"`, `"baseModels"`, `"cardData"`, `"childrenModelCount"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"gguf"`, `"inference"`, `"inferenceProviderMapping"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"`, `"trendingScore"`, `"widgetData"`, `"usedStorage"`, `"resourceGroup"` and `"xetEnabled"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`huggingface_hub.hf_api.ModelInfo`]: The model repository information. - - > [!TIP] - > Raises the following errors: - > - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - > - [`~utils.RevisionNotFoundError`] - > If the revision to download from cannot be found. - """ - if expand and (securityStatus or files_metadata): - raise ValueError("`expand` cannot be used if `securityStatus` or `files_metadata` are set.") - - headers = self._build_hf_headers(token=token) - path = ( - f"{self.endpoint}/api/models/{repo_id}" - if revision is None - else (f"{self.endpoint}/api/models/{repo_id}/revision/{quote(revision, safe='')}") - ) - params: Dict = {} - if securityStatus: - params["securityStatus"] = True - if files_metadata: - params["blobs"] = True - if expand: - params["expand"] = expand - r = get_session().get(path, headers=headers, timeout=timeout, params=params) - hf_raise_for_status(r) - data = r.json() - return ModelInfo(**data) - - @validate_hf_hub_args - def dataset_info( - self, - repo_id: str, - *, - revision: Optional[str] = None, - timeout: Optional[float] = None, - files_metadata: bool = False, - expand: Optional[List[ExpandDatasetProperty_T]] = None, - token: Union[bool, str, None] = None, - ) -> DatasetInfo: - """ - Get info on one specific dataset on huggingface.co. - - Dataset can be private if you pass an acceptable token. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - revision (`str`, *optional*): - The revision of the dataset repository from which to get the - information. - timeout (`float`, *optional*): - Whether to set a timeout for the request to the Hub. - files_metadata (`bool`, *optional*): - Whether or not to retrieve metadata for files in the repository - (size, LFS metadata, etc). Defaults to `False`. - expand (`List[ExpandDatasetProperty_T]`, *optional*): - List properties to return in the response. When used, only the properties in the list will be returned. - This parameter cannot be used if `files_metadata` is passed. - Possible values are `"author"`, `"cardData"`, `"citation"`, `"createdAt"`, `"disabled"`, `"description"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"lastModified"`, `"likes"`, `"paperswithcode_id"`, `"private"`, `"siblings"`, `"sha"`, `"tags"`, `"trendingScore"`,`"usedStorage"`, `"resourceGroup"` and `"xetEnabled"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`hf_api.DatasetInfo`]: The dataset repository information. - - > [!TIP] - > Raises the following errors: - > - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - > - [`~utils.RevisionNotFoundError`] - > If the revision to download from cannot be found. - """ - if expand and files_metadata: - raise ValueError("`expand` cannot be used if `files_metadata` is set.") - - headers = self._build_hf_headers(token=token) - path = ( - f"{self.endpoint}/api/datasets/{repo_id}" - if revision is None - else (f"{self.endpoint}/api/datasets/{repo_id}/revision/{quote(revision, safe='')}") - ) - params: Dict = {} - if files_metadata: - params["blobs"] = True - if expand: - params["expand"] = expand - - r = get_session().get(path, headers=headers, timeout=timeout, params=params) - hf_raise_for_status(r) - data = r.json() - return DatasetInfo(**data) - - @validate_hf_hub_args - def space_info( - self, - repo_id: str, - *, - revision: Optional[str] = None, - timeout: Optional[float] = None, - files_metadata: bool = False, - expand: Optional[List[ExpandSpaceProperty_T]] = None, - token: Union[bool, str, None] = None, - ) -> SpaceInfo: - """ - Get info on one specific Space on huggingface.co. - - Space can be private if you pass an acceptable token. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - revision (`str`, *optional*): - The revision of the space repository from which to get the - information. - timeout (`float`, *optional*): - Whether to set a timeout for the request to the Hub. - files_metadata (`bool`, *optional*): - Whether or not to retrieve metadata for files in the repository - (size, LFS metadata, etc). Defaults to `False`. - expand (`List[ExpandSpaceProperty_T]`, *optional*): - List properties to return in the response. When used, only the properties in the list will be returned. - This parameter cannot be used if `full` is passed. - Possible values are `"author"`, `"cardData"`, `"createdAt"`, `"datasets"`, `"disabled"`, `"lastModified"`, `"likes"`, `"models"`, `"private"`, `"runtime"`, `"sdk"`, `"siblings"`, `"sha"`, `"subdomain"`, `"tags"`, `"trendingScore"`, `"usedStorage"`, `"resourceGroup"` and `"xetEnabled"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`~hf_api.SpaceInfo`]: The space repository information. - - > [!TIP] - > Raises the following errors: - > - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - > - [`~utils.RevisionNotFoundError`] - > If the revision to download from cannot be found. - """ - if expand and files_metadata: - raise ValueError("`expand` cannot be used if `files_metadata` is set.") - - headers = self._build_hf_headers(token=token) - path = ( - f"{self.endpoint}/api/spaces/{repo_id}" - if revision is None - else (f"{self.endpoint}/api/spaces/{repo_id}/revision/{quote(revision, safe='')}") - ) - params: Dict = {} - if files_metadata: - params["blobs"] = True - if expand: - params["expand"] = expand - - r = get_session().get(path, headers=headers, timeout=timeout, params=params) - hf_raise_for_status(r) - data = r.json() - return SpaceInfo(**data) - - @validate_hf_hub_args - def repo_info( - self, - repo_id: str, - *, - revision: Optional[str] = None, - repo_type: Optional[str] = None, - timeout: Optional[float] = None, - files_metadata: bool = False, - expand: Optional[Union[ExpandModelProperty_T, ExpandDatasetProperty_T, ExpandSpaceProperty_T]] = None, - token: Union[bool, str, None] = None, - ) -> Union[ModelInfo, DatasetInfo, SpaceInfo]: - """ - Get the info object for a given repo of a given type. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - revision (`str`, *optional*): - The revision of the repository from which to get the - information. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if getting repository info from a dataset or a space, - `None` or `"model"` if getting repository info from a model. Default is `None`. - timeout (`float`, *optional*): - Whether to set a timeout for the request to the Hub. - expand (`ExpandModelProperty_T` or `ExpandDatasetProperty_T` or `ExpandSpaceProperty_T`, *optional*): - List properties to return in the response. When used, only the properties in the list will be returned. - This parameter cannot be used if `files_metadata` is passed. - For an exhaustive list of available properties, check out [`model_info`], [`dataset_info`] or [`space_info`]. - files_metadata (`bool`, *optional*): - Whether or not to retrieve metadata for files in the repository - (size, LFS metadata, etc). Defaults to `False`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Union[SpaceInfo, DatasetInfo, ModelInfo]`: The repository information, as a - [`huggingface_hub.hf_api.DatasetInfo`], [`huggingface_hub.hf_api.ModelInfo`] - or [`huggingface_hub.hf_api.SpaceInfo`] object. - - > [!TIP] - > Raises the following errors: - > - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - > - [`~utils.RevisionNotFoundError`] - > If the revision to download from cannot be found. - """ - if repo_type is None or repo_type == "model": - method = self.model_info - elif repo_type == "dataset": - method = self.dataset_info # type: ignore - elif repo_type == "space": - method = self.space_info # type: ignore - else: - raise ValueError("Unsupported repo type.") - return method( - repo_id, - revision=revision, - token=token, - timeout=timeout, - expand=expand, # type: ignore[arg-type] - files_metadata=files_metadata, - ) - - @validate_hf_hub_args - def repo_exists( - self, - repo_id: str, - *, - repo_type: Optional[str] = None, - token: Union[str, bool, None] = None, - ) -> bool: - """ - Checks if a repository exists on the Hugging Face Hub. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if getting repository info from a dataset or a space, - `None` or `"model"` if getting repository info from a model. Default is `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - True if the repository exists, False otherwise. - - Examples: - ```py - >>> from huggingface_hub import repo_exists - >>> repo_exists("google/gemma-7b") - True - >>> repo_exists("google/not-a-repo") - False - ``` - """ - try: - self.repo_info(repo_id=repo_id, repo_type=repo_type, token=token) - return True - except GatedRepoError: - return True # we don't have access but it exists - except RepositoryNotFoundError: - return False - - @validate_hf_hub_args - def revision_exists( - self, - repo_id: str, - revision: str, - *, - repo_type: Optional[str] = None, - token: Union[str, bool, None] = None, - ) -> bool: - """ - Checks if a specific revision exists on a repo on the Hugging Face Hub. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - revision (`str`): - The revision of the repository to check. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if getting repository info from a dataset or a space, - `None` or `"model"` if getting repository info from a model. Default is `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - True if the repository and the revision exists, False otherwise. - - Examples: - ```py - >>> from huggingface_hub import revision_exists - >>> revision_exists("google/gemma-7b", "float16") - True - >>> revision_exists("google/gemma-7b", "not-a-revision") - False - ``` - """ - try: - self.repo_info(repo_id=repo_id, revision=revision, repo_type=repo_type, token=token) - return True - except RevisionNotFoundError: - return False - except RepositoryNotFoundError: - return False - - @validate_hf_hub_args - def file_exists( - self, - repo_id: str, - filename: str, - *, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - token: Union[str, bool, None] = None, - ) -> bool: - """ - Checks if a file exists in a repository on the Hugging Face Hub. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - filename (`str`): - The name of the file to check, for example: - `"config.json"` - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if getting repository info from a dataset or a space, - `None` or `"model"` if getting repository info from a model. Default is `None`. - revision (`str`, *optional*): - The revision of the repository from which to get the information. Defaults to `"main"` branch. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - True if the file exists, False otherwise. - - Examples: - ```py - >>> from huggingface_hub import file_exists - >>> file_exists("bigcode/starcoder", "config.json") - True - >>> file_exists("bigcode/starcoder", "not-a-file") - False - >>> file_exists("bigcode/not-a-repo", "config.json") - False - ``` - """ - url = hf_hub_url( - repo_id=repo_id, repo_type=repo_type, revision=revision, filename=filename, endpoint=self.endpoint - ) - try: - if token is None: - token = self.token - get_hf_file_metadata(url, token=token) - return True - except GatedRepoError: # raise specifically on gated repo - raise - except (RepositoryNotFoundError, EntryNotFoundError, RevisionNotFoundError): - return False - - @validate_hf_hub_args - def list_repo_files( - self, - repo_id: str, - *, - revision: Optional[str] = None, - repo_type: Optional[str] = None, - token: Union[str, bool, None] = None, - ) -> List[str]: - """ - Get the list of files in a given repo. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated by a `/`. - revision (`str`, *optional*): - The revision of the repository from which to get the information. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or space, `None` or `"model"` if uploading to - a model. Default is `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `List[str]`: the list of files in a given repository. - """ - return [ - f.rfilename - for f in self.list_repo_tree( - repo_id=repo_id, recursive=True, revision=revision, repo_type=repo_type, token=token - ) - if isinstance(f, RepoFile) - ] - - @validate_hf_hub_args - def list_repo_tree( - self, - repo_id: str, - path_in_repo: Optional[str] = None, - *, - recursive: bool = False, - expand: bool = False, - revision: Optional[str] = None, - repo_type: Optional[str] = None, - token: Union[str, bool, None] = None, - ) -> Iterable[Union[RepoFile, RepoFolder]]: - """ - List a repo tree's files and folders and get information about them. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated by a `/`. - path_in_repo (`str`, *optional*): - Relative path of the tree (folder) in the repo, for example: - `"checkpoints/1fec34a/results"`. Will default to the root tree (folder) of the repository. - recursive (`bool`, *optional*, defaults to `False`): - Whether to list tree's files and folders recursively. - expand (`bool`, *optional*, defaults to `False`): - Whether to fetch more information about the tree's files and folders (e.g. last commit and files' security scan results). This - operation is more expensive for the server so only 50 results are returned per page (instead of 1000). - As pagination is implemented in `huggingface_hub`, this is transparent for you except for the time it - takes to get the results. - revision (`str`, *optional*): - The revision of the repository from which to get the tree. Defaults to `"main"` branch. - repo_type (`str`, *optional*): - The type of the repository from which to get the tree (`"model"`, `"dataset"` or `"space"`. - Defaults to `"model"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Iterable[Union[RepoFile, RepoFolder]]`: - The information about the tree's files and folders, as an iterable of [`RepoFile`] and [`RepoFolder`] objects. The order of the files and folders is - not guaranteed. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo - does not exist. - [`~utils.RevisionNotFoundError`]: - If revision is not found (error 404) on the repo. - [`~utils.EntryNotFoundError`]: - If the tree (folder) does not exist (error 404) on the repo. - - Examples: - - Get information about a repo's tree. - ```py - >>> from huggingface_hub import list_repo_tree - >>> repo_tree = list_repo_tree("lysandre/arxiv-nlp") - >>> repo_tree - - >>> list(repo_tree) - [ - RepoFile(path='.gitattributes', size=391, blob_id='ae8c63daedbd4206d7d40126955d4e6ab1c80f8f', lfs=None, last_commit=None, security=None), - RepoFile(path='README.md', size=391, blob_id='43bd404b159de6fba7c2f4d3264347668d43af25', lfs=None, last_commit=None, security=None), - RepoFile(path='config.json', size=554, blob_id='2f9618c3a19b9a61add74f70bfb121335aeef666', lfs=None, last_commit=None, security=None), - RepoFile( - path='flax_model.msgpack', size=497764107, blob_id='8095a62ccb4d806da7666fcda07467e2d150218e', - lfs={'size': 497764107, 'sha256': 'd88b0d6a6ff9c3f8151f9d3228f57092aaea997f09af009eefd7373a77b5abb9', 'pointer_size': 134}, last_commit=None, security=None - ), - RepoFile(path='merges.txt', size=456318, blob_id='226b0752cac7789c48f0cb3ec53eda48b7be36cc', lfs=None, last_commit=None, security=None), - RepoFile( - path='pytorch_model.bin', size=548123560, blob_id='64eaa9c526867e404b68f2c5d66fd78e27026523', - lfs={'size': 548123560, 'sha256': '9be78edb5b928eba33aa88f431551348f7466ba9f5ef3daf1d552398722a5436', 'pointer_size': 134}, last_commit=None, security=None - ), - RepoFile(path='vocab.json', size=898669, blob_id='b00361fece0387ca34b4b8b8539ed830d644dbeb', lfs=None, last_commit=None, security=None)] - ] - ``` - - Get even more information about a repo's tree (last commit and files' security scan results) - ```py - >>> from huggingface_hub import list_repo_tree - >>> repo_tree = list_repo_tree("prompthero/openjourney-v4", expand=True) - >>> list(repo_tree) - [ - RepoFolder( - path='feature_extractor', - tree_id='aa536c4ea18073388b5b0bc791057a7296a00398', - last_commit={ - 'oid': '47b62b20b20e06b9de610e840282b7e6c3d51190', - 'title': 'Upload diffusers weights (#48)', - 'date': datetime.datetime(2023, 3, 21, 9, 5, 27, tzinfo=datetime.timezone.utc) - } - ), - RepoFolder( - path='safety_checker', - tree_id='65aef9d787e5557373fdf714d6c34d4fcdd70440', - last_commit={ - 'oid': '47b62b20b20e06b9de610e840282b7e6c3d51190', - 'title': 'Upload diffusers weights (#48)', - 'date': datetime.datetime(2023, 3, 21, 9, 5, 27, tzinfo=datetime.timezone.utc) - } - ), - RepoFile( - path='model_index.json', - size=582, - blob_id='d3d7c1e8c3e78eeb1640b8e2041ee256e24c9ee1', - lfs=None, - last_commit={ - 'oid': 'b195ed2d503f3eb29637050a886d77bd81d35f0e', - 'title': 'Fix deprecation warning by changing `CLIPFeatureExtractor` to `CLIPImageProcessor`. (#54)', - 'date': datetime.datetime(2023, 5, 15, 21, 41, 59, tzinfo=datetime.timezone.utc) - }, - security={ - 'safe': True, - 'av_scan': {'virusFound': False, 'virusNames': None}, - 'pickle_import_scan': None - } - ) - ... - ] - ``` - """ - repo_type = repo_type or constants.REPO_TYPE_MODEL - revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION - headers = self._build_hf_headers(token=token) - - encoded_path_in_repo = "/" + quote(path_in_repo, safe="") if path_in_repo else "" - tree_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tree/{revision}{encoded_path_in_repo}" - for path_info in paginate(path=tree_url, headers=headers, params={"recursive": recursive, "expand": expand}): - yield (RepoFile(**path_info) if path_info["type"] == "file" else RepoFolder(**path_info)) - - @validate_hf_hub_args - def list_repo_refs( - self, - repo_id: str, - *, - repo_type: Optional[str] = None, - include_pull_requests: bool = False, - token: Union[str, bool, None] = None, - ) -> GitRefs: - """ - Get the list of refs of a given repo (both tags and branches). - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if listing refs from a dataset or a Space, - `None` or `"model"` if listing from a model. Default is `None`. - include_pull_requests (`bool`, *optional*): - Whether to include refs from pull requests in the list. Defaults to `False`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Example: - ```py - >>> from huggingface_hub import HfApi - >>> api = HfApi() - >>> api.list_repo_refs("gpt2") - GitRefs(branches=[GitRefInfo(name='main', ref='refs/heads/main', target_commit='e7da7f221d5bf496a48136c0cd264e630fe9fcc8')], converts=[], tags=[]) - - >>> api.list_repo_refs("bigcode/the-stack", repo_type='dataset') - GitRefs( - branches=[ - GitRefInfo(name='main', ref='refs/heads/main', target_commit='18edc1591d9ce72aa82f56c4431b3c969b210ae3'), - GitRefInfo(name='v1.1.a1', ref='refs/heads/v1.1.a1', target_commit='f9826b862d1567f3822d3d25649b0d6d22ace714') - ], - converts=[], - tags=[ - GitRefInfo(name='v1.0', ref='refs/tags/v1.0', target_commit='c37a8cd1e382064d8aced5e05543c5f7753834da') - ] - ) - ``` - - Returns: - [`GitRefs`]: object containing all information about branches and tags for a - repo on the Hub. - """ - repo_type = repo_type or constants.REPO_TYPE_MODEL - response = get_session().get( - f"{self.endpoint}/api/{repo_type}s/{repo_id}/refs", - headers=self._build_hf_headers(token=token), - params={"include_prs": 1} if include_pull_requests else {}, - ) - hf_raise_for_status(response) - data = response.json() - - def _format_as_git_ref_info(item: Dict) -> GitRefInfo: - return GitRefInfo(name=item["name"], ref=item["ref"], target_commit=item["targetCommit"]) - - return GitRefs( - branches=[_format_as_git_ref_info(item) for item in data["branches"]], - converts=[_format_as_git_ref_info(item) for item in data["converts"]], - tags=[_format_as_git_ref_info(item) for item in data["tags"]], - pull_requests=[_format_as_git_ref_info(item) for item in data["pullRequests"]] - if include_pull_requests - else None, - ) - - @validate_hf_hub_args - def list_repo_commits( - self, - repo_id: str, - *, - repo_type: Optional[str] = None, - token: Union[bool, str, None] = None, - revision: Optional[str] = None, - formatted: bool = False, - ) -> List[GitCommitInfo]: - """ - Get the list of commits of a given revision for a repo on the Hub. - - Commits are sorted by date (last commit first). - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated by a `/`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if listing commits from a dataset or a Space, `None` or `"model"` if - listing from a model. Default is `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - revision (`str`, *optional*): - The git revision to commit from. Defaults to the head of the `"main"` branch. - formatted (`bool`): - Whether to return the HTML-formatted title and description of the commits. Defaults to False. - - Example: - ```py - >>> from huggingface_hub import HfApi - >>> api = HfApi() - - # Commits are sorted by date (last commit first) - >>> initial_commit = api.list_repo_commits("gpt2")[-1] - - # Initial commit is always a system commit containing the `.gitattributes` file. - >>> initial_commit - GitCommitInfo( - commit_id='9b865efde13a30c13e0a33e536cf3e4a5a9d71d8', - authors=['system'], - created_at=datetime.datetime(2019, 2, 18, 10, 36, 15, tzinfo=datetime.timezone.utc), - title='initial commit', - message='', - formatted_title=None, - formatted_message=None - ) - - # Create an empty branch by deriving from initial commit - >>> api.create_branch("gpt2", "new_empty_branch", revision=initial_commit.commit_id) - ``` - - Returns: - List[[`GitCommitInfo`]]: list of objects containing information about the commits for a repo on the Hub. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo - does not exist. - [`~utils.RevisionNotFoundError`]: - If revision is not found (error 404) on the repo. - """ - repo_type = repo_type or constants.REPO_TYPE_MODEL - revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION - - # Paginate over results and return the list of commits. - return [ - GitCommitInfo( - commit_id=item["id"], - authors=[author["user"] for author in item["authors"]], - created_at=parse_datetime(item["date"]), - title=item["title"], - message=item["message"], - formatted_title=item.get("formatted", {}).get("title"), - formatted_message=item.get("formatted", {}).get("message"), - ) - for item in paginate( - f"{self.endpoint}/api/{repo_type}s/{repo_id}/commits/{revision}", - headers=self._build_hf_headers(token=token), - params={"expand[]": "formatted"} if formatted else {}, - ) - ] - - @validate_hf_hub_args - def get_paths_info( - self, - repo_id: str, - paths: Union[List[str], str], - *, - expand: bool = False, - revision: Optional[str] = None, - repo_type: Optional[str] = None, - token: Union[str, bool, None] = None, - ) -> List[Union[RepoFile, RepoFolder]]: - """ - Get information about a repo's paths. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated by a `/`. - paths (`Union[List[str], str]`, *optional*): - The paths to get information about. If a path do not exist, it is ignored without raising - an exception. - expand (`bool`, *optional*, defaults to `False`): - Whether to fetch more information about the paths (e.g. last commit and files' security scan results). This - operation is more expensive for the server so only 50 results are returned per page (instead of 1000). - As pagination is implemented in `huggingface_hub`, this is transparent for you except for the time it - takes to get the results. - revision (`str`, *optional*): - The revision of the repository from which to get the information. Defaults to `"main"` branch. - repo_type (`str`, *optional*): - The type of the repository from which to get the information (`"model"`, `"dataset"` or `"space"`. - Defaults to `"model"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `List[Union[RepoFile, RepoFolder]]`: - The information about the paths, as a list of [`RepoFile`] and [`RepoFolder`] objects. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo - does not exist. - [`~utils.RevisionNotFoundError`]: - If revision is not found (error 404) on the repo. - - Example: - ```py - >>> from huggingface_hub import get_paths_info - >>> paths_info = get_paths_info("allenai/c4", ["README.md", "en"], repo_type="dataset") - >>> paths_info - [ - RepoFile(path='README.md', size=2379, blob_id='f84cb4c97182890fc1dbdeaf1a6a468fd27b4fff', lfs=None, last_commit=None, security=None), - RepoFolder(path='en', tree_id='dc943c4c40f53d02b31ced1defa7e5f438d5862e', last_commit=None) - ] - ``` - """ - repo_type = repo_type or constants.REPO_TYPE_MODEL - revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION - headers = self._build_hf_headers(token=token) - - response = get_session().post( - f"{self.endpoint}/api/{repo_type}s/{repo_id}/paths-info/{revision}", - data={ - "paths": paths if isinstance(paths, list) else [paths], - "expand": expand, - }, - headers=headers, - ) - hf_raise_for_status(response) - paths_info = response.json() - return [ - RepoFile(**path_info) if path_info["type"] == "file" else RepoFolder(**path_info) - for path_info in paths_info - ] - - @validate_hf_hub_args - def super_squash_history( - self, - repo_id: str, - *, - branch: Optional[str] = None, - commit_message: Optional[str] = None, - repo_type: Optional[str] = None, - token: Union[str, bool, None] = None, - ) -> None: - """Squash commit history on a branch for a repo on the Hub. - - Squashing the repo history is useful when you know you'll make hundreds of commits and you don't want to - clutter the history. Squashing commits can only be performed from the head of a branch. - - > [!WARNING] - > Once squashed, the commit history cannot be retrieved. This is a non-revertible operation. - - > [!WARNING] - > Once the history of a branch has been squashed, it is not possible to merge it back into another branch since - > their history will have diverged. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated by a `/`. - branch (`str`, *optional*): - The branch to squash. Defaults to the head of the `"main"` branch. - commit_message (`str`, *optional*): - The commit message to use for the squashed commit. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if listing commits from a dataset or a Space, `None` or `"model"` if - listing from a model. Default is `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo - does not exist. - [`~utils.RevisionNotFoundError`]: - If the branch to squash cannot be found. - [`~utils.BadRequestError`]: - If invalid reference for a branch. You cannot squash history on tags. - - Example: - ```py - >>> from huggingface_hub import HfApi - >>> api = HfApi() - - # Create repo - >>> repo_id = api.create_repo("test-squash").repo_id - - # Make a lot of commits. - >>> api.upload_file(repo_id=repo_id, path_in_repo="file.txt", path_or_fileobj=b"content") - >>> api.upload_file(repo_id=repo_id, path_in_repo="lfs.bin", path_or_fileobj=b"content") - >>> api.upload_file(repo_id=repo_id, path_in_repo="file.txt", path_or_fileobj=b"another_content") - - # Squash history - >>> api.super_squash_history(repo_id=repo_id) - ``` - """ - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - if repo_type not in constants.REPO_TYPES: - raise ValueError("Invalid repo type") - if branch is None: - branch = constants.DEFAULT_REVISION - - # Prepare request - url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/super-squash/{quote(branch, safe='')}" - headers = self._build_hf_headers(token=token) - commit_message = commit_message or f"Super-squash branch '{branch}' using huggingface_hub" - - # Super-squash - response = get_session().post(url=url, headers=headers, json={"message": commit_message}) - hf_raise_for_status(response) - - @validate_hf_hub_args - def list_lfs_files( - self, - repo_id: str, - *, - repo_type: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> Iterable[LFSFileInfo]: - """ - List all LFS files in a repo on the Hub. - - This is primarily useful to count how much storage a repo is using and to eventually clean up large files - with [`permanently_delete_lfs_files`]. Note that this would be a permanent action that will affect all commits - referencing this deleted files and that cannot be undone. - - Args: - repo_id (`str`): - The repository for which you are listing LFS files. - repo_type (`str`, *optional*): - Type of repository. Set to `"dataset"` or `"space"` if listing from a dataset or space, `None` or - `"model"` if listing from a model. Default is `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Iterable[LFSFileInfo]`: An iterator of [`LFSFileInfo`] objects. - - Example: - ```py - >>> from huggingface_hub import HfApi - >>> api = HfApi() - >>> lfs_files = api.list_lfs_files("username/my-cool-repo") - - # Filter files files to delete based on a combination of `filename`, `pushed_at`, `ref` or `size`. - # e.g. select only LFS files in the "checkpoints" folder - >>> lfs_files_to_delete = (lfs_file for lfs_file in lfs_files if lfs_file.filename.startswith("checkpoints/")) - - # Permanently delete LFS files - >>> api.permanently_delete_lfs_files("username/my-cool-repo", lfs_files_to_delete) - ``` - """ - # Prepare request - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/lfs-files" - headers = self._build_hf_headers(token=token) - - # Paginate over LFS items - for item in paginate(url, params={}, headers=headers): - yield LFSFileInfo(**item) - - @validate_hf_hub_args - def permanently_delete_lfs_files( - self, - repo_id: str, - lfs_files: Iterable[LFSFileInfo], - *, - rewrite_history: bool = True, - repo_type: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> None: - """ - Permanently delete LFS files from a repo on the Hub. - - > [!WARNING] - > This is a permanent action that will affect all commits referencing the deleted files and might corrupt your - > repository. This is a non-revertible operation. Use it only if you know what you are doing. - - Args: - repo_id (`str`): - The repository for which you are listing LFS files. - lfs_files (`Iterable[LFSFileInfo]`): - An iterable of [`LFSFileInfo`] items to permanently delete from the repo. Use [`list_lfs_files`] to list - all LFS files from a repo. - rewrite_history (`bool`, *optional*, default to `True`): - Whether to rewrite repository history to remove file pointers referencing the deleted LFS files (recommended). - repo_type (`str`, *optional*): - Type of repository. Set to `"dataset"` or `"space"` if listing from a dataset or space, `None` or - `"model"` if listing from a model. Default is `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Example: - ```py - >>> from huggingface_hub import HfApi - >>> api = HfApi() - >>> lfs_files = api.list_lfs_files("username/my-cool-repo") - - # Filter files files to delete based on a combination of `filename`, `pushed_at`, `ref` or `size`. - # e.g. select only LFS files in the "checkpoints" folder - >>> lfs_files_to_delete = (lfs_file for lfs_file in lfs_files if lfs_file.filename.startswith("checkpoints/")) - - # Permanently delete LFS files - >>> api.permanently_delete_lfs_files("username/my-cool-repo", lfs_files_to_delete) - ``` - """ - # Prepare request - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/lfs-files/batch" - headers = self._build_hf_headers(token=token) - - # Delete LFS items by batches of 1000 - for batch in chunk_iterable(lfs_files, 1000): - shas = [item.file_oid for item in batch] - if len(shas) == 0: - return - payload = { - "deletions": { - "sha": shas, - "rewriteHistory": rewrite_history, - } - } - response = get_session().post(url, headers=headers, json=payload) - hf_raise_for_status(response) - - @validate_hf_hub_args - def create_repo( - self, - repo_id: str, - *, - token: Union[str, bool, None] = None, - private: Optional[bool] = None, - repo_type: Optional[str] = None, - exist_ok: bool = False, - resource_group_id: Optional[str] = None, - space_sdk: Optional[str] = None, - space_hardware: Optional[SpaceHardware] = None, - space_storage: Optional[SpaceStorage] = None, - space_sleep_time: Optional[int] = None, - space_secrets: Optional[List[Dict[str, str]]] = None, - space_variables: Optional[List[Dict[str, str]]] = None, - ) -> RepoUrl: - """Create an empty repo on the HuggingFace Hub. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - private (`bool`, *optional*): - Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - exist_ok (`bool`, *optional*, defaults to `False`): - If `True`, do not raise an error if repo already exists. - resource_group_id (`str`, *optional*): - Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations and - allow to define which members of the organization can access the resource. The ID of a resource group - can be found in the URL of the resource's page on the Hub (e.g. `"66670e5163145ca562cb1988"`). - To learn more about resource groups, see https://huggingface.co/docs/hub/en/security-resource-groups. - space_sdk (`str`, *optional*): - Choice of SDK to use if repo_type is "space". Can be "streamlit", "gradio", "docker", or "static". - space_hardware (`SpaceHardware` or `str`, *optional*): - Choice of Hardware if repo_type is "space". See [`SpaceHardware`] for a complete list. - space_storage (`SpaceStorage` or `str`, *optional*): - Choice of persistent storage tier. Example: `"small"`. See [`SpaceStorage`] for a complete list. - space_sleep_time (`int`, *optional*): - Number of seconds of inactivity to wait before a Space is put to sleep. Set to `-1` if you don't want - your Space to sleep (default behavior for upgraded hardware). For free hardware, you can't configure - the sleep time (value is fixed to 48 hours of inactivity). - See https://huggingface.co/docs/hub/spaces-gpus#sleep-time for more details. - space_secrets (`List[Dict[str, str]]`, *optional*): - A list of secret keys to set in your Space. Each item is in the form `{"key": ..., "value": ..., "description": ...}` where description is optional. - For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets. - space_variables (`List[Dict[str, str]]`, *optional*): - A list of public environment variables to set in your Space. Each item is in the form `{"key": ..., "value": ..., "description": ...}` where description is optional. - For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets-and-environment-variables. - - Returns: - [`RepoUrl`]: URL to the newly created repo. Value is a subclass of `str` containing - attributes like `endpoint`, `repo_type` and `repo_id`. - """ - organization, name = repo_id.split("/") if "/" in repo_id else (None, repo_id) - - path = f"{self.endpoint}/api/repos/create" - - if repo_type not in constants.REPO_TYPES: - raise ValueError("Invalid repo type") - - json: Dict[str, Any] = {"name": name, "organization": organization} - if private is not None: - json["private"] = private - if repo_type is not None: - json["type"] = repo_type - if repo_type == "space": - if space_sdk is None: - raise ValueError( - "No space_sdk provided. `create_repo` expects space_sdk to be one" - f" of {constants.SPACES_SDK_TYPES} when repo_type is 'space'`" - ) - if space_sdk not in constants.SPACES_SDK_TYPES: - raise ValueError(f"Invalid space_sdk. Please choose one of {constants.SPACES_SDK_TYPES}.") - json["sdk"] = space_sdk - - if space_sdk is not None and repo_type != "space": - warnings.warn("Ignoring provided space_sdk because repo_type is not 'space'.") - - function_args = [ - "space_hardware", - "space_storage", - "space_sleep_time", - "space_secrets", - "space_variables", - ] - json_keys = ["hardware", "storageTier", "sleepTimeSeconds", "secrets", "variables"] - values = [space_hardware, space_storage, space_sleep_time, space_secrets, space_variables] - - if repo_type == "space": - json.update({k: v for k, v in zip(json_keys, values) if v is not None}) - else: - provided_space_args = [key for key, value in zip(function_args, values) if value is not None] - - if provided_space_args: - warnings.warn(f"Ignoring provided {', '.join(provided_space_args)} because repo_type is not 'space'.") - - if getattr(self, "_lfsmultipartthresh", None): - # Testing purposes only. - # See https://github.com/huggingface/huggingface_hub/pull/733/files#r820604472 - json["lfsmultipartthresh"] = self._lfsmultipartthresh # type: ignore - - if resource_group_id is not None: - json["resourceGroupId"] = resource_group_id - - headers = self._build_hf_headers(token=token) - while True: - r = get_session().post(path, headers=headers, json=json) - if r.status_code == 409 and "Cannot create repo: another conflicting operation is in progress" in r.text: - # Since https://github.com/huggingface/moon-landing/pull/7272 (private repo), it is not possible to - # concurrently create repos on the Hub for a same user. This is rarely an issue, except when running - # tests. To avoid any inconvenience, we retry to create the repo for this specific error. - # NOTE: This could have being fixed directly in the tests but adding it here should fixed CIs for all - # dependent libraries. - # NOTE: If a fix is implemented server-side, we should be able to remove this retry mechanism. - logger.debug("Create repo failed due to a concurrency issue. Retrying...") - continue - break - - try: - hf_raise_for_status(r) - except HTTPError as err: - if exist_ok and err.response.status_code == 409: - # Repo already exists and `exist_ok=True` - pass - elif exist_ok and err.response.status_code == 403: - # No write permission on the namespace but repo might already exist - try: - self.repo_info(repo_id=repo_id, repo_type=repo_type, token=token) - if repo_type is None or repo_type == constants.REPO_TYPE_MODEL: - return RepoUrl(f"{self.endpoint}/{repo_id}") - return RepoUrl(f"{self.endpoint}/{repo_type}/{repo_id}") - except HfHubHTTPError: - raise err - else: - raise - - d = r.json() - return RepoUrl(d["url"], endpoint=self.endpoint) - - @validate_hf_hub_args - def delete_repo( - self, - repo_id: str, - *, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - missing_ok: bool = False, - ) -> None: - """ - Delete a repo from the HuggingFace Hub. CAUTION: this is irreversible. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. - missing_ok (`bool`, *optional*, defaults to `False`): - If `True`, do not raise an error if repo does not exist. - - Raises: - [`~utils.RepositoryNotFoundError`] - If the repository to delete from cannot be found and `missing_ok` is set to False (default). - """ - organization, name = repo_id.split("/") if "/" in repo_id else (None, repo_id) - - path = f"{self.endpoint}/api/repos/delete" - - if repo_type not in constants.REPO_TYPES: - raise ValueError("Invalid repo type") - - json = {"name": name, "organization": organization} - if repo_type is not None: - json["type"] = repo_type - - headers = self._build_hf_headers(token=token) - r = get_session().delete(path, headers=headers, json=json) - try: - hf_raise_for_status(r) - except RepositoryNotFoundError: - if not missing_ok: - raise - - @_deprecate_method(version="0.32", message="Please use `update_repo_settings` instead.") - @validate_hf_hub_args - def update_repo_visibility( - self, - repo_id: str, - private: bool = False, - *, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - ) -> Dict[str, bool]: - """Update the visibility setting of a repository. - - Deprecated. Use `update_repo_settings` instead. - - Args: - repo_id (`str`, *optional*): - A namespace (user or an organization) and a repo name separated by a `/`. - private (`bool`, *optional*, defaults to `False`): - Whether the repository should be private. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - - Returns: - The HTTP response in json. - - > [!TIP] - > Raises the following errors: - > - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - """ - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL # default repo type - - r = get_session().put( - url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/settings", - headers=self._build_hf_headers(token=token), - json={"private": private}, - ) - hf_raise_for_status(r) - return r.json() - - @validate_hf_hub_args - def update_repo_settings( - self, - repo_id: str, - *, - gated: Optional[Literal["auto", "manual", False]] = None, - private: Optional[bool] = None, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - xet_enabled: Optional[bool] = None, - ) -> None: - """ - Update the settings of a repository, including gated access and visibility. - - To give more control over how repos are used, the Hub allows repo authors to enable - access requests for their repos, and also to set the visibility of the repo to private. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated by a /. - gated (`Literal["auto", "manual", False]`, *optional*): - The gated status for the repository. If set to `None` (default), the `gated` setting of the repository won't be updated. - * "auto": The repository is gated, and access requests are automatically approved or denied based on predefined criteria. - * "manual": The repository is gated, and access requests require manual approval. - * False : The repository is not gated, and anyone can access it. - private (`bool`, *optional*): - Whether the repository should be private. - token (`Union[str, bool, None]`, *optional*): - A valid user access token (string). Defaults to the locally saved token, - which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass False. - repo_type (`str`, *optional*): - The type of the repository to update settings from (`"model"`, `"dataset"` or `"space"`). - Defaults to `"model"`. - xet_enabled (`bool`, *optional*): - Whether the repository should be enabled for Xet Storage. - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If gated is not one of "auto", "manual", or False. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If repo_type is not one of the values in constants.REPO_TYPES. - [`~utils.HfHubHTTPError`]: - If the request to the Hugging Face Hub API fails. - [`~utils.RepositoryNotFoundError`] - If the repository to download from cannot be found. This may be because it doesn't exist, - or because it is set to `private` and you do not have access. - """ - - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL # default repo type - - # Prepare the JSON payload for the PUT request - payload: Dict = {} - - if gated is not None: - if gated not in ["auto", "manual", False]: - raise ValueError(f"Invalid gated status, must be one of 'auto', 'manual', or False. Got '{gated}'.") - payload["gated"] = gated - - if private is not None: - payload["private"] = private - - if xet_enabled is not None: - payload["xetEnabled"] = xet_enabled - - if len(payload) == 0: - raise ValueError("At least one setting must be updated.") - - # Build headers - headers = self._build_hf_headers(token=token) - - r = get_session().put( - url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/settings", - headers=headers, - json=payload, - ) - hf_raise_for_status(r) - - def move_repo( - self, - from_id: str, - to_id: str, - *, - repo_type: Optional[str] = None, - token: Union[str, bool, None] = None, - ): - """ - Moving a repository from namespace1/repo_name1 to namespace2/repo_name2 - - Note there are certain limitations. For more information about moving - repositories, please see - https://hf.co/docs/hub/repositories-settings#renaming-or-transferring-a-repo. - - Args: - from_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. Original repository identifier. - to_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. Final repository identifier. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - > [!TIP] - > Raises the following errors: - > - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - """ - if len(from_id.split("/")) != 2: - raise ValueError(f"Invalid repo_id: {from_id}. It should have a namespace (:namespace:/:repo_name:)") - - if len(to_id.split("/")) != 2: - raise ValueError(f"Invalid repo_id: {to_id}. It should have a namespace (:namespace:/:repo_name:)") - - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL # Hub won't accept `None`. - - json = {"fromRepo": from_id, "toRepo": to_id, "type": repo_type} - - path = f"{self.endpoint}/api/repos/move" - headers = self._build_hf_headers(token=token) - r = get_session().post(path, headers=headers, json=json) - try: - hf_raise_for_status(r) - except HfHubHTTPError as e: - e.append_to_message( - "\nFor additional documentation please see" - " https://hf.co/docs/hub/repositories-settings#renaming-or-transferring-a-repo." - ) - raise - - @overload - def create_commit( # type: ignore - self, - repo_id: str, - operations: Iterable[CommitOperation], - *, - commit_message: str, - commit_description: Optional[str] = None, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - create_pr: Optional[bool] = None, - num_threads: int = 5, - parent_commit: Optional[str] = None, - run_as_future: Literal[False] = ..., - ) -> CommitInfo: ... - - @overload - def create_commit( - self, - repo_id: str, - operations: Iterable[CommitOperation], - *, - commit_message: str, - commit_description: Optional[str] = None, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - create_pr: Optional[bool] = None, - num_threads: int = 5, - parent_commit: Optional[str] = None, - run_as_future: Literal[True] = ..., - ) -> Future[CommitInfo]: ... - - @validate_hf_hub_args - @future_compatible - def create_commit( - self, - repo_id: str, - operations: Iterable[CommitOperation], - *, - commit_message: str, - commit_description: Optional[str] = None, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - create_pr: Optional[bool] = None, - num_threads: int = 5, - parent_commit: Optional[str] = None, - run_as_future: bool = False, - ) -> Union[CommitInfo, Future[CommitInfo]]: - """ - Creates a commit in the given repo, deleting & uploading files as needed. - - > [!WARNING] - > The input list of `CommitOperation` will be mutated during the commit process. Do not reuse the same objects - > for multiple commits. - - > [!WARNING] - > `create_commit` assumes that the repo already exists on the Hub. If you get a - > Client error 404, please make sure you are authenticated and that `repo_id` and - > `repo_type` are set correctly. If repo does not exist, create it first using - > [`~hf_api.create_repo`]. - - > [!WARNING] - > `create_commit` is limited to 25k LFS files and a 1GB payload for regular files. - - Args: - repo_id (`str`): - The repository in which the commit will be created, for example: - `"username/custom_transformers"` - - operations (`Iterable` of [`~hf_api.CommitOperation`]): - An iterable of operations to include in the commit, either: - - - [`~hf_api.CommitOperationAdd`] to upload a file - - [`~hf_api.CommitOperationDelete`] to delete a file - - [`~hf_api.CommitOperationCopy`] to copy a file - - Operation objects will be mutated to include information relative to the upload. Do not reuse the - same objects for multiple commits. - - commit_message (`str`): - The summary (first line) of the commit that will be created. - - commit_description (`str`, *optional*): - The description of the commit that will be created - - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - - revision (`str`, *optional*): - The git revision to commit from. Defaults to the head of the `"main"` branch. - - create_pr (`boolean`, *optional*): - Whether or not to create a Pull Request with that commit. Defaults to `False`. - If `revision` is not set, PR is opened against the `"main"` branch. If - `revision` is set and is a branch, PR is opened against this branch. If - `revision` is set and is not a branch name (example: a commit oid), an - `RevisionNotFoundError` is returned by the server. - - num_threads (`int`, *optional*): - Number of concurrent threads for uploading files. Defaults to 5. - Setting it to 2 means at most 2 files will be uploaded concurrently. - - parent_commit (`str`, *optional*): - The OID / SHA of the parent commit, as a hexadecimal string. - Shorthands (7 first characters) are also supported. If specified and `create_pr` is `False`, - the commit will fail if `revision` does not point to `parent_commit`. If specified and `create_pr` - is `True`, the pull request will be created from `parent_commit`. Specifying `parent_commit` - ensures the repo has not changed before committing the changes, and can be especially useful - if the repo is updated / committed to concurrently. - run_as_future (`bool`, *optional*): - Whether or not to run this method in the background. Background jobs are run sequentially without - blocking the main thread. Passing `run_as_future=True` will return a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) - object. Defaults to `False`. - - Returns: - [`CommitInfo`] or `Future`: - Instance of [`CommitInfo`] containing information about the newly created commit (commit hash, commit - url, pr url, commit message,...). If `run_as_future=True` is passed, returns a Future object which will - contain the result when executed. - - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If commit message is empty. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If parent commit is not a valid commit OID. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If a README.md file with an invalid metadata section is committed. In this case, the commit will fail - early, before trying to upload any file. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If `create_pr` is `True` and revision is neither `None` nor `"main"`. - [`~utils.RepositoryNotFoundError`]: - If repository is not found (error 404): wrong repo_id/repo_type, private - but not authenticated or repo does not exist. - """ - if parent_commit is not None and not constants.REGEX_COMMIT_OID.fullmatch(parent_commit): - raise ValueError( - f"`parent_commit` is not a valid commit OID. It must match the following regex: {constants.REGEX_COMMIT_OID}" - ) - - if commit_message is None or len(commit_message) == 0: - raise ValueError("`commit_message` can't be empty, please pass a value.") - - commit_description = commit_description if commit_description is not None else "" - repo_type = repo_type if repo_type is not None else constants.REPO_TYPE_MODEL - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - unquoted_revision = revision or constants.DEFAULT_REVISION - revision = quote(unquoted_revision, safe="") - create_pr = create_pr if create_pr is not None else False - - headers = self._build_hf_headers(token=token) - - operations = list(operations) - additions = [op for op in operations if isinstance(op, CommitOperationAdd)] - copies = [op for op in operations if isinstance(op, CommitOperationCopy)] - nb_additions = len(additions) - nb_copies = len(copies) - nb_deletions = len(operations) - nb_additions - nb_copies - - for addition in additions: - if addition._is_committed: - raise ValueError( - f"CommitOperationAdd {addition} has already being committed and cannot be reused. Please create a" - " new CommitOperationAdd object if you want to create a new commit." - ) - - if repo_type != "dataset": - for addition in additions: - if addition.path_in_repo.endswith((".arrow", ".parquet")): - warnings.warn( - f"It seems that you are about to commit a data file ({addition.path_in_repo}) to a {repo_type}" - " repository. You are sure this is intended? If you are trying to upload a dataset, please" - " set `repo_type='dataset'` or `--repo-type=dataset` in a CLI." - ) - - logger.debug( - f"About to commit to the hub: {len(additions)} addition(s), {len(copies)} copie(s) and" - f" {nb_deletions} deletion(s)." - ) - - # If updating a README.md file, make sure the metadata format is valid - # It's better to fail early than to fail after all the files have been uploaded. - for addition in additions: - if addition.path_in_repo == "README.md": - with addition.as_file() as file: - content = file.read().decode() - self._validate_yaml(content, repo_type=repo_type, token=token) - # Skip other additions after `README.md` has been processed - break - - # If updating twice the same file or update then delete a file in a single commit - _warn_on_overwriting_operations(operations) - - self.preupload_lfs_files( - repo_id=repo_id, - additions=additions, - token=token, - repo_type=repo_type, - revision=unquoted_revision, # first-class methods take unquoted revision - create_pr=create_pr, - num_threads=num_threads, - free_memory=False, # do not remove `CommitOperationAdd.path_or_fileobj` on LFS files for "normal" users - ) - - files_to_copy = _fetch_files_to_copy( - copies=copies, - repo_type=repo_type, - repo_id=repo_id, - headers=headers, - revision=unquoted_revision, - endpoint=self.endpoint, - ) - # Remove no-op operations (files that have not changed) - operations_without_no_op = [] - for operation in operations: - if ( - isinstance(operation, CommitOperationAdd) - and operation._remote_oid is not None - and operation._remote_oid == operation._local_oid - ): - # File already exists on the Hub and has not changed: we can skip it. - logger.debug(f"Skipping upload for '{operation.path_in_repo}' as the file has not changed.") - continue - if ( - isinstance(operation, CommitOperationCopy) - and operation._dest_oid is not None - and operation._dest_oid == operation._src_oid - ): - # Source and destination files are identical - skip - logger.debug( - f"Skipping copy for '{operation.src_path_in_repo}' -> '{operation.path_in_repo}' as the content of the source file is the same as the destination file." - ) - continue - operations_without_no_op.append(operation) - if len(operations) != len(operations_without_no_op): - logger.info( - f"Removing {len(operations) - len(operations_without_no_op)} file(s) from commit that have not changed." - ) - - # Return early if empty commit - if len(operations_without_no_op) == 0: - logger.warning("No files have been modified since last commit. Skipping to prevent empty commit.") - - # Get latest commit info - try: - info = self.repo_info(repo_id=repo_id, repo_type=repo_type, revision=unquoted_revision, token=token) - except RepositoryNotFoundError as e: - e.append_to_message(_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE) - raise - - # Return commit info based on latest commit - url_prefix = self.endpoint - if repo_type is not None and repo_type != constants.REPO_TYPE_MODEL: - url_prefix = f"{url_prefix}/{repo_type}s" - return CommitInfo( - commit_url=f"{url_prefix}/{repo_id}/commit/{info.sha}", - commit_message=commit_message, - commit_description=commit_description, - oid=info.sha, # type: ignore[arg-type] - ) - - commit_payload = _prepare_commit_payload( - operations=operations, - files_to_copy=files_to_copy, - commit_message=commit_message, - commit_description=commit_description, - parent_commit=parent_commit, - ) - commit_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/commit/{revision}" - - def _payload_as_ndjson() -> Iterable[bytes]: - for item in commit_payload: - yield json.dumps(item).encode() - yield b"\n" - - headers = { - # See https://github.com/huggingface/huggingface_hub/issues/1085#issuecomment-1265208073 - "Content-Type": "application/x-ndjson", - **headers, - } - data = b"".join(_payload_as_ndjson()) - params = {"create_pr": "1"} if create_pr else None - - try: - commit_resp = get_session().post(url=commit_url, headers=headers, data=data, params=params) - hf_raise_for_status(commit_resp, endpoint_name="commit") - except RepositoryNotFoundError as e: - e.append_to_message(_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE) - raise - except EntryNotFoundError as e: - if nb_deletions > 0 and "A file with this name doesn't exist" in str(e): - e.append_to_message( - "\nMake sure to differentiate file and folder paths in delete" - " operations with a trailing '/' or using `is_folder=True/False`." - ) - raise - - # Mark additions as committed (cannot be reused in another commit) - for addition in additions: - addition._is_committed = True - - commit_data = commit_resp.json() - return CommitInfo( - commit_url=commit_data["commitUrl"], - commit_message=commit_message, - commit_description=commit_description, - oid=commit_data["commitOid"], - pr_url=commit_data["pullRequestUrl"] if create_pr else None, - ) - - def preupload_lfs_files( - self, - repo_id: str, - additions: Iterable[CommitOperationAdd], - *, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - create_pr: Optional[bool] = None, - num_threads: int = 5, - free_memory: bool = True, - gitignore_content: Optional[str] = None, - ): - """Pre-upload LFS files to S3 in preparation on a future commit. - - This method is useful if you are generating the files to upload on-the-fly and you don't want to store them - in memory before uploading them all at once. - - > [!WARNING] - > This is a power-user method. You shouldn't need to call it directly to make a normal commit. - > Use [`create_commit`] directly instead. - - > [!WARNING] - > Commit operations will be mutated during the process. In particular, the attached `path_or_fileobj` will be - > removed after the upload to save memory (and replaced by an empty `bytes` object). Do not reuse the same - > objects except to pass them to [`create_commit`]. If you don't want to remove the attached content from the - > commit operation object, pass `free_memory=False`. - - Args: - repo_id (`str`): - The repository in which you will commit the files, for example: `"username/custom_transformers"`. - - operations (`Iterable` of [`CommitOperationAdd`]): - The list of files to upload. Warning: the objects in this list will be mutated to include information - relative to the upload. Do not reuse the same objects for multiple commits. - - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - repo_type (`str`, *optional*): - The type of repository to upload to (e.g. `"model"` -default-, `"dataset"` or `"space"`). - - revision (`str`, *optional*): - The git revision to commit from. Defaults to the head of the `"main"` branch. - - create_pr (`boolean`, *optional*): - Whether or not you plan to create a Pull Request with that commit. Defaults to `False`. - - num_threads (`int`, *optional*): - Number of concurrent threads for uploading files. Defaults to 5. - Setting it to 2 means at most 2 files will be uploaded concurrently. - - gitignore_content (`str`, *optional*): - The content of the `.gitignore` file to know which files should be ignored. The order of priority - is to first check if `gitignore_content` is passed, then check if the `.gitignore` file is present - in the list of files to commit and finally default to the `.gitignore` file already hosted on the Hub - (if any). - - Example: - ```py - >>> from huggingface_hub import CommitOperationAdd, preupload_lfs_files, create_commit, create_repo - - >>> repo_id = create_repo("test_preupload").repo_id - - # Generate and preupload LFS files one by one - >>> operations = [] # List of all `CommitOperationAdd` objects that will be generated - >>> for i in range(5): - ... content = ... # generate binary content - ... addition = CommitOperationAdd(path_in_repo=f"shard_{i}_of_5.bin", path_or_fileobj=content) - ... preupload_lfs_files(repo_id, additions=[addition]) # upload + free memory - ... operations.append(addition) - - # Create commit - >>> create_commit(repo_id, operations=operations, commit_message="Commit all shards") - ``` - """ - repo_type = repo_type if repo_type is not None else constants.REPO_TYPE_MODEL - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION - create_pr = create_pr if create_pr is not None else False - headers = self._build_hf_headers(token=token) - - # Check if a `gitignore` file is being committed to the Hub. - additions = list(additions) - if gitignore_content is None: - for addition in additions: - if addition.path_in_repo == ".gitignore": - with addition.as_file() as f: - gitignore_content = f.read().decode() - break - - # Filter out already uploaded files - new_additions = [addition for addition in additions if not addition._is_uploaded] - - # Check which new files are LFS - # For some items, we might have already fetched the upload mode (in case of upload_large_folder) - additions_no_upload_mode = [addition for addition in new_additions if addition._upload_mode is None] - if len(additions_no_upload_mode) > 0: - try: - _fetch_upload_modes( - additions=additions_no_upload_mode, - repo_type=repo_type, - repo_id=repo_id, - headers=headers, - revision=revision, - endpoint=self.endpoint, - create_pr=create_pr or False, - gitignore_content=gitignore_content, - ) - except RepositoryNotFoundError as e: - e.append_to_message(_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE) - raise - - # Filter out regular files - new_lfs_additions = [addition for addition in new_additions if addition._upload_mode == "lfs"] - - # Filter out files listed in .gitignore - new_lfs_additions_to_upload = [] - for addition in new_lfs_additions: - if addition._should_ignore: - logger.debug(f"Skipping upload for LFS file '{addition.path_in_repo}' (ignored by gitignore file).") - else: - new_lfs_additions_to_upload.append(addition) - if len(new_lfs_additions) != len(new_lfs_additions_to_upload): - logger.info( - f"Skipped upload for {len(new_lfs_additions) - len(new_lfs_additions_to_upload)} LFS file(s) " - "(ignored by gitignore file)." - ) - # If no LFS files remain to upload, keep previous behavior and log explicitly - if len(new_lfs_additions_to_upload) == 0: - logger.debug("No LFS files to upload.") - return - # Prepare upload parameters - upload_kwargs = { - "additions": new_lfs_additions_to_upload, - "repo_type": repo_type, - "repo_id": repo_id, - "headers": headers, - "endpoint": self.endpoint, - # If `create_pr`, we don't want to check user permission on the revision as users with read permission - # should still be able to create PRs even if they don't have write permission on the target branch of the - # PR (i.e. `revision`). - "revision": revision if not create_pr else None, - } - _upload_files(**upload_kwargs, num_threads=num_threads, create_pr=create_pr) # type: ignore [arg-type] - for addition in new_lfs_additions_to_upload: - addition._is_uploaded = True - if free_memory: - addition.path_or_fileobj = b"" - - @overload - def upload_file( # type: ignore - self, - *, - path_or_fileobj: Union[str, Path, bytes, BinaryIO], - path_in_repo: str, - repo_id: str, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - commit_message: Optional[str] = None, - commit_description: Optional[str] = None, - create_pr: Optional[bool] = None, - parent_commit: Optional[str] = None, - run_as_future: Literal[False] = ..., - ) -> CommitInfo: ... - - @overload - def upload_file( - self, - *, - path_or_fileobj: Union[str, Path, bytes, BinaryIO], - path_in_repo: str, - repo_id: str, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - commit_message: Optional[str] = None, - commit_description: Optional[str] = None, - create_pr: Optional[bool] = None, - parent_commit: Optional[str] = None, - run_as_future: Literal[True] = ..., - ) -> Future[CommitInfo]: ... - - @validate_hf_hub_args - @future_compatible - def upload_file( - self, - *, - path_or_fileobj: Union[str, Path, bytes, BinaryIO], - path_in_repo: str, - repo_id: str, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - commit_message: Optional[str] = None, - commit_description: Optional[str] = None, - create_pr: Optional[bool] = None, - parent_commit: Optional[str] = None, - run_as_future: bool = False, - ) -> Union[CommitInfo, Future[CommitInfo]]: - """ - Upload a local file (up to 50 GB) to the given repo. The upload is done - through a HTTP post request, and doesn't require git or git-lfs to be - installed. - - Args: - path_or_fileobj (`str`, `Path`, `bytes`, or `IO`): - Path to a file on the local machine or binary data stream / - fileobj / buffer. - path_in_repo (`str`): - Relative filepath in the repo, for example: - `"checkpoints/1fec34a/weights.bin"` - repo_id (`str`): - The repository to which the file will be uploaded, for example: - `"username/custom_transformers"` - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - revision (`str`, *optional*): - The git revision to commit from. Defaults to the head of the `"main"` branch. - commit_message (`str`, *optional*): - The summary / title / first line of the generated commit - commit_description (`str` *optional*) - The description of the generated commit - create_pr (`boolean`, *optional*): - Whether or not to create a Pull Request with that commit. Defaults to `False`. - If `revision` is not set, PR is opened against the `"main"` branch. If - `revision` is set and is a branch, PR is opened against this branch. If - `revision` is set and is not a branch name (example: a commit oid), an - `RevisionNotFoundError` is returned by the server. - parent_commit (`str`, *optional*): - The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. - If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. - If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. - Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be - especially useful if the repo is updated / committed to concurrently. - run_as_future (`bool`, *optional*): - Whether or not to run this method in the background. Background jobs are run sequentially without - blocking the main thread. Passing `run_as_future=True` will return a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) - object. Defaults to `False`. - - - Returns: - [`CommitInfo`] or `Future`: - Instance of [`CommitInfo`] containing information about the newly created commit (commit hash, commit - url, pr url, commit message,...). If `run_as_future=True` is passed, returns a Future object which will - contain the result when executed. - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - > - [`~utils.RevisionNotFoundError`] - > If the revision to download from cannot be found. - - > [!WARNING] - > `upload_file` assumes that the repo already exists on the Hub. If you get a - > Client error 404, please make sure you are authenticated and that `repo_id` and - > `repo_type` are set correctly. If repo does not exist, create it first using - > [`~hf_api.create_repo`]. - - Example: - - ```python - >>> from huggingface_hub import upload_file - - >>> with open("./local/filepath", "rb") as fobj: - ... upload_file( - ... path_or_fileobj=fileobj, - ... path_in_repo="remote/file/path.h5", - ... repo_id="username/my-dataset", - ... repo_type="dataset", - ... token="my_token", - ... ) - "https://huggingface.co/datasets/username/my-dataset/blob/main/remote/file/path.h5" - - >>> upload_file( - ... path_or_fileobj=".\\\\local\\\\file\\\\path", - ... path_in_repo="remote/file/path.h5", - ... repo_id="username/my-model", - ... token="my_token", - ... ) - "https://huggingface.co/username/my-model/blob/main/remote/file/path.h5" - - >>> upload_file( - ... path_or_fileobj=".\\\\local\\\\file\\\\path", - ... path_in_repo="remote/file/path.h5", - ... repo_id="username/my-model", - ... token="my_token", - ... create_pr=True, - ... ) - "https://huggingface.co/username/my-model/blob/refs%2Fpr%2F1/remote/file/path.h5" - ``` - """ - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - - commit_message = ( - commit_message if commit_message is not None else f"Upload {path_in_repo} with huggingface_hub" - ) - operation = CommitOperationAdd( - path_or_fileobj=path_or_fileobj, - path_in_repo=path_in_repo, - ) - - commit_info = self.create_commit( - repo_id=repo_id, - repo_type=repo_type, - operations=[operation], - commit_message=commit_message, - commit_description=commit_description, - token=token, - revision=revision, - create_pr=create_pr, - parent_commit=parent_commit, - ) - - if commit_info.pr_url is not None: - revision = quote(_parse_revision_from_pr_url(commit_info.pr_url), safe="") - if repo_type in constants.REPO_TYPES_URL_PREFIXES: - repo_id = constants.REPO_TYPES_URL_PREFIXES[repo_type] + repo_id - revision = revision if revision is not None else constants.DEFAULT_REVISION - - return CommitInfo( - commit_url=commit_info.commit_url, - commit_message=commit_info.commit_message, - commit_description=commit_info.commit_description, - oid=commit_info.oid, - pr_url=commit_info.pr_url, - # Similar to `hf_hub_url` but it's "blob" instead of "resolve" - # TODO: remove this in v1.0 - _url=f"{self.endpoint}/{repo_id}/blob/{revision}/{path_in_repo}", - ) - - @overload - def upload_folder( # type: ignore - self, - *, - repo_id: str, - folder_path: Union[str, Path], - path_in_repo: Optional[str] = None, - commit_message: Optional[str] = None, - commit_description: Optional[str] = None, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - create_pr: Optional[bool] = None, - parent_commit: Optional[str] = None, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - delete_patterns: Optional[Union[List[str], str]] = None, - run_as_future: Literal[False] = ..., - ) -> CommitInfo: ... - - @overload - def upload_folder( # type: ignore - self, - *, - repo_id: str, - folder_path: Union[str, Path], - path_in_repo: Optional[str] = None, - commit_message: Optional[str] = None, - commit_description: Optional[str] = None, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - create_pr: Optional[bool] = None, - parent_commit: Optional[str] = None, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - delete_patterns: Optional[Union[List[str], str]] = None, - run_as_future: Literal[True] = ..., - ) -> Future[CommitInfo]: ... - - @validate_hf_hub_args - @future_compatible - def upload_folder( - self, - *, - repo_id: str, - folder_path: Union[str, Path], - path_in_repo: Optional[str] = None, - commit_message: Optional[str] = None, - commit_description: Optional[str] = None, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - create_pr: Optional[bool] = None, - parent_commit: Optional[str] = None, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - delete_patterns: Optional[Union[List[str], str]] = None, - run_as_future: bool = False, - ) -> Union[CommitInfo, Future[CommitInfo]]: - """ - Upload a local folder to the given repo. The upload is done through a HTTP requests, and doesn't require git or - git-lfs to be installed. - - The structure of the folder will be preserved. Files with the same name already present in the repository will - be overwritten. Others will be left untouched. - - Use the `allow_patterns` and `ignore_patterns` arguments to specify which files to upload. These parameters - accept either a single pattern or a list of patterns. Patterns are Standard Wildcards (globbing patterns) as - documented [here](https://tldp.org/LDP/GNU-Linux-Tools-Summary/html/x11655.htm). If both `allow_patterns` and - `ignore_patterns` are provided, both constraints apply. By default, all files from the folder are uploaded. - - Use the `delete_patterns` argument to specify remote files you want to delete. Input type is the same as for - `allow_patterns` (see above). If `path_in_repo` is also provided, the patterns are matched against paths - relative to this folder. For example, `upload_folder(..., path_in_repo="experiment", delete_patterns="logs/*")` - will delete any remote file under `./experiment/logs/`. Note that the `.gitattributes` file will not be deleted - even if it matches the patterns. - - Any `.git/` folder present in any subdirectory will be ignored. However, please be aware that the `.gitignore` - file is not taken into account. - - Uses `HfApi.create_commit` under the hood. - - Args: - repo_id (`str`): - The repository to which the file will be uploaded, for example: - `"username/custom_transformers"` - folder_path (`str` or `Path`): - Path to the folder to upload on the local file system - path_in_repo (`str`, *optional*): - Relative path of the directory in the repo, for example: - `"checkpoints/1fec34a/results"`. Will default to the root folder of the repository. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - revision (`str`, *optional*): - The git revision to commit from. Defaults to the head of the `"main"` branch. - commit_message (`str`, *optional*): - The summary / title / first line of the generated commit. Defaults to: - `f"Upload {path_in_repo} with huggingface_hub"` - commit_description (`str` *optional*): - The description of the generated commit - create_pr (`boolean`, *optional*): - Whether or not to create a Pull Request with that commit. Defaults to `False`. If `revision` is not - set, PR is opened against the `"main"` branch. If `revision` is set and is a branch, PR is opened - against this branch. If `revision` is set and is not a branch name (example: a commit oid), an - `RevisionNotFoundError` is returned by the server. - parent_commit (`str`, *optional*): - The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. - If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. - If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. - Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be - especially useful if the repo is updated / committed to concurrently. - allow_patterns (`List[str]` or `str`, *optional*): - If provided, only files matching at least one pattern are uploaded. - ignore_patterns (`List[str]` or `str`, *optional*): - If provided, files matching any of the patterns are not uploaded. - delete_patterns (`List[str]` or `str`, *optional*): - If provided, remote files matching any of the patterns will be deleted from the repo while committing - new files. This is useful if you don't know which files have already been uploaded. - Note: to avoid discrepancies the `.gitattributes` file is not deleted even if it matches the pattern. - run_as_future (`bool`, *optional*): - Whether or not to run this method in the background. Background jobs are run sequentially without - blocking the main thread. Passing `run_as_future=True` will return a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) - object. Defaults to `False`. - - Returns: - [`CommitInfo`] or `Future`: - Instance of [`CommitInfo`] containing information about the newly created commit (commit hash, commit - url, pr url, commit message,...). If `run_as_future=True` is passed, returns a Future object which will - contain the result when executed. - - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - - > [!WARNING] - > `upload_folder` assumes that the repo already exists on the Hub. If you get a Client error 404, please make - > sure you are authenticated and that `repo_id` and `repo_type` are set correctly. If repo does not exist, create - > it first using [`~hf_api.create_repo`]. - - > [!TIP] - > When dealing with a large folder (thousands of files or hundreds of GB), we recommend using [`~hf_api.upload_large_folder`] instead. - - Example: - - ```python - # Upload checkpoints folder except the log files - >>> upload_folder( - ... folder_path="local/checkpoints", - ... path_in_repo="remote/experiment/checkpoints", - ... repo_id="username/my-dataset", - ... repo_type="datasets", - ... token="my_token", - ... ignore_patterns="**/logs/*.txt", - ... ) - # "https://huggingface.co/datasets/username/my-dataset/tree/main/remote/experiment/checkpoints" - - # Upload checkpoints folder including logs while deleting existing logs from the repo - # Useful if you don't know exactly which log files have already being pushed - >>> upload_folder( - ... folder_path="local/checkpoints", - ... path_in_repo="remote/experiment/checkpoints", - ... repo_id="username/my-dataset", - ... repo_type="datasets", - ... token="my_token", - ... delete_patterns="**/logs/*.txt", - ... ) - "https://huggingface.co/datasets/username/my-dataset/tree/main/remote/experiment/checkpoints" - - # Upload checkpoints folder while creating a PR - >>> upload_folder( - ... folder_path="local/checkpoints", - ... path_in_repo="remote/experiment/checkpoints", - ... repo_id="username/my-dataset", - ... repo_type="datasets", - ... token="my_token", - ... create_pr=True, - ... ) - "https://huggingface.co/datasets/username/my-dataset/tree/refs%2Fpr%2F1/remote/experiment/checkpoints" - - ``` - """ - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - - # By default, upload folder to the root directory in repo. - if path_in_repo is None: - path_in_repo = "" - - # Do not upload .git folder - if ignore_patterns is None: - ignore_patterns = [] - elif isinstance(ignore_patterns, str): - ignore_patterns = [ignore_patterns] - ignore_patterns += DEFAULT_IGNORE_PATTERNS - - delete_operations = self._prepare_folder_deletions( - repo_id=repo_id, - repo_type=repo_type, - revision=constants.DEFAULT_REVISION if create_pr else revision, - token=token, - path_in_repo=path_in_repo, - delete_patterns=delete_patterns, - ) - add_operations = self._prepare_upload_folder_additions( - folder_path, - path_in_repo, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - token=token, - repo_type=repo_type, - ) - - # Optimize operations: if some files will be overwritten, we don't need to delete them first - if len(add_operations) > 0: - added_paths = set(op.path_in_repo for op in add_operations) - delete_operations = [ - delete_op for delete_op in delete_operations if delete_op.path_in_repo not in added_paths - ] - commit_operations = delete_operations + add_operations - - commit_message = commit_message or "Upload folder using huggingface_hub" - - commit_info = self.create_commit( - repo_type=repo_type, - repo_id=repo_id, - operations=commit_operations, - commit_message=commit_message, - commit_description=commit_description, - token=token, - revision=revision, - create_pr=create_pr, - parent_commit=parent_commit, - ) - - # Create url to uploaded folder (for legacy return value) - if create_pr and commit_info.pr_url is not None: - revision = quote(_parse_revision_from_pr_url(commit_info.pr_url), safe="") - if repo_type in constants.REPO_TYPES_URL_PREFIXES: - repo_id = constants.REPO_TYPES_URL_PREFIXES[repo_type] + repo_id - revision = revision if revision is not None else constants.DEFAULT_REVISION - - return CommitInfo( - commit_url=commit_info.commit_url, - commit_message=commit_info.commit_message, - commit_description=commit_info.commit_description, - oid=commit_info.oid, - pr_url=commit_info.pr_url, - # Similar to `hf_hub_url` but it's "tree" instead of "resolve" - # TODO: remove this in v1.0 - _url=f"{self.endpoint}/{repo_id}/tree/{revision}/{path_in_repo}", - ) - - @validate_hf_hub_args - def delete_file( - self, - path_in_repo: str, - repo_id: str, - *, - token: Union[str, bool, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - commit_message: Optional[str] = None, - commit_description: Optional[str] = None, - create_pr: Optional[bool] = None, - parent_commit: Optional[str] = None, - ) -> CommitInfo: - """ - Deletes a file in the given repo. - - Args: - path_in_repo (`str`): - Relative filepath in the repo, for example: - `"checkpoints/1fec34a/weights.bin"` - repo_id (`str`): - The repository from which the file will be deleted, for example: - `"username/custom_transformers"` - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if the file is in a dataset or - space, `None` or `"model"` if in a model. Default is `None`. - revision (`str`, *optional*): - The git revision to commit from. Defaults to the head of the `"main"` branch. - commit_message (`str`, *optional*): - The summary / title / first line of the generated commit. Defaults to - `f"Delete {path_in_repo} with huggingface_hub"`. - commit_description (`str` *optional*) - The description of the generated commit - create_pr (`boolean`, *optional*): - Whether or not to create a Pull Request with that commit. Defaults to `False`. - If `revision` is not set, PR is opened against the `"main"` branch. If - `revision` is set and is a branch, PR is opened against this branch. If - `revision` is set and is not a branch name (example: a commit oid), an - `RevisionNotFoundError` is returned by the server. - parent_commit (`str`, *optional*): - The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. - If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. - If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. - Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be - especially useful if the repo is updated / committed to concurrently. - - - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - > - [`~utils.RevisionNotFoundError`] - > If the revision to download from cannot be found. - > - [`~utils.EntryNotFoundError`] - > If the file to download cannot be found. - - """ - commit_message = ( - commit_message if commit_message is not None else f"Delete {path_in_repo} with huggingface_hub" - ) - - operations = [CommitOperationDelete(path_in_repo=path_in_repo)] - - return self.create_commit( - repo_id=repo_id, - repo_type=repo_type, - token=token, - operations=operations, - revision=revision, - commit_message=commit_message, - commit_description=commit_description, - create_pr=create_pr, - parent_commit=parent_commit, - ) - - @validate_hf_hub_args - def delete_files( - self, - repo_id: str, - delete_patterns: List[str], - *, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - commit_message: Optional[str] = None, - commit_description: Optional[str] = None, - create_pr: Optional[bool] = None, - parent_commit: Optional[str] = None, - ) -> CommitInfo: - """ - Delete files from a repository on the Hub. - - If a folder path is provided, the entire folder is deleted as well as - all files it contained. - - Args: - repo_id (`str`): - The repository from which the folder will be deleted, for example: - `"username/custom_transformers"` - delete_patterns (`List[str]`): - List of files or folders to delete. Each string can either be - a file path, a folder path or a Unix shell-style wildcard. - E.g. `["file.txt", "folder/", "data/*.parquet"]` - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - to the stored token. - repo_type (`str`, *optional*): - Type of the repo to delete files from. Can be `"model"`, - `"dataset"` or `"space"`. Defaults to `"model"`. - revision (`str`, *optional*): - The git revision to commit from. Defaults to the head of the `"main"` branch. - commit_message (`str`, *optional*): - The summary (first line) of the generated commit. Defaults to - `f"Delete files using huggingface_hub"`. - commit_description (`str` *optional*) - The description of the generated commit. - create_pr (`boolean`, *optional*): - Whether or not to create a Pull Request with that commit. Defaults to `False`. - If `revision` is not set, PR is opened against the `"main"` branch. If - `revision` is set and is a branch, PR is opened against this branch. If - `revision` is set and is not a branch name (example: a commit oid), an - `RevisionNotFoundError` is returned by the server. - parent_commit (`str`, *optional*): - The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. - If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. - If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. - Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be - especially useful if the repo is updated / committed to concurrently. - """ - operations = self._prepare_folder_deletions( - repo_id=repo_id, repo_type=repo_type, delete_patterns=delete_patterns, path_in_repo="", revision=revision - ) - - if commit_message is None: - commit_message = f"Delete files {' '.join(delete_patterns)} with huggingface_hub" - - return self.create_commit( - repo_id=repo_id, - repo_type=repo_type, - token=token, - operations=operations, - revision=revision, - commit_message=commit_message, - commit_description=commit_description, - create_pr=create_pr, - parent_commit=parent_commit, - ) - - @validate_hf_hub_args - def delete_folder( - self, - path_in_repo: str, - repo_id: str, - *, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - commit_message: Optional[str] = None, - commit_description: Optional[str] = None, - create_pr: Optional[bool] = None, - parent_commit: Optional[str] = None, - ) -> CommitInfo: - """ - Deletes a folder in the given repo. - - Simple wrapper around [`create_commit`] method. - - Args: - path_in_repo (`str`): - Relative folder path in the repo, for example: `"checkpoints/1fec34a"`. - repo_id (`str`): - The repository from which the folder will be deleted, for example: - `"username/custom_transformers"` - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - to the stored token. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if the folder is in a dataset or - space, `None` or `"model"` if in a model. Default is `None`. - revision (`str`, *optional*): - The git revision to commit from. Defaults to the head of the `"main"` branch. - commit_message (`str`, *optional*): - The summary / title / first line of the generated commit. Defaults to - `f"Delete folder {path_in_repo} with huggingface_hub"`. - commit_description (`str` *optional*) - The description of the generated commit. - create_pr (`boolean`, *optional*): - Whether or not to create a Pull Request with that commit. Defaults to `False`. - If `revision` is not set, PR is opened against the `"main"` branch. If - `revision` is set and is a branch, PR is opened against this branch. If - `revision` is set and is not a branch name (example: a commit oid), an - `RevisionNotFoundError` is returned by the server. - parent_commit (`str`, *optional*): - The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. - If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. - If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. - Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be - especially useful if the repo is updated / committed to concurrently. - """ - return self.create_commit( - repo_id=repo_id, - repo_type=repo_type, - token=token, - operations=[CommitOperationDelete(path_in_repo=path_in_repo, is_folder=True)], - revision=revision, - commit_message=( - commit_message if commit_message is not None else f"Delete folder {path_in_repo} with huggingface_hub" - ), - commit_description=commit_description, - create_pr=create_pr, - parent_commit=parent_commit, - ) - - def upload_large_folder( - self, - repo_id: str, - folder_path: Union[str, Path], - *, - repo_type: str, # Repo type is required! - revision: Optional[str] = None, - private: Optional[bool] = None, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - num_workers: Optional[int] = None, - print_report: bool = True, - print_report_every: int = 60, - ) -> None: - """Upload a large folder to the Hub in the most resilient way possible. - - Several workers are started to upload files in an optimized way. Before being committed to a repo, files must be - hashed and be pre-uploaded if they are LFS files. Workers will perform these tasks for each file in the folder. - At each step, some metadata information about the upload process is saved in the folder under `.cache/.huggingface/` - to be able to resume the process if interrupted. The whole process might result in several commits. - - Args: - repo_id (`str`): - The repository to which the file will be uploaded. - E.g. `"HuggingFaceTB/smollm-corpus"`. - folder_path (`str` or `Path`): - Path to the folder to upload on the local file system. - repo_type (`str`): - Type of the repository. Must be one of `"model"`, `"dataset"` or `"space"`. - Unlike in all other `HfApi` methods, `repo_type` is explicitly required here. This is to avoid - any mistake when uploading a large folder to the Hub, and therefore prevent from having to re-upload - everything. - revision (`str`, `optional`): - The branch to commit to. If not provided, the `main` branch will be used. - private (`bool`, `optional`): - Whether the repository should be private. - If `None` (default), the repo will be public unless the organization's default is private. - allow_patterns (`List[str]` or `str`, *optional*): - If provided, only files matching at least one pattern are uploaded. - ignore_patterns (`List[str]` or `str`, *optional*): - If provided, files matching any of the patterns are not uploaded. - num_workers (`int`, *optional*): - Number of workers to start. Defaults to `os.cpu_count() - 2` (minimum 2). - A higher number of workers may speed up the process if your machine allows it. However, on machines with a - slower connection, it is recommended to keep the number of workers low to ensure better resumability. - Indeed, partially uploaded files will have to be completely re-uploaded if the process is interrupted. - print_report (`bool`, *optional*): - Whether to print a report of the upload progress. Defaults to True. - Report is printed to `sys.stdout` every X seconds (60 by defaults) and overwrites the previous report. - print_report_every (`int`, *optional*): - Frequency at which the report is printed. Defaults to 60 seconds. - - > [!TIP] - > A few things to keep in mind: - > - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations - > - Do not start several processes in parallel. - > - You can interrupt and resume the process at any time. - > - Do not upload the same folder to several repositories. If you need to do so, you must delete the local `.cache/.huggingface/` folder first. - - > [!WARNING] - > While being much more robust to upload large folders, `upload_large_folder` is more limited than [`upload_folder`] feature-wise. In practice: - > - you cannot set a custom `path_in_repo`. If you want to upload to a subfolder, you need to set the proper structure locally. - > - you cannot set a custom `commit_message` and `commit_description` since multiple commits are created. - > - you cannot delete from the repo while uploading. Please make a separate commit first. - > - you cannot create a PR directly. Please create a PR first (from the UI or using [`create_pull_request`]) and then commit to it by passing `revision`. - - **Technical details:** - - `upload_large_folder` process is as follow: - 1. (Check parameters and setup.) - 2. Create repo if missing. - 3. List local files to upload. - 4. Run validation checks and display warnings if repository limits might be exceeded: - - Warns if the total number of files exceeds 100k (recommended limit). - - Warns if any folder contains more than 10k files (recommended limit). - - Warns about files larger than 20GB (recommended) or 50GB (hard limit). - 5. Start workers. Workers can perform the following tasks: - - Hash a file. - - Get upload mode (regular or LFS) for a list of files. - - Pre-upload an LFS file. - - Commit a bunch of files. - Once a worker finishes a task, it will move on to the next task based on the priority list (see below) until - all files are uploaded and committed. - 6. While workers are up, regularly print a report to sys.stdout. - - Order of priority: - 1. Commit if more than 5 minutes since last commit attempt (and at least 1 file). - 2. Commit if at least 150 files are ready to commit. - 3. Get upload mode if at least 10 files have been hashed. - 4. Pre-upload LFS file if at least 1 file and no worker is pre-uploading. - 5. Hash file if at least 1 file and no worker is hashing. - 6. Get upload mode if at least 1 file and no worker is getting upload mode. - 7. Pre-upload LFS file if at least 1 file (exception: if hf_transfer is enabled, only 1 worker can preupload LFS at a time). - 8. Hash file if at least 1 file to hash. - 9. Get upload mode if at least 1 file to get upload mode. - 10. Commit if at least 1 file to commit and at least 1 min since last commit attempt. - 11. Commit if at least 1 file to commit and all other queues are empty. - - Special rules: - - If `hf_transfer` is enabled, only 1 LFS uploader at a time. Otherwise the CPU would be bloated by `hf_transfer`. - - Only one worker can commit at a time. - - If no tasks are available, the worker waits for 10 seconds before checking again. - """ - return upload_large_folder_internal( - self, - repo_id=repo_id, - folder_path=folder_path, - repo_type=repo_type, - revision=revision, - private=private, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - num_workers=num_workers, - print_report=print_report, - print_report_every=print_report_every, - ) - - @validate_hf_hub_args - def get_hf_file_metadata( - self, - *, - url: str, - token: Union[bool, str, None] = None, - proxies: Optional[Dict] = None, - timeout: Optional[float] = constants.DEFAULT_REQUEST_TIMEOUT, - ) -> HfFileMetadata: - """Fetch metadata of a file versioned on the Hub for a given url. - - Args: - url (`str`): - File url, for example returned by [`hf_hub_url`]. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - proxies (`dict`, *optional*): - Dictionary mapping protocol to the URL of the proxy passed to `requests.request`. - timeout (`float`, *optional*, defaults to 10): - How many seconds to wait for the server to send metadata before giving up. - - Returns: - A [`HfFileMetadata`] object containing metadata such as location, etag, size and commit_hash. - """ - if token is None: - # Cannot do `token = token or self.token` as token can be `False`. - token = self.token - - return get_hf_file_metadata( - url=url, - token=token, - proxies=proxies, - timeout=timeout, - library_name=self.library_name, - library_version=self.library_version, - user_agent=self.user_agent, - endpoint=self.endpoint, - ) - - @validate_hf_hub_args - def hf_hub_download( - self, - repo_id: str, - filename: str, - *, - subfolder: Optional[str] = None, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - cache_dir: Union[str, Path, None] = None, - local_dir: Union[str, Path, None] = None, - force_download: bool = False, - proxies: Optional[Dict] = None, - etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, - token: Union[bool, str, None] = None, - local_files_only: bool = False, - # Deprecated args - resume_download: Optional[bool] = None, - force_filename: Optional[str] = None, - local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto", - ) -> str: - """Download a given file if it's not already present in the local cache. - - The new cache file layout looks like this: - - The cache directory contains one subfolder per repo_id (namespaced by repo type) - - inside each repo folder: - - refs is a list of the latest known revision => commit_hash pairs - - blobs contains the actual file blobs (identified by their git-sha or sha256, depending on - whether they're LFS files or not) - - snapshots contains one subfolder per commit, each "commit" contains the subset of the files - that have been resolved at that particular commit. Each filename is a symlink to the blob - at that particular commit. - - ``` - [ 96] . - └── [ 160] models--julien-c--EsperBERTo-small - ├── [ 160] blobs - │ ├── [321M] 403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd - │ ├── [ 398] 7cb18dc9bafbfcf74629a4b760af1b160957a83e - │ └── [1.4K] d7edf6bd2a681fb0175f7735299831ee1b22b812 - ├── [ 96] refs - │ └── [ 40] main - └── [ 128] snapshots - ├── [ 128] 2439f60ef33a0d46d85da5001d52aeda5b00ce9f - │ ├── [ 52] README.md -> ../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812 - │ └── [ 76] pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd - └── [ 128] bbc77c8132af1cc5cf678da3f1ddf2de43606d48 - ├── [ 52] README.md -> ../../blobs/7cb18dc9bafbfcf74629a4b760af1b160957a83e - └── [ 76] pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd - ``` - - If `local_dir` is provided, the file structure from the repo will be replicated in this location. When using this - option, the `cache_dir` will not be used and a `.cache/huggingface/` folder will be created at the root of `local_dir` - to store some metadata related to the downloaded files. While this mechanism is not as robust as the main - cache-system, it's optimized for regularly pulling the latest version of a repository. - - Args: - repo_id (`str`): - A user or an organization name and a repo name separated by a `/`. - filename (`str`): - The name of the file in the repo. - subfolder (`str`, *optional*): - An optional value corresponding to a folder inside the repository. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if downloading from a dataset or space, - `None` or `"model"` if downloading from a model. Default is `None`. - revision (`str`, *optional*): - An optional Git revision id which can be a branch name, a tag, or a - commit hash. - cache_dir (`str`, `Path`, *optional*): - Path to the folder where cached files are stored. - local_dir (`str` or `Path`, *optional*): - If provided, the downloaded file will be placed under this directory. - force_download (`bool`, *optional*, defaults to `False`): - Whether the file should be downloaded even if it already exists in - the local cache. - proxies (`dict`, *optional*): - Dictionary mapping protocol to the URL of the proxy passed to - `requests.request`. - etag_timeout (`float`, *optional*, defaults to `10`): - When fetching ETag, how many seconds to wait for the server to send - data before giving up which is passed to `requests.request`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - local_files_only (`bool`, *optional*, defaults to `False`): - If `True`, avoid downloading the file and return the path to the - local cached file if it exists. - - Returns: - `str`: Local path of file or if networking is off, last version of file cached on disk. - - Raises: - [`~utils.RepositoryNotFoundError`] - If the repository to download from cannot be found. This may be because it doesn't exist, - or because it is set to `private` and you do not have access. - [`~utils.RevisionNotFoundError`] - If the revision to download from cannot be found. - [`~utils.EntryNotFoundError`] - If the file to download cannot be found. - [`~utils.LocalEntryNotFoundError`] - If network is disabled or unavailable and file is not found in cache. - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) - If `token=True` but the token cannot be found. - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) - If ETag cannot be determined. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If some parameter value is invalid. - """ - from .file_download import hf_hub_download - - if token is None: - # Cannot do `token = token or self.token` as token can be `False`. - token = self.token - - return hf_hub_download( - repo_id=repo_id, - filename=filename, - subfolder=subfolder, - repo_type=repo_type, - revision=revision, - endpoint=self.endpoint, - library_name=self.library_name, - library_version=self.library_version, - cache_dir=cache_dir, - local_dir=local_dir, - local_dir_use_symlinks=local_dir_use_symlinks, - user_agent=self.user_agent, - force_download=force_download, - force_filename=force_filename, - proxies=proxies, - etag_timeout=etag_timeout, - resume_download=resume_download, - token=token, - headers=self.headers, - local_files_only=local_files_only, - ) - - @validate_hf_hub_args - def snapshot_download( - self, - repo_id: str, - *, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - cache_dir: Union[str, Path, None] = None, - local_dir: Union[str, Path, None] = None, - proxies: Optional[Dict] = None, - etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, - force_download: bool = False, - token: Union[bool, str, None] = None, - local_files_only: bool = False, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - max_workers: int = 8, - tqdm_class: Optional[Type[base_tqdm]] = None, - # Deprecated args - local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto", - resume_download: Optional[bool] = None, - ) -> str: - """Download repo files. - - Download a whole snapshot of a repo's files at the specified revision. This is useful when you want all files from - a repo, because you don't know which ones you will need a priori. All files are nested inside a folder in order - to keep their actual filename relative to that folder. You can also filter which files to download using - `allow_patterns` and `ignore_patterns`. - - If `local_dir` is provided, the file structure from the repo will be replicated in this location. When using this - option, the `cache_dir` will not be used and a `.cache/huggingface/` folder will be created at the root of `local_dir` - to store some metadata related to the downloaded files.While this mechanism is not as robust as the main - cache-system, it's optimized for regularly pulling the latest version of a repository. - - An alternative would be to clone the repo but this requires git and git-lfs to be installed and properly - configured. It is also not possible to filter which files to download when cloning a repository using git. - - Args: - repo_id (`str`): - A user or an organization name and a repo name separated by a `/`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if downloading from a dataset or space, - `None` or `"model"` if downloading from a model. Default is `None`. - revision (`str`, *optional*): - An optional Git revision id which can be a branch name, a tag, or a - commit hash. - cache_dir (`str`, `Path`, *optional*): - Path to the folder where cached files are stored. - local_dir (`str` or `Path`, *optional*): - If provided, the downloaded files will be placed under this directory. - proxies (`dict`, *optional*): - Dictionary mapping protocol to the URL of the proxy passed to - `requests.request`. - etag_timeout (`float`, *optional*, defaults to `10`): - When fetching ETag, how many seconds to wait for the server to send - data before giving up which is passed to `requests.request`. - force_download (`bool`, *optional*, defaults to `False`): - Whether the file should be downloaded even if it already exists in the local cache. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - local_files_only (`bool`, *optional*, defaults to `False`): - If `True`, avoid downloading the file and return the path to the - local cached file if it exists. - allow_patterns (`List[str]` or `str`, *optional*): - If provided, only files matching at least one pattern are downloaded. - ignore_patterns (`List[str]` or `str`, *optional*): - If provided, files matching any of the patterns are not downloaded. - max_workers (`int`, *optional*): - Number of concurrent threads to download files (1 thread = 1 file download). - Defaults to 8. - tqdm_class (`tqdm`, *optional*): - If provided, overwrites the default behavior for the progress bar. Passed - argument must inherit from `tqdm.auto.tqdm` or at least mimic its behavior. - Note that the `tqdm_class` is not passed to each individual download. - Defaults to the custom HF progress bar that can be disabled by setting - `HF_HUB_DISABLE_PROGRESS_BARS` environment variable. - - Returns: - `str`: folder path of the repo snapshot. - - Raises: - [`~utils.RepositoryNotFoundError`] - If the repository to download from cannot be found. This may be because it doesn't exist, - or because it is set to `private` and you do not have access. - [`~utils.RevisionNotFoundError`] - If the revision to download from cannot be found. - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) - If `token=True` and the token cannot be found. - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if - ETag cannot be determined. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - if some parameter value is invalid. - """ - from ._snapshot_download import snapshot_download - - if token is None: - # Cannot do `token = token or self.token` as token can be `False`. - token = self.token - - return snapshot_download( - repo_id=repo_id, - repo_type=repo_type, - revision=revision, - endpoint=self.endpoint, - cache_dir=cache_dir, - local_dir=local_dir, - local_dir_use_symlinks=local_dir_use_symlinks, - library_name=self.library_name, - library_version=self.library_version, - user_agent=self.user_agent, - proxies=proxies, - etag_timeout=etag_timeout, - resume_download=resume_download, - force_download=force_download, - token=token, - local_files_only=local_files_only, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - max_workers=max_workers, - tqdm_class=tqdm_class, - ) - - def get_safetensors_metadata( - self, - repo_id: str, - *, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> SafetensorsRepoMetadata: - """ - Parse metadata for a safetensors repo on the Hub. - - We first check if the repo has a single safetensors file or a sharded safetensors repo. If it's a single - safetensors file, we parse the metadata from this file. If it's a sharded safetensors repo, we parse the - metadata from the index file and then parse the metadata from each shard. - - To parse metadata from a single safetensors file, use [`parse_safetensors_file_metadata`]. - - For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. - - Args: - repo_id (`str`): - A user or an organization name and a repo name separated by a `/`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if the file is in a dataset or space, `None` or `"model"` if in a - model. Default is `None`. - revision (`str`, *optional*): - The git revision to fetch the file from. Can be a branch name, a tag, or a commit hash. Defaults to the - head of the `"main"` branch. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`SafetensorsRepoMetadata`]: information related to safetensors repo. - - Raises: - [`NotASafetensorsRepoError`] - If the repo is not a safetensors repo i.e. doesn't have either a - `model.safetensors` or a `model.safetensors.index.json` file. - [`SafetensorsParsingError`] - If a safetensors file header couldn't be parsed correctly. - - Example: - ```py - # Parse repo with single weights file - >>> metadata = get_safetensors_metadata("bigscience/bloomz-560m") - >>> metadata - SafetensorsRepoMetadata( - metadata=None, - sharded=False, - weight_map={'h.0.input_layernorm.bias': 'model.safetensors', ...}, - files_metadata={'model.safetensors': SafetensorsFileMetadata(...)} - ) - >>> metadata.files_metadata["model.safetensors"].metadata - {'format': 'pt'} - - # Parse repo with sharded model - >>> metadata = get_safetensors_metadata("bigscience/bloom") - Parse safetensors files: 100%|██████████████████████████████████████████| 72/72 [00:12<00:00, 5.78it/s] - >>> metadata - SafetensorsRepoMetadata(metadata={'total_size': 352494542848}, sharded=True, weight_map={...}, files_metadata={...}) - >>> len(metadata.files_metadata) - 72 # All safetensors files have been fetched - - # Parse repo with sharded model - >>> get_safetensors_metadata("runwayml/stable-diffusion-v1-5") - NotASafetensorsRepoError: 'runwayml/stable-diffusion-v1-5' is not a safetensors repo. Couldn't find 'model.safetensors.index.json' or 'model.safetensors' files. - ``` - """ - if self.file_exists( # Single safetensors file => non-sharded model - repo_id=repo_id, - filename=constants.SAFETENSORS_SINGLE_FILE, - repo_type=repo_type, - revision=revision, - token=token, - ): - file_metadata = self.parse_safetensors_file_metadata( - repo_id=repo_id, - filename=constants.SAFETENSORS_SINGLE_FILE, - repo_type=repo_type, - revision=revision, - token=token, - ) - return SafetensorsRepoMetadata( - metadata=None, - sharded=False, - weight_map={ - tensor_name: constants.SAFETENSORS_SINGLE_FILE for tensor_name in file_metadata.tensors.keys() - }, - files_metadata={constants.SAFETENSORS_SINGLE_FILE: file_metadata}, - ) - elif self.file_exists( # Multiple safetensors files => sharded with index - repo_id=repo_id, - filename=constants.SAFETENSORS_INDEX_FILE, - repo_type=repo_type, - revision=revision, - token=token, - ): - # Fetch index - index_file = self.hf_hub_download( - repo_id=repo_id, - filename=constants.SAFETENSORS_INDEX_FILE, - repo_type=repo_type, - revision=revision, - token=token, - ) - with open(index_file) as f: - index = json.load(f) - - weight_map = index.get("weight_map", {}) - - # Fetch metadata per shard - files_metadata = {} - - def _parse(filename: str) -> None: - files_metadata[filename] = self.parse_safetensors_file_metadata( - repo_id=repo_id, filename=filename, repo_type=repo_type, revision=revision, token=token - ) - - thread_map( - _parse, - set(weight_map.values()), - desc="Parse safetensors files", - tqdm_class=hf_tqdm, - ) - - return SafetensorsRepoMetadata( - metadata=index.get("metadata", None), - sharded=True, - weight_map=weight_map, - files_metadata=files_metadata, - ) - else: - # Not a safetensors repo - raise NotASafetensorsRepoError( - f"'{repo_id}' is not a safetensors repo. Couldn't find '{constants.SAFETENSORS_INDEX_FILE}' or '{constants.SAFETENSORS_SINGLE_FILE}' files." - ) - - def parse_safetensors_file_metadata( - self, - repo_id: str, - filename: str, - *, - repo_type: Optional[str] = None, - revision: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> SafetensorsFileMetadata: - """ - Parse metadata from a safetensors file on the Hub. - - To parse metadata from all safetensors files in a repo at once, use [`get_safetensors_metadata`]. - - For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. - - Args: - repo_id (`str`): - A user or an organization name and a repo name separated by a `/`. - filename (`str`): - The name of the file in the repo. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if the file is in a dataset or space, `None` or `"model"` if in a - model. Default is `None`. - revision (`str`, *optional*): - The git revision to fetch the file from. Can be a branch name, a tag, or a commit hash. Defaults to the - head of the `"main"` branch. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`SafetensorsFileMetadata`]: information related to a safetensors file. - - Raises: - [`NotASafetensorsRepoError`]: - If the repo is not a safetensors repo i.e. doesn't have either a - `model.safetensors` or a `model.safetensors.index.json` file. - [`SafetensorsParsingError`]: - If a safetensors file header couldn't be parsed correctly. - """ - url = hf_hub_url( - repo_id=repo_id, filename=filename, repo_type=repo_type, revision=revision, endpoint=self.endpoint - ) - _headers = self._build_hf_headers(token=token) - - # 1. Fetch first 100kb - # Empirically, 97% of safetensors files have a metadata size < 100kb (over the top 1000 models on the Hub). - # We assume fetching 100kb is faster than making 2 GET requests. Therefore we always fetch the first 100kb to - # avoid the 2nd GET in most cases. - # See https://github.com/huggingface/huggingface_hub/pull/1855#discussion_r1404286419. - response = get_session().get(url, headers={**_headers, "range": "bytes=0-100000"}) - hf_raise_for_status(response) - - # 2. Parse metadata size - metadata_size = struct.unpack(" constants.SAFETENSORS_MAX_HEADER_LENGTH: - raise SafetensorsParsingError( - f"Failed to parse safetensors header for '{filename}' (repo '{repo_id}', revision " - f"'{revision or constants.DEFAULT_REVISION}'): safetensors header is too big. Maximum supported size is " - f"{constants.SAFETENSORS_MAX_HEADER_LENGTH} bytes (got {metadata_size})." - ) - - # 3.a. Get metadata from payload - if metadata_size <= 100000: - metadata_as_bytes = response.content[8 : 8 + metadata_size] - else: # 3.b. Request full metadata - response = get_session().get(url, headers={**_headers, "range": f"bytes=8-{metadata_size + 7}"}) - hf_raise_for_status(response) - metadata_as_bytes = response.content - - # 4. Parse json header - try: - metadata_as_dict = json.loads(metadata_as_bytes.decode(errors="ignore")) - except json.JSONDecodeError as e: - raise SafetensorsParsingError( - f"Failed to parse safetensors header for '{filename}' (repo '{repo_id}', revision " - f"'{revision or constants.DEFAULT_REVISION}'): header is not json-encoded string. Please make sure this is a " - "correctly formatted safetensors file." - ) from e - - try: - return SafetensorsFileMetadata( - metadata=metadata_as_dict.get("__metadata__", {}), - tensors={ - key: TensorInfo( - dtype=tensor["dtype"], - shape=tensor["shape"], - data_offsets=tuple(tensor["data_offsets"]), # type: ignore - ) - for key, tensor in metadata_as_dict.items() - if key != "__metadata__" - }, - ) - except (KeyError, IndexError) as e: - raise SafetensorsParsingError( - f"Failed to parse safetensors header for '{filename}' (repo '{repo_id}', revision " - f"'{revision or constants.DEFAULT_REVISION}'): header format not recognized. Please make sure this is a correctly" - " formatted safetensors file." - ) from e - - @validate_hf_hub_args - def create_branch( - self, - repo_id: str, - *, - branch: str, - revision: Optional[str] = None, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - exist_ok: bool = False, - ) -> None: - """ - Create a new branch for a repo on the Hub, starting from the specified revision (defaults to `main`). - To find a revision suiting your needs, you can use [`list_repo_refs`] or [`list_repo_commits`]. - - Args: - repo_id (`str`): - The repository in which the branch will be created. - Example: `"user/my-cool-model"`. - - branch (`str`): - The name of the branch to create. - - revision (`str`, *optional*): - The git revision to create the branch from. It can be a branch name or - the OID/SHA of a commit, as a hexadecimal string. Defaults to the head - of the `"main"` branch. - - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if creating a branch on a dataset or - space, `None` or `"model"` if tagging a model. Default is `None`. - - exist_ok (`bool`, *optional*, defaults to `False`): - If `True`, do not raise an error if branch already exists. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If repository is not found (error 404): wrong repo_id/repo_type, private - but not authenticated or repo does not exist. - [`~utils.BadRequestError`]: - If invalid reference for a branch. Ex: `refs/pr/5` or 'refs/foo/bar'. - [`~utils.HfHubHTTPError`]: - If the branch already exists on the repo (error 409) and `exist_ok` is - set to `False`. - """ - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - branch = quote(branch, safe="") - - # Prepare request - branch_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/branch/{branch}" - headers = self._build_hf_headers(token=token) - payload = {} - if revision is not None: - payload["startingPoint"] = revision - - # Create branch - response = get_session().post(url=branch_url, headers=headers, json=payload) - try: - hf_raise_for_status(response) - except HfHubHTTPError as e: - if exist_ok and e.response.status_code == 409: - return - elif exist_ok and e.response.status_code == 403: - # No write permission on the namespace but branch might already exist - try: - refs = self.list_repo_refs(repo_id=repo_id, repo_type=repo_type, token=token) - for branch_ref in refs.branches: - if branch_ref.name == branch: - return # Branch already exists => do not raise - except HfHubHTTPError: - pass # We raise the original error if the branch does not exist - raise - - @validate_hf_hub_args - def delete_branch( - self, - repo_id: str, - *, - branch: str, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - ) -> None: - """ - Delete a branch from a repo on the Hub. - - Args: - repo_id (`str`): - The repository in which a branch will be deleted. - Example: `"user/my-cool-model"`. - - branch (`str`): - The name of the branch to delete. - - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if creating a branch on a dataset or - space, `None` or `"model"` if tagging a model. Default is `None`. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If repository is not found (error 404): wrong repo_id/repo_type, private - but not authenticated or repo does not exist. - [`~utils.HfHubHTTPError`]: - If trying to delete a protected branch. Ex: `main` cannot be deleted. - [`~utils.HfHubHTTPError`]: - If trying to delete a branch that does not exist. - - """ - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - branch = quote(branch, safe="") - - # Prepare request - branch_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/branch/{branch}" - headers = self._build_hf_headers(token=token) - - # Delete branch - response = get_session().delete(url=branch_url, headers=headers) - hf_raise_for_status(response) - - @validate_hf_hub_args - def create_tag( - self, - repo_id: str, - *, - tag: str, - tag_message: Optional[str] = None, - revision: Optional[str] = None, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - exist_ok: bool = False, - ) -> None: - """ - Tag a given commit of a repo on the Hub. - - Args: - repo_id (`str`): - The repository in which a commit will be tagged. - Example: `"user/my-cool-model"`. - - tag (`str`): - The name of the tag to create. - - tag_message (`str`, *optional*): - The description of the tag to create. - - revision (`str`, *optional*): - The git revision to tag. It can be a branch name or the OID/SHA of a - commit, as a hexadecimal string. Shorthands (7 first characters) are - also supported. Defaults to the head of the `"main"` branch. - - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if tagging a dataset or - space, `None` or `"model"` if tagging a model. Default is - `None`. - - exist_ok (`bool`, *optional*, defaults to `False`): - If `True`, do not raise an error if tag already exists. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If repository is not found (error 404): wrong repo_id/repo_type, private - but not authenticated or repo does not exist. - [`~utils.RevisionNotFoundError`]: - If revision is not found (error 404) on the repo. - [`~utils.HfHubHTTPError`]: - If the branch already exists on the repo (error 409) and `exist_ok` is - set to `False`. - """ - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION - - # Prepare request - tag_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tag/{revision}" - headers = self._build_hf_headers(token=token) - payload = {"tag": tag} - if tag_message is not None: - payload["message"] = tag_message - - # Tag - response = get_session().post(url=tag_url, headers=headers, json=payload) - try: - hf_raise_for_status(response) - except HfHubHTTPError as e: - if not (e.response.status_code == 409 and exist_ok): - raise - - @validate_hf_hub_args - def delete_tag( - self, - repo_id: str, - *, - tag: str, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - ) -> None: - """ - Delete a tag from a repo on the Hub. - - Args: - repo_id (`str`): - The repository in which a tag will be deleted. - Example: `"user/my-cool-model"`. - - tag (`str`): - The name of the tag to delete. - - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if tagging a dataset or space, `None` or - `"model"` if tagging a model. Default is `None`. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If repository is not found (error 404): wrong repo_id/repo_type, private - but not authenticated or repo does not exist. - [`~utils.RevisionNotFoundError`]: - If tag is not found. - """ - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - tag = quote(tag, safe="") - - # Prepare request - tag_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tag/{tag}" - headers = self._build_hf_headers(token=token) - - # Un-tag - response = get_session().delete(url=tag_url, headers=headers) - hf_raise_for_status(response) - - @validate_hf_hub_args - def get_full_repo_name( - self, - model_id: str, - *, - organization: Optional[str] = None, - token: Union[bool, str, None] = None, - ): - """ - Returns the repository name for a given model ID and optional - organization. - - Args: - model_id (`str`): - The name of the model. - organization (`str`, *optional*): - If passed, the repository name will be in the organization - namespace instead of the user namespace. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `str`: The repository name in the user's namespace - ({username}/{model_id}) if no organization is passed, and under the - organization namespace ({organization}/{model_id}) otherwise. - """ - if organization is None: - if "/" in model_id: - username = model_id.split("/")[0] - else: - username = self.whoami(token=token)["name"] # type: ignore - return f"{username}/{model_id}" - else: - return f"{organization}/{model_id}" - - @validate_hf_hub_args - def get_repo_discussions( - self, - repo_id: str, - *, - author: Optional[str] = None, - discussion_type: Optional[constants.DiscussionTypeFilter] = None, - discussion_status: Optional[constants.DiscussionStatusFilter] = None, - repo_type: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> Iterator[Discussion]: - """ - Fetches Discussions and Pull Requests for the given repo. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - author (`str`, *optional*): - Pass a value to filter by discussion author. `None` means no filter. - Default is `None`. - discussion_type (`str`, *optional*): - Set to `"pull_request"` to fetch only pull requests, `"discussion"` - to fetch only discussions. Set to `"all"` or `None` to fetch both. - Default is `None`. - discussion_status (`str`, *optional*): - Set to `"open"` (respectively `"closed"`) to fetch only open - (respectively closed) discussions. Set to `"all"` or `None` - to fetch both. - Default is `None`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if fetching from a dataset or - space, `None` or `"model"` if fetching from a model. Default is - `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Iterator[Discussion]`: An iterator of [`Discussion`] objects. - - Example: - Collecting all discussions of a repo in a list: - - ```python - >>> from huggingface_hub import get_repo_discussions - >>> discussions_list = list(get_repo_discussions(repo_id="bert-base-uncased")) - ``` - - Iterating over discussions of a repo: - - ```python - >>> from huggingface_hub import get_repo_discussions - >>> for discussion in get_repo_discussions(repo_id="bert-base-uncased"): - ... print(discussion.num, discussion.title) - ``` - """ - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - if discussion_type is not None and discussion_type not in constants.DISCUSSION_TYPES: - raise ValueError(f"Invalid discussion_type, must be one of {constants.DISCUSSION_TYPES}") - if discussion_status is not None and discussion_status not in constants.DISCUSSION_STATUS: - raise ValueError(f"Invalid discussion_status, must be one of {constants.DISCUSSION_STATUS}") - - headers = self._build_hf_headers(token=token) - path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions" - - params: Dict[str, Union[str, int]] = {} - if discussion_type is not None: - params["type"] = discussion_type - if discussion_status is not None: - params["status"] = discussion_status - if author is not None: - params["author"] = author - - def _fetch_discussion_page(page_index: int): - params["p"] = page_index - resp = get_session().get(path, headers=headers, params=params) - hf_raise_for_status(resp) - paginated_discussions = resp.json() - total = paginated_discussions["count"] - start = paginated_discussions["start"] - discussions = paginated_discussions["discussions"] - has_next = (start + len(discussions)) < total - return discussions, has_next - - has_next, page_index = True, 0 - - while has_next: - discussions, has_next = _fetch_discussion_page(page_index=page_index) - for discussion in discussions: - yield Discussion( - title=discussion["title"], - num=discussion["num"], - author=discussion.get("author", {}).get("name", "deleted"), - created_at=parse_datetime(discussion["createdAt"]), - status=discussion["status"], - repo_id=discussion["repo"]["name"], - repo_type=discussion["repo"]["type"], - is_pull_request=discussion["isPullRequest"], - endpoint=self.endpoint, - ) - page_index = page_index + 1 - - @validate_hf_hub_args - def get_discussion_details( - self, - repo_id: str, - discussion_num: int, - *, - repo_type: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> DiscussionWithDetails: - """Fetches a Discussion's / Pull Request 's details from the Hub. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - discussion_num (`int`): - The number of the Discussion or Pull Request . Must be a strictly positive integer. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: [`DiscussionWithDetails`] - - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - """ - if not isinstance(discussion_num, int) or discussion_num <= 0: - raise ValueError("Invalid discussion_num, must be a positive integer") - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - - path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions/{discussion_num}" - headers = self._build_hf_headers(token=token) - resp = get_session().get(path, params={"diff": "1"}, headers=headers) - hf_raise_for_status(resp) - - discussion_details = resp.json() - is_pull_request = discussion_details["isPullRequest"] - - target_branch = discussion_details["changes"]["base"] if is_pull_request else None - conflicting_files = discussion_details["filesWithConflicts"] if is_pull_request else None - merge_commit_oid = discussion_details["changes"].get("mergeCommitId", None) if is_pull_request else None - - return DiscussionWithDetails( - title=discussion_details["title"], - num=discussion_details["num"], - author=discussion_details.get("author", {}).get("name", "deleted"), - created_at=parse_datetime(discussion_details["createdAt"]), - status=discussion_details["status"], - repo_id=discussion_details["repo"]["name"], - repo_type=discussion_details["repo"]["type"], - is_pull_request=discussion_details["isPullRequest"], - events=[deserialize_event(evt) for evt in discussion_details["events"]], - conflicting_files=conflicting_files, - target_branch=target_branch, - merge_commit_oid=merge_commit_oid, - diff=discussion_details.get("diff"), - endpoint=self.endpoint, - ) - - @validate_hf_hub_args - def create_discussion( - self, - repo_id: str, - title: str, - *, - token: Union[bool, str, None] = None, - description: Optional[str] = None, - repo_type: Optional[str] = None, - pull_request: bool = False, - ) -> DiscussionWithDetails: - """Creates a Discussion or Pull Request. - - Pull Requests created programmatically will be in `"draft"` status. - - Creating a Pull Request with changes can also be done at once with [`HfApi.create_commit`]. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - title (`str`): - The title of the discussion. It can be up to 200 characters long, - and must be at least 3 characters long. Leading and trailing whitespaces - will be stripped. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - description (`str`, *optional*): - An optional description for the Pull Request. - Defaults to `"Discussion opened with the huggingface_hub Python library"` - pull_request (`bool`, *optional*): - Whether to create a Pull Request or discussion. If `True`, creates a Pull Request. - If `False`, creates a discussion. Defaults to `False`. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - - Returns: [`DiscussionWithDetails`] - - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access.""" - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - - if description is not None: - description = description.strip() - description = ( - description - if description - else ( - f"{'Pull Request' if pull_request else 'Discussion'} opened with the" - " [huggingface_hub Python" - " library](https://huggingface.co/docs/huggingface_hub)" - ) - ) - - headers = self._build_hf_headers(token=token) - resp = get_session().post( - f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions", - json={ - "title": title.strip(), - "description": description, - "pullRequest": pull_request, - }, - headers=headers, - ) - hf_raise_for_status(resp) - num = resp.json()["num"] - return self.get_discussion_details( - repo_id=repo_id, - repo_type=repo_type, - discussion_num=num, - token=token, - ) - - @validate_hf_hub_args - def create_pull_request( - self, - repo_id: str, - title: str, - *, - token: Union[bool, str, None] = None, - description: Optional[str] = None, - repo_type: Optional[str] = None, - ) -> DiscussionWithDetails: - """Creates a Pull Request . Pull Requests created programmatically will be in `"draft"` status. - - Creating a Pull Request with changes can also be done at once with [`HfApi.create_commit`]; - - This is a wrapper around [`HfApi.create_discussion`]. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - title (`str`): - The title of the discussion. It can be up to 200 characters long, - and must be at least 3 characters long. Leading and trailing whitespaces - will be stripped. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - description (`str`, *optional*): - An optional description for the Pull Request. - Defaults to `"Discussion opened with the huggingface_hub Python library"` - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - - Returns: [`DiscussionWithDetails`] - - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access.""" - return self.create_discussion( - repo_id=repo_id, - title=title, - token=token, - description=description, - repo_type=repo_type, - pull_request=True, - ) - - def _post_discussion_changes( - self, - *, - repo_id: str, - discussion_num: int, - resource: str, - body: Optional[dict] = None, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - ) -> requests.Response: - """Internal utility to POST changes to a Discussion or Pull Request""" - if not isinstance(discussion_num, int) or discussion_num <= 0: - raise ValueError("Invalid discussion_num, must be a positive integer") - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - repo_id = f"{repo_type}s/{repo_id}" - - path = f"{self.endpoint}/api/{repo_id}/discussions/{discussion_num}/{resource}" - - headers = self._build_hf_headers(token=token) - resp = requests.post(path, headers=headers, json=body) - hf_raise_for_status(resp) - return resp - - @validate_hf_hub_args - def comment_discussion( - self, - repo_id: str, - discussion_num: int, - comment: str, - *, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - ) -> DiscussionComment: - """Creates a new comment on the given Discussion. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - discussion_num (`int`): - The number of the Discussion or Pull Request . Must be a strictly positive integer. - comment (`str`): - The content of the comment to create. Comments support markdown formatting. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`DiscussionComment`]: the newly created comment - - - Examples: - ```python - - >>> comment = \"\"\" - ... Hello @otheruser! - ... - ... # This is a title - ... - ... **This is bold**, *this is italic* and ~this is strikethrough~ - ... And [this](http://url) is a link - ... \"\"\" - - >>> HfApi().comment_discussion( - ... repo_id="username/repo_name", - ... discussion_num=34 - ... comment=comment - ... ) - # DiscussionComment(id='deadbeef0000000', type='comment', ...) - - ``` - - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - """ - resp = self._post_discussion_changes( - repo_id=repo_id, - repo_type=repo_type, - discussion_num=discussion_num, - token=token, - resource="comment", - body={"comment": comment}, - ) - return deserialize_event(resp.json()["newMessage"]) # type: ignore - - @validate_hf_hub_args - def rename_discussion( - self, - repo_id: str, - discussion_num: int, - new_title: str, - *, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - ) -> DiscussionTitleChange: - """Renames a Discussion. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - discussion_num (`int`): - The number of the Discussion or Pull Request . Must be a strictly positive integer. - new_title (`str`): - The new title for the discussion - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`DiscussionTitleChange`]: the title change event - - - Examples: - ```python - >>> new_title = "New title, fixing a typo" - >>> HfApi().rename_discussion( - ... repo_id="username/repo_name", - ... discussion_num=34 - ... new_title=new_title - ... ) - # DiscussionTitleChange(id='deadbeef0000000', type='title-change', ...) - - ``` - - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - """ - resp = self._post_discussion_changes( - repo_id=repo_id, - repo_type=repo_type, - discussion_num=discussion_num, - token=token, - resource="title", - body={"title": new_title}, - ) - return deserialize_event(resp.json()["newTitle"]) # type: ignore - - @validate_hf_hub_args - def change_discussion_status( - self, - repo_id: str, - discussion_num: int, - new_status: Literal["open", "closed"], - *, - token: Union[bool, str, None] = None, - comment: Optional[str] = None, - repo_type: Optional[str] = None, - ) -> DiscussionStatusChange: - """Closes or re-opens a Discussion or Pull Request. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - discussion_num (`int`): - The number of the Discussion or Pull Request . Must be a strictly positive integer. - new_status (`str`): - The new status for the discussion, either `"open"` or `"closed"`. - comment (`str`, *optional*): - An optional comment to post with the status change. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`DiscussionStatusChange`]: the status change event - - - Examples: - ```python - >>> new_title = "New title, fixing a typo" - >>> HfApi().rename_discussion( - ... repo_id="username/repo_name", - ... discussion_num=34 - ... new_title=new_title - ... ) - # DiscussionStatusChange(id='deadbeef0000000', type='status-change', ...) - - ``` - - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - """ - if new_status not in ["open", "closed"]: - raise ValueError("Invalid status, valid statuses are: 'open' and 'closed'") - body: Dict[str, str] = {"status": new_status} - if comment and comment.strip(): - body["comment"] = comment.strip() - resp = self._post_discussion_changes( - repo_id=repo_id, - repo_type=repo_type, - discussion_num=discussion_num, - token=token, - resource="status", - body=body, - ) - return deserialize_event(resp.json()["newStatus"]) # type: ignore - - @validate_hf_hub_args - def merge_pull_request( - self, - repo_id: str, - discussion_num: int, - *, - token: Union[bool, str, None] = None, - comment: Optional[str] = None, - repo_type: Optional[str] = None, - ): - """Merges a Pull Request. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - discussion_num (`int`): - The number of the Discussion or Pull Request . Must be a strictly positive integer. - comment (`str`, *optional*): - An optional comment to post with the status change. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`DiscussionStatusChange`]: the status change event - - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - """ - self._post_discussion_changes( - repo_id=repo_id, - repo_type=repo_type, - discussion_num=discussion_num, - token=token, - resource="merge", - body={"comment": comment.strip()} if comment and comment.strip() else None, - ) - - @validate_hf_hub_args - def edit_discussion_comment( - self, - repo_id: str, - discussion_num: int, - comment_id: str, - new_content: str, - *, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - ) -> DiscussionComment: - """Edits a comment on a Discussion / Pull Request. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - discussion_num (`int`): - The number of the Discussion or Pull Request . Must be a strictly positive integer. - comment_id (`str`): - The ID of the comment to edit. - new_content (`str`): - The new content of the comment. Comments support markdown formatting. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`DiscussionComment`]: the edited comment - - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - """ - resp = self._post_discussion_changes( - repo_id=repo_id, - repo_type=repo_type, - discussion_num=discussion_num, - token=token, - resource=f"comment/{comment_id.lower()}/edit", - body={"content": new_content}, - ) - return deserialize_event(resp.json()["updatedComment"]) # type: ignore - - @validate_hf_hub_args - def hide_discussion_comment( - self, - repo_id: str, - discussion_num: int, - comment_id: str, - *, - token: Union[bool, str, None] = None, - repo_type: Optional[str] = None, - ) -> DiscussionComment: - """Hides a comment on a Discussion / Pull Request. - - > [!WARNING] - > Hidden comments' content cannot be retrieved anymore. Hiding a comment is irreversible. - - Args: - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - discussion_num (`int`): - The number of the Discussion or Pull Request . Must be a strictly positive integer. - comment_id (`str`): - The ID of the comment to edit. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if uploading to a dataset or - space, `None` or `"model"` if uploading to a model. Default is - `None`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`DiscussionComment`]: the hidden comment - - > [!TIP] - > Raises the following errors: - > - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the HuggingFace API returned an error - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if some parameter value is invalid - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it doesn't exist, - > or because it is set to `private` and you do not have access. - """ - warnings.warn( - "Hidden comments' content cannot be retrieved anymore. Hiding a comment is irreversible.", - UserWarning, - ) - resp = self._post_discussion_changes( - repo_id=repo_id, - repo_type=repo_type, - discussion_num=discussion_num, - token=token, - resource=f"comment/{comment_id.lower()}/hide", - ) - return deserialize_event(resp.json()["updatedComment"]) # type: ignore - - @validate_hf_hub_args - def add_space_secret( - self, - repo_id: str, - key: str, - value: str, - *, - description: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> None: - """Adds or updates a secret in a Space. - - Secrets allow to set secret keys or tokens to a Space without hardcoding them. - For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets. - - Args: - repo_id (`str`): - ID of the repo to update. Example: `"bigcode/in-the-stack"`. - key (`str`): - Secret key. Example: `"GITHUB_API_KEY"` - value (`str`): - Secret value. Example: `"your_github_api_key"`. - description (`str`, *optional*): - Secret description. Example: `"Github API key to access the Github API"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - """ - payload = {"key": key, "value": value} - if description is not None: - payload["description"] = description - r = get_session().post( - f"{self.endpoint}/api/spaces/{repo_id}/secrets", - headers=self._build_hf_headers(token=token), - json=payload, - ) - hf_raise_for_status(r) - - @validate_hf_hub_args - def delete_space_secret(self, repo_id: str, key: str, *, token: Union[bool, str, None] = None) -> None: - """Deletes a secret from a Space. - - Secrets allow to set secret keys or tokens to a Space without hardcoding them. - For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets. - - Args: - repo_id (`str`): - ID of the repo to update. Example: `"bigcode/in-the-stack"`. - key (`str`): - Secret key. Example: `"GITHUB_API_KEY"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - """ - r = get_session().delete( - f"{self.endpoint}/api/spaces/{repo_id}/secrets", - headers=self._build_hf_headers(token=token), - json={"key": key}, - ) - hf_raise_for_status(r) - - @validate_hf_hub_args - def get_space_variables(self, repo_id: str, *, token: Union[bool, str, None] = None) -> Dict[str, SpaceVariable]: - """Gets all variables from a Space. - - Variables allow to set environment variables to a Space without hardcoding them. - For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets-and-environment-variables - - Args: - repo_id (`str`): - ID of the repo to query. Example: `"bigcode/in-the-stack"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - """ - r = get_session().get( - f"{self.endpoint}/api/spaces/{repo_id}/variables", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(r) - return {k: SpaceVariable(k, v) for k, v in r.json().items()} - - @validate_hf_hub_args - def add_space_variable( - self, - repo_id: str, - key: str, - value: str, - *, - description: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> Dict[str, SpaceVariable]: - """Adds or updates a variable in a Space. - - Variables allow to set environment variables to a Space without hardcoding them. - For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets-and-environment-variables - - Args: - repo_id (`str`): - ID of the repo to update. Example: `"bigcode/in-the-stack"`. - key (`str`): - Variable key. Example: `"MODEL_REPO_ID"` - value (`str`): - Variable value. Example: `"the_model_repo_id"`. - description (`str`): - Description of the variable. Example: `"Model Repo ID of the implemented model"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - """ - payload = {"key": key, "value": value} - if description is not None: - payload["description"] = description - r = get_session().post( - f"{self.endpoint}/api/spaces/{repo_id}/variables", - headers=self._build_hf_headers(token=token), - json=payload, - ) - hf_raise_for_status(r) - return {k: SpaceVariable(k, v) for k, v in r.json().items()} - - @validate_hf_hub_args - def delete_space_variable( - self, repo_id: str, key: str, *, token: Union[bool, str, None] = None - ) -> Dict[str, SpaceVariable]: - """Deletes a variable from a Space. - - Variables allow to set environment variables to a Space without hardcoding them. - For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets-and-environment-variables - - Args: - repo_id (`str`): - ID of the repo to update. Example: `"bigcode/in-the-stack"`. - key (`str`): - Variable key. Example: `"MODEL_REPO_ID"` - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - """ - r = get_session().delete( - f"{self.endpoint}/api/spaces/{repo_id}/variables", - headers=self._build_hf_headers(token=token), - json={"key": key}, - ) - hf_raise_for_status(r) - return {k: SpaceVariable(k, v) for k, v in r.json().items()} - - @validate_hf_hub_args - def get_space_runtime(self, repo_id: str, *, token: Union[bool, str, None] = None) -> SpaceRuntime: - """Gets runtime information about a Space. - - Args: - repo_id (`str`): - ID of the repo to update. Example: `"bigcode/in-the-stack"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - Returns: - [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware. - """ - r = get_session().get( - f"{self.endpoint}/api/spaces/{repo_id}/runtime", headers=self._build_hf_headers(token=token) - ) - hf_raise_for_status(r) - return SpaceRuntime(r.json()) - - @validate_hf_hub_args - def request_space_hardware( - self, - repo_id: str, - hardware: SpaceHardware, - *, - token: Union[bool, str, None] = None, - sleep_time: Optional[int] = None, - ) -> SpaceRuntime: - """Request new hardware for a Space. - - Args: - repo_id (`str`): - ID of the repo to update. Example: `"bigcode/in-the-stack"`. - hardware (`str` or [`SpaceHardware`]): - Hardware on which to run the Space. Example: `"t4-medium"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - sleep_time (`int`, *optional*): - Number of seconds of inactivity to wait before a Space is put to sleep. Set to `-1` if you don't want - your Space to sleep (default behavior for upgraded hardware). For free hardware, you can't configure - the sleep time (value is fixed to 48 hours of inactivity). - See https://huggingface.co/docs/hub/spaces-gpus#sleep-time for more details. - Returns: - [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware. - - > [!TIP] - > It is also possible to request hardware directly when creating the Space repo! See [`create_repo`] for details. - """ - if sleep_time is not None and hardware == SpaceHardware.CPU_BASIC: - warnings.warn( - "If your Space runs on the default 'cpu-basic' hardware, it will go to sleep if inactive for more" - " than 48 hours. This value is not configurable. If you don't want your Space to deactivate or if" - " you want to set a custom sleep time, you need to upgrade to a paid Hardware.", - UserWarning, - ) - payload: Dict[str, Any] = {"flavor": hardware} - if sleep_time is not None: - payload["sleepTimeSeconds"] = sleep_time - r = get_session().post( - f"{self.endpoint}/api/spaces/{repo_id}/hardware", - headers=self._build_hf_headers(token=token), - json=payload, - ) - hf_raise_for_status(r) - return SpaceRuntime(r.json()) - - @validate_hf_hub_args - def set_space_sleep_time( - self, repo_id: str, sleep_time: int, *, token: Union[bool, str, None] = None - ) -> SpaceRuntime: - """Set a custom sleep time for a Space running on upgraded hardware.. - - Your Space will go to sleep after X seconds of inactivity. You are not billed when your Space is in "sleep" - mode. If a new visitor lands on your Space, it will "wake it up". Only upgraded hardware can have a - configurable sleep time. To know more about the sleep stage, please refer to - https://huggingface.co/docs/hub/spaces-gpus#sleep-time. - - Args: - repo_id (`str`): - ID of the repo to update. Example: `"bigcode/in-the-stack"`. - sleep_time (`int`, *optional*): - Number of seconds of inactivity to wait before a Space is put to sleep. Set to `-1` if you don't want - your Space to pause (default behavior for upgraded hardware). For free hardware, you can't configure - the sleep time (value is fixed to 48 hours of inactivity). - See https://huggingface.co/docs/hub/spaces-gpus#sleep-time for more details. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - Returns: - [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware. - - > [!TIP] - > It is also possible to set a custom sleep time when requesting hardware with [`request_space_hardware`]. - """ - r = get_session().post( - f"{self.endpoint}/api/spaces/{repo_id}/sleeptime", - headers=self._build_hf_headers(token=token), - json={"seconds": sleep_time}, - ) - hf_raise_for_status(r) - runtime = SpaceRuntime(r.json()) - - hardware = runtime.requested_hardware or runtime.hardware - if hardware == SpaceHardware.CPU_BASIC: - warnings.warn( - "If your Space runs on the default 'cpu-basic' hardware, it will go to sleep if inactive for more" - " than 48 hours. This value is not configurable. If you don't want your Space to deactivate or if" - " you want to set a custom sleep time, you need to upgrade to a paid Hardware.", - UserWarning, - ) - return runtime - - @validate_hf_hub_args - def pause_space(self, repo_id: str, *, token: Union[bool, str, None] = None) -> SpaceRuntime: - """Pause your Space. - - A paused Space stops executing until manually restarted by its owner. This is different from the sleeping - state in which free Spaces go after 48h of inactivity. Paused time is not billed to your account, no matter the - hardware you've selected. To restart your Space, use [`restart_space`] and go to your Space settings page. - - For more details, please visit [the docs](https://huggingface.co/docs/hub/spaces-gpus#pause). - - Args: - repo_id (`str`): - ID of the Space to pause. Example: `"Salesforce/BLIP2"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`SpaceRuntime`]: Runtime information about your Space including `stage=PAUSED` and requested hardware. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If your Space is not found (error 404). Most probably wrong repo_id or your space is private but you - are not authenticated. - [`~utils.HfHubHTTPError`]: - 403 Forbidden: only the owner of a Space can pause it. If you want to manage a Space that you don't - own, either ask the owner by opening a Discussion or duplicate the Space. - [`~utils.BadRequestError`]: - If your Space is a static Space. Static Spaces are always running and never billed. If you want to hide - a static Space, you can set it to private. - """ - r = get_session().post( - f"{self.endpoint}/api/spaces/{repo_id}/pause", headers=self._build_hf_headers(token=token) - ) - hf_raise_for_status(r) - return SpaceRuntime(r.json()) - - @validate_hf_hub_args - def restart_space( - self, repo_id: str, *, token: Union[bool, str, None] = None, factory_reboot: bool = False - ) -> SpaceRuntime: - """Restart your Space. - - This is the only way to programmatically restart a Space if you've put it on Pause (see [`pause_space`]). You - must be the owner of the Space to restart it. If you are using an upgraded hardware, your account will be - billed as soon as the Space is restarted. You can trigger a restart no matter the current state of a Space. - - For more details, please visit [the docs](https://huggingface.co/docs/hub/spaces-gpus#pause). - - Args: - repo_id (`str`): - ID of the Space to restart. Example: `"Salesforce/BLIP2"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - factory_reboot (`bool`, *optional*): - If `True`, the Space will be rebuilt from scratch without caching any requirements. - - Returns: - [`SpaceRuntime`]: Runtime information about your Space. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If your Space is not found (error 404). Most probably wrong repo_id or your space is private but you - are not authenticated. - [`~utils.HfHubHTTPError`]: - 403 Forbidden: only the owner of a Space can restart it. If you want to restart a Space that you don't - own, either ask the owner by opening a Discussion or duplicate the Space. - [`~utils.BadRequestError`]: - If your Space is a static Space. Static Spaces are always running and never billed. If you want to hide - a static Space, you can set it to private. - """ - params = {} - if factory_reboot: - params["factory"] = "true" - r = get_session().post( - f"{self.endpoint}/api/spaces/{repo_id}/restart", headers=self._build_hf_headers(token=token), params=params - ) - hf_raise_for_status(r) - return SpaceRuntime(r.json()) - - @validate_hf_hub_args - def duplicate_space( - self, - from_id: str, - to_id: Optional[str] = None, - *, - private: Optional[bool] = None, - token: Union[bool, str, None] = None, - exist_ok: bool = False, - hardware: Optional[SpaceHardware] = None, - storage: Optional[SpaceStorage] = None, - sleep_time: Optional[int] = None, - secrets: Optional[List[Dict[str, str]]] = None, - variables: Optional[List[Dict[str, str]]] = None, - ) -> RepoUrl: - """Duplicate a Space. - - Programmatically duplicate a Space. The new Space will be created in your account and will be in the same state - as the original Space (running or paused). You can duplicate a Space no matter the current state of a Space. - - Args: - from_id (`str`): - ID of the Space to duplicate. Example: `"pharma/CLIP-Interrogator"`. - to_id (`str`, *optional*): - ID of the new Space. Example: `"dog/CLIP-Interrogator"`. If not provided, the new Space will have the same - name as the original Space, but in your account. - private (`bool`, *optional*): - Whether the new Space should be private or not. Defaults to the same privacy as the original Space. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - exist_ok (`bool`, *optional*, defaults to `False`): - If `True`, do not raise an error if repo already exists. - hardware (`SpaceHardware` or `str`, *optional*): - Choice of Hardware. Example: `"t4-medium"`. See [`SpaceHardware`] for a complete list. - storage (`SpaceStorage` or `str`, *optional*): - Choice of persistent storage tier. Example: `"small"`. See [`SpaceStorage`] for a complete list. - sleep_time (`int`, *optional*): - Number of seconds of inactivity to wait before a Space is put to sleep. Set to `-1` if you don't want - your Space to sleep (default behavior for upgraded hardware). For free hardware, you can't configure - the sleep time (value is fixed to 48 hours of inactivity). - See https://huggingface.co/docs/hub/spaces-gpus#sleep-time for more details. - secrets (`List[Dict[str, str]]`, *optional*): - A list of secret keys to set in your Space. Each item is in the form `{"key": ..., "value": ..., "description": ...}` where description is optional. - For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets. - variables (`List[Dict[str, str]]`, *optional*): - A list of public environment variables to set in your Space. Each item is in the form `{"key": ..., "value": ..., "description": ...}` where description is optional. - For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets-and-environment-variables. - - Returns: - [`RepoUrl`]: URL to the newly created repo. Value is a subclass of `str` containing - attributes like `endpoint`, `repo_type` and `repo_id`. - - Raises: - [`~utils.RepositoryNotFoundError`]: - If one of `from_id` or `to_id` cannot be found. This may be because it doesn't exist, - or because it is set to `private` and you do not have access. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - If the HuggingFace API returned an error - - Example: - ```python - >>> from huggingface_hub import duplicate_space - - # Duplicate a Space to your account - >>> duplicate_space("multimodalart/dreambooth-training") - RepoUrl('https://huggingface.co/spaces/nateraw/dreambooth-training',...) - - # Can set custom destination id and visibility flag. - >>> duplicate_space("multimodalart/dreambooth-training", to_id="my-dreambooth", private=True) - RepoUrl('https://huggingface.co/spaces/nateraw/my-dreambooth',...) - ``` - """ - # Parse to_id if provided - parsed_to_id = RepoUrl(to_id) if to_id is not None else None - - # Infer target repo_id - to_namespace = ( # set namespace manually or default to username - parsed_to_id.namespace - if parsed_to_id is not None and parsed_to_id.namespace is not None - else self.whoami(token)["name"] - ) - to_repo_name = parsed_to_id.repo_name if to_id is not None else RepoUrl(from_id).repo_name # type: ignore - - # repository must be a valid repo_id (namespace/repo_name). - payload: Dict[str, Any] = {"repository": f"{to_namespace}/{to_repo_name}"} - - keys = ["private", "hardware", "storageTier", "sleepTimeSeconds", "secrets", "variables"] - values = [private, hardware, storage, sleep_time, secrets, variables] - payload.update({k: v for k, v in zip(keys, values) if v is not None}) - - if sleep_time is not None and hardware == SpaceHardware.CPU_BASIC: - warnings.warn( - "If your Space runs on the default 'cpu-basic' hardware, it will go to sleep if inactive for more" - " than 48 hours. This value is not configurable. If you don't want your Space to deactivate or if" - " you want to set a custom sleep time, you need to upgrade to a paid Hardware.", - UserWarning, - ) - - r = get_session().post( - f"{self.endpoint}/api/spaces/{from_id}/duplicate", - headers=self._build_hf_headers(token=token), - json=payload, - ) - - try: - hf_raise_for_status(r) - except HTTPError as err: - if exist_ok and err.response.status_code == 409: - # Repo already exists and `exist_ok=True` - pass - else: - raise - - return RepoUrl(r.json()["url"], endpoint=self.endpoint) - - @validate_hf_hub_args - def request_space_storage( - self, - repo_id: str, - storage: SpaceStorage, - *, - token: Union[bool, str, None] = None, - ) -> SpaceRuntime: - """Request persistent storage for a Space. - - Args: - repo_id (`str`): - ID of the Space to update. Example: `"open-llm-leaderboard/open_llm_leaderboard"`. - storage (`str` or [`SpaceStorage`]): - Storage tier. Either 'small', 'medium', or 'large'. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - Returns: - [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware. - - > [!TIP] - > It is not possible to decrease persistent storage after its granted. To do so, you must delete it - > via [`delete_space_storage`]. - """ - payload: Dict[str, SpaceStorage] = {"tier": storage} - r = get_session().post( - f"{self.endpoint}/api/spaces/{repo_id}/storage", - headers=self._build_hf_headers(token=token), - json=payload, - ) - hf_raise_for_status(r) - return SpaceRuntime(r.json()) - - @validate_hf_hub_args - def delete_space_storage( - self, - repo_id: str, - *, - token: Union[bool, str, None] = None, - ) -> SpaceRuntime: - """Delete persistent storage for a Space. - - Args: - repo_id (`str`): - ID of the Space to update. Example: `"open-llm-leaderboard/open_llm_leaderboard"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - Returns: - [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware. - Raises: - [`BadRequestError`] - If space has no persistent storage. - - """ - r = get_session().delete( - f"{self.endpoint}/api/spaces/{repo_id}/storage", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(r) - return SpaceRuntime(r.json()) - - ####################### - # Inference Endpoints # - ####################### - - def list_inference_endpoints( - self, namespace: Optional[str] = None, *, token: Union[bool, str, None] = None - ) -> List[InferenceEndpoint]: - """Lists all inference endpoints for the given namespace. - - Args: - namespace (`str`, *optional*): - The namespace to list endpoints for. Defaults to the current user. Set to `"*"` to list all endpoints - from all namespaces (i.e. personal namespace and all orgs the user belongs to). - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - List[`InferenceEndpoint`]: A list of all inference endpoints for the given namespace. - - Example: - ```python - >>> from huggingface_hub import HfApi - >>> api = HfApi() - >>> api.list_inference_endpoints() - [InferenceEndpoint(name='my-endpoint', ...), ...] - ``` - """ - # Special case: list all endpoints for all namespaces the user has access to - if namespace == "*": - user = self.whoami(token=token) - - # List personal endpoints first - endpoints: List[InferenceEndpoint] = list_inference_endpoints(namespace=self._get_namespace(token=token)) - - # Then list endpoints for all orgs the user belongs to and ignore 401 errors (no billing or no access) - for org in user.get("orgs", []): - try: - endpoints += list_inference_endpoints(namespace=org["name"], token=token) - except HfHubHTTPError as error: - if error.response.status_code == 401: # Either no billing or user don't have access) - logger.debug("Cannot list Inference Endpoints for org '%s': %s", org["name"], error) - pass - - return endpoints - - # Normal case: list endpoints for a specific namespace - namespace = namespace or self._get_namespace(token=token) - - response = get_session().get( - f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - - return [ - InferenceEndpoint.from_raw(endpoint, namespace=namespace, token=token) - for endpoint in response.json()["items"] - ] - - def create_inference_endpoint( - self, - name: str, - *, - repository: str, - framework: str, - accelerator: str, - instance_size: str, - instance_type: str, - region: str, - vendor: str, - account_id: Optional[str] = None, - min_replica: int = 1, - max_replica: int = 1, - scale_to_zero_timeout: Optional[int] = None, - revision: Optional[str] = None, - task: Optional[str] = None, - custom_image: Optional[Dict] = None, - env: Optional[Dict[str, str]] = None, - secrets: Optional[Dict[str, str]] = None, - type: InferenceEndpointType = InferenceEndpointType.PROTECTED, - domain: Optional[str] = None, - path: Optional[str] = None, - cache_http_responses: Optional[bool] = None, - tags: Optional[List[str]] = None, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> InferenceEndpoint: - """Create a new Inference Endpoint. - - Args: - name (`str`): - The unique name for the new Inference Endpoint. - repository (`str`): - The name of the model repository associated with the Inference Endpoint (e.g. `"gpt2"`). - framework (`str`): - The machine learning framework used for the model (e.g. `"custom"`). - accelerator (`str`): - The hardware accelerator to be used for inference (e.g. `"cpu"`). - instance_size (`str`): - The size or type of the instance to be used for hosting the model (e.g. `"x4"`). - instance_type (`str`): - The cloud instance type where the Inference Endpoint will be deployed (e.g. `"intel-icl"`). - region (`str`): - The cloud region in which the Inference Endpoint will be created (e.g. `"us-east-1"`). - vendor (`str`): - The cloud provider or vendor where the Inference Endpoint will be hosted (e.g. `"aws"`). - account_id (`str`, *optional*): - The account ID used to link a VPC to a private Inference Endpoint (if applicable). - min_replica (`int`, *optional*): - The minimum number of replicas (instances) to keep running for the Inference Endpoint. To enable - scaling to zero, set this value to 0 and adjust `scale_to_zero_timeout` accordingly. Defaults to 1. - max_replica (`int`, *optional*): - The maximum number of replicas (instances) to scale to for the Inference Endpoint. Defaults to 1. - scale_to_zero_timeout (`int`, *optional*): - The duration in minutes before an inactive endpoint is scaled to zero, or no scaling to zero if - set to None and `min_replica` is not 0. Defaults to None. - revision (`str`, *optional*): - The specific model revision to deploy on the Inference Endpoint (e.g. `"6c0e6080953db56375760c0471a8c5f2929baf11"`). - task (`str`, *optional*): - The task on which to deploy the model (e.g. `"text-classification"`). - custom_image (`Dict`, *optional*): - A custom Docker image to use for the Inference Endpoint. This is useful if you want to deploy an - Inference Endpoint running on the `text-generation-inference` (TGI) framework (see examples). - env (`Dict[str, str]`, *optional*): - Non-secret environment variables to inject in the container environment. - secrets (`Dict[str, str]`, *optional*): - Secret values to inject in the container environment. - type ([`InferenceEndpointType]`, *optional*): - The type of the Inference Endpoint, which can be `"protected"` (default), `"public"` or `"private"`. - domain (`str`, *optional*): - The custom domain for the Inference Endpoint deployment, if setup the inference endpoint will be available at this domain (e.g. `"my-new-domain.cool-website.woof"`). - path (`str`, *optional*): - The custom path to the deployed model, should start with a `/` (e.g. `"/models/google-bert/bert-base-uncased"`). - cache_http_responses (`bool`, *optional*): - Whether to cache HTTP responses from the Inference Endpoint. Defaults to `False`. - tags (`List[str]`, *optional*): - A list of tags to associate with the Inference Endpoint. - namespace (`str`, *optional*): - The namespace where the Inference Endpoint will be created. Defaults to the current user's namespace. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`InferenceEndpoint`]: information about the updated Inference Endpoint. - - Example: - ```python - >>> from huggingface_hub import HfApi - >>> api = HfApi() - >>> endpoint = api.create_inference_endpoint( - ... "my-endpoint-name", - ... repository="gpt2", - ... framework="pytorch", - ... task="text-generation", - ... accelerator="cpu", - ... vendor="aws", - ... region="us-east-1", - ... type="protected", - ... instance_size="x2", - ... instance_type="intel-icl", - ... ) - >>> endpoint - InferenceEndpoint(name='my-endpoint-name', status="pending",...) - - # Run inference on the endpoint - >>> endpoint.client.text_generation(...) - "..." - ``` - - ```python - # Start an Inference Endpoint running Zephyr-7b-beta on TGI - >>> from huggingface_hub import HfApi - >>> api = HfApi() - >>> endpoint = api.create_inference_endpoint( - ... "aws-zephyr-7b-beta-0486", - ... repository="HuggingFaceH4/zephyr-7b-beta", - ... framework="pytorch", - ... task="text-generation", - ... accelerator="gpu", - ... vendor="aws", - ... region="us-east-1", - ... type="protected", - ... instance_size="x1", - ... instance_type="nvidia-a10g", - ... env={ - ... "MAX_BATCH_PREFILL_TOKENS": "2048", - ... "MAX_INPUT_LENGTH": "1024", - ... "MAX_TOTAL_TOKENS": "1512", - ... "MODEL_ID": "/repository" - ... }, - ... custom_image={ - ... "health_route": "/health", - ... "url": "ghcr.io/huggingface/text-generation-inference:1.1.0", - ... }, - ... secrets={"MY_SECRET_KEY": "secret_value"}, - ... tags=["dev", "text-generation"], - ... ) - ``` - - ```python - # Start an Inference Endpoint running ProsusAI/finbert while scaling to zero in 15 minutes - >>> from huggingface_hub import HfApi - >>> api = HfApi() - >>> endpoint = api.create_inference_endpoint( - ... "finbert-classifier", - ... repository="ProsusAI/finbert", - ... framework="pytorch", - ... task="text-classification", - ... min_replica=0, - ... scale_to_zero_timeout=15, - ... accelerator="cpu", - ... vendor="aws", - ... region="us-east-1", - ... type="protected", - ... instance_size="x2", - ... instance_type="intel-icl", - ... ) - >>> endpoint.wait(timeout=300) - # Run inference on the endpoint - >>> endpoint.client.text_generation(...) - TextClassificationOutputElement(label='positive', score=0.8983615040779114) - ``` - - """ - namespace = namespace or self._get_namespace(token=token) - - if custom_image is not None: - image = ( - custom_image - if next(iter(custom_image)) in constants.INFERENCE_ENDPOINT_IMAGE_KEYS - else {"custom": custom_image} - ) - else: - image = {"huggingface": {}} - - payload: Dict = { - "accountId": account_id, - "compute": { - "accelerator": accelerator, - "instanceSize": instance_size, - "instanceType": instance_type, - "scaling": { - "maxReplica": max_replica, - "minReplica": min_replica, - "scaleToZeroTimeout": scale_to_zero_timeout, - }, - }, - "model": { - "framework": framework, - "repository": repository, - "revision": revision, - "task": task, - "image": image, - }, - "name": name, - "provider": { - "region": region, - "vendor": vendor, - }, - "type": type, - } - if env: - payload["model"]["env"] = env - if secrets: - payload["model"]["secrets"] = secrets - if domain is not None or path is not None: - payload["route"] = {} - if domain is not None: - payload["route"]["domain"] = domain - if path is not None: - payload["route"]["path"] = path - if cache_http_responses is not None: - payload["cacheHttpResponses"] = cache_http_responses - if tags is not None: - payload["tags"] = tags - - response = get_session().post( - f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}", - headers=self._build_hf_headers(token=token), - json=payload, - ) - hf_raise_for_status(response) - - return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) - - @experimental - @validate_hf_hub_args - def create_inference_endpoint_from_catalog( - self, - repo_id: str, - *, - name: Optional[str] = None, - token: Union[bool, str, None] = None, - namespace: Optional[str] = None, - ) -> InferenceEndpoint: - """Create a new Inference Endpoint from a model in the Hugging Face Inference Catalog. - - The goal of the Inference Catalog is to provide a curated list of models that are optimized for inference - and for which default configurations have been tested. See https://endpoints.huggingface.co/catalog for a list - of available models in the catalog. - - Args: - repo_id (`str`): - The ID of the model in the catalog to deploy as an Inference Endpoint. - name (`str`, *optional*): - The unique name for the new Inference Endpoint. If not provided, a random name will be generated. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - namespace (`str`, *optional*): - The namespace where the Inference Endpoint will be created. Defaults to the current user's namespace. - - Returns: - [`InferenceEndpoint`]: information about the new Inference Endpoint. - - > [!WARNING] - > `create_inference_endpoint_from_catalog` is experimental. Its API is subject to change in the future. Please provide feedback - > if you have any suggestions or requests. - """ - token = token or self.token or get_token() - payload: Dict = { - "namespace": namespace or self._get_namespace(token=token), - "repoId": repo_id, - } - if name is not None: - payload["endpointName"] = name - - response = get_session().post( - f"{constants.INFERENCE_CATALOG_ENDPOINT}/deploy", - headers=self._build_hf_headers(token=token), - json=payload, - ) - hf_raise_for_status(response) - data = response.json()["endpoint"] - return InferenceEndpoint.from_raw(data, namespace=data["name"], token=token) - - @experimental - @validate_hf_hub_args - def list_inference_catalog(self, *, token: Union[bool, str, None] = None) -> List[str]: - """List models available in the Hugging Face Inference Catalog. - - The goal of the Inference Catalog is to provide a curated list of models that are optimized for inference - and for which default configurations have been tested. See https://endpoints.huggingface.co/catalog for a list - of available models in the catalog. - - Use [`create_inference_endpoint_from_catalog`] to deploy a model from the catalog. - - Args: - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - - Returns: - List[`str`]: A list of model IDs available in the catalog. - > [!WARNING] - > `list_inference_catalog` is experimental. Its API is subject to change in the future. Please provide feedback - > if you have any suggestions or requests. - """ - response = get_session().get( - f"{constants.INFERENCE_CATALOG_ENDPOINT}/repo-list", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - return response.json()["models"] - - def get_inference_endpoint( - self, name: str, *, namespace: Optional[str] = None, token: Union[bool, str, None] = None - ) -> InferenceEndpoint: - """Get information about an Inference Endpoint. - - Args: - name (`str`): - The name of the Inference Endpoint to retrieve information about. - namespace (`str`, *optional*): - The namespace in which the Inference Endpoint is located. Defaults to the current user. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`InferenceEndpoint`]: information about the requested Inference Endpoint. - - Example: - ```python - >>> from huggingface_hub import HfApi - >>> api = HfApi() - >>> endpoint = api.get_inference_endpoint("my-text-to-image") - >>> endpoint - InferenceEndpoint(name='my-text-to-image', ...) - - # Get status - >>> endpoint.status - 'running' - >>> endpoint.url - 'https://my-text-to-image.region.vendor.endpoints.huggingface.cloud' - - # Run inference - >>> endpoint.client.text_to_image(...) - ``` - """ - namespace = namespace or self._get_namespace(token=token) - - response = get_session().get( - f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - - return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) - - def update_inference_endpoint( - self, - name: str, - *, - # Compute update - accelerator: Optional[str] = None, - instance_size: Optional[str] = None, - instance_type: Optional[str] = None, - min_replica: Optional[int] = None, - max_replica: Optional[int] = None, - scale_to_zero_timeout: Optional[int] = None, - # Model update - repository: Optional[str] = None, - framework: Optional[str] = None, - revision: Optional[str] = None, - task: Optional[str] = None, - custom_image: Optional[Dict] = None, - env: Optional[Dict[str, str]] = None, - secrets: Optional[Dict[str, str]] = None, - # Route update - domain: Optional[str] = None, - path: Optional[str] = None, - # Other - cache_http_responses: Optional[bool] = None, - tags: Optional[List[str]] = None, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> InferenceEndpoint: - """Update an Inference Endpoint. - - This method allows the update of either the compute configuration, the deployed model, the route, or any combination. - All arguments are optional but at least one must be provided. - - For convenience, you can also update an Inference Endpoint using [`InferenceEndpoint.update`]. - - Args: - name (`str`): - The name of the Inference Endpoint to update. - - accelerator (`str`, *optional*): - The hardware accelerator to be used for inference (e.g. `"cpu"`). - instance_size (`str`, *optional*): - The size or type of the instance to be used for hosting the model (e.g. `"x4"`). - instance_type (`str`, *optional*): - The cloud instance type where the Inference Endpoint will be deployed (e.g. `"intel-icl"`). - min_replica (`int`, *optional*): - The minimum number of replicas (instances) to keep running for the Inference Endpoint. - max_replica (`int`, *optional*): - The maximum number of replicas (instances) to scale to for the Inference Endpoint. - scale_to_zero_timeout (`int`, *optional*): - The duration in minutes before an inactive endpoint is scaled to zero. - - repository (`str`, *optional*): - The name of the model repository associated with the Inference Endpoint (e.g. `"gpt2"`). - framework (`str`, *optional*): - The machine learning framework used for the model (e.g. `"custom"`). - revision (`str`, *optional*): - The specific model revision to deploy on the Inference Endpoint (e.g. `"6c0e6080953db56375760c0471a8c5f2929baf11"`). - task (`str`, *optional*): - The task on which to deploy the model (e.g. `"text-classification"`). - custom_image (`Dict`, *optional*): - A custom Docker image to use for the Inference Endpoint. This is useful if you want to deploy an - Inference Endpoint running on the `text-generation-inference` (TGI) framework (see examples). - env (`Dict[str, str]`, *optional*): - Non-secret environment variables to inject in the container environment - secrets (`Dict[str, str]`, *optional*): - Secret values to inject in the container environment. - - domain (`str`, *optional*): - The custom domain for the Inference Endpoint deployment, if setup the inference endpoint will be available at this domain (e.g. `"my-new-domain.cool-website.woof"`). - path (`str`, *optional*): - The custom path to the deployed model, should start with a `/` (e.g. `"/models/google-bert/bert-base-uncased"`). - - cache_http_responses (`bool`, *optional*): - Whether to cache HTTP responses from the Inference Endpoint. - tags (`List[str]`, *optional*): - A list of tags to associate with the Inference Endpoint. - - namespace (`str`, *optional*): - The namespace where the Inference Endpoint will be updated. Defaults to the current user's namespace. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`InferenceEndpoint`]: information about the updated Inference Endpoint. - """ - namespace = namespace or self._get_namespace(token=token) - - # Populate only the fields that are not None - payload: Dict = defaultdict(lambda: defaultdict(dict)) - if accelerator is not None: - payload["compute"]["accelerator"] = accelerator - if instance_size is not None: - payload["compute"]["instanceSize"] = instance_size - if instance_type is not None: - payload["compute"]["instanceType"] = instance_type - if max_replica is not None: - payload["compute"]["scaling"]["maxReplica"] = max_replica - if min_replica is not None: - payload["compute"]["scaling"]["minReplica"] = min_replica - if scale_to_zero_timeout is not None: - payload["compute"]["scaling"]["scaleToZeroTimeout"] = scale_to_zero_timeout - if repository is not None: - payload["model"]["repository"] = repository - if framework is not None: - payload["model"]["framework"] = framework - if revision is not None: - payload["model"]["revision"] = revision - if task is not None: - payload["model"]["task"] = task - if custom_image is not None: - payload["model"]["image"] = {"custom": custom_image} - if env is not None: - payload["model"]["env"] = env - if secrets is not None: - payload["model"]["secrets"] = secrets - if domain is not None: - payload["route"]["domain"] = domain - if path is not None: - payload["route"]["path"] = path - if cache_http_responses is not None: - payload["cacheHttpResponses"] = cache_http_responses - if tags is not None: - payload["tags"] = tags - - response = get_session().put( - f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}", - headers=self._build_hf_headers(token=token), - json=payload, - ) - hf_raise_for_status(response) - - return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) - - def delete_inference_endpoint( - self, name: str, *, namespace: Optional[str] = None, token: Union[bool, str, None] = None - ) -> None: - """Delete an Inference Endpoint. - - This operation is not reversible. If you don't want to be charged for an Inference Endpoint, it is preferable - to pause it with [`pause_inference_endpoint`] or scale it to zero with [`scale_to_zero_inference_endpoint`]. - - For convenience, you can also delete an Inference Endpoint using [`InferenceEndpoint.delete`]. - - Args: - name (`str`): - The name of the Inference Endpoint to delete. - namespace (`str`, *optional*): - The namespace in which the Inference Endpoint is located. Defaults to the current user. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - """ - namespace = namespace or self._get_namespace(token=token) - response = get_session().delete( - f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - - def pause_inference_endpoint( - self, name: str, *, namespace: Optional[str] = None, token: Union[bool, str, None] = None - ) -> InferenceEndpoint: - """Pause an Inference Endpoint. - - A paused Inference Endpoint will not be charged. It can be resumed at any time using [`resume_inference_endpoint`]. - This is different than scaling the Inference Endpoint to zero with [`scale_to_zero_inference_endpoint`], which - would be automatically restarted when a request is made to it. - - For convenience, you can also pause an Inference Endpoint using [`pause_inference_endpoint`]. - - Args: - name (`str`): - The name of the Inference Endpoint to pause. - namespace (`str`, *optional*): - The namespace in which the Inference Endpoint is located. Defaults to the current user. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`InferenceEndpoint`]: information about the paused Inference Endpoint. - """ - namespace = namespace or self._get_namespace(token=token) - - response = get_session().post( - f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/pause", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - - return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) - - def resume_inference_endpoint( - self, - name: str, - *, - namespace: Optional[str] = None, - running_ok: bool = True, - token: Union[bool, str, None] = None, - ) -> InferenceEndpoint: - """Resume an Inference Endpoint. - - For convenience, you can also resume an Inference Endpoint using [`InferenceEndpoint.resume`]. - - Args: - name (`str`): - The name of the Inference Endpoint to resume. - namespace (`str`, *optional*): - The namespace in which the Inference Endpoint is located. Defaults to the current user. - running_ok (`bool`, *optional*): - If `True`, the method will not raise an error if the Inference Endpoint is already running. Defaults to - `True`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`InferenceEndpoint`]: information about the resumed Inference Endpoint. - """ - namespace = namespace or self._get_namespace(token=token) - - response = get_session().post( - f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/resume", - headers=self._build_hf_headers(token=token), - ) - try: - hf_raise_for_status(response) - except HfHubHTTPError as error: - # If already running (and it's ok), then fetch current status and return - if running_ok and error.response.status_code == 400 and "already running" in error.response.text: - return self.get_inference_endpoint(name, namespace=namespace, token=token) - # Otherwise, raise the error - raise - - return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) - - def scale_to_zero_inference_endpoint( - self, name: str, *, namespace: Optional[str] = None, token: Union[bool, str, None] = None - ) -> InferenceEndpoint: - """Scale Inference Endpoint to zero. - - An Inference Endpoint scaled to zero will not be charged. It will be resume on the next request to it, with a - cold start delay. This is different than pausing the Inference Endpoint with [`pause_inference_endpoint`], which - would require a manual resume with [`resume_inference_endpoint`]. - - For convenience, you can also scale an Inference Endpoint to zero using [`InferenceEndpoint.scale_to_zero`]. - - Args: - name (`str`): - The name of the Inference Endpoint to scale to zero. - namespace (`str`, *optional*): - The namespace in which the Inference Endpoint is located. Defaults to the current user. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`InferenceEndpoint`]: information about the scaled-to-zero Inference Endpoint. - """ - namespace = namespace or self._get_namespace(token=token) - - response = get_session().post( - f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/scale-to-zero", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - - return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) - - def _get_namespace(self, token: Union[bool, str, None] = None) -> str: - """Get the default namespace for the current user.""" - me = self.whoami(token=token) - if me["type"] == "user": - return me["name"] - else: - raise ValueError( - "Cannot determine default namespace. You must provide a 'namespace' as input or be logged in as a" - " user." - ) - - ######################## - # Collection Endpoints # - ######################## - @validate_hf_hub_args - def list_collections( - self, - *, - owner: Union[List[str], str, None] = None, - item: Union[List[str], str, None] = None, - sort: Optional[Literal["lastModified", "trending", "upvotes"]] = None, - limit: Optional[int] = None, - token: Union[bool, str, None] = None, - ) -> Iterable[Collection]: - """List collections on the Huggingface Hub, given some filters. - - > [!WARNING] - > When listing collections, the item list per collection is truncated to 4 items maximum. To retrieve all items - > from a collection, you must use [`get_collection`]. - - Args: - owner (`List[str]` or `str`, *optional*): - Filter by owner's username. - item (`List[str]` or `str`, *optional*): - Filter collections containing a particular items. Example: `"models/teknium/OpenHermes-2.5-Mistral-7B"`, `"datasets/squad"` or `"papers/2311.12983"`. - sort (`Literal["lastModified", "trending", "upvotes"]`, *optional*): - Sort collections by last modified, trending or upvotes. - limit (`int`, *optional*): - Maximum number of collections to be returned. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Iterable[Collection]`: an iterable of [`Collection`] objects. - """ - # Construct the API endpoint - path = f"{self.endpoint}/api/collections" - headers = self._build_hf_headers(token=token) - params: Dict = {} - if owner is not None: - params.update({"owner": owner}) - if item is not None: - params.update({"item": item}) - if sort is not None: - params.update({"sort": sort}) - if limit is not None: - params.update({"limit": limit}) - - # Paginate over the results until limit is reached - items = paginate(path, headers=headers, params=params) - if limit is not None: - items = islice(items, limit) # Do not iterate over all pages - - # Parse as Collection and return - for position, collection_data in enumerate(items): - yield Collection(position=position, **collection_data) - - def get_collection(self, collection_slug: str, *, token: Union[bool, str, None] = None) -> Collection: - """Gets information about a Collection on the Hub. - - Args: - collection_slug (`str`): - Slug of the collection of the Hub. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: [`Collection`] - - Example: - - ```py - >>> from huggingface_hub import get_collection - >>> collection = get_collection("TheBloke/recent-models-64f9a55bb3115b4f513ec026") - >>> collection.title - 'Recent models' - >>> len(collection.items) - 37 - >>> collection.items[0] - CollectionItem( - item_object_id='651446103cd773a050bf64c2', - item_id='TheBloke/U-Amethyst-20B-AWQ', - item_type='model', - position=88, - note=None - ) - ``` - """ - r = get_session().get( - f"{self.endpoint}/api/collections/{collection_slug}", headers=self._build_hf_headers(token=token) - ) - hf_raise_for_status(r) - return Collection(**{**r.json(), "endpoint": self.endpoint}) - - def create_collection( - self, - title: str, - *, - namespace: Optional[str] = None, - description: Optional[str] = None, - private: bool = False, - exists_ok: bool = False, - token: Union[bool, str, None] = None, - ) -> Collection: - """Create a new Collection on the Hub. - - Args: - title (`str`): - Title of the collection to create. Example: `"Recent models"`. - namespace (`str`, *optional*): - Namespace of the collection to create (username or org). Will default to the owner name. - description (`str`, *optional*): - Description of the collection to create. - private (`bool`, *optional*): - Whether the collection should be private or not. Defaults to `False` (i.e. public collection). - exists_ok (`bool`, *optional*): - If `True`, do not raise an error if collection already exists. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: [`Collection`] - - Example: - - ```py - >>> from huggingface_hub import create_collection - >>> collection = create_collection( - ... title="ICCV 2023", - ... description="Portfolio of models, papers and demos I presented at ICCV 2023", - ... ) - >>> collection.slug - "username/iccv-2023-64f9a55bb3115b4f513ec026" - ``` - """ - if namespace is None: - namespace = self.whoami(token)["name"] - - payload = { - "title": title, - "namespace": namespace, - "private": private, - } - if description is not None: - payload["description"] = description - - r = get_session().post( - f"{self.endpoint}/api/collections", headers=self._build_hf_headers(token=token), json=payload - ) - try: - hf_raise_for_status(r) - except HTTPError as err: - if exists_ok and err.response.status_code == 409: - # Collection already exists and `exists_ok=True` - slug = r.json()["slug"] - return self.get_collection(slug, token=token) - else: - raise - return Collection(**{**r.json(), "endpoint": self.endpoint}) - - def update_collection_metadata( - self, - collection_slug: str, - *, - title: Optional[str] = None, - description: Optional[str] = None, - position: Optional[int] = None, - private: Optional[bool] = None, - theme: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> Collection: - """Update metadata of a collection on the Hub. - - All arguments are optional. Only provided metadata will be updated. - - Args: - collection_slug (`str`): - Slug of the collection to update. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. - title (`str`): - Title of the collection to update. - description (`str`, *optional*): - Description of the collection to update. - position (`int`, *optional*): - New position of the collection in the list of collections of the user. - private (`bool`, *optional*): - Whether the collection should be private or not. - theme (`str`, *optional*): - Theme of the collection on the Hub. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: [`Collection`] - - Example: - - ```py - >>> from huggingface_hub import update_collection_metadata - >>> collection = update_collection_metadata( - ... collection_slug="username/iccv-2023-64f9a55bb3115b4f513ec026", - ... title="ICCV Oct. 2023" - ... description="Portfolio of models, datasets, papers and demos I presented at ICCV Oct. 2023", - ... private=False, - ... theme="pink", - ... ) - >>> collection.slug - "username/iccv-oct-2023-64f9a55bb3115b4f513ec026" - # ^collection slug got updated but not the trailing ID - ``` - """ - payload = { - "position": position, - "private": private, - "theme": theme, - "title": title, - "description": description, - } - r = get_session().patch( - f"{self.endpoint}/api/collections/{collection_slug}", - headers=self._build_hf_headers(token=token), - # Only send not-none values to the API - json={key: value for key, value in payload.items() if value is not None}, - ) - hf_raise_for_status(r) - return Collection(**{**r.json()["data"], "endpoint": self.endpoint}) - - def delete_collection( - self, collection_slug: str, *, missing_ok: bool = False, token: Union[bool, str, None] = None - ) -> None: - """Delete a collection on the Hub. - - Args: - collection_slug (`str`): - Slug of the collection to delete. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. - missing_ok (`bool`, *optional*): - If `True`, do not raise an error if collection doesn't exists. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Example: - - ```py - >>> from huggingface_hub import delete_collection - >>> collection = delete_collection("username/useless-collection-64f9a55bb3115b4f513ec026", missing_ok=True) - ``` - - > [!WARNING] - > This is a non-revertible action. A deleted collection cannot be restored. - """ - r = get_session().delete( - f"{self.endpoint}/api/collections/{collection_slug}", headers=self._build_hf_headers(token=token) - ) - try: - hf_raise_for_status(r) - except HTTPError as err: - if missing_ok and err.response.status_code == 404: - # Collection doesn't exists and `missing_ok=True` - return - else: - raise - - def add_collection_item( - self, - collection_slug: str, - item_id: str, - item_type: CollectionItemType_T, - *, - note: Optional[str] = None, - exists_ok: bool = False, - token: Union[bool, str, None] = None, - ) -> Collection: - """Add an item to a collection on the Hub. - - Args: - collection_slug (`str`): - Slug of the collection to update. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. - item_id (`str`): - ID of the item to add to the collection. It can be the ID of a repo on the Hub (e.g. `"facebook/bart-large-mnli"`) - or a paper id (e.g. `"2307.09288"`). - item_type (`str`): - Type of the item to add. Can be one of `"model"`, `"dataset"`, `"space"` or `"paper"`. - note (`str`, *optional*): - A note to attach to the item in the collection. The maximum size for a note is 500 characters. - exists_ok (`bool`, *optional*): - If `True`, do not raise an error if item already exists. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: [`Collection`] - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` - or `admin` role in the organization the repo belongs to or if you passed a `read` token. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 if the item you try to add to the collection does not exist on the Hub. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 409 if the item you try to add to the collection is already in the collection (and exists_ok=False) - - Example: - - ```py - >>> from huggingface_hub import add_collection_item - >>> collection = add_collection_item( - ... collection_slug="davanstrien/climate-64f99dc2a5067f6b65531bab", - ... item_id="pierre-loic/climate-news-articles", - ... item_type="dataset" - ... ) - >>> collection.items[-1].item_id - "pierre-loic/climate-news-articles" - # ^item got added to the collection on last position - - # Add item with a note - >>> add_collection_item( - ... collection_slug="davanstrien/climate-64f99dc2a5067f6b65531bab", - ... item_id="datasets/climate_fever", - ... item_type="dataset" - ... note="This dataset adopts the FEVER methodology that consists of 1,535 real-world claims regarding climate-change collected on the internet." - ... ) - (...) - ``` - """ - payload: Dict[str, Any] = {"item": {"id": item_id, "type": item_type}} - if note is not None: - payload["note"] = note - r = get_session().post( - f"{self.endpoint}/api/collections/{collection_slug}/items", - headers=self._build_hf_headers(token=token), - json=payload, - ) - try: - hf_raise_for_status(r) - except HTTPError as err: - if exists_ok and err.response.status_code == 409: - # Item already exists and `exists_ok=True` - return self.get_collection(collection_slug, token=token) - else: - raise - return Collection(**{**r.json(), "endpoint": self.endpoint}) - - def update_collection_item( - self, - collection_slug: str, - item_object_id: str, - *, - note: Optional[str] = None, - position: Optional[int] = None, - token: Union[bool, str, None] = None, - ) -> None: - """Update an item in a collection. - - Args: - collection_slug (`str`): - Slug of the collection to update. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. - item_object_id (`str`): - ID of the item in the collection. This is not the id of the item on the Hub (repo_id or paper id). - It must be retrieved from a [`CollectionItem`] object. Example: `collection.items[0].item_object_id`. - note (`str`, *optional*): - A note to attach to the item in the collection. The maximum size for a note is 500 characters. - position (`int`, *optional*): - New position of the item in the collection. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Example: - - ```py - >>> from huggingface_hub import get_collection, update_collection_item - - # Get collection first - >>> collection = get_collection("TheBloke/recent-models-64f9a55bb3115b4f513ec026") - - # Update item based on its ID (add note + update position) - >>> update_collection_item( - ... collection_slug="TheBloke/recent-models-64f9a55bb3115b4f513ec026", - ... item_object_id=collection.items[-1].item_object_id, - ... note="Newly updated model!" - ... position=0, - ... ) - ``` - """ - payload = {"position": position, "note": note} - r = get_session().patch( - f"{self.endpoint}/api/collections/{collection_slug}/items/{item_object_id}", - headers=self._build_hf_headers(token=token), - # Only send not-none values to the API - json={key: value for key, value in payload.items() if value is not None}, - ) - hf_raise_for_status(r) - - def delete_collection_item( - self, - collection_slug: str, - item_object_id: str, - *, - missing_ok: bool = False, - token: Union[bool, str, None] = None, - ) -> None: - """Delete an item from a collection. - - Args: - collection_slug (`str`): - Slug of the collection to update. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. - item_object_id (`str`): - ID of the item in the collection. This is not the id of the item on the Hub (repo_id or paper id). - It must be retrieved from a [`CollectionItem`] object. Example: `collection.items[0].item_object_id`. - missing_ok (`bool`, *optional*): - If `True`, do not raise an error if item doesn't exists. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Example: - - ```py - >>> from huggingface_hub import get_collection, delete_collection_item - - # Get collection first - >>> collection = get_collection("TheBloke/recent-models-64f9a55bb3115b4f513ec026") - - # Delete item based on its ID - >>> delete_collection_item( - ... collection_slug="TheBloke/recent-models-64f9a55bb3115b4f513ec026", - ... item_object_id=collection.items[-1].item_object_id, - ... ) - ``` - """ - r = get_session().delete( - f"{self.endpoint}/api/collections/{collection_slug}/items/{item_object_id}", - headers=self._build_hf_headers(token=token), - ) - try: - hf_raise_for_status(r) - except HTTPError as err: - if missing_ok and err.response.status_code == 404: - # Item already deleted and `missing_ok=True` - return - else: - raise - - ########################## - # Manage access requests # - ########################## - - @validate_hf_hub_args - def list_pending_access_requests( - self, repo_id: str, *, repo_type: Optional[str] = None, token: Union[bool, str, None] = None - ) -> List[AccessRequest]: - """ - Get pending access requests for a given gated repo. - - A pending request means the user has requested access to the repo but the request has not been processed yet. - If the approval mode is automatic, this list should be empty. Pending requests can be accepted or rejected - using [`accept_access_request`] and [`reject_access_request`]. - - For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. - - Args: - repo_id (`str`): - The id of the repo to get access requests for. - repo_type (`str`, *optional*): - The type of the repo to get access requests for. Must be one of `model`, `dataset` or `space`. - Defaults to `model`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `List[AccessRequest]`: A list of [`AccessRequest`] objects. Each time contains a `username`, `email`, - `status` and `timestamp` attribute. If the gated repo has a custom form, the `fields` attribute will - be populated with user's answers. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 400 if the repo is not gated. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` - or `admin` role in the organization the repo belongs to or if you passed a `read` token. - - Example: - ```py - >>> from huggingface_hub import list_pending_access_requests, accept_access_request - - # List pending requests - >>> requests = list_pending_access_requests("meta-llama/Llama-2-7b") - >>> len(requests) - 411 - >>> requests[0] - [ - AccessRequest( - username='clem', - fullname='Clem 🤗', - email='***', - timestamp=datetime.datetime(2023, 11, 23, 18, 4, 53, 828000, tzinfo=datetime.timezone.utc), - status='pending', - fields=None, - ), - ... - ] - - # Accept Clem's request - >>> accept_access_request("meta-llama/Llama-2-7b", "clem") - ``` - """ - return self._list_access_requests(repo_id, "pending", repo_type=repo_type, token=token) - - @validate_hf_hub_args - def list_accepted_access_requests( - self, repo_id: str, *, repo_type: Optional[str] = None, token: Union[bool, str, None] = None - ) -> List[AccessRequest]: - """ - Get accepted access requests for a given gated repo. - - An accepted request means the user has requested access to the repo and the request has been accepted. The user - can download any file of the repo. If the approval mode is automatic, this list should contains by default all - requests. Accepted requests can be cancelled or rejected at any time using [`cancel_access_request`] and - [`reject_access_request`]. A cancelled request will go back to the pending list while a rejected request will - go to the rejected list. In both cases, the user will lose access to the repo. - - For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. - - Args: - repo_id (`str`): - The id of the repo to get access requests for. - repo_type (`str`, *optional*): - The type of the repo to get access requests for. Must be one of `model`, `dataset` or `space`. - Defaults to `model`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `List[AccessRequest]`: A list of [`AccessRequest`] objects. Each time contains a `username`, `email`, - `status` and `timestamp` attribute. If the gated repo has a custom form, the `fields` attribute will - be populated with user's answers. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 400 if the repo is not gated. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` - or `admin` role in the organization the repo belongs to or if you passed a `read` token. - - Example: - ```py - >>> from huggingface_hub import list_accepted_access_requests - - >>> requests = list_accepted_access_requests("meta-llama/Llama-2-7b") - >>> len(requests) - 411 - >>> requests[0] - [ - AccessRequest( - username='clem', - fullname='Clem 🤗', - email='***', - timestamp=datetime.datetime(2023, 11, 23, 18, 4, 53, 828000, tzinfo=datetime.timezone.utc), - status='accepted', - fields=None, - ), - ... - ] - ``` - """ - return self._list_access_requests(repo_id, "accepted", repo_type=repo_type, token=token) - - @validate_hf_hub_args - def list_rejected_access_requests( - self, repo_id: str, *, repo_type: Optional[str] = None, token: Union[bool, str, None] = None - ) -> List[AccessRequest]: - """ - Get rejected access requests for a given gated repo. - - A rejected request means the user has requested access to the repo and the request has been explicitly rejected - by a repo owner (either you or another user from your organization). The user cannot download any file of the - repo. Rejected requests can be accepted or cancelled at any time using [`accept_access_request`] and - [`cancel_access_request`]. A cancelled request will go back to the pending list while an accepted request will - go to the accepted list. - - For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. - - Args: - repo_id (`str`): - The id of the repo to get access requests for. - repo_type (`str`, *optional*): - The type of the repo to get access requests for. Must be one of `model`, `dataset` or `space`. - Defaults to `model`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `List[AccessRequest]`: A list of [`AccessRequest`] objects. Each time contains a `username`, `email`, - `status` and `timestamp` attribute. If the gated repo has a custom form, the `fields` attribute will - be populated with user's answers. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 400 if the repo is not gated. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` - or `admin` role in the organization the repo belongs to or if you passed a `read` token. - - Example: - ```py - >>> from huggingface_hub import list_rejected_access_requests - - >>> requests = list_rejected_access_requests("meta-llama/Llama-2-7b") - >>> len(requests) - 411 - >>> requests[0] - [ - AccessRequest( - username='clem', - fullname='Clem 🤗', - email='***', - timestamp=datetime.datetime(2023, 11, 23, 18, 4, 53, 828000, tzinfo=datetime.timezone.utc), - status='rejected', - fields=None, - ), - ... - ] - ``` - """ - return self._list_access_requests(repo_id, "rejected", repo_type=repo_type, token=token) - - def _list_access_requests( - self, - repo_id: str, - status: Literal["accepted", "rejected", "pending"], - repo_type: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> List[AccessRequest]: - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - - response = get_session().get( - f"{constants.ENDPOINT}/api/{repo_type}s/{repo_id}/user-access-request/{status}", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - return [ - AccessRequest( - username=request["user"]["user"], - fullname=request["user"]["fullname"], - email=request["user"].get("email"), - status=request["status"], - timestamp=parse_datetime(request["timestamp"]), - fields=request.get("fields"), # only if custom fields in form - ) - for request in response.json() - ] - - @validate_hf_hub_args - def cancel_access_request( - self, repo_id: str, user: str, *, repo_type: Optional[str] = None, token: Union[bool, str, None] = None - ) -> None: - """ - Cancel an access request from a user for a given gated repo. - - A cancelled request will go back to the pending list and the user will lose access to the repo. - - For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. - - Args: - repo_id (`str`): - The id of the repo to cancel access request for. - user (`str`): - The username of the user which access request should be cancelled. - repo_type (`str`, *optional*): - The type of the repo to cancel access request for. Must be one of `model`, `dataset` or `space`. - Defaults to `model`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 400 if the repo is not gated. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` - or `admin` role in the organization the repo belongs to or if you passed a `read` token. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 if the user does not exist on the Hub. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 if the user access request cannot be found. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 if the user access request is already in the pending list. - """ - self._handle_access_request(repo_id, user, "pending", repo_type=repo_type, token=token) - - @validate_hf_hub_args - def accept_access_request( - self, repo_id: str, user: str, *, repo_type: Optional[str] = None, token: Union[bool, str, None] = None - ) -> None: - """ - Accept an access request from a user for a given gated repo. - - Once the request is accepted, the user will be able to download any file of the repo and access the community - tab. If the approval mode is automatic, you don't have to accept requests manually. An accepted request can be - cancelled or rejected at any time using [`cancel_access_request`] and [`reject_access_request`]. - - For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. - - Args: - repo_id (`str`): - The id of the repo to accept access request for. - user (`str`): - The username of the user which access request should be accepted. - repo_type (`str`, *optional*): - The type of the repo to accept access request for. Must be one of `model`, `dataset` or `space`. - Defaults to `model`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 400 if the repo is not gated. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` - or `admin` role in the organization the repo belongs to or if you passed a `read` token. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 if the user does not exist on the Hub. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 if the user access request cannot be found. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 if the user access request is already in the accepted list. - """ - self._handle_access_request(repo_id, user, "accepted", repo_type=repo_type, token=token) - - @validate_hf_hub_args - def reject_access_request( - self, - repo_id: str, - user: str, - *, - repo_type: Optional[str] = None, - rejection_reason: Optional[str], - token: Union[bool, str, None] = None, - ) -> None: - """ - Reject an access request from a user for a given gated repo. - - A rejected request will go to the rejected list. The user cannot download any file of the repo. Rejected - requests can be accepted or cancelled at any time using [`accept_access_request`] and [`cancel_access_request`]. - A cancelled request will go back to the pending list while an accepted request will go to the accepted list. - - For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. - - Args: - repo_id (`str`): - The id of the repo to reject access request for. - user (`str`): - The username of the user which access request should be rejected. - repo_type (`str`, *optional*): - The type of the repo to reject access request for. Must be one of `model`, `dataset` or `space`. - Defaults to `model`. - rejection_reason (`str`, *optional*): - Optional rejection reason that will be visible to the user (max 200 characters). - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 400 if the repo is not gated. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` - or `admin` role in the organization the repo belongs to or if you passed a `read` token. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 if the user does not exist on the Hub. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 if the user access request cannot be found. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 if the user access request is already in the rejected list. - """ - self._handle_access_request( - repo_id, user, "rejected", repo_type=repo_type, rejection_reason=rejection_reason, token=token - ) - - @validate_hf_hub_args - def _handle_access_request( - self, - repo_id: str, - user: str, - status: Literal["accepted", "rejected", "pending"], - repo_type: Optional[str] = None, - rejection_reason: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> None: - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - - payload = {"user": user, "status": status} - - if rejection_reason is not None: - if status != "rejected": - raise ValueError("`rejection_reason` can only be passed when rejecting an access request.") - payload["rejectionReason"] = rejection_reason - - response = get_session().post( - f"{constants.ENDPOINT}/api/{repo_type}s/{repo_id}/user-access-request/handle", - headers=self._build_hf_headers(token=token), - json=payload, - ) - hf_raise_for_status(response) - - @validate_hf_hub_args - def grant_access( - self, repo_id: str, user: str, *, repo_type: Optional[str] = None, token: Union[bool, str, None] = None - ) -> None: - """ - Grant access to a user for a given gated repo. - - Granting access don't require for the user to send an access request by themselves. The user is automatically - added to the accepted list meaning they can download the files You can revoke the granted access at any time - using [`cancel_access_request`] or [`reject_access_request`]. - - For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. - - Args: - repo_id (`str`): - The id of the repo to grant access to. - user (`str`): - The username of the user to grant access. - repo_type (`str`, *optional*): - The type of the repo to grant access to. Must be one of `model`, `dataset` or `space`. - Defaults to `model`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 400 if the repo is not gated. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 400 if the user already has access to the repo. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` - or `admin` role in the organization the repo belongs to or if you passed a `read` token. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 if the user does not exist on the Hub. - """ - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - - response = get_session().post( - f"{constants.ENDPOINT}/api/{repo_type}s/{repo_id}/user-access-request/grant", - headers=self._build_hf_headers(token=token), - json={"user": user}, - ) - hf_raise_for_status(response) - return response.json() - - ################### - # Manage webhooks # - ################### - - @validate_hf_hub_args - def get_webhook(self, webhook_id: str, *, token: Union[bool, str, None] = None) -> WebhookInfo: - """Get a webhook by its id. - - Args: - webhook_id (`str`): - The unique identifier of the webhook to get. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved token, which is the recommended - method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`WebhookInfo`]: - Info about the webhook. - - Example: - ```python - >>> from huggingface_hub import get_webhook - >>> webhook = get_webhook("654bbbc16f2ec14d77f109cc") - >>> print(webhook) - WebhookInfo( - id="654bbbc16f2ec14d77f109cc", - job=None, - watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], - url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", - secret="my-secret", - domains=["repo", "discussion"], - disabled=False, - ) - ``` - """ - response = get_session().get( - f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - webhook_data = response.json()["webhook"] - - watched_items = [WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook_data["watched"]] - - webhook = WebhookInfo( - id=webhook_data["id"], - url=webhook_data.get("url"), - job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None, - watched=watched_items, - domains=webhook_data["domains"], - secret=webhook_data.get("secret"), - disabled=webhook_data["disabled"], - ) - - return webhook - - @validate_hf_hub_args - def list_webhooks(self, *, token: Union[bool, str, None] = None) -> List[WebhookInfo]: - """List all configured webhooks. - - Args: - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved token, which is the recommended - method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `List[WebhookInfo]`: - List of webhook info objects. - - Example: - ```python - >>> from huggingface_hub import list_webhooks - >>> webhooks = list_webhooks() - >>> len(webhooks) - 2 - >>> webhooks[0] - WebhookInfo( - id="654bbbc16f2ec14d77f109cc", - watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], - url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", - secret="my-secret", - domains=["repo", "discussion"], - disabled=False, - ) - ``` - """ - response = get_session().get( - f"{constants.ENDPOINT}/api/settings/webhooks", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - webhooks_data = response.json() - - return [ - WebhookInfo( - id=webhook["id"], - url=webhook.get("url"), - job=JobSpec(**webhook["job"]) if webhook.get("job") else None, - watched=[WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook["watched"]], - domains=webhook["domains"], - secret=webhook.get("secret"), - disabled=webhook["disabled"], - ) - for webhook in webhooks_data - ] - - @validate_hf_hub_args - def create_webhook( - self, - *, - url: Optional[str] = None, - job_id: Optional[str] = None, - watched: List[Union[Dict, WebhookWatchedItem]], - domains: Optional[List[constants.WEBHOOK_DOMAIN_T]] = None, - secret: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> WebhookInfo: - """Create a new webhook. - - The webhook can either send a payload to a URL, or trigger a Job to run on Hugging Face infrastructure. - This function should be called with one of `url` or `job_id`, but not both. - - Args: - url (`str`): - URL to send the payload to. - job_id (`str`): - ID of the source Job to trigger with the webhook payload in the environment variable WEBHOOK_PAYLOAD. - Additional environment variables are available for convenience: WEBHOOK_REPO_ID, WEBHOOK_REPO_TYPE and WEBHOOK_SECRET. - watched (`List[WebhookWatchedItem]`): - List of [`WebhookWatchedItem`] to be watched by the webhook. It can be users, orgs, models, datasets or spaces. - Watched items can also be provided as plain dictionaries. - domains (`List[Literal["repo", "discussion"]]`, optional): - List of domains to watch. It can be "repo", "discussion" or both. - secret (`str`, optional): - A secret to sign the payload with. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved token, which is the recommended - method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`WebhookInfo`]: - Info about the newly created webhook. - - Example: - - Create a webhook that sends a payload to a URL - ```python - >>> from huggingface_hub import create_webhook - >>> payload = create_webhook( - ... watched=[{"type": "user", "name": "julien-c"}, {"type": "org", "name": "HuggingFaceH4"}], - ... url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", - ... domains=["repo", "discussion"], - ... secret="my-secret", - ... ) - >>> print(payload) - WebhookInfo( - id="654bbbc16f2ec14d77f109cc", - url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", - job=None, - watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], - domains=["repo", "discussion"], - secret="my-secret", - disabled=False, - ) - ``` - - Run a Job and then create a webhook that triggers this Job - ```python - >>> from huggingface_hub import create_webhook, run_job - >>> job = run_job( - ... image="ubuntu", - ... command=["bash", "-c", r"echo An event occured in $WEBHOOK_REPO_ID: $WEBHOOK_PAYLOAD"], - ... ) - >>> payload = create_webhook( - ... watched=[{"type": "user", "name": "julien-c"}, {"type": "org", "name": "HuggingFaceH4"}], - ... job_id=job.id, - ... domains=["repo", "discussion"], - ... secret="my-secret", - ... ) - >>> print(payload) - WebhookInfo( - id="654bbbc16f2ec14d77f109cc", - url=None, - job=JobSpec( - docker_image='ubuntu', - space_id=None, - command=['bash', '-c', 'echo An event occured in $WEBHOOK_REPO_ID: $WEBHOOK_PAYLOAD'], - arguments=[], - environment={}, - secrets=[], - flavor='cpu-basic', - timeout=None, - tags=None, - arch=None - ), - watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], - domains=["repo", "discussion"], - secret="my-secret", - disabled=False, - ) - ``` - """ - watched_dicts = [asdict(item) if isinstance(item, WebhookWatchedItem) else item for item in watched] - - post_webhooks_json = {"watched": watched_dicts, "domains": domains, "secret": secret} - if url is not None and job_id is not None: - raise ValueError("Set `url` or `job_id` but not both.") - elif url is not None: - post_webhooks_json["url"] = url - elif job_id is not None: - post_webhooks_json["jobSourceId"] = job_id - else: - raise ValueError("Missing argument for webhook: `url` or `job_id`.") - - response = get_session().post( - f"{constants.ENDPOINT}/api/settings/webhooks", - json=post_webhooks_json, - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - webhook_data = response.json()["webhook"] - watched_items = [WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook_data["watched"]] - - webhook = WebhookInfo( - id=webhook_data["id"], - url=webhook_data.get("url"), - job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None, - watched=watched_items, - domains=webhook_data["domains"], - secret=webhook_data.get("secret"), - disabled=webhook_data["disabled"], - ) - - return webhook - - @validate_hf_hub_args - def update_webhook( - self, - webhook_id: str, - *, - url: Optional[str] = None, - watched: Optional[List[Union[Dict, WebhookWatchedItem]]] = None, - domains: Optional[List[constants.WEBHOOK_DOMAIN_T]] = None, - secret: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> WebhookInfo: - """Update an existing webhook. - - Args: - webhook_id (`str`): - The unique identifier of the webhook to be updated. - url (`str`, optional): - The URL to which the payload will be sent. - watched (`List[WebhookWatchedItem]`, optional): - List of items to watch. It can be users, orgs, models, datasets, or spaces. - Refer to [`WebhookWatchedItem`] for more details. Watched items can also be provided as plain dictionaries. - domains (`List[Literal["repo", "discussion"]]`, optional): - The domains to watch. This can include "repo", "discussion", or both. - secret (`str`, optional): - A secret to sign the payload with, providing an additional layer of security. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved token, which is the recommended - method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`WebhookInfo`]: - Info about the updated webhook. - - Example: - ```python - >>> from huggingface_hub import update_webhook - >>> updated_payload = update_webhook( - ... webhook_id="654bbbc16f2ec14d77f109cc", - ... url="https://new.webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", - ... watched=[{"type": "user", "name": "julien-c"}, {"type": "org", "name": "HuggingFaceH4"}], - ... domains=["repo"], - ... secret="my-secret", - ... ) - >>> print(updated_payload) - WebhookInfo( - id="654bbbc16f2ec14d77f109cc", - job=None, - url="https://new.webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", - watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], - domains=["repo"], - secret="my-secret", - disabled=False, - ``` - """ - if watched is None: - watched = [] - watched_dicts = [asdict(item) if isinstance(item, WebhookWatchedItem) else item for item in watched] - - response = get_session().post( - f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}", - json={"watched": watched_dicts, "url": url, "domains": domains, "secret": secret}, - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - webhook_data = response.json()["webhook"] - - watched_items = [WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook_data["watched"]] - - webhook = WebhookInfo( - id=webhook_data["id"], - url=webhook_data.get("url"), - job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None, - watched=watched_items, - domains=webhook_data["domains"], - secret=webhook_data.get("secret"), - disabled=webhook_data["disabled"], - ) - - return webhook - - @validate_hf_hub_args - def enable_webhook(self, webhook_id: str, *, token: Union[bool, str, None] = None) -> WebhookInfo: - """Enable a webhook (makes it "active"). - - Args: - webhook_id (`str`): - The unique identifier of the webhook to enable. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved token, which is the recommended - method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`WebhookInfo`]: - Info about the enabled webhook. - - Example: - ```python - >>> from huggingface_hub import enable_webhook - >>> enabled_webhook = enable_webhook("654bbbc16f2ec14d77f109cc") - >>> enabled_webhook - WebhookInfo( - id="654bbbc16f2ec14d77f109cc", - job=None, - url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", - watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], - domains=["repo", "discussion"], - secret="my-secret", - disabled=False, - ) - ``` - """ - response = get_session().post( - f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}/enable", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - webhook_data = response.json()["webhook"] - - watched_items = [WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook_data["watched"]] - - webhook = WebhookInfo( - id=webhook_data["id"], - url=webhook_data.get("url"), - job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None, - watched=watched_items, - domains=webhook_data["domains"], - secret=webhook_data.get("secret"), - disabled=webhook_data["disabled"], - ) - - return webhook - - @validate_hf_hub_args - def disable_webhook(self, webhook_id: str, *, token: Union[bool, str, None] = None) -> WebhookInfo: - """Disable a webhook (makes it "disabled"). - - Args: - webhook_id (`str`): - The unique identifier of the webhook to disable. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved token, which is the recommended - method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - [`WebhookInfo`]: - Info about the disabled webhook. - - Example: - ```python - >>> from huggingface_hub import disable_webhook - >>> disabled_webhook = disable_webhook("654bbbc16f2ec14d77f109cc") - >>> disabled_webhook - WebhookInfo( - id="654bbbc16f2ec14d77f109cc", - url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", - jon=None, - watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], - domains=["repo", "discussion"], - secret="my-secret", - disabled=True, - ) - ``` - """ - response = get_session().post( - f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}/disable", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - webhook_data = response.json()["webhook"] - - watched_items = [WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook_data["watched"]] - - webhook = WebhookInfo( - id=webhook_data["id"], - url=webhook_data.get("url"), - job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None, - watched=watched_items, - domains=webhook_data["domains"], - secret=webhook_data.get("secret"), - disabled=webhook_data["disabled"], - ) - - return webhook - - @validate_hf_hub_args - def delete_webhook(self, webhook_id: str, *, token: Union[bool, str, None] = None) -> None: - """Delete a webhook. - - Args: - webhook_id (`str`): - The unique identifier of the webhook to delete. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved token, which is the recommended - method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `None` - - Example: - ```python - >>> from huggingface_hub import delete_webhook - >>> delete_webhook("654bbbc16f2ec14d77f109cc") - ``` - """ - response = get_session().delete( - f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - - ############# - # Internals # - ############# - - def _build_hf_headers( - self, - token: Union[bool, str, None] = None, - library_name: Optional[str] = None, - library_version: Optional[str] = None, - user_agent: Union[Dict, str, None] = None, - ) -> Dict[str, str]: - """ - Alias for [`build_hf_headers`] that uses the token from [`HfApi`] client - when `token` is not provided. - """ - if token is None: - # Cannot do `token = token or self.token` as token can be `False`. - token = self.token - return build_hf_headers( - token=token, - library_name=library_name or self.library_name, - library_version=library_version or self.library_version, - user_agent=user_agent or self.user_agent, - headers=self.headers, - ) - - def _prepare_folder_deletions( - self, - repo_id: str, - repo_type: Optional[str], - revision: Optional[str], - path_in_repo: str, - delete_patterns: Optional[Union[List[str], str]], - token: Union[bool, str, None] = None, - ) -> List[CommitOperationDelete]: - """Generate the list of Delete operations for a commit to delete files from a repo. - - List remote files and match them against the `delete_patterns` constraints. Returns a list of [`CommitOperationDelete`] - with the matching items. - - Note: `.gitattributes` file is essential to make a repo work properly on the Hub. This file will always be - kept even if it matches the `delete_patterns` constraints. - """ - if delete_patterns is None: - # If no delete patterns, no need to list and filter remote files - return [] - - # List remote files - filenames = self.list_repo_files(repo_id=repo_id, revision=revision, repo_type=repo_type, token=token) - - # Compute relative path in repo - if path_in_repo and path_in_repo not in (".", "./"): - path_in_repo = path_in_repo.strip("/") + "/" # harmonize - relpath_to_abspath = { - file[len(path_in_repo) :]: file for file in filenames if file.startswith(path_in_repo) - } - else: - relpath_to_abspath = {file: file for file in filenames} - - # Apply filter on relative paths and return - return [ - CommitOperationDelete(path_in_repo=relpath_to_abspath[relpath], is_folder=False) - for relpath in filter_repo_objects(relpath_to_abspath.keys(), allow_patterns=delete_patterns) - if relpath_to_abspath[relpath] != ".gitattributes" - ] - - def _prepare_upload_folder_additions( - self, - folder_path: Union[str, Path], - path_in_repo: str, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - repo_type: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> List[CommitOperationAdd]: - """Generate the list of Add operations for a commit to upload a folder. - - Files not matching the `allow_patterns` (allowlist) and `ignore_patterns` (denylist) - constraints are discarded. - """ - - folder_path = Path(folder_path).expanduser().resolve() - if not folder_path.is_dir(): - raise ValueError(f"Provided path: '{folder_path}' is not a directory") - - # List files from folder - relpath_to_abspath = { - path.relative_to(folder_path).as_posix(): path - for path in sorted(folder_path.glob("**/*")) # sorted to be deterministic - if path.is_file() - } - - # Filter files - # Patterns are applied on the path relative to `folder_path`. `path_in_repo` is prefixed after the filtering. - filtered_repo_objects = list( - filter_repo_objects( - relpath_to_abspath.keys(), allow_patterns=allow_patterns, ignore_patterns=ignore_patterns - ) - ) - - prefix = f"{path_in_repo.strip('/')}/" if path_in_repo else "" - - # If updating a README.md file, make sure the metadata format is valid - # It's better to fail early than to fail after all the files have been hashed. - if "README.md" in filtered_repo_objects: - self._validate_yaml( - content=relpath_to_abspath["README.md"].read_text(encoding="utf8"), - repo_type=repo_type, - token=token, - ) - if len(filtered_repo_objects) > 30: - log = logger.warning if len(filtered_repo_objects) > 200 else logger.info - log( - "It seems you are trying to upload a large folder at once. This might take some time and then fail if " - "the folder is too large. For such cases, it is recommended to upload in smaller batches or to use " - "`HfApi().upload_large_folder(...)`/`hf upload-large-folder` instead. For more details, " - "check out https://huggingface.co/docs/huggingface_hub/main/en/guides/upload#upload-a-large-folder." - ) - - logger.info(f"Start hashing {len(filtered_repo_objects)} files.") - operations = [ - CommitOperationAdd( - path_or_fileobj=relpath_to_abspath[relpath], # absolute path on disk - path_in_repo=prefix + relpath, # "absolute" path in repo - ) - for relpath in filtered_repo_objects - ] - logger.info(f"Finished hashing {len(filtered_repo_objects)} files.") - return operations - - def _validate_yaml(self, content: str, *, repo_type: Optional[str] = None, token: Union[bool, str, None] = None): - """ - Validate YAML from `README.md`, used before file hashing and upload. - - Args: - content (`str`): - Content of `README.md` to validate. - repo_type (`str`, *optional*): - The type of the repo to grant access to. Must be one of `model`, `dataset` or `space`. - Defaults to `model`. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Raises: - - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - if YAML is invalid - """ - repo_type = repo_type if repo_type is not None else constants.REPO_TYPE_MODEL - headers = self._build_hf_headers(token=token) - - response = get_session().post( - f"{self.endpoint}/api/validate-yaml", - json={"content": content, "repoType": repo_type}, - headers=headers, - ) - # Handle warnings (example: empty metadata) - response_content = response.json() - message = "\n".join([f"- {warning.get('message')}" for warning in response_content.get("warnings", [])]) - if message: - warnings.warn(f"Warnings while validating metadata in README.md:\n{message}") - - # Raise on errors - try: - hf_raise_for_status(response) - except BadRequestError as e: - errors = response_content.get("errors", []) - message = "\n".join([f"- {error.get('message')}" for error in errors]) - raise ValueError(f"Invalid metadata in README.md.\n{message}") from e - - def get_user_overview(self, username: str, token: Union[bool, str, None] = None) -> User: - """ - Get an overview of a user on the Hub. - - Args: - username (`str`): - Username of the user to get an overview of. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `User`: A [`User`] object with the user's overview. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 If the user does not exist on the Hub. - """ - r = get_session().get( - f"{constants.ENDPOINT}/api/users/{username}/overview", headers=self._build_hf_headers(token=token) - ) - hf_raise_for_status(r) - return User(**r.json()) - - @validate_hf_hub_args - def get_organization_overview(self, organization: str, token: Union[bool, str, None] = None) -> Organization: - """ - Get an overview of an organization on the Hub. - - Args: - organization (`str`): - Name of the organization to get an overview of. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved token, which is the recommended method - for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Organization`: An [`Organization`] object with the organization's overview. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 If the organization does not exist on the Hub. - """ - r = get_session().get( - f"{constants.ENDPOINT}/api/organizations/{organization}/overview", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(r) - return Organization(**r.json()) - - def list_organization_members(self, organization: str, token: Union[bool, str, None] = None) -> Iterable[User]: - """ - List of members of an organization on the Hub. - - Args: - organization (`str`): - Name of the organization to get the members of. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Iterable[User]`: A list of [`User`] objects with the members of the organization. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 If the organization does not exist on the Hub. - - """ - for member in paginate( - path=f"{constants.ENDPOINT}/api/organizations/{organization}/members", - params={}, - headers=self._build_hf_headers(token=token), - ): - yield User(**member) - - def list_user_followers(self, username: str, token: Union[bool, str, None] = None) -> Iterable[User]: - """ - Get the list of followers of a user on the Hub. - - Args: - username (`str`): - Username of the user to get the followers of. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Iterable[User]`: A list of [`User`] objects with the followers of the user. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 If the user does not exist on the Hub. - - """ - for follower in paginate( - path=f"{constants.ENDPOINT}/api/users/{username}/followers", - params={}, - headers=self._build_hf_headers(token=token), - ): - yield User(**follower) - - def list_user_following(self, username: str, token: Union[bool, str, None] = None) -> Iterable[User]: - """ - Get the list of users followed by a user on the Hub. - - Args: - username (`str`): - Username of the user to get the users followed by. - token (`bool` or `str`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Iterable[User]`: A list of [`User`] objects with the users followed by the user. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 If the user does not exist on the Hub. - - """ - for followed_user in paginate( - path=f"{constants.ENDPOINT}/api/users/{username}/following", - params={}, - headers=self._build_hf_headers(token=token), - ): - yield User(**followed_user) - - def list_papers( - self, - *, - query: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> Iterable[PaperInfo]: - """ - List daily papers on the Hugging Face Hub given a search query. - - Args: - query (`str`, *optional*): - A search query string to find papers. - If provided, returns papers that match the query. - token (Union[bool, str, None], *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - - Returns: - `Iterable[PaperInfo]`: an iterable of [`huggingface_hub.hf_api.PaperInfo`] objects. - - Example: - - ```python - >>> from huggingface_hub import HfApi - - >>> api = HfApi() - - # List all papers with "attention" in their title - >>> api.list_papers(query="attention") - ``` - """ - path = f"{self.endpoint}/api/papers/search" - params = {} - if query: - params["q"] = query - r = get_session().get( - path, - params=params, - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(r) - for paper in r.json(): - yield PaperInfo(**paper) - - def paper_info(self, id: str) -> PaperInfo: - """ - Get information for a paper on the Hub. - - Args: - id (`str`, **optional**): - ArXiv id of the paper. - - Returns: - `PaperInfo`: A `PaperInfo` object. - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): - HTTP 404 If the paper does not exist on the Hub. - """ - path = f"{self.endpoint}/api/papers/{id}" - r = get_session().get(path) - hf_raise_for_status(r) - return PaperInfo(**r.json()) - - def auth_check( - self, repo_id: str, *, repo_type: Optional[str] = None, token: Union[bool, str, None] = None - ) -> None: - """ - Check if the provided user token has access to a specific repository on the Hugging Face Hub. - - This method verifies whether the user, authenticated via the provided token, has access to the specified - repository. If the repository is not found or if the user lacks the required permissions to access it, - the method raises an appropriate exception. - - Args: - repo_id (`str`): - The repository to check for access. Format should be `"user/repo_name"`. - Example: `"user/my-cool-model"`. - - repo_type (`str`, *optional*): - The type of the repository. Should be one of `"model"`, `"dataset"`, or `"space"`. - If not specified, the default is `"model"`. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - - Raises: - [`~utils.RepositoryNotFoundError`]: - Raised if the repository does not exist, is private, or the user does not have access. This can - occur if the `repo_id` or `repo_type` is incorrect or if the repository is private but the user - is not authenticated. - - [`~utils.GatedRepoError`]: - Raised if the repository exists but is gated and the user is not authorized to access it. - - Example: - Check if the user has access to a repository: - - ```python - >>> from huggingface_hub import auth_check - >>> from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError - - try: - auth_check("user/my-cool-model") - except GatedRepoError: - # Handle gated repository error - print("You do not have permission to access this gated repository.") - except RepositoryNotFoundError: - # Handle repository not found error - print("The repository was not found or you do not have access.") - ``` - - In this example: - - If the user has access, the method completes successfully. - - If the repository is gated or does not exist, appropriate exceptions are raised, allowing the user - to handle them accordingly. - """ - headers = self._build_hf_headers(token=token) - if repo_type is None: - repo_type = constants.REPO_TYPE_MODEL - if repo_type not in constants.REPO_TYPES: - raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") - path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/auth-check" - r = get_session().get(path, headers=headers) - hf_raise_for_status(r) - - def run_job( - self, - *, - image: str, - command: List[str], - env: Optional[Dict[str, Any]] = None, - secrets: Optional[Dict[str, Any]] = None, - flavor: Optional[SpaceHardware] = None, - timeout: Optional[Union[int, float, str]] = None, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> JobInfo: - """ - Run compute Jobs on Hugging Face infrastructure. - - Args: - image (`str`): - The Docker image to use. - Examples: `"ubuntu"`, `"python:3.12"`, `"pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel"`. - Example with an image from a Space: `"hf.co/spaces/lhoestq/duckdb"`. - - command (`List[str]`): - The command to run. Example: `["echo", "hello"]`. - - env (`Dict[str, Any]`, *optional*): - Defines the environment variables for the Job. - - secrets (`Dict[str, Any]`, *optional*): - Defines the secret environment variables for the Job. - - flavor (`str`, *optional*): - Flavor for the hardware, as in Hugging Face Spaces. See [`SpaceHardware`] for possible values. - Defaults to `"cpu-basic"`. - - timeout (`Union[int, float, str]`, *optional*): - Max duration for the Job: int/float with s (seconds, default), m (minutes), h (hours) or d (days). - Example: `300` or `"5m"` for 5 minutes. - - namespace (`str`, *optional*): - The namespace where the Job will be created. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - - Example: - Run your first Job: - - ```python - >>> from huggingface_hub import run_job - >>> run_job(image="python:3.12", command=["python", "-c" ,"print('Hello from HF compute!')"]) - ``` - - Run a GPU Job: - - ```python - >>> from huggingface_hub import run_job - >>> image = "pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel" - >>> command = ["python", "-c", "import torch; print(f"This code ran with the following GPU: {torch.cuda.get_device_name()}")"] - >>> run_job(image=image, command=command, flavor="a10g-small") - ``` - - """ - if namespace is None: - namespace = self.whoami(token=token)["name"] - job_spec = _create_job_spec( - image=image, - command=command, - env=env, - secrets=secrets, - flavor=flavor, - timeout=timeout, - ) - response = get_session().post( - f"https://huggingface.co/api/jobs/{namespace}", - json=job_spec, - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - job_info = response.json() - return JobInfo(**job_info, endpoint=self.endpoint) - - def fetch_job_logs( - self, - *, - job_id: str, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> Iterable[str]: - """ - Fetch all the logs from a compute Job on Hugging Face infrastructure. - - Args: - job_id (`str`): - ID of the Job. - - namespace (`str`, *optional*): - The namespace where the Job is running. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - - Example: - - ```python - >>> from huggingface_hub import fetch_job_logs, run_job - >>> job = run_job(image="python:3.12", command=["python", "-c" ,"print('Hello from HF compute!')"]) - >>> for log in fetch_job_logs(job.id): - ... print(log) - Hello from HF compute! - ``` - """ - if namespace is None: - namespace = self.whoami(token=token)["name"] - logging_finished = logging_started = False - job_finished = False - # - We need to retry because sometimes the /logs doesn't return logs when the job just started. - # (for example it can return only two lines: one for "Job started" and one empty line) - # - Timeouts can happen in case of build errors - # - ChunkedEncodingError can happen in case of stopped logging in the middle of streaming - # - Infinite empty log stream can happen in case of build error - # (the logs stream is infinite and empty except for the Job started message) - # - there is a ": keep-alive" every 30 seconds - - # We don't use http_backoff since we need to check ourselves if ConnectionError.__context__ is a TimeoutError - max_retries = 5 - min_wait_time = 1 - max_wait_time = 10 - sleep_time = 0 - for _ in range(max_retries): - time.sleep(sleep_time) - sleep_time = min(max_wait_time, max(min_wait_time, sleep_time * 2)) - try: - resp = get_session().get( - f"https://huggingface.co/api/jobs/{namespace}/{job_id}/logs", - headers=self._build_hf_headers(token=token), - stream=True, - timeout=120, - ) - log = None - for line in resp.iter_lines(chunk_size=1): - line = line.decode("utf-8") - if line and line.startswith("data: {"): - data = json.loads(line[len("data: ") :]) - # timestamp = data["timestamp"] - if not data["data"].startswith("===== Job started"): - logging_started = True - log = data["data"] - yield log - logging_finished = logging_started - except requests.exceptions.ChunkedEncodingError: - # Response ended prematurely - break - except KeyboardInterrupt: - break - except requests.exceptions.ConnectionError as err: - is_timeout = err.__context__ and isinstance(getattr(err.__context__, "__cause__", None), TimeoutError) - if logging_started or not is_timeout: - raise - if logging_finished or job_finished: - break - job_status = ( - get_session() - .get( - f"https://huggingface.co/api/jobs/{namespace}/{job_id}", - headers=self._build_hf_headers(token=token), - ) - .json() - ) - if "status" in job_status and job_status["status"]["stage"] not in ("RUNNING", "UPDATING"): - job_finished = True - - def list_jobs( - self, - *, - timeout: Optional[int] = None, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> List[JobInfo]: - """ - List compute Jobs on Hugging Face infrastructure. - - Args: - timeout (`float`, *optional*): - Whether to set a timeout for the request to the Hub. - - namespace (`str`, *optional*): - The namespace from where it lists the jobs. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - """ - if namespace is None: - namespace = whoami(token=token)["name"] - response = get_session().get( - f"{self.endpoint}/api/jobs/{namespace}", - headers=self._build_hf_headers(token=token), - timeout=timeout, - ) - response.raise_for_status() - return [JobInfo(**job_info, endpoint=self.endpoint) for job_info in response.json()] - - def inspect_job( - self, - *, - job_id: str, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> JobInfo: - """ - Inspect a compute Job on Hugging Face infrastructure. - - Args: - job_id (`str`): - ID of the Job. - - namespace (`str`, *optional*): - The namespace where the Job is running. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - - Example: - - ```python - >>> from huggingface_hub import inspect_job, run_job - >>> job = run_job(image="python:3.12", command=["python", "-c" ,"print('Hello from HF compute!')"]) - >>> inspect_job(job.id) - JobInfo( - id='68780d00bbe36d38803f645f', - created_at=datetime.datetime(2025, 7, 16, 20, 35, 12, 808000, tzinfo=datetime.timezone.utc), - docker_image='python:3.12', - space_id=None, - command=['python', '-c', "print('Hello from HF compute!')"], - arguments=[], - environment={}, - secrets={}, - flavor='cpu-basic', - status=JobStatus(stage='RUNNING', message=None) - ) - ``` - """ - if namespace is None: - namespace = self.whoami(token=token)["name"] - response = get_session().get( - f"{self.endpoint}/api/jobs/{namespace}/{job_id}", - headers=self._build_hf_headers(token=token), - ) - response.raise_for_status() - return JobInfo(**response.json(), endpoint=self.endpoint) - - def cancel_job( - self, - *, - job_id: str, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> None: - """ - Cancel a compute Job on Hugging Face infrastructure. - - Args: - job_id (`str`): - ID of the Job. - - namespace (`str`, *optional*): - The namespace where the Job is running. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - """ - if namespace is None: - namespace = self.whoami(token=token)["name"] - get_session().post( - f"{self.endpoint}/api/jobs/{namespace}/{job_id}/cancel", - headers=self._build_hf_headers(token=token), - ).raise_for_status() - - @experimental - def run_uv_job( - self, - script: str, - *, - script_args: Optional[List[str]] = None, - dependencies: Optional[List[str]] = None, - python: Optional[str] = None, - image: Optional[str] = None, - env: Optional[Dict[str, Any]] = None, - secrets: Optional[Dict[str, Any]] = None, - flavor: Optional[SpaceHardware] = None, - timeout: Optional[Union[int, float, str]] = None, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - _repo: Optional[str] = None, - ) -> JobInfo: - """ - Run a UV script Job on Hugging Face infrastructure. - - Args: - script (`str`): - Path or URL of the UV script, or a command. - - script_args (`List[str]`, *optional*) - Arguments to pass to the script or command. - - dependencies (`List[str]`, *optional*) - Dependencies to use to run the UV script. - - python (`str`, *optional*) - Use a specific Python version. Default is 3.12. - - image (`str`, *optional*, defaults to "ghcr.io/astral-sh/uv:python3.12-bookworm"): - Use a custom Docker image with `uv` installed. - - env (`Dict[str, Any]`, *optional*): - Defines the environment variables for the Job. - - secrets (`Dict[str, Any]`, *optional*): - Defines the secret environment variables for the Job. - - flavor (`str`, *optional*): - Flavor for the hardware, as in Hugging Face Spaces. See [`SpaceHardware`] for possible values. - Defaults to `"cpu-basic"`. - - timeout (`Union[int, float, str]`, *optional*): - Max duration for the Job: int/float with s (seconds, default), m (minutes), h (hours) or d (days). - Example: `300` or `"5m"` for 5 minutes. - - namespace (`str`, *optional*): - The namespace where the Job will be created. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - - Example: - - Run a script from a URL: - - ```python - >>> from huggingface_hub import run_uv_job - >>> script = "https://raw.githubusercontent.com/huggingface/trl/refs/heads/main/trl/scripts/sft.py" - >>> script_args = ["--model_name_or_path", "Qwen/Qwen2-0.5B", "--dataset_name", "trl-lib/Capybara", "--push_to_hub"] - >>> run_uv_job(script, script_args=script_args, dependencies=["trl"], flavor="a10g-small") - ``` - - Run a local script: - - ```python - >>> from huggingface_hub import run_uv_job - >>> script = "my_sft.py" - >>> script_args = ["--model_name_or_path", "Qwen/Qwen2-0.5B", "--dataset_name", "trl-lib/Capybara", "--push_to_hub"] - >>> run_uv_job(script, script_args=script_args, dependencies=["trl"], flavor="a10g-small") - ``` - - Run a command: - - ```python - >>> from huggingface_hub import run_uv_job - >>> script = "lighteval" - >>> script_args= ["endpoint", "inference-providers", "model_name=openai/gpt-oss-20b,provider=auto", "lighteval|gsm8k|0|0"] - >>> run_uv_job(script, script_args=script_args, dependencies=["lighteval"], flavor="a10g-small") - ``` - """ - image = image or "ghcr.io/astral-sh/uv:python3.12-bookworm" - env = env or {} - secrets = secrets or {} - - # Build command - command, env, secrets = self._create_uv_command_env_and_secrets( - script=script, - script_args=script_args, - dependencies=dependencies, - python=python, - env=env, - secrets=secrets, - namespace=namespace, - token=token, - _repo=_repo, - ) - # Create RunCommand args - return self.run_job( - image=image, - command=command, - env=env, - secrets=secrets, - flavor=flavor, - timeout=timeout, - namespace=namespace, - token=token, - ) - - def create_scheduled_job( - self, - *, - image: str, - command: List[str], - schedule: str, - suspend: Optional[bool] = None, - concurrency: Optional[bool] = None, - env: Optional[Dict[str, Any]] = None, - secrets: Optional[Dict[str, Any]] = None, - flavor: Optional[SpaceHardware] = None, - timeout: Optional[Union[int, float, str]] = None, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> ScheduledJobInfo: - """ - Create scheduled compute Jobs on Hugging Face infrastructure. - - Args: - image (`str`): - The Docker image to use. - Examples: `"ubuntu"`, `"python:3.12"`, `"pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel"`. - Example with an image from a Space: `"hf.co/spaces/lhoestq/duckdb"`. - - command (`List[str]`): - The command to run. Example: `["echo", "hello"]`. - - schedule (`str`): - One of "@annually", "@yearly", "@monthly", "@weekly", "@daily", "@hourly", or a - CRON schedule expression (e.g., '0 9 * * 1' for 9 AM every Monday). - - suspend (`bool`, *optional*): - If True, the scheduled Job is suspended (paused). Defaults to False. - - concurrency (`bool`, *optional*): - If True, multiple instances of this Job can run concurrently. Defaults to False. - - env (`Dict[str, Any]`, *optional*): - Defines the environment variables for the Job. - - secrets (`Dict[str, Any]`, *optional*): - Defines the secret environment variables for the Job. - - flavor (`str`, *optional*): - Flavor for the hardware, as in Hugging Face Spaces. See [`SpaceHardware`] for possible values. - Defaults to `"cpu-basic"`. - - timeout (`Union[int, float, str]`, *optional*): - Max duration for the Job: int/float with s (seconds, default), m (minutes), h (hours) or d (days). - Example: `300` or `"5m"` for 5 minutes. - - namespace (`str`, *optional*): - The namespace where the Job will be created. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - - Example: - Create your first scheduled Job: - - ```python - >>> from huggingface_hub import create_scheduled_job - >>> create_scheduled_job(image="python:3.12", command=["python", "-c" ,"print('Hello from HF compute!')"], schedule="@hourly") - ``` - - Use a CRON schedule expression: - - ```python - >>> from huggingface_hub import create_scheduled_job - >>> create_scheduled_job(image="python:3.12", command=["python", "-c" ,"print('this runs every 5min')"], schedule="*/5 * * * *") - ``` - - Create a scheduled GPU Job: - - ```python - >>> from huggingface_hub import create_scheduled_job - >>> image = "pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel" - >>> command = ["python", "-c", "import torch; print(f"This code ran with the following GPU: {torch.cuda.get_device_name()}")"] - >>> create_scheduled_job(image, command, flavor="a10g-small", schedule="@hourly") - ``` - - """ - if namespace is None: - namespace = self.whoami(token=token)["name"] - - # prepare payload to send to HF Jobs API - job_spec = _create_job_spec( - image=image, - command=command, - env=env, - secrets=secrets, - flavor=flavor, - timeout=timeout, - ) - input_json: Dict[str, Any] = { - "jobSpec": job_spec, - "schedule": schedule, - } - if concurrency is not None: - input_json["concurrency"] = concurrency - if suspend is not None: - input_json["suspend"] = suspend - response = get_session().post( - f"https://huggingface.co/api/scheduled-jobs/{namespace}", - json=input_json, - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - scheduled_job_info = response.json() - return ScheduledJobInfo(**scheduled_job_info) - - def list_scheduled_jobs( - self, - *, - timeout: Optional[int] = None, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> List[ScheduledJobInfo]: - """ - List scheduled compute Jobs on Hugging Face infrastructure. - - Args: - timeout (`float`, *optional*): - Whether to set a timeout for the request to the Hub. - - namespace (`str`, *optional*): - The namespace from where it lists the jobs. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - """ - if namespace is None: - namespace = self.whoami(token=token)["name"] - response = get_session().get( - f"{self.endpoint}/api/scheduled-jobs/{namespace}", - headers=self._build_hf_headers(token=token), - timeout=timeout, - ) - hf_raise_for_status(response) - return [ScheduledJobInfo(**scheduled_job_info) for scheduled_job_info in response.json()] - - def inspect_scheduled_job( - self, - *, - scheduled_job_id: str, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> ScheduledJobInfo: - """ - Inspect a scheduled compute Job on Hugging Face infrastructure. - - Args: - scheduled_job_id (`str`): - ID of the scheduled Job. - - namespace (`str`, *optional*): - The namespace where the scheduled Job is. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - - Example: - - ```python - >>> from huggingface_hub import inspect_job, create_scheduled_job - >>> scheduled_job = create_scheduled_job(image="python:3.12", command=["python", "-c" ,"print('Hello from HF compute!')"], schedule="@hourly") - >>> inspect_scheduled_job(scheduled_job.id) - ``` - """ - if namespace is None: - namespace = self.whoami(token=token)["name"] - response = get_session().get( - f"{self.endpoint}/api/scheduled-jobs/{namespace}/{scheduled_job_id}", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - return ScheduledJobInfo(**response.json()) - - def delete_scheduled_job( - self, - *, - scheduled_job_id: str, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> None: - """ - Delete a scheduled compute Job on Hugging Face infrastructure. - - Args: - scheduled_job_id (`str`): - ID of the scheduled Job. - - namespace (`str`, *optional*): - The namespace where the scheduled Job is. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - """ - if namespace is None: - namespace = self.whoami(token=token)["name"] - response = get_session().delete( - f"{self.endpoint}/api/scheduled-jobs/{namespace}/{scheduled_job_id}", - headers=self._build_hf_headers(token=token), - ) - hf_raise_for_status(response) - - def suspend_scheduled_job( - self, - *, - scheduled_job_id: str, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> None: - """ - Suspend (pause) a scheduled compute Job on Hugging Face infrastructure. - - Args: - scheduled_job_id (`str`): - ID of the scheduled Job. - - namespace (`str`, *optional*): - The namespace where the scheduled Job is. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - """ - if namespace is None: - namespace = self.whoami(token=token)["name"] - get_session().post( - f"{self.endpoint}/api/scheduled-jobs/{namespace}/{scheduled_job_id}/suspend", - headers=self._build_hf_headers(token=token), - ).raise_for_status() - - def resume_scheduled_job( - self, - *, - scheduled_job_id: str, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - ) -> None: - """ - Resume (unpause) a scheduled compute Job on Hugging Face infrastructure. - - Args: - scheduled_job_id (`str`): - ID of the scheduled Job. - - namespace (`str`, *optional*): - The namespace where the scheduled Job is. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - """ - if namespace is None: - namespace = self.whoami(token=token)["name"] - get_session().post( - f"{self.endpoint}/api/scheduled-jobs/{namespace}/{scheduled_job_id}/resume", - headers=self._build_hf_headers(token=token), - ).raise_for_status() - - @experimental - def create_scheduled_uv_job( - self, - script: str, - *, - script_args: Optional[List[str]] = None, - schedule: str, - suspend: Optional[bool] = None, - concurrency: Optional[bool] = None, - dependencies: Optional[List[str]] = None, - python: Optional[str] = None, - image: Optional[str] = None, - env: Optional[Dict[str, Any]] = None, - secrets: Optional[Dict[str, Any]] = None, - flavor: Optional[SpaceHardware] = None, - timeout: Optional[Union[int, float, str]] = None, - namespace: Optional[str] = None, - token: Union[bool, str, None] = None, - _repo: Optional[str] = None, - ) -> ScheduledJobInfo: - """ - Run a UV script Job on Hugging Face infrastructure. - - Args: - script (`str`): - Path or URL of the UV script, or a command. - - script_args (`List[str]`, *optional*) - Arguments to pass to the script, or a command. - - schedule (`str`): - One of "@annually", "@yearly", "@monthly", "@weekly", "@daily", "@hourly", or a - CRON schedule expression (e.g., '0 9 * * 1' for 9 AM every Monday). - - suspend (`bool`, *optional*): - If True, the scheduled Job is suspended (paused). Defaults to False. - - concurrency (`bool`, *optional*): - If True, multiple instances of this Job can run concurrently. Defaults to False. - - dependencies (`List[str]`, *optional*) - Dependencies to use to run the UV script. - - python (`str`, *optional*) - Use a specific Python version. Default is 3.12. - - image (`str`, *optional*, defaults to "ghcr.io/astral-sh/uv:python3.12-bookworm"): - Use a custom Docker image with `uv` installed. - - env (`Dict[str, Any]`, *optional*): - Defines the environment variables for the Job. - - secrets (`Dict[str, Any]`, *optional*): - Defines the secret environment variables for the Job. - - flavor (`str`, *optional*): - Flavor for the hardware, as in Hugging Face Spaces. See [`SpaceHardware`] for possible values. - Defaults to `"cpu-basic"`. - - timeout (`Union[int, float, str]`, *optional*): - Max duration for the Job: int/float with s (seconds, default), m (minutes), h (hours) or d (days). - Example: `300` or `"5m"` for 5 minutes. - - namespace (`str`, *optional*): - The namespace where the Job will be created. Defaults to the current user's namespace. - - token `(Union[bool, str, None]`, *optional*): - A valid user access token. If not provided, the locally saved token will be used, which is the - recommended authentication method. Set to `False` to disable authentication. - Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. - - Example: - - Schedule a script from a URL: - - ```python - >>> from huggingface_hub import create_scheduled_uv_job - >>> script = "https://raw.githubusercontent.com/huggingface/trl/refs/heads/main/trl/scripts/sft.py" - >>> script_args = ["--model_name_or_path", "Qwen/Qwen2-0.5B", "--dataset_name", "trl-lib/Capybara", "--push_to_hub"] - >>> create_scheduled_uv_job(script, script_args=script_args, dependencies=["trl"], flavor="a10g-small", schedule="@weekly") - ``` - - Schedule a local script: - - ```python - >>> from huggingface_hub import create_scheduled_uv_job - >>> script = "my_sft.py" - >>> script_args = ["--model_name_or_path", "Qwen/Qwen2-0.5B", "--dataset_name", "trl-lib/Capybara", "--push_to_hub"] - >>> create_scheduled_uv_job(script, script_args=script_args, dependencies=["trl"], flavor="a10g-small", schedule="@weekly") - ``` - - Schedule a command: - - ```python - >>> from huggingface_hub import create_scheduled_uv_job - >>> script = "lighteval" - >>> script_args= ["endpoint", "inference-providers", "model_name=openai/gpt-oss-20b,provider=auto", "lighteval|gsm8k|0|0"] - >>> create_scheduled_uv_job(script, script_args=script_args, dependencies=["lighteval"], flavor="a10g-small", schedule="@weekly") - ``` - """ - image = image or "ghcr.io/astral-sh/uv:python3.12-bookworm" - # Build command - command, env, secrets = self._create_uv_command_env_and_secrets( - script=script, - script_args=script_args, - dependencies=dependencies, - python=python, - env=env, - secrets=secrets, - namespace=namespace, - token=token, - _repo=_repo, - ) - # Create RunCommand args - return self.create_scheduled_job( - image=image, - command=command, - schedule=schedule, - suspend=suspend, - concurrency=concurrency, - env=env, - secrets=secrets, - flavor=flavor, - timeout=timeout, - namespace=namespace, - token=token, - ) - - def _create_uv_command_env_and_secrets( - self, - *, - script: str, - script_args: Optional[List[str]], - dependencies: Optional[List[str]], - python: Optional[str], - env: Optional[Dict[str, Any]], - secrets: Optional[Dict[str, Any]], - namespace: Optional[str], - token: Union[bool, str, None], - _repo: Optional[str], - ) -> Tuple[List[str], Dict[str, Any], Dict[str, Any]]: - env = env or {} - secrets = secrets or {} - - # Build command - uv_args = [] - if dependencies: - for dependency in dependencies: - uv_args += ["--with", dependency] - if python: - uv_args += ["--python", python] - script_args = script_args or [] - - if namespace is None: - namespace = self.whoami(token=token)["name"] - - is_url = script.startswith("http://") or script.startswith("https://") - if is_url or not Path(script).is_file(): - # Direct URL execution or command - no upload needed - command = ["uv", "run"] + uv_args + [script] + script_args - else: - # Local file - upload to HF - script_path = Path(script) - filename = script_path.name - # Parse repo - if _repo: - repo_id = _repo - if "/" not in repo_id: - repo_id = f"{namespace}/{repo_id}" - else: - repo_id = f"{namespace}/hf-cli-jobs-uv-run-scripts" - - # Create repo if needed - try: - self.repo_info(repo_id, repo_type="dataset") - logger.debug(f"Using existing repository: {repo_id}") - except RepositoryNotFoundError: - logger.info(f"Creating repository: {repo_id}") - create_repo(repo_id, repo_type="dataset", private=True, exist_ok=True) - - # Upload script - logger.info(f"Uploading {script_path.name} to {repo_id}...") - with open(script_path, "r") as f: - script_content = f.read() - - commit_hash = self.upload_file( - path_or_fileobj=script_content.encode(), - path_in_repo=filename, - repo_id=repo_id, - repo_type="dataset", - ).oid - - script_url = f"{self.endpoint}/datasets/{repo_id}/resolve/{commit_hash}/{filename}" - repo_url = f"{self.endpoint}/datasets/{repo_id}" - - logger.debug(f"✓ Script uploaded to: {repo_url}/blob/main/{filename}") - - # Create and upload minimal README - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S UTC") - readme_content = dedent( - f""" - --- - tags: - - hf-cli-jobs-uv-script - - ephemeral - viewer: false - --- - - # UV Script: {filename} - - Executed via `hf jobs uv run` on {timestamp} - - ## Run this script - - ```bash - hf jobs uv run {filename} - ``` - - --- - *Created with [hf jobs](https://huggingface.co/docs/huggingface_hub/main/en/guides/jobs)* - """ - ) - self.upload_file( - path_or_fileobj=readme_content.encode(), - path_in_repo="README.md", - repo_id=repo_id, - repo_type="dataset", - ) - - secrets["UV_SCRIPT_HF_TOKEN"] = token or self.token or get_token() - env["UV_SCRIPT_URL"] = script_url - - pre_command = ( - dedent( - """ - import urllib.request - import os - from pathlib import Path - o = urllib.request.build_opener() - o.addheaders = [("Authorization", "Bearer " + os.environ["UV_SCRIPT_HF_TOKEN"])] - Path("/tmp/script.py").write_bytes(o.open(os.environ["UV_SCRIPT_URL"]).read()) - """ - ) - .strip() - .replace('"', r"\"") - .split("\n") - ) - pre_command = ["python", "-c", '"' + "; ".join(pre_command) + '"'] - command = ["uv", "run"] + uv_args + ["/tmp/script.py"] + script_args - command = ["bash", "-c", " ".join(pre_command) + " && " + " ".join(command)] - return command, env, secrets - - -def _parse_revision_from_pr_url(pr_url: str) -> str: - """Safely parse revision number from a PR url. - - Example: - ```py - >>> _parse_revision_from_pr_url("https://huggingface.co/bigscience/bloom/discussions/2") - "refs/pr/2" - ``` - """ - re_match = re.match(_REGEX_DISCUSSION_URL, pr_url) - if re_match is None: - raise RuntimeError(f"Unexpected response from the hub, expected a Pull Request URL but got: '{pr_url}'") - return f"refs/pr/{re_match[1]}" - - -api = HfApi() - -whoami = api.whoami -auth_check = api.auth_check -get_token_permission = api.get_token_permission - -list_models = api.list_models -model_info = api.model_info - -list_datasets = api.list_datasets -dataset_info = api.dataset_info - -list_spaces = api.list_spaces -space_info = api.space_info - -list_papers = api.list_papers -paper_info = api.paper_info - -repo_exists = api.repo_exists -revision_exists = api.revision_exists -file_exists = api.file_exists -repo_info = api.repo_info -list_repo_files = api.list_repo_files -list_repo_refs = api.list_repo_refs -list_repo_commits = api.list_repo_commits -list_repo_tree = api.list_repo_tree -get_paths_info = api.get_paths_info - -get_model_tags = api.get_model_tags -get_dataset_tags = api.get_dataset_tags - -create_commit = api.create_commit -create_repo = api.create_repo -delete_repo = api.delete_repo -update_repo_visibility = api.update_repo_visibility -update_repo_settings = api.update_repo_settings -move_repo = api.move_repo -upload_file = api.upload_file -upload_folder = api.upload_folder -delete_file = api.delete_file -delete_folder = api.delete_folder -delete_files = api.delete_files -upload_large_folder = api.upload_large_folder -preupload_lfs_files = api.preupload_lfs_files -create_branch = api.create_branch -delete_branch = api.delete_branch -create_tag = api.create_tag -delete_tag = api.delete_tag -get_full_repo_name = api.get_full_repo_name - -# Danger-zone API -super_squash_history = api.super_squash_history -list_lfs_files = api.list_lfs_files -permanently_delete_lfs_files = api.permanently_delete_lfs_files - -# Safetensors helpers -get_safetensors_metadata = api.get_safetensors_metadata -parse_safetensors_file_metadata = api.parse_safetensors_file_metadata - -# Background jobs -run_as_future = api.run_as_future - -# Activity API -list_liked_repos = api.list_liked_repos -list_repo_likers = api.list_repo_likers -unlike = api.unlike - -# Community API -get_discussion_details = api.get_discussion_details -get_repo_discussions = api.get_repo_discussions -create_discussion = api.create_discussion -create_pull_request = api.create_pull_request -change_discussion_status = api.change_discussion_status -comment_discussion = api.comment_discussion -edit_discussion_comment = api.edit_discussion_comment -rename_discussion = api.rename_discussion -merge_pull_request = api.merge_pull_request - -# Space API -add_space_secret = api.add_space_secret -delete_space_secret = api.delete_space_secret -get_space_variables = api.get_space_variables -add_space_variable = api.add_space_variable -delete_space_variable = api.delete_space_variable -get_space_runtime = api.get_space_runtime -request_space_hardware = api.request_space_hardware -set_space_sleep_time = api.set_space_sleep_time -pause_space = api.pause_space -restart_space = api.restart_space -duplicate_space = api.duplicate_space -request_space_storage = api.request_space_storage -delete_space_storage = api.delete_space_storage - -# Inference Endpoint API -list_inference_endpoints = api.list_inference_endpoints -create_inference_endpoint = api.create_inference_endpoint -get_inference_endpoint = api.get_inference_endpoint -update_inference_endpoint = api.update_inference_endpoint -delete_inference_endpoint = api.delete_inference_endpoint -pause_inference_endpoint = api.pause_inference_endpoint -resume_inference_endpoint = api.resume_inference_endpoint -scale_to_zero_inference_endpoint = api.scale_to_zero_inference_endpoint -create_inference_endpoint_from_catalog = api.create_inference_endpoint_from_catalog -list_inference_catalog = api.list_inference_catalog - -# Collections API -get_collection = api.get_collection -list_collections = api.list_collections -create_collection = api.create_collection -update_collection_metadata = api.update_collection_metadata -delete_collection = api.delete_collection -add_collection_item = api.add_collection_item -update_collection_item = api.update_collection_item -delete_collection_item = api.delete_collection_item -delete_collection_item = api.delete_collection_item - -# Access requests API -list_pending_access_requests = api.list_pending_access_requests -list_accepted_access_requests = api.list_accepted_access_requests -list_rejected_access_requests = api.list_rejected_access_requests -cancel_access_request = api.cancel_access_request -accept_access_request = api.accept_access_request -reject_access_request = api.reject_access_request -grant_access = api.grant_access - -# Webhooks API -create_webhook = api.create_webhook -disable_webhook = api.disable_webhook -delete_webhook = api.delete_webhook -enable_webhook = api.enable_webhook -get_webhook = api.get_webhook -list_webhooks = api.list_webhooks -update_webhook = api.update_webhook - - -# User API -get_user_overview = api.get_user_overview -get_organization_overview = api.get_organization_overview -list_organization_members = api.list_organization_members -list_user_followers = api.list_user_followers -list_user_following = api.list_user_following - -# Jobs API -run_job = api.run_job -fetch_job_logs = api.fetch_job_logs -list_jobs = api.list_jobs -inspect_job = api.inspect_job -cancel_job = api.cancel_job -run_uv_job = api.run_uv_job -create_scheduled_job = api.create_scheduled_job -list_scheduled_jobs = api.list_scheduled_jobs -inspect_scheduled_job = api.inspect_scheduled_job -delete_scheduled_job = api.delete_scheduled_job -suspend_scheduled_job = api.suspend_scheduled_job -resume_scheduled_job = api.resume_scheduled_job -create_scheduled_uv_job = api.create_scheduled_uv_job diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/hf_file_system.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/hf_file_system.py deleted file mode 100644 index a29d38a92ee4ddc9348e2575769ca36de6ceab08..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/hf_file_system.py +++ /dev/null @@ -1,1150 +0,0 @@ -import os -import re -import tempfile -from collections import deque -from dataclasses import dataclass, field -from datetime import datetime -from itertools import chain -from pathlib import Path -from typing import Any, Dict, Iterator, List, NoReturn, Optional, Tuple, Union -from urllib.parse import quote, unquote - -import fsspec -from fsspec.callbacks import _DEFAULT_CALLBACK, NoOpCallback, TqdmCallback -from fsspec.utils import isfilelike -from requests import Response - -from . import constants -from ._commit_api import CommitOperationCopy, CommitOperationDelete -from .errors import EntryNotFoundError, RepositoryNotFoundError, RevisionNotFoundError -from .file_download import hf_hub_url, http_get -from .hf_api import HfApi, LastCommitInfo, RepoFile -from .utils import HFValidationError, hf_raise_for_status, http_backoff - - -# Regex used to match special revisions with "/" in them (see #1710) -SPECIAL_REFS_REVISION_REGEX = re.compile( - r""" - (^refs\/convert\/\w+) # `refs/convert/parquet` revisions - | - (^refs\/pr\/\d+) # PR revisions - """, - re.VERBOSE, -) - - -@dataclass -class HfFileSystemResolvedPath: - """Data structure containing information about a resolved Hugging Face file system path.""" - - repo_type: str - repo_id: str - revision: str - path_in_repo: str - # The part placed after '@' in the initial path. It can be a quoted or unquoted refs revision. - # Used to reconstruct the unresolved path to return to the user. - _raw_revision: Optional[str] = field(default=None, repr=False) - - def unresolve(self) -> str: - repo_path = constants.REPO_TYPES_URL_PREFIXES.get(self.repo_type, "") + self.repo_id - if self._raw_revision: - return f"{repo_path}@{self._raw_revision}/{self.path_in_repo}".rstrip("/") - elif self.revision != constants.DEFAULT_REVISION: - return f"{repo_path}@{safe_revision(self.revision)}/{self.path_in_repo}".rstrip("/") - else: - return f"{repo_path}/{self.path_in_repo}".rstrip("/") - - -class HfFileSystem(fsspec.AbstractFileSystem): - """ - Access a remote Hugging Face Hub repository as if were a local file system. - - > [!WARNING] - > [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading - > Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility - > layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. - - Args: - token (`str` or `bool`, *optional*): - A valid user access token (string). Defaults to the locally saved - token, which is the recommended method for authentication (see - https://huggingface.co/docs/huggingface_hub/quick-start#authentication). - To disable authentication, pass `False`. - endpoint (`str`, *optional*): - Endpoint of the Hub. Defaults to . - Usage: - - ```python - >>> from huggingface_hub import HfFileSystem - - >>> fs = HfFileSystem() - - >>> # List files - >>> fs.glob("my-username/my-model/*.bin") - ['my-username/my-model/pytorch_model.bin'] - >>> fs.ls("datasets/my-username/my-dataset", detail=False) - ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] - - >>> # Read/write files - >>> with fs.open("my-username/my-model/pytorch_model.bin") as f: - ... data = f.read() - >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f: - ... f.write(data) - ``` - """ - - root_marker = "" - protocol = "hf" - - def __init__( - self, - *args, - endpoint: Optional[str] = None, - token: Union[bool, str, None] = None, - block_size: Optional[int] = None, - **storage_options, - ): - super().__init__(*args, **storage_options) - self.endpoint = endpoint or constants.ENDPOINT - self.token = token - self._api = HfApi(endpoint=endpoint, token=token) - self.block_size = block_size - # Maps (repo_type, repo_id, revision) to a 2-tuple with: - # * the 1st element indicating whether the repositoy and the revision exist - # * the 2nd element being the exception raised if the repository or revision doesn't exist - self._repo_and_revision_exists_cache: Dict[ - Tuple[str, str, Optional[str]], Tuple[bool, Optional[Exception]] - ] = {} - # Maps parent directory path to path infos - self.dircache: Dict[str, List[Dict[str, Any]]] = {} - - def _repo_and_revision_exist( - self, repo_type: str, repo_id: str, revision: Optional[str] - ) -> Tuple[bool, Optional[Exception]]: - if (repo_type, repo_id, revision) not in self._repo_and_revision_exists_cache: - try: - self._api.repo_info( - repo_id, revision=revision, repo_type=repo_type, timeout=constants.HF_HUB_ETAG_TIMEOUT - ) - except (RepositoryNotFoundError, HFValidationError) as e: - self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)] = False, e - self._repo_and_revision_exists_cache[(repo_type, repo_id, None)] = False, e - except RevisionNotFoundError as e: - self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)] = False, e - self._repo_and_revision_exists_cache[(repo_type, repo_id, None)] = True, None - else: - self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)] = True, None - self._repo_and_revision_exists_cache[(repo_type, repo_id, None)] = True, None - return self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)] - - def resolve_path(self, path: str, revision: Optional[str] = None) -> HfFileSystemResolvedPath: - """ - Resolve a Hugging Face file system path into its components. - - Args: - path (`str`): - Path to resolve. - revision (`str`, *optional*): - The revision of the repo to resolve. Defaults to the revision specified in the path. - - Returns: - [`HfFileSystemResolvedPath`]: Resolved path information containing `repo_type`, `repo_id`, `revision` and `path_in_repo`. - - Raises: - `ValueError`: - If path contains conflicting revision information. - `NotImplementedError`: - If trying to list repositories. - """ - - def _align_revision_in_path_with_revision( - revision_in_path: Optional[str], revision: Optional[str] - ) -> Optional[str]: - if revision is not None: - if revision_in_path is not None and revision_in_path != revision: - raise ValueError( - f'Revision specified in path ("{revision_in_path}") and in `revision` argument ("{revision}")' - " are not the same." - ) - else: - revision = revision_in_path - return revision - - path = self._strip_protocol(path) - if not path: - # can't list repositories at root - raise NotImplementedError("Access to repositories lists is not implemented.") - elif path.split("/")[0] + "/" in constants.REPO_TYPES_URL_PREFIXES.values(): - if "/" not in path: - # can't list repositories at the repository type level - raise NotImplementedError("Access to repositories lists is not implemented.") - repo_type, path = path.split("/", 1) - repo_type = constants.REPO_TYPES_MAPPING[repo_type] - else: - repo_type = constants.REPO_TYPE_MODEL - if path.count("/") > 0: - if "@" in path: - repo_id, revision_in_path = path.split("@", 1) - if "/" in revision_in_path: - match = SPECIAL_REFS_REVISION_REGEX.search(revision_in_path) - if match is not None and revision in (None, match.group()): - # Handle `refs/convert/parquet` and PR revisions separately - path_in_repo = SPECIAL_REFS_REVISION_REGEX.sub("", revision_in_path).lstrip("/") - revision_in_path = match.group() - else: - revision_in_path, path_in_repo = revision_in_path.split("/", 1) - else: - path_in_repo = "" - revision = _align_revision_in_path_with_revision(unquote(revision_in_path), revision) - repo_and_revision_exist, err = self._repo_and_revision_exist(repo_type, repo_id, revision) - if not repo_and_revision_exist: - _raise_file_not_found(path, err) - else: - revision_in_path = None - repo_id_with_namespace = "/".join(path.split("/")[:2]) - path_in_repo_with_namespace = "/".join(path.split("/")[2:]) - repo_id_without_namespace = path.split("/")[0] - path_in_repo_without_namespace = "/".join(path.split("/")[1:]) - repo_id = repo_id_with_namespace - path_in_repo = path_in_repo_with_namespace - repo_and_revision_exist, err = self._repo_and_revision_exist(repo_type, repo_id, revision) - if not repo_and_revision_exist: - if isinstance(err, (RepositoryNotFoundError, HFValidationError)): - repo_id = repo_id_without_namespace - path_in_repo = path_in_repo_without_namespace - repo_and_revision_exist, _ = self._repo_and_revision_exist(repo_type, repo_id, revision) - if not repo_and_revision_exist: - _raise_file_not_found(path, err) - else: - _raise_file_not_found(path, err) - else: - repo_id = path - path_in_repo = "" - if "@" in path: - repo_id, revision_in_path = path.split("@", 1) - revision = _align_revision_in_path_with_revision(unquote(revision_in_path), revision) - else: - revision_in_path = None - repo_and_revision_exist, _ = self._repo_and_revision_exist(repo_type, repo_id, revision) - if not repo_and_revision_exist: - raise NotImplementedError("Access to repositories lists is not implemented.") - - revision = revision if revision is not None else constants.DEFAULT_REVISION - return HfFileSystemResolvedPath(repo_type, repo_id, revision, path_in_repo, _raw_revision=revision_in_path) - - def invalidate_cache(self, path: Optional[str] = None) -> None: - """ - Clear the cache for a given path. - - For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.invalidate_cache). - - Args: - path (`str`, *optional*): - Path to clear from cache. If not provided, clear the entire cache. - - """ - if not path: - self.dircache.clear() - self._repo_and_revision_exists_cache.clear() - else: - resolved_path = self.resolve_path(path) - path = resolved_path.unresolve() - while path: - self.dircache.pop(path, None) - path = self._parent(path) - - # Only clear repo cache if path is to repo root - if not resolved_path.path_in_repo: - self._repo_and_revision_exists_cache.pop((resolved_path.repo_type, resolved_path.repo_id, None), None) - self._repo_and_revision_exists_cache.pop( - (resolved_path.repo_type, resolved_path.repo_id, resolved_path.revision), None - ) - - def _open( - self, - path: str, - mode: str = "rb", - revision: Optional[str] = None, - block_size: Optional[int] = None, - **kwargs, - ) -> "HfFileSystemFile": - block_size = block_size if block_size is not None else self.block_size - if block_size is not None: - kwargs["block_size"] = block_size - if "a" in mode: - raise NotImplementedError("Appending to remote files is not yet supported.") - if block_size == 0: - return HfFileSystemStreamFile(self, path, mode=mode, revision=revision, **kwargs) - else: - return HfFileSystemFile(self, path, mode=mode, revision=revision, **kwargs) - - def _rm(self, path: str, revision: Optional[str] = None, **kwargs) -> None: - resolved_path = self.resolve_path(path, revision=revision) - self._api.delete_file( - path_in_repo=resolved_path.path_in_repo, - repo_id=resolved_path.repo_id, - token=self.token, - repo_type=resolved_path.repo_type, - revision=resolved_path.revision, - commit_message=kwargs.get("commit_message"), - commit_description=kwargs.get("commit_description"), - ) - self.invalidate_cache(path=resolved_path.unresolve()) - - def rm( - self, - path: str, - recursive: bool = False, - maxdepth: Optional[int] = None, - revision: Optional[str] = None, - **kwargs, - ) -> None: - """ - Delete files from a repository. - - For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm). - - > [!WARNING] - > Note: When possible, use `HfApi.delete_file()` for better performance. - - Args: - path (`str`): - Path to delete. - recursive (`bool`, *optional*): - If True, delete directory and all its contents. Defaults to False. - maxdepth (`int`, *optional*): - Maximum number of subdirectories to visit when deleting recursively. - revision (`str`, *optional*): - The git revision to delete from. - - """ - resolved_path = self.resolve_path(path, revision=revision) - paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth, revision=revision) - paths_in_repo = [self.resolve_path(path).path_in_repo for path in paths if not self.isdir(path)] - operations = [CommitOperationDelete(path_in_repo=path_in_repo) for path_in_repo in paths_in_repo] - commit_message = f"Delete {path} " - commit_message += "recursively " if recursive else "" - commit_message += f"up to depth {maxdepth} " if maxdepth is not None else "" - # TODO: use `commit_description` to list all the deleted paths? - self._api.create_commit( - repo_id=resolved_path.repo_id, - repo_type=resolved_path.repo_type, - token=self.token, - operations=operations, - revision=resolved_path.revision, - commit_message=kwargs.get("commit_message", commit_message), - commit_description=kwargs.get("commit_description"), - ) - self.invalidate_cache(path=resolved_path.unresolve()) - - def ls( - self, path: str, detail: bool = True, refresh: bool = False, revision: Optional[str] = None, **kwargs - ) -> List[Union[str, Dict[str, Any]]]: - """ - List the contents of a directory. - - For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.ls). - - > [!WARNING] - > Note: When possible, use `HfApi.list_repo_tree()` for better performance. - - Args: - path (`str`): - Path to the directory. - detail (`bool`, *optional*): - If True, returns a list of dictionaries containing file information. If False, - returns a list of file paths. Defaults to True. - refresh (`bool`, *optional*): - If True, bypass the cache and fetch the latest data. Defaults to False. - revision (`str`, *optional*): - The git revision to list from. - - Returns: - `List[Union[str, Dict[str, Any]]]`: List of file paths (if detail=False) or list of file information - dictionaries (if detail=True). - """ - resolved_path = self.resolve_path(path, revision=revision) - path = resolved_path.unresolve() - try: - out = self._ls_tree(path, refresh=refresh, revision=revision, **kwargs) - except EntryNotFoundError: - # Path could be a file - if not resolved_path.path_in_repo: - _raise_file_not_found(path, None) - out = self._ls_tree(self._parent(path), refresh=refresh, revision=revision, **kwargs) - out = [o for o in out if o["name"] == path] - if len(out) == 0: - _raise_file_not_found(path, None) - return out if detail else [o["name"] for o in out] - - def _ls_tree( - self, - path: str, - recursive: bool = False, - refresh: bool = False, - revision: Optional[str] = None, - expand_info: bool = False, - maxdepth: Optional[int] = None, - ): - resolved_path = self.resolve_path(path, revision=revision) - path = resolved_path.unresolve() - root_path = HfFileSystemResolvedPath( - resolved_path.repo_type, - resolved_path.repo_id, - resolved_path.revision, - path_in_repo="", - _raw_revision=resolved_path._raw_revision, - ).unresolve() - - out = [] - if path in self.dircache and not refresh: - cached_path_infos = self.dircache[path] - out.extend(cached_path_infos) - dirs_not_in_dircache = [] - if recursive: - # Use BFS to traverse the cache and build the "recursive "output - # (The Hub uses a so-called "tree first" strategy for the tree endpoint but we sort the output to follow the spec so the result is (eventually) the same) - depth = 2 - dirs_to_visit = deque( - [(depth, path_info) for path_info in cached_path_infos if path_info["type"] == "directory"] - ) - while dirs_to_visit: - depth, dir_info = dirs_to_visit.popleft() - if maxdepth is None or depth <= maxdepth: - if dir_info["name"] not in self.dircache: - dirs_not_in_dircache.append(dir_info["name"]) - else: - cached_path_infos = self.dircache[dir_info["name"]] - out.extend(cached_path_infos) - dirs_to_visit.extend( - [ - (depth + 1, path_info) - for path_info in cached_path_infos - if path_info["type"] == "directory" - ] - ) - - dirs_not_expanded = [] - if expand_info: - # Check if there are directories with non-expanded entries - dirs_not_expanded = [self._parent(o["name"]) for o in out if o["last_commit"] is None] - - if (recursive and dirs_not_in_dircache) or (expand_info and dirs_not_expanded): - # If the dircache is incomplete, find the common path of the missing and non-expanded entries - # and extend the output with the result of `_ls_tree(common_path, recursive=True)` - common_prefix = os.path.commonprefix(dirs_not_in_dircache + dirs_not_expanded) - # Get the parent directory if the common prefix itself is not a directory - common_path = ( - common_prefix.rstrip("/") - if common_prefix.endswith("/") - or common_prefix == root_path - or common_prefix in chain(dirs_not_in_dircache, dirs_not_expanded) - else self._parent(common_prefix) - ) - if maxdepth is not None: - common_path_depth = common_path[len(path) :].count("/") - maxdepth -= common_path_depth - out = [o for o in out if not o["name"].startswith(common_path + "/")] - for cached_path in list(self.dircache): - if cached_path.startswith(common_path + "/"): - self.dircache.pop(cached_path, None) - self.dircache.pop(common_path, None) - out.extend( - self._ls_tree( - common_path, - recursive=recursive, - refresh=True, - revision=revision, - expand_info=expand_info, - maxdepth=maxdepth, - ) - ) - else: - tree = self._api.list_repo_tree( - resolved_path.repo_id, - resolved_path.path_in_repo, - recursive=recursive, - expand=expand_info, - revision=resolved_path.revision, - repo_type=resolved_path.repo_type, - ) - for path_info in tree: - cache_path = root_path + "/" + path_info.path - if isinstance(path_info, RepoFile): - cache_path_info = { - "name": cache_path, - "size": path_info.size, - "type": "file", - "blob_id": path_info.blob_id, - "lfs": path_info.lfs, - "last_commit": path_info.last_commit, - "security": path_info.security, - } - else: - cache_path_info = { - "name": cache_path, - "size": 0, - "type": "directory", - "tree_id": path_info.tree_id, - "last_commit": path_info.last_commit, - } - parent_path = self._parent(cache_path_info["name"]) - self.dircache.setdefault(parent_path, []).append(cache_path_info) - depth = cache_path[len(path) :].count("/") - if maxdepth is None or depth <= maxdepth: - out.append(cache_path_info) - return out - - def walk(self, path: str, *args, **kwargs) -> Iterator[Tuple[str, List[str], List[str]]]: - """ - Return all files below the given path. - - For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.walk). - - Args: - path (`str`): - Root path to list files from. - - Returns: - `Iterator[Tuple[str, List[str], List[str]]]`: An iterator of (path, list of directory names, list of file names) tuples. - """ - path = self.resolve_path(path, revision=kwargs.get("revision")).unresolve() - yield from super().walk(path, *args, **kwargs) - - def glob(self, path: str, **kwargs) -> List[str]: - """ - Find files by glob-matching. - - For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob). - - Args: - path (`str`): - Path pattern to match. - - Returns: - `List[str]`: List of paths matching the pattern. - """ - path = self.resolve_path(path, revision=kwargs.get("revision")).unresolve() - return super().glob(path, **kwargs) - - def find( - self, - path: str, - maxdepth: Optional[int] = None, - withdirs: bool = False, - detail: bool = False, - refresh: bool = False, - revision: Optional[str] = None, - **kwargs, - ) -> Union[List[str], Dict[str, Dict[str, Any]]]: - """ - List all files below path. - - For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). - - Args: - path (`str`): - Root path to list files from. - maxdepth (`int`, *optional*): - Maximum depth to descend into subdirectories. - withdirs (`bool`, *optional*): - Include directory paths in the output. Defaults to False. - detail (`bool`, *optional*): - If True, returns a dict mapping paths to file information. Defaults to False. - refresh (`bool`, *optional*): - If True, bypass the cache and fetch the latest data. Defaults to False. - revision (`str`, *optional*): - The git revision to list from. - - Returns: - `Union[List[str], Dict[str, Dict[str, Any]]]`: List of paths or dict of file information. - """ - if maxdepth is not None and maxdepth < 1: - raise ValueError("maxdepth must be at least 1") - resolved_path = self.resolve_path(path, revision=revision) - path = resolved_path.unresolve() - try: - out = self._ls_tree( - path, recursive=True, refresh=refresh, revision=resolved_path.revision, maxdepth=maxdepth, **kwargs - ) - except EntryNotFoundError: - # Path could be a file - try: - if self.info(path, revision=revision, **kwargs)["type"] == "file": - out = {path: {}} - else: - out = {} - except FileNotFoundError: - out = {} - else: - if not withdirs: - out = [o for o in out if o["type"] != "directory"] - else: - # If `withdirs=True`, include the directory itself to be consistent with the spec - path_info = self.info(path, revision=resolved_path.revision, **kwargs) - out = [path_info] + out if path_info["type"] == "directory" else out - out = {o["name"]: o for o in out} - names = sorted(out) - if not detail: - return names - else: - return {name: out[name] for name in names} - - def cp_file(self, path1: str, path2: str, revision: Optional[str] = None, **kwargs) -> None: - """ - Copy a file within or between repositories. - - > [!WARNING] - > Note: When possible, use `HfApi.upload_file()` for better performance. - - Args: - path1 (`str`): - Source path to copy from. - path2 (`str`): - Destination path to copy to. - revision (`str`, *optional*): - The git revision to copy from. - - """ - resolved_path1 = self.resolve_path(path1, revision=revision) - resolved_path2 = self.resolve_path(path2, revision=revision) - - same_repo = ( - resolved_path1.repo_type == resolved_path2.repo_type and resolved_path1.repo_id == resolved_path2.repo_id - ) - - if same_repo: - commit_message = f"Copy {path1} to {path2}" - self._api.create_commit( - repo_id=resolved_path1.repo_id, - repo_type=resolved_path1.repo_type, - revision=resolved_path2.revision, - commit_message=kwargs.get("commit_message", commit_message), - commit_description=kwargs.get("commit_description", ""), - operations=[ - CommitOperationCopy( - src_path_in_repo=resolved_path1.path_in_repo, - path_in_repo=resolved_path2.path_in_repo, - src_revision=resolved_path1.revision, - ) - ], - ) - else: - with self.open(path1, "rb", revision=resolved_path1.revision) as f: - content = f.read() - commit_message = f"Copy {path1} to {path2}" - self._api.upload_file( - path_or_fileobj=content, - path_in_repo=resolved_path2.path_in_repo, - repo_id=resolved_path2.repo_id, - token=self.token, - repo_type=resolved_path2.repo_type, - revision=resolved_path2.revision, - commit_message=kwargs.get("commit_message", commit_message), - commit_description=kwargs.get("commit_description"), - ) - self.invalidate_cache(path=resolved_path1.unresolve()) - self.invalidate_cache(path=resolved_path2.unresolve()) - - def modified(self, path: str, **kwargs) -> datetime: - """ - Get the last modified time of a file. - - For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). - - Args: - path (`str`): - Path to the file. - - Returns: - `datetime`: Last commit date of the file. - """ - info = self.info(path, **{**kwargs, "expand_info": True}) - return info["last_commit"]["date"] - - def info(self, path: str, refresh: bool = False, revision: Optional[str] = None, **kwargs) -> Dict[str, Any]: - """ - Get information about a file or directory. - - For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). - - > [!WARNING] - > Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance. - - Args: - path (`str`): - Path to get info for. - refresh (`bool`, *optional*): - If True, bypass the cache and fetch the latest data. Defaults to False. - revision (`str`, *optional*): - The git revision to get info from. - - Returns: - `Dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). - - """ - resolved_path = self.resolve_path(path, revision=revision) - path = resolved_path.unresolve() - expand_info = kwargs.get( - "expand_info", False - ) # don't expose it as a parameter in the public API to follow the spec - if not resolved_path.path_in_repo: - # Path is the root directory - out = { - "name": path, - "size": 0, - "type": "directory", - "last_commit": None, - } - if expand_info: - last_commit = self._api.list_repo_commits( - resolved_path.repo_id, repo_type=resolved_path.repo_type, revision=resolved_path.revision - )[-1] - out = { - **out, - "tree_id": None, # TODO: tree_id of the root directory? - "last_commit": LastCommitInfo( - oid=last_commit.commit_id, title=last_commit.title, date=last_commit.created_at - ), - } - else: - out = None - parent_path = self._parent(path) - if not expand_info and parent_path not in self.dircache: - # Fill the cache with cheap call - self.ls(parent_path) - if parent_path in self.dircache: - # Check if the path is in the cache - out1 = [o for o in self.dircache[parent_path] if o["name"] == path] - if not out1: - _raise_file_not_found(path, None) - out = out1[0] - if refresh or out is None or (expand_info and out and out["last_commit"] is None): - paths_info = self._api.get_paths_info( - resolved_path.repo_id, - resolved_path.path_in_repo, - expand=expand_info, - revision=resolved_path.revision, - repo_type=resolved_path.repo_type, - ) - if not paths_info: - _raise_file_not_found(path, None) - path_info = paths_info[0] - root_path = HfFileSystemResolvedPath( - resolved_path.repo_type, - resolved_path.repo_id, - resolved_path.revision, - path_in_repo="", - _raw_revision=resolved_path._raw_revision, - ).unresolve() - if isinstance(path_info, RepoFile): - out = { - "name": root_path + "/" + path_info.path, - "size": path_info.size, - "type": "file", - "blob_id": path_info.blob_id, - "lfs": path_info.lfs, - "last_commit": path_info.last_commit, - "security": path_info.security, - } - else: - out = { - "name": root_path + "/" + path_info.path, - "size": 0, - "type": "directory", - "tree_id": path_info.tree_id, - "last_commit": path_info.last_commit, - } - if not expand_info: - out = {k: out[k] for k in ["name", "size", "type"]} - assert out is not None - return out - - def exists(self, path, **kwargs): - """ - Check if a file exists. - - For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). - - > [!WARNING] - > Note: When possible, use `HfApi.file_exists()` for better performance. - - Args: - path (`str`): - Path to check. - - Returns: - `bool`: True if file exists, False otherwise. - """ - try: - if kwargs.get("refresh", False): - self.invalidate_cache(path) - - self.info(path, **kwargs) - return True - except: # noqa: E722 - return False - - def isdir(self, path): - """ - Check if a path is a directory. - - For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). - - Args: - path (`str`): - Path to check. - - Returns: - `bool`: True if path is a directory, False otherwise. - """ - try: - return self.info(path)["type"] == "directory" - except OSError: - return False - - def isfile(self, path): - """ - Check if a path is a file. - - For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). - - Args: - path (`str`): - Path to check. - - Returns: - `bool`: True if path is a file, False otherwise. - """ - try: - return self.info(path)["type"] == "file" - except: # noqa: E722 - return False - - def url(self, path: str) -> str: - """ - Get the HTTP URL of the given path. - - Args: - path (`str`): - Path to get URL for. - - Returns: - `str`: HTTP URL to access the file or directory on the Hub. - """ - resolved_path = self.resolve_path(path) - url = hf_hub_url( - resolved_path.repo_id, - resolved_path.path_in_repo, - repo_type=resolved_path.repo_type, - revision=resolved_path.revision, - endpoint=self.endpoint, - ) - if self.isdir(path): - url = url.replace("/resolve/", "/tree/", 1) - return url - - def get_file(self, rpath, lpath, callback=_DEFAULT_CALLBACK, outfile=None, **kwargs) -> None: - """ - Copy single remote file to local. - - > [!WARNING] - > Note: When possible, use `HfApi.hf_hub_download()` for better performance. - - Args: - rpath (`str`): - Remote path to download from. - lpath (`str`): - Local path to download to. - callback (`Callback`, *optional*): - Optional callback to track download progress. Defaults to no callback. - outfile (`IO`, *optional*): - Optional file-like object to write to. If provided, `lpath` is ignored. - - """ - revision = kwargs.get("revision") - unhandled_kwargs = set(kwargs.keys()) - {"revision"} - if not isinstance(callback, (NoOpCallback, TqdmCallback)) or len(unhandled_kwargs) > 0: - # for now, let's not handle custom callbacks - # and let's not handle custom kwargs - return super().get_file(rpath, lpath, callback=callback, outfile=outfile, **kwargs) - - # Taken from https://github.com/fsspec/filesystem_spec/blob/47b445ae4c284a82dd15e0287b1ffc410e8fc470/fsspec/spec.py#L883 - if isfilelike(lpath): - outfile = lpath - elif self.isdir(rpath): - os.makedirs(lpath, exist_ok=True) - return None - - if isinstance(lpath, (str, Path)): # otherwise, let's assume it's a file-like object - os.makedirs(os.path.dirname(lpath), exist_ok=True) - - # Open file if not already open - close_file = False - if outfile is None: - outfile = open(lpath, "wb") - close_file = True - initial_pos = outfile.tell() - - # Custom implementation of `get_file` to use `http_get`. - resolve_remote_path = self.resolve_path(rpath, revision=revision) - expected_size = self.info(rpath, revision=revision)["size"] - callback.set_size(expected_size) - try: - http_get( - url=hf_hub_url( - repo_id=resolve_remote_path.repo_id, - revision=resolve_remote_path.revision, - filename=resolve_remote_path.path_in_repo, - repo_type=resolve_remote_path.repo_type, - endpoint=self.endpoint, - ), - temp_file=outfile, # type: ignore[arg-type] - displayed_filename=rpath, - expected_size=expected_size, - resume_size=0, - headers=self._api._build_hf_headers(), - _tqdm_bar=callback.tqdm if isinstance(callback, TqdmCallback) else None, - ) - outfile.seek(initial_pos) - finally: - # Close file only if we opened it ourselves - if close_file: - outfile.close() - - @property - def transaction(self): - """A context within which files are committed together upon exit - - Requires the file class to implement `.commit()` and `.discard()` - for the normal and exception cases. - """ - # Taken from https://github.com/fsspec/filesystem_spec/blob/3fbb6fee33b46cccb015607630843dea049d3243/fsspec/spec.py#L231 - # See https://github.com/huggingface/huggingface_hub/issues/1733 - raise NotImplementedError("Transactional commits are not supported.") - - def start_transaction(self): - """Begin write transaction for deferring files, non-context version""" - # Taken from https://github.com/fsspec/filesystem_spec/blob/3fbb6fee33b46cccb015607630843dea049d3243/fsspec/spec.py#L241 - # See https://github.com/huggingface/huggingface_hub/issues/1733 - raise NotImplementedError("Transactional commits are not supported.") - - def __reduce__(self): - # re-populate the instance cache at HfFileSystem._cache and re-populate the cache attributes of every instance - return make_instance, ( - type(self), - self.storage_args, - self.storage_options, - { - "dircache": self.dircache, - "_repo_and_revision_exists_cache": self._repo_and_revision_exists_cache, - }, - ) - - -class HfFileSystemFile(fsspec.spec.AbstractBufferedFile): - def __init__(self, fs: HfFileSystem, path: str, revision: Optional[str] = None, **kwargs): - try: - self.resolved_path = fs.resolve_path(path, revision=revision) - except FileNotFoundError as e: - if "w" in kwargs.get("mode", ""): - raise FileNotFoundError( - f"{e}.\nMake sure the repository and revision exist before writing data." - ) from e - raise - super().__init__(fs, self.resolved_path.unresolve(), **kwargs) - self.fs: HfFileSystem - - def __del__(self): - if not hasattr(self, "resolved_path"): - # Means that the constructor failed. Nothing to do. - return - return super().__del__() - - def _fetch_range(self, start: int, end: int) -> bytes: - headers = { - "range": f"bytes={start}-{end - 1}", - **self.fs._api._build_hf_headers(), - } - url = hf_hub_url( - repo_id=self.resolved_path.repo_id, - revision=self.resolved_path.revision, - filename=self.resolved_path.path_in_repo, - repo_type=self.resolved_path.repo_type, - endpoint=self.fs.endpoint, - ) - r = http_backoff("GET", url, headers=headers, timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT) - hf_raise_for_status(r) - return r.content - - def _initiate_upload(self) -> None: - self.temp_file = tempfile.NamedTemporaryFile(prefix="hffs-", delete=False) - - def _upload_chunk(self, final: bool = False) -> None: - self.buffer.seek(0) - block = self.buffer.read() - self.temp_file.write(block) - if final: - self.temp_file.close() - self.fs._api.upload_file( - path_or_fileobj=self.temp_file.name, - path_in_repo=self.resolved_path.path_in_repo, - repo_id=self.resolved_path.repo_id, - token=self.fs.token, - repo_type=self.resolved_path.repo_type, - revision=self.resolved_path.revision, - commit_message=self.kwargs.get("commit_message"), - commit_description=self.kwargs.get("commit_description"), - ) - os.remove(self.temp_file.name) - self.fs.invalidate_cache( - path=self.resolved_path.unresolve(), - ) - - def read(self, length=-1): - """Read remote file. - - If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems and if - `hf_transfer` is not enabled, the file is loaded in memory directly. Otherwise, the file is downloaded to a - temporary file and read from there. - """ - if self.mode == "rb" and (length is None or length == -1) and self.loc == 0: - with self.fs.open(self.path, "rb", block_size=0) as f: # block_size=0 enables fast streaming - out = f.read() - self.loc += len(out) - return out - return super().read(length) - - def url(self) -> str: - return self.fs.url(self.path) - - -class HfFileSystemStreamFile(fsspec.spec.AbstractBufferedFile): - def __init__( - self, - fs: HfFileSystem, - path: str, - mode: str = "rb", - revision: Optional[str] = None, - block_size: int = 0, - cache_type: str = "none", - **kwargs, - ): - if block_size != 0: - raise ValueError(f"HfFileSystemStreamFile only supports block_size=0 but got {block_size}") - if cache_type != "none": - raise ValueError(f"HfFileSystemStreamFile only supports cache_type='none' but got {cache_type}") - if "w" in mode: - raise ValueError(f"HfFileSystemStreamFile only supports reading but got mode='{mode}'") - try: - self.resolved_path = fs.resolve_path(path, revision=revision) - except FileNotFoundError as e: - if "w" in kwargs.get("mode", ""): - raise FileNotFoundError( - f"{e}.\nMake sure the repository and revision exist before writing data." - ) from e - # avoid an unnecessary .info() call to instantiate .details - self.details = {"name": self.resolved_path.unresolve(), "size": None} - super().__init__( - fs, self.resolved_path.unresolve(), mode=mode, block_size=block_size, cache_type=cache_type, **kwargs - ) - self.response: Optional[Response] = None - self.fs: HfFileSystem - - def seek(self, loc: int, whence: int = 0): - if loc == 0 and whence == 1: - return - if loc == self.loc and whence == 0: - return - raise ValueError("Cannot seek streaming HF file") - - def read(self, length: int = -1): - read_args = (length,) if length >= 0 else () - if self.response is None: - url = hf_hub_url( - repo_id=self.resolved_path.repo_id, - revision=self.resolved_path.revision, - filename=self.resolved_path.path_in_repo, - repo_type=self.resolved_path.repo_type, - endpoint=self.fs.endpoint, - ) - self.response = http_backoff( - "GET", - url, - headers=self.fs._api._build_hf_headers(), - stream=True, - timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT, - ) - hf_raise_for_status(self.response) - try: - self.response.raw.decode_content = True - out = self.response.raw.read(*read_args) - except Exception: - self.response.close() - - # Retry by recreating the connection - url = hf_hub_url( - repo_id=self.resolved_path.repo_id, - revision=self.resolved_path.revision, - filename=self.resolved_path.path_in_repo, - repo_type=self.resolved_path.repo_type, - endpoint=self.fs.endpoint, - ) - self.response = http_backoff( - "GET", - url, - headers={"Range": "bytes=%d-" % self.loc, **self.fs._api._build_hf_headers()}, - stream=True, - timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT, - ) - hf_raise_for_status(self.response) - try: - self.response.raw.decode_content = True - out = self.response.raw.read(*read_args) - except Exception: - self.response.close() - raise - self.loc += len(out) - return out - - def url(self) -> str: - return self.fs.url(self.path) - - def __del__(self): - if not hasattr(self, "resolved_path"): - # Means that the constructor failed. Nothing to do. - return - return super().__del__() - - def __reduce__(self): - return reopen, (self.fs, self.path, self.mode, self.blocksize, self.cache.name) - - -def safe_revision(revision: str) -> str: - return revision if SPECIAL_REFS_REVISION_REGEX.match(revision) else safe_quote(revision) - - -def safe_quote(s: str) -> str: - return quote(s, safe="") - - -def _raise_file_not_found(path: str, err: Optional[Exception]) -> NoReturn: - msg = path - if isinstance(err, RepositoryNotFoundError): - msg = f"{path} (repository not found)" - elif isinstance(err, RevisionNotFoundError): - msg = f"{path} (revision not found)" - elif isinstance(err, HFValidationError): - msg = f"{path} (invalid repository id)" - raise FileNotFoundError(msg) from err - - -def reopen(fs: HfFileSystem, path: str, mode: str, block_size: int, cache_type: str): - return fs.open(path, mode=mode, block_size=block_size, cache_type=cache_type) - - -def make_instance(cls, args, kwargs, instance_cache_attributes_dict): - fs = cls(*args, **kwargs) - for attr, cached_value in instance_cache_attributes_dict.items(): - setattr(fs, attr, cached_value) - return fs diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/hub_mixin.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/hub_mixin.py deleted file mode 100644 index 9fa702ceda97318a817cb1a325223e26a78e2710..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/hub_mixin.py +++ /dev/null @@ -1,853 +0,0 @@ -import inspect -import json -import os -from dataclasses import Field, asdict, dataclass, is_dataclass -from pathlib import Path -from typing import Any, Callable, ClassVar, Dict, List, Optional, Protocol, Tuple, Type, TypeVar, Union - -import packaging.version - -from . import constants -from .errors import EntryNotFoundError, HfHubHTTPError -from .file_download import hf_hub_download -from .hf_api import HfApi -from .repocard import ModelCard, ModelCardData -from .utils import ( - SoftTemporaryDirectory, - is_jsonable, - is_safetensors_available, - is_simple_optional_type, - is_torch_available, - logging, - unwrap_simple_optional_type, - validate_hf_hub_args, -) - - -if is_torch_available(): - import torch # type: ignore - -if is_safetensors_available(): - import safetensors - from safetensors.torch import load_model as load_model_as_safetensor - from safetensors.torch import save_model as save_model_as_safetensor - - -logger = logging.get_logger(__name__) - - -# Type alias for dataclass instances, copied from https://github.com/python/typeshed/blob/9f28171658b9ca6c32a7cb93fbb99fc92b17858b/stdlib/_typeshed/__init__.pyi#L349 -class DataclassInstance(Protocol): - __dataclass_fields__: ClassVar[Dict[str, Field]] - - -# Generic variable that is either ModelHubMixin or a subclass thereof -T = TypeVar("T", bound="ModelHubMixin") -# Generic variable to represent an args type -ARGS_T = TypeVar("ARGS_T") -ENCODER_T = Callable[[ARGS_T], Any] -DECODER_T = Callable[[Any], ARGS_T] -CODER_T = Tuple[ENCODER_T, DECODER_T] - - -DEFAULT_MODEL_CARD = """ ---- -# For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 -# Doc / guide: https://huggingface.co/docs/hub/model-cards -{{ card_data }} ---- - -This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: -- Code: {{ repo_url | default("[More Information Needed]", true) }} -- Paper: {{ paper_url | default("[More Information Needed]", true) }} -- Docs: {{ docs_url | default("[More Information Needed]", true) }} -""" - - -@dataclass -class MixinInfo: - model_card_template: str - model_card_data: ModelCardData - docs_url: Optional[str] = None - paper_url: Optional[str] = None - repo_url: Optional[str] = None - - -class ModelHubMixin: - """ - A generic mixin to integrate ANY machine learning framework with the Hub. - - To integrate your framework, your model class must inherit from this class. Custom logic for saving/loading models - have to be overwritten in [`_from_pretrained`] and [`_save_pretrained`]. [`PyTorchModelHubMixin`] is a good example - of mixin integration with the Hub. Check out our [integration guide](../guides/integrations) for more instructions. - - When inheriting from [`ModelHubMixin`], you can define class-level attributes. These attributes are not passed to - `__init__` but to the class definition itself. This is useful to define metadata about the library integrating - [`ModelHubMixin`]. - - For more details on how to integrate the mixin with your library, checkout the [integration guide](../guides/integrations). - - Args: - repo_url (`str`, *optional*): - URL of the library repository. Used to generate model card. - paper_url (`str`, *optional*): - URL of the library paper. Used to generate model card. - docs_url (`str`, *optional*): - URL of the library documentation. Used to generate model card. - model_card_template (`str`, *optional*): - Template of the model card. Used to generate model card. Defaults to a generic template. - language (`str` or `List[str]`, *optional*): - Language supported by the library. Used to generate model card. - library_name (`str`, *optional*): - Name of the library integrating ModelHubMixin. Used to generate model card. - license (`str`, *optional*): - License of the library integrating ModelHubMixin. Used to generate model card. - E.g: "apache-2.0" - license_name (`str`, *optional*): - Name of the library integrating ModelHubMixin. Used to generate model card. - Only used if `license` is set to `other`. - E.g: "coqui-public-model-license". - license_link (`str`, *optional*): - URL to the license of the library integrating ModelHubMixin. Used to generate model card. - Only used if `license` is set to `other` and `license_name` is set. - E.g: "https://coqui.ai/cpml". - pipeline_tag (`str`, *optional*): - Tag of the pipeline. Used to generate model card. E.g. "text-classification". - tags (`List[str]`, *optional*): - Tags to be added to the model card. Used to generate model card. E.g. ["computer-vision"] - coders (`Dict[Type, Tuple[Callable, Callable]]`, *optional*): - Dictionary of custom types and their encoders/decoders. Used to encode/decode arguments that are not - jsonable by default. E.g dataclasses, argparse.Namespace, OmegaConf, etc. - - Example: - - ```python - >>> from huggingface_hub import ModelHubMixin - - # Inherit from ModelHubMixin - >>> class MyCustomModel( - ... ModelHubMixin, - ... library_name="my-library", - ... tags=["computer-vision"], - ... repo_url="https://github.com/huggingface/my-cool-library", - ... paper_url="https://arxiv.org/abs/2304.12244", - ... docs_url="https://huggingface.co/docs/my-cool-library", - ... # ^ optional metadata to generate model card - ... ): - ... def __init__(self, size: int = 512, device: str = "cpu"): - ... # define how to initialize your model - ... super().__init__() - ... ... - ... - ... def _save_pretrained(self, save_directory: Path) -> None: - ... # define how to serialize your model - ... ... - ... - ... @classmethod - ... def from_pretrained( - ... cls: Type[T], - ... pretrained_model_name_or_path: Union[str, Path], - ... *, - ... force_download: bool = False, - ... resume_download: Optional[bool] = None, - ... proxies: Optional[Dict] = None, - ... token: Optional[Union[str, bool]] = None, - ... cache_dir: Optional[Union[str, Path]] = None, - ... local_files_only: bool = False, - ... revision: Optional[str] = None, - ... **model_kwargs, - ... ) -> T: - ... # define how to deserialize your model - ... ... - - >>> model = MyCustomModel(size=256, device="gpu") - - # Save model weights to local directory - >>> model.save_pretrained("my-awesome-model") - - # Push model weights to the Hub - >>> model.push_to_hub("my-awesome-model") - - # Download and initialize weights from the Hub - >>> reloaded_model = MyCustomModel.from_pretrained("username/my-awesome-model") - >>> reloaded_model.size - 256 - - # Model card has been correctly populated - >>> from huggingface_hub import ModelCard - >>> card = ModelCard.load("username/my-awesome-model") - >>> card.data.tags - ["x-custom-tag", "pytorch_model_hub_mixin", "model_hub_mixin"] - >>> card.data.library_name - "my-library" - ``` - """ - - _hub_mixin_config: Optional[Union[dict, DataclassInstance]] = None - # ^ optional config attribute automatically set in `from_pretrained` - _hub_mixin_info: MixinInfo - # ^ information about the library integrating ModelHubMixin (used to generate model card) - _hub_mixin_inject_config: bool # whether `_from_pretrained` expects `config` or not - _hub_mixin_init_parameters: Dict[str, inspect.Parameter] # __init__ parameters - _hub_mixin_jsonable_default_values: Dict[str, Any] # default values for __init__ parameters - _hub_mixin_jsonable_custom_types: Tuple[Type, ...] # custom types that can be encoded/decoded - _hub_mixin_coders: Dict[Type, CODER_T] # encoders/decoders for custom types - # ^ internal values to handle config - - def __init_subclass__( - cls, - *, - # Generic info for model card - repo_url: Optional[str] = None, - paper_url: Optional[str] = None, - docs_url: Optional[str] = None, - # Model card template - model_card_template: str = DEFAULT_MODEL_CARD, - # Model card metadata - language: Optional[List[str]] = None, - library_name: Optional[str] = None, - license: Optional[str] = None, - license_name: Optional[str] = None, - license_link: Optional[str] = None, - pipeline_tag: Optional[str] = None, - tags: Optional[List[str]] = None, - # How to encode/decode arguments with custom type into a JSON config? - coders: Optional[ - Dict[Type, CODER_T] - # Key is a type. - # Value is a tuple (encoder, decoder). - # Example: {MyCustomType: (lambda x: x.value, lambda data: MyCustomType(data))} - ] = None, - ) -> None: - """Inspect __init__ signature only once when subclassing + handle modelcard.""" - super().__init_subclass__() - - # Will be reused when creating modelcard - tags = tags or [] - tags.append("model_hub_mixin") - - # Initialize MixinInfo if not existent - info = MixinInfo(model_card_template=model_card_template, model_card_data=ModelCardData()) - - # If parent class has a MixinInfo, inherit from it as a copy - if hasattr(cls, "_hub_mixin_info"): - # Inherit model card template from parent class if not explicitly set - if model_card_template == DEFAULT_MODEL_CARD: - info.model_card_template = cls._hub_mixin_info.model_card_template - - # Inherit from parent model card data - info.model_card_data = ModelCardData(**cls._hub_mixin_info.model_card_data.to_dict()) - - # Inherit other info - info.docs_url = cls._hub_mixin_info.docs_url - info.paper_url = cls._hub_mixin_info.paper_url - info.repo_url = cls._hub_mixin_info.repo_url - cls._hub_mixin_info = info - - # Update MixinInfo with metadata - if model_card_template is not None and model_card_template != DEFAULT_MODEL_CARD: - info.model_card_template = model_card_template - if repo_url is not None: - info.repo_url = repo_url - if paper_url is not None: - info.paper_url = paper_url - if docs_url is not None: - info.docs_url = docs_url - if language is not None: - info.model_card_data.language = language - if library_name is not None: - info.model_card_data.library_name = library_name - if license is not None: - info.model_card_data.license = license - if license_name is not None: - info.model_card_data.license_name = license_name - if license_link is not None: - info.model_card_data.license_link = license_link - if pipeline_tag is not None: - info.model_card_data.pipeline_tag = pipeline_tag - if tags is not None: - normalized_tags = list(tags) - if info.model_card_data.tags is not None: - info.model_card_data.tags.extend(normalized_tags) - else: - info.model_card_data.tags = normalized_tags - - if info.model_card_data.tags is not None: - info.model_card_data.tags = sorted(set(info.model_card_data.tags)) - - # Handle encoders/decoders for args - cls._hub_mixin_coders = coders or {} - cls._hub_mixin_jsonable_custom_types = tuple(cls._hub_mixin_coders.keys()) - - # Inspect __init__ signature to handle config - cls._hub_mixin_init_parameters = dict(inspect.signature(cls.__init__).parameters) - cls._hub_mixin_jsonable_default_values = { - param.name: cls._encode_arg(param.default) - for param in cls._hub_mixin_init_parameters.values() - if param.default is not inspect.Parameter.empty and cls._is_jsonable(param.default) - } - cls._hub_mixin_inject_config = "config" in inspect.signature(cls._from_pretrained).parameters - - def __new__(cls: Type[T], *args, **kwargs) -> T: - """Create a new instance of the class and handle config. - - 3 cases: - - If `self._hub_mixin_config` is already set, do nothing. - - If `config` is passed as a dataclass, set it as `self._hub_mixin_config`. - - Otherwise, build `self._hub_mixin_config` from default values and passed values. - """ - instance = super().__new__(cls) - - # If `config` is already set, return early - if instance._hub_mixin_config is not None: - return instance - - # Infer passed values - passed_values = { - **{ - key: value - for key, value in zip( - # [1:] to skip `self` parameter - list(cls._hub_mixin_init_parameters)[1:], - args, - ) - }, - **kwargs, - } - - # If config passed as dataclass => set it and return early - if is_dataclass(passed_values.get("config")): - instance._hub_mixin_config = passed_values["config"] - return instance - - # Otherwise, build config from default + passed values - init_config = { - # default values - **cls._hub_mixin_jsonable_default_values, - # passed values - **{ - key: cls._encode_arg(value) # Encode custom types as jsonable value - for key, value in passed_values.items() - if instance._is_jsonable(value) # Only if jsonable or we have a custom encoder - }, - } - passed_config = init_config.pop("config", {}) - - # Populate `init_config` with provided config - if isinstance(passed_config, dict): - init_config.update(passed_config) - - # Set `config` attribute and return - if init_config != {}: - instance._hub_mixin_config = init_config - return instance - - @classmethod - def _is_jsonable(cls, value: Any) -> bool: - """Check if a value is JSON serializable.""" - if is_dataclass(value): - return True - if isinstance(value, cls._hub_mixin_jsonable_custom_types): - return True - return is_jsonable(value) - - @classmethod - def _encode_arg(cls, arg: Any) -> Any: - """Encode an argument into a JSON serializable format.""" - if is_dataclass(arg): - return asdict(arg) # type: ignore[arg-type] - for type_, (encoder, _) in cls._hub_mixin_coders.items(): - if isinstance(arg, type_): - if arg is None: - return None - return encoder(arg) - return arg - - @classmethod - def _decode_arg(cls, expected_type: Type[ARGS_T], value: Any) -> Optional[ARGS_T]: - """Decode a JSON serializable value into an argument.""" - if is_simple_optional_type(expected_type): - if value is None: - return None - expected_type = unwrap_simple_optional_type(expected_type) - # Dataclass => handle it - if is_dataclass(expected_type): - return _load_dataclass(expected_type, value) # type: ignore[return-value] - # Otherwise => check custom decoders - for type_, (_, decoder) in cls._hub_mixin_coders.items(): - if inspect.isclass(expected_type) and issubclass(expected_type, type_): - return decoder(value) - # Otherwise => don't decode - return value - - def save_pretrained( - self, - save_directory: Union[str, Path], - *, - config: Optional[Union[dict, DataclassInstance]] = None, - repo_id: Optional[str] = None, - push_to_hub: bool = False, - model_card_kwargs: Optional[Dict[str, Any]] = None, - **push_to_hub_kwargs, - ) -> Optional[str]: - """ - Save weights in local directory. - - Args: - save_directory (`str` or `Path`): - Path to directory in which the model weights and configuration will be saved. - config (`dict` or `DataclassInstance`, *optional*): - Model configuration specified as a key/value dictionary or a dataclass instance. - push_to_hub (`bool`, *optional*, defaults to `False`): - Whether or not to push your model to the Huggingface Hub after saving it. - repo_id (`str`, *optional*): - ID of your repository on the Hub. Used only if `push_to_hub=True`. Will default to the folder name if - not provided. - model_card_kwargs (`Dict[str, Any]`, *optional*): - Additional arguments passed to the model card template to customize the model card. - push_to_hub_kwargs: - Additional key word arguments passed along to the [`~ModelHubMixin.push_to_hub`] method. - Returns: - `str` or `None`: url of the commit on the Hub if `push_to_hub=True`, `None` otherwise. - """ - save_directory = Path(save_directory) - save_directory.mkdir(parents=True, exist_ok=True) - - # Remove config.json if already exists. After `_save_pretrained` we don't want to overwrite config.json - # as it might have been saved by the custom `_save_pretrained` already. However we do want to overwrite - # an existing config.json if it was not saved by `_save_pretrained`. - config_path = save_directory / constants.CONFIG_NAME - config_path.unlink(missing_ok=True) - - # save model weights/files (framework-specific) - self._save_pretrained(save_directory) - - # save config (if provided and if not serialized yet in `_save_pretrained`) - if config is None: - config = self._hub_mixin_config - if config is not None: - if is_dataclass(config): - config = asdict(config) # type: ignore[arg-type] - if not config_path.exists(): - config_str = json.dumps(config, sort_keys=True, indent=2) - config_path.write_text(config_str) - - # save model card - model_card_path = save_directory / "README.md" - model_card_kwargs = model_card_kwargs if model_card_kwargs is not None else {} - if not model_card_path.exists(): # do not overwrite if already exists - self.generate_model_card(**model_card_kwargs).save(save_directory / "README.md") - - # push to the Hub if required - if push_to_hub: - kwargs = push_to_hub_kwargs.copy() # soft-copy to avoid mutating input - if config is not None: # kwarg for `push_to_hub` - kwargs["config"] = config - if repo_id is None: - repo_id = save_directory.name # Defaults to `save_directory` name - return self.push_to_hub(repo_id=repo_id, model_card_kwargs=model_card_kwargs, **kwargs) - return None - - def _save_pretrained(self, save_directory: Path) -> None: - """ - Overwrite this method in subclass to define how to save your model. - Check out our [integration guide](../guides/integrations) for instructions. - - Args: - save_directory (`str` or `Path`): - Path to directory in which the model weights and configuration will be saved. - """ - raise NotImplementedError - - @classmethod - @validate_hf_hub_args - def from_pretrained( - cls: Type[T], - pretrained_model_name_or_path: Union[str, Path], - *, - force_download: bool = False, - resume_download: Optional[bool] = None, - proxies: Optional[Dict] = None, - token: Optional[Union[str, bool]] = None, - cache_dir: Optional[Union[str, Path]] = None, - local_files_only: bool = False, - revision: Optional[str] = None, - **model_kwargs, - ) -> T: - """ - Download a model from the Huggingface Hub and instantiate it. - - Args: - pretrained_model_name_or_path (`str`, `Path`): - - Either the `model_id` (string) of a model hosted on the Hub, e.g. `bigscience/bloom`. - - Or a path to a `directory` containing model weights saved using - [`~transformers.PreTrainedModel.save_pretrained`], e.g., `../path/to/my_model_directory/`. - revision (`str`, *optional*): - Revision of the model on the Hub. Can be a branch name, a git tag or any commit id. - Defaults to the latest commit on `main` branch. - force_download (`bool`, *optional*, defaults to `False`): - Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding - the existing cache. - proxies (`Dict[str, str]`, *optional*): - A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', - 'http://hostname': 'foo.bar:4012'}`. The proxies are used on every request. - token (`str` or `bool`, *optional*): - The token to use as HTTP bearer authorization for remote files. By default, it will use the token - cached when running `hf auth login`. - cache_dir (`str`, `Path`, *optional*): - Path to the folder where cached files are stored. - local_files_only (`bool`, *optional*, defaults to `False`): - If `True`, avoid downloading the file and return the path to the local cached file if it exists. - model_kwargs (`Dict`, *optional*): - Additional kwargs to pass to the model during initialization. - """ - model_id = str(pretrained_model_name_or_path) - config_file: Optional[str] = None - if os.path.isdir(model_id): - if constants.CONFIG_NAME in os.listdir(model_id): - config_file = os.path.join(model_id, constants.CONFIG_NAME) - else: - logger.warning(f"{constants.CONFIG_NAME} not found in {Path(model_id).resolve()}") - else: - try: - config_file = hf_hub_download( - repo_id=model_id, - filename=constants.CONFIG_NAME, - revision=revision, - cache_dir=cache_dir, - force_download=force_download, - proxies=proxies, - resume_download=resume_download, - token=token, - local_files_only=local_files_only, - ) - except HfHubHTTPError as e: - logger.info(f"{constants.CONFIG_NAME} not found on the HuggingFace Hub: {str(e)}") - - # Read config - config = None - if config_file is not None: - with open(config_file, "r", encoding="utf-8") as f: - config = json.load(f) - - # Decode custom types in config - for key, value in config.items(): - if key in cls._hub_mixin_init_parameters: - expected_type = cls._hub_mixin_init_parameters[key].annotation - if expected_type is not inspect.Parameter.empty: - config[key] = cls._decode_arg(expected_type, value) - - # Populate model_kwargs from config - for param in cls._hub_mixin_init_parameters.values(): - if param.name not in model_kwargs and param.name in config: - model_kwargs[param.name] = config[param.name] - - # Check if `config` argument was passed at init - if "config" in cls._hub_mixin_init_parameters and "config" not in model_kwargs: - # Decode `config` argument if it was passed - config_annotation = cls._hub_mixin_init_parameters["config"].annotation - config = cls._decode_arg(config_annotation, config) - - # Forward config to model initialization - model_kwargs["config"] = config - - # Inject config if `**kwargs` are expected - if is_dataclass(cls): - for key in cls.__dataclass_fields__: - if key not in model_kwargs and key in config: - model_kwargs[key] = config[key] - elif any(param.kind == inspect.Parameter.VAR_KEYWORD for param in cls._hub_mixin_init_parameters.values()): - for key, value in config.items(): - if key not in model_kwargs: - model_kwargs[key] = value - - # Finally, also inject if `_from_pretrained` expects it - if cls._hub_mixin_inject_config and "config" not in model_kwargs: - model_kwargs["config"] = config - - instance = cls._from_pretrained( - model_id=str(model_id), - revision=revision, - cache_dir=cache_dir, - force_download=force_download, - proxies=proxies, - resume_download=resume_download, - local_files_only=local_files_only, - token=token, - **model_kwargs, - ) - - # Implicitly set the config as instance attribute if not already set by the class - # This way `config` will be available when calling `save_pretrained` or `push_to_hub`. - if config is not None and (getattr(instance, "_hub_mixin_config", None) in (None, {})): - instance._hub_mixin_config = config - - return instance - - @classmethod - def _from_pretrained( - cls: Type[T], - *, - model_id: str, - revision: Optional[str], - cache_dir: Optional[Union[str, Path]], - force_download: bool, - proxies: Optional[Dict], - resume_download: Optional[bool], - local_files_only: bool, - token: Optional[Union[str, bool]], - **model_kwargs, - ) -> T: - """Overwrite this method in subclass to define how to load your model from pretrained. - - Use [`hf_hub_download`] or [`snapshot_download`] to download files from the Hub before loading them. Most - args taken as input can be directly passed to those 2 methods. If needed, you can add more arguments to this - method using "model_kwargs". For example [`PyTorchModelHubMixin._from_pretrained`] takes as input a `map_location` - parameter to set on which device the model should be loaded. - - Check out our [integration guide](../guides/integrations) for more instructions. - - Args: - model_id (`str`): - ID of the model to load from the Huggingface Hub (e.g. `bigscience/bloom`). - revision (`str`, *optional*): - Revision of the model on the Hub. Can be a branch name, a git tag or any commit id. Defaults to the - latest commit on `main` branch. - force_download (`bool`, *optional*, defaults to `False`): - Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding - the existing cache. - proxies (`Dict[str, str]`, *optional*): - A dictionary of proxy servers to use by protocol or endpoint (e.g., `{'http': 'foo.bar:3128', - 'http://hostname': 'foo.bar:4012'}`). - token (`str` or `bool`, *optional*): - The token to use as HTTP bearer authorization for remote files. By default, it will use the token - cached when running `hf auth login`. - cache_dir (`str`, `Path`, *optional*): - Path to the folder where cached files are stored. - local_files_only (`bool`, *optional*, defaults to `False`): - If `True`, avoid downloading the file and return the path to the local cached file if it exists. - model_kwargs: - Additional keyword arguments passed along to the [`~ModelHubMixin._from_pretrained`] method. - """ - raise NotImplementedError - - @validate_hf_hub_args - def push_to_hub( - self, - repo_id: str, - *, - config: Optional[Union[dict, DataclassInstance]] = None, - commit_message: str = "Push model using huggingface_hub.", - private: Optional[bool] = None, - token: Optional[str] = None, - branch: Optional[str] = None, - create_pr: Optional[bool] = None, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - delete_patterns: Optional[Union[List[str], str]] = None, - model_card_kwargs: Optional[Dict[str, Any]] = None, - ) -> str: - """ - Upload model checkpoint to the Hub. - - Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be pushed to the hub. Use - `delete_patterns` to delete existing remote files in the same commit. See [`upload_folder`] reference for more - details. - - Args: - repo_id (`str`): - ID of the repository to push to (example: `"username/my-model"`). - config (`dict` or `DataclassInstance`, *optional*): - Model configuration specified as a key/value dictionary or a dataclass instance. - commit_message (`str`, *optional*): - Message to commit while pushing. - private (`bool`, *optional*): - Whether the repository created should be private. - If `None` (default), the repo will be public unless the organization's default is private. - token (`str`, *optional*): - The token to use as HTTP bearer authorization for remote files. By default, it will use the token - cached when running `hf auth login`. - branch (`str`, *optional*): - The git branch on which to push the model. This defaults to `"main"`. - create_pr (`boolean`, *optional*): - Whether or not to create a Pull Request from `branch` with that commit. Defaults to `False`. - allow_patterns (`List[str]` or `str`, *optional*): - If provided, only files matching at least one pattern are pushed. - ignore_patterns (`List[str]` or `str`, *optional*): - If provided, files matching any of the patterns are not pushed. - delete_patterns (`List[str]` or `str`, *optional*): - If provided, remote files matching any of the patterns will be deleted from the repo. - model_card_kwargs (`Dict[str, Any]`, *optional*): - Additional arguments passed to the model card template to customize the model card. - - Returns: - The url of the commit of your model in the given repository. - """ - api = HfApi(token=token) - repo_id = api.create_repo(repo_id=repo_id, private=private, exist_ok=True).repo_id - - # Push the files to the repo in a single commit - with SoftTemporaryDirectory() as tmp: - saved_path = Path(tmp) / repo_id - self.save_pretrained(saved_path, config=config, model_card_kwargs=model_card_kwargs) - return api.upload_folder( - repo_id=repo_id, - repo_type="model", - folder_path=saved_path, - commit_message=commit_message, - revision=branch, - create_pr=create_pr, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - delete_patterns=delete_patterns, - ) - - def generate_model_card(self, *args, **kwargs) -> ModelCard: - card = ModelCard.from_template( - card_data=self._hub_mixin_info.model_card_data, - template_str=self._hub_mixin_info.model_card_template, - repo_url=self._hub_mixin_info.repo_url, - paper_url=self._hub_mixin_info.paper_url, - docs_url=self._hub_mixin_info.docs_url, - **kwargs, - ) - return card - - -class PyTorchModelHubMixin(ModelHubMixin): - """ - Implementation of [`ModelHubMixin`] to provide model Hub upload/download capabilities to PyTorch models. The model - is set in evaluation mode by default using `model.eval()` (dropout modules are deactivated). To train the model, - you should first set it back in training mode with `model.train()`. - - See [`ModelHubMixin`] for more details on how to use the mixin. - - Example: - - ```python - >>> import torch - >>> import torch.nn as nn - >>> from huggingface_hub import PyTorchModelHubMixin - - >>> class MyModel( - ... nn.Module, - ... PyTorchModelHubMixin, - ... library_name="keras-nlp", - ... repo_url="https://github.com/keras-team/keras-nlp", - ... paper_url="https://arxiv.org/abs/2304.12244", - ... docs_url="https://keras.io/keras_nlp/", - ... # ^ optional metadata to generate model card - ... ): - ... def __init__(self, hidden_size: int = 512, vocab_size: int = 30000, output_size: int = 4): - ... super().__init__() - ... self.param = nn.Parameter(torch.rand(hidden_size, vocab_size)) - ... self.linear = nn.Linear(output_size, vocab_size) - - ... def forward(self, x): - ... return self.linear(x + self.param) - >>> model = MyModel(hidden_size=256) - - # Save model weights to local directory - >>> model.save_pretrained("my-awesome-model") - - # Push model weights to the Hub - >>> model.push_to_hub("my-awesome-model") - - # Download and initialize weights from the Hub - >>> model = MyModel.from_pretrained("username/my-awesome-model") - >>> model.hidden_size - 256 - ``` - """ - - def __init_subclass__(cls, *args, tags: Optional[List[str]] = None, **kwargs) -> None: - tags = tags or [] - tags.append("pytorch_model_hub_mixin") - kwargs["tags"] = tags - return super().__init_subclass__(*args, **kwargs) - - def _save_pretrained(self, save_directory: Path) -> None: - """Save weights from a Pytorch model to a local directory.""" - model_to_save = self.module if hasattr(self, "module") else self # type: ignore - save_model_as_safetensor(model_to_save, str(save_directory / constants.SAFETENSORS_SINGLE_FILE)) # type: ignore [arg-type] - - @classmethod - def _from_pretrained( - cls, - *, - model_id: str, - revision: Optional[str], - cache_dir: Optional[Union[str, Path]], - force_download: bool, - proxies: Optional[Dict], - resume_download: Optional[bool], - local_files_only: bool, - token: Union[str, bool, None], - map_location: str = "cpu", - strict: bool = False, - **model_kwargs, - ): - """Load Pytorch pretrained weights and return the loaded model.""" - model = cls(**model_kwargs) - if os.path.isdir(model_id): - print("Loading weights from local directory") - model_file = os.path.join(model_id, constants.SAFETENSORS_SINGLE_FILE) - return cls._load_as_safetensor(model, model_file, map_location, strict) - else: - try: - model_file = hf_hub_download( - repo_id=model_id, - filename=constants.SAFETENSORS_SINGLE_FILE, - revision=revision, - cache_dir=cache_dir, - force_download=force_download, - proxies=proxies, - resume_download=resume_download, - token=token, - local_files_only=local_files_only, - ) - return cls._load_as_safetensor(model, model_file, map_location, strict) - except EntryNotFoundError: - model_file = hf_hub_download( - repo_id=model_id, - filename=constants.PYTORCH_WEIGHTS_NAME, - revision=revision, - cache_dir=cache_dir, - force_download=force_download, - proxies=proxies, - resume_download=resume_download, - token=token, - local_files_only=local_files_only, - ) - return cls._load_as_pickle(model, model_file, map_location, strict) - - @classmethod - def _load_as_pickle(cls, model: T, model_file: str, map_location: str, strict: bool) -> T: - state_dict = torch.load(model_file, map_location=torch.device(map_location), weights_only=True) - model.load_state_dict(state_dict, strict=strict) # type: ignore - model.eval() # type: ignore - return model - - @classmethod - def _load_as_safetensor(cls, model: T, model_file: str, map_location: str, strict: bool) -> T: - if packaging.version.parse(safetensors.__version__) < packaging.version.parse("0.4.3"): # type: ignore [attr-defined] - load_model_as_safetensor(model, model_file, strict=strict) # type: ignore [arg-type] - if map_location != "cpu": - logger.warning( - "Loading model weights on other devices than 'cpu' is not supported natively in your version of safetensors." - " This means that the model is loaded on 'cpu' first and then copied to the device." - " This leads to a slower loading time." - " Please update safetensors to version 0.4.3 or above for improved performance." - ) - model.to(map_location) # type: ignore [attr-defined] - else: - safetensors.torch.load_model(model, model_file, strict=strict, device=map_location) # type: ignore [arg-type] - return model - - -def _load_dataclass(datacls: Type[DataclassInstance], data: dict) -> DataclassInstance: - """Load a dataclass instance from a dictionary. - - Fields not expected by the dataclass are ignored. - """ - return datacls(**{k: v for k, v in data.items() if k in datacls.__dataclass_fields__}) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/__init__.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_client.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_client.py deleted file mode 100644 index f50e7d56eb9fd5b0362fb60819d97ee406ded94f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_client.py +++ /dev/null @@ -1,3368 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Related resources: -# https://huggingface.co/tasks -# https://huggingface.co/docs/huggingface.js/inference/README -# https://github.com/huggingface/huggingface.js/tree/main/packages/inference/src -# https://github.com/huggingface/text-generation-inference/tree/main/clients/python -# https://github.com/huggingface/text-generation-inference/blob/main/clients/python/text_generation/client.py -# https://huggingface.slack.com/archives/C03E4DQ9LAJ/p1680169099087869 -# https://github.com/huggingface/unity-api#tasks -# -# Some TODO: -# - add all tasks -# -# NOTE: the philosophy of this client is "let's make it as easy as possible to use it, even if less optimized". Some -# examples of how it translates: -# - Timeout / Server unavailable is handled by the client in a single "timeout" parameter. -# - Files can be provided as bytes, file paths, or URLs and the client will try to "guess" the type. -# - Images are parsed as PIL.Image for easier manipulation. -# - Provides a "recommended model" for each task => suboptimal but user-wise quicker to get a first script running. -# - Only the main parameters are publicly exposed. Power users can always read the docs for more options. -import base64 -import logging -import re -import warnings -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Literal, Optional, Union, overload - -from requests import HTTPError - -from huggingface_hub import constants -from huggingface_hub.errors import BadRequestError, InferenceTimeoutError -from huggingface_hub.inference._common import ( - TASKS_EXPECTING_IMAGES, - ContentT, - RequestParameters, - _b64_encode, - _b64_to_image, - _bytes_to_dict, - _bytes_to_image, - _bytes_to_list, - _get_unsupported_text_generation_kwargs, - _import_numpy, - _set_unsupported_text_generation_kwargs, - _stream_chat_completion_response, - _stream_text_generation_response, - raise_text_generation_error, -) -from huggingface_hub.inference._generated.types import ( - AudioClassificationOutputElement, - AudioClassificationOutputTransform, - AudioToAudioOutputElement, - AutomaticSpeechRecognitionOutput, - ChatCompletionInputGrammarType, - ChatCompletionInputMessage, - ChatCompletionInputStreamOptions, - ChatCompletionInputTool, - ChatCompletionInputToolChoiceClass, - ChatCompletionInputToolChoiceEnum, - ChatCompletionOutput, - ChatCompletionStreamOutput, - DocumentQuestionAnsweringOutputElement, - FillMaskOutputElement, - ImageClassificationOutputElement, - ImageClassificationOutputTransform, - ImageSegmentationOutputElement, - ImageSegmentationSubtask, - ImageToImageTargetSize, - ImageToTextOutput, - ImageToVideoTargetSize, - ObjectDetectionOutputElement, - Padding, - QuestionAnsweringOutputElement, - SummarizationOutput, - SummarizationTruncationStrategy, - TableQuestionAnsweringOutputElement, - TextClassificationOutputElement, - TextClassificationOutputTransform, - TextGenerationInputGrammarType, - TextGenerationOutput, - TextGenerationStreamOutput, - TextToSpeechEarlyStoppingEnum, - TokenClassificationAggregationStrategy, - TokenClassificationOutputElement, - TranslationOutput, - TranslationTruncationStrategy, - VisualQuestionAnsweringOutputElement, - ZeroShotClassificationOutputElement, - ZeroShotImageClassificationOutputElement, -) -from huggingface_hub.inference._providers import PROVIDER_OR_POLICY_T, get_provider_helper -from huggingface_hub.utils import build_hf_headers, get_session, hf_raise_for_status -from huggingface_hub.utils._auth import get_token - - -if TYPE_CHECKING: - import numpy as np - from PIL.Image import Image - -logger = logging.getLogger(__name__) - - -MODEL_KWARGS_NOT_USED_REGEX = re.compile(r"The following `model_kwargs` are not used by the model: \[(.*?)\]") - - -class InferenceClient: - """ - Initialize a new Inference Client. - - [`InferenceClient`] aims to provide a unified experience to perform inference. The client can be used - seamlessly with either the (free) Inference API, self-hosted Inference Endpoints, or third-party Inference Providers. - - Args: - model (`str`, `optional`): - The model to run inference with. Can be a model id hosted on the Hugging Face Hub, e.g. `meta-llama/Meta-Llama-3-8B-Instruct` - or a URL to a deployed Inference Endpoint. Defaults to None, in which case a recommended model is - automatically selected for the task. - Note: for better compatibility with OpenAI's client, `model` has been aliased as `base_url`. Those 2 - arguments are mutually exclusive. If a URL is passed as `model` or `base_url` for chat completion, the `(/v1)/chat/completions` suffix path will be appended to the URL. - provider (`str`, *optional*): - Name of the provider to use for inference. Can be `"black-forest-labs"`, `"cerebras"`, `"clarifai"`, `"cohere"`, `"fal-ai"`, `"featherless-ai"`, `"fireworks-ai"`, `"groq"`, `"hf-inference"`, `"hyperbolic"`, `"nebius"`, `"novita"`, `"nscale"`, `"openai"`, `publicai`, `"replicate"`, `"sambanova"`, `"scaleway"`, `"together"` or `"zai-org"`. - Defaults to "auto" i.e. the first of the providers available for the model, sorted by the user's order in https://hf.co/settings/inference-providers. - If model is a URL or `base_url` is passed, then `provider` is not used. - token (`str`, *optional*): - Hugging Face token. Will default to the locally saved token if not provided. - Note: for better compatibility with OpenAI's client, `token` has been aliased as `api_key`. Those 2 - arguments are mutually exclusive and have the exact same behavior. - timeout (`float`, `optional`): - The maximum number of seconds to wait for a response from the server. Defaults to None, meaning it will loop until the server is available. - headers (`Dict[str, str]`, `optional`): - Additional headers to send to the server. By default only the authorization and user-agent headers are sent. - Values in this dictionary will override the default values. - bill_to (`str`, `optional`): - The billing account to use for the requests. By default the requests are billed on the user's account. - Requests can only be billed to an organization the user is a member of, and which has subscribed to Enterprise Hub. - cookies (`Dict[str, str]`, `optional`): - Additional cookies to send to the server. - proxies (`Any`, `optional`): - Proxies to use for the request. - base_url (`str`, `optional`): - Base URL to run inference. This is a duplicated argument from `model` to make [`InferenceClient`] - follow the same pattern as `openai.OpenAI` client. Cannot be used if `model` is set. Defaults to None. - api_key (`str`, `optional`): - Token to use for authentication. This is a duplicated argument from `token` to make [`InferenceClient`] - follow the same pattern as `openai.OpenAI` client. Cannot be used if `token` is set. Defaults to None. - """ - - def __init__( - self, - model: Optional[str] = None, - *, - provider: Optional[PROVIDER_OR_POLICY_T] = None, - token: Optional[str] = None, - timeout: Optional[float] = None, - headers: Optional[Dict[str, str]] = None, - cookies: Optional[Dict[str, str]] = None, - proxies: Optional[Any] = None, - bill_to: Optional[str] = None, - # OpenAI compatibility - base_url: Optional[str] = None, - api_key: Optional[str] = None, - ) -> None: - if model is not None and base_url is not None: - raise ValueError( - "Received both `model` and `base_url` arguments. Please provide only one of them." - " `base_url` is an alias for `model` to make the API compatible with OpenAI's client." - " If using `base_url` for chat completion, the `/chat/completions` suffix path will be appended to the base url." - " When passing a URL as `model`, the client will not append any suffix path to it." - ) - if token is not None and api_key is not None: - raise ValueError( - "Received both `token` and `api_key` arguments. Please provide only one of them." - " `api_key` is an alias for `token` to make the API compatible with OpenAI's client." - " It has the exact same behavior as `token`." - ) - token = token if token is not None else api_key - if isinstance(token, bool): - # Legacy behavior: previously is was possible to pass `token=False` to disable authentication. This is not - # supported anymore as authentication is required. Better to explicitly raise here rather than risking - # sending the locally saved token without the user knowing about it. - if token is False: - raise ValueError( - "Cannot use `token=False` to disable authentication as authentication is required to run Inference." - ) - warnings.warn( - "Using `token=True` to automatically use the locally saved token is deprecated and will be removed in a future release. " - "Please use `token=None` instead (default).", - DeprecationWarning, - ) - token = get_token() - - self.model: Optional[str] = base_url or model - self.token: Optional[str] = token - - self.headers = {**headers} if headers is not None else {} - if bill_to is not None: - if ( - constants.HUGGINGFACE_HEADER_X_BILL_TO in self.headers - and self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO] != bill_to - ): - warnings.warn( - f"Overriding existing '{self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO]}' value in headers with '{bill_to}'.", - UserWarning, - ) - self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO] = bill_to - - if token is not None and not token.startswith("hf_"): - warnings.warn( - "You've provided an external provider's API key, so requests will be billed directly by the provider. " - "The `bill_to` parameter is only applicable for Hugging Face billing and will be ignored.", - UserWarning, - ) - - # Configure provider - self.provider = provider - - self.cookies = cookies - self.timeout = timeout - self.proxies = proxies - - def __repr__(self): - return f"" - - @overload - def _inner_post( # type: ignore[misc] - self, request_parameters: RequestParameters, *, stream: Literal[False] = ... - ) -> bytes: ... - - @overload - def _inner_post( # type: ignore[misc] - self, request_parameters: RequestParameters, *, stream: Literal[True] = ... - ) -> Iterable[bytes]: ... - - @overload - def _inner_post( - self, request_parameters: RequestParameters, *, stream: bool = False - ) -> Union[bytes, Iterable[bytes]]: ... - - def _inner_post( - self, request_parameters: RequestParameters, *, stream: bool = False - ) -> Union[bytes, Iterable[bytes]]: - """Make a request to the inference server.""" - # TODO: this should be handled in provider helpers directly - if request_parameters.task in TASKS_EXPECTING_IMAGES and "Accept" not in request_parameters.headers: - request_parameters.headers["Accept"] = "image/png" - - try: - response = get_session().post( - request_parameters.url, - json=request_parameters.json, - data=request_parameters.data, - headers=request_parameters.headers, - cookies=self.cookies, - timeout=self.timeout, - stream=stream, - proxies=self.proxies, - ) - except TimeoutError as error: - # Convert any `TimeoutError` to a `InferenceTimeoutError` - raise InferenceTimeoutError(f"Inference call timed out: {request_parameters.url}") from error # type: ignore - - try: - hf_raise_for_status(response) - return response.iter_lines() if stream else response.content - except HTTPError as error: - if error.response.status_code == 422 and request_parameters.task != "unknown": - msg = str(error.args[0]) - if len(error.response.text) > 0: - msg += f"\n{error.response.text}\n" - error.args = (msg,) + error.args[1:] - raise - - def audio_classification( - self, - audio: ContentT, - *, - model: Optional[str] = None, - top_k: Optional[int] = None, - function_to_apply: Optional["AudioClassificationOutputTransform"] = None, - ) -> List[AudioClassificationOutputElement]: - """ - Perform audio classification on the provided audio content. - - Args: - audio (Union[str, Path, bytes, BinaryIO]): - The audio content to classify. It can be raw audio bytes, a local audio file, or a URL pointing to an - audio file. - model (`str`, *optional*): - The model to use for audio classification. Can be a model ID hosted on the Hugging Face Hub - or a URL to a deployed Inference Endpoint. If not provided, the default recommended model for - audio classification will be used. - top_k (`int`, *optional*): - When specified, limits the output to the top K most probable classes. - function_to_apply (`"AudioClassificationOutputTransform"`, *optional*): - The function to apply to the model outputs in order to retrieve the scores. - - Returns: - `List[AudioClassificationOutputElement]`: List of [`AudioClassificationOutputElement`] items containing the predicted labels and their confidence. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.audio_classification("audio.flac") - [ - AudioClassificationOutputElement(score=0.4976358711719513, label='hap'), - AudioClassificationOutputElement(score=0.3677836060523987, label='neu'), - ... - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="audio-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=audio, - parameters={"function_to_apply": function_to_apply, "top_k": top_k}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return AudioClassificationOutputElement.parse_obj_as_list(response) - - def audio_to_audio( - self, - audio: ContentT, - *, - model: Optional[str] = None, - ) -> List[AudioToAudioOutputElement]: - """ - Performs multiple tasks related to audio-to-audio depending on the model (eg: speech enhancement, source separation). - - Args: - audio (Union[str, Path, bytes, BinaryIO]): - The audio content for the model. It can be raw audio bytes, a local audio file, or a URL pointing to an - audio file. - model (`str`, *optional*): - The model can be any model which takes an audio file and returns another audio file. Can be a model ID hosted on the Hugging Face Hub - or a URL to a deployed Inference Endpoint. If not provided, the default recommended model for - audio_to_audio will be used. - - Returns: - `List[AudioToAudioOutputElement]`: A list of [`AudioToAudioOutputElement`] items containing audios label, content-type, and audio content in blob. - - Raises: - `InferenceTimeoutError`: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> audio_output = client.audio_to_audio("audio.flac") - >>> for i, item in enumerate(audio_output): - >>> with open(f"output_{i}.flac", "wb") as f: - f.write(item.blob) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="audio-to-audio", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=audio, - parameters={}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - audio_output = AudioToAudioOutputElement.parse_obj_as_list(response) - for item in audio_output: - item.blob = base64.b64decode(item.blob) - return audio_output - - def automatic_speech_recognition( - self, - audio: ContentT, - *, - model: Optional[str] = None, - extra_body: Optional[Dict] = None, - ) -> AutomaticSpeechRecognitionOutput: - """ - Perform automatic speech recognition (ASR or audio-to-text) on the given audio content. - - Args: - audio (Union[str, Path, bytes, BinaryIO]): - The content to transcribe. It can be raw audio bytes, local audio file, or a URL to an audio file. - model (`str`, *optional*): - The model to use for ASR. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended model for ASR will be used. - extra_body (`Dict`, *optional*): - Additional provider-specific parameters to pass to the model. Refer to the provider's documentation - for supported parameters. - Returns: - [`AutomaticSpeechRecognitionOutput`]: An item containing the transcribed text and optionally the timestamp chunks. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.automatic_speech_recognition("hello_world.flac").text - "hello world" - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="automatic-speech-recognition", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=audio, - parameters={**(extra_body or {})}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return AutomaticSpeechRecognitionOutput.parse_obj_as_instance(response) - - @overload - def chat_completion( # type: ignore - self, - messages: List[Union[Dict, ChatCompletionInputMessage]], - *, - model: Optional[str] = None, - stream: Literal[False] = False, - frequency_penalty: Optional[float] = None, - logit_bias: Optional[List[float]] = None, - logprobs: Optional[bool] = None, - max_tokens: Optional[int] = None, - n: Optional[int] = None, - presence_penalty: Optional[float] = None, - response_format: Optional[ChatCompletionInputGrammarType] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stream_options: Optional[ChatCompletionInputStreamOptions] = None, - temperature: Optional[float] = None, - tool_choice: Optional[Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"]] = None, - tool_prompt: Optional[str] = None, - tools: Optional[List[ChatCompletionInputTool]] = None, - top_logprobs: Optional[int] = None, - top_p: Optional[float] = None, - extra_body: Optional[Dict] = None, - ) -> ChatCompletionOutput: ... - - @overload - def chat_completion( # type: ignore - self, - messages: List[Union[Dict, ChatCompletionInputMessage]], - *, - model: Optional[str] = None, - stream: Literal[True] = True, - frequency_penalty: Optional[float] = None, - logit_bias: Optional[List[float]] = None, - logprobs: Optional[bool] = None, - max_tokens: Optional[int] = None, - n: Optional[int] = None, - presence_penalty: Optional[float] = None, - response_format: Optional[ChatCompletionInputGrammarType] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stream_options: Optional[ChatCompletionInputStreamOptions] = None, - temperature: Optional[float] = None, - tool_choice: Optional[Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"]] = None, - tool_prompt: Optional[str] = None, - tools: Optional[List[ChatCompletionInputTool]] = None, - top_logprobs: Optional[int] = None, - top_p: Optional[float] = None, - extra_body: Optional[Dict] = None, - ) -> Iterable[ChatCompletionStreamOutput]: ... - - @overload - def chat_completion( - self, - messages: List[Union[Dict, ChatCompletionInputMessage]], - *, - model: Optional[str] = None, - stream: bool = False, - frequency_penalty: Optional[float] = None, - logit_bias: Optional[List[float]] = None, - logprobs: Optional[bool] = None, - max_tokens: Optional[int] = None, - n: Optional[int] = None, - presence_penalty: Optional[float] = None, - response_format: Optional[ChatCompletionInputGrammarType] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stream_options: Optional[ChatCompletionInputStreamOptions] = None, - temperature: Optional[float] = None, - tool_choice: Optional[Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"]] = None, - tool_prompt: Optional[str] = None, - tools: Optional[List[ChatCompletionInputTool]] = None, - top_logprobs: Optional[int] = None, - top_p: Optional[float] = None, - extra_body: Optional[Dict] = None, - ) -> Union[ChatCompletionOutput, Iterable[ChatCompletionStreamOutput]]: ... - - def chat_completion( - self, - messages: List[Union[Dict, ChatCompletionInputMessage]], - *, - model: Optional[str] = None, - stream: bool = False, - # Parameters from ChatCompletionInput (handled manually) - frequency_penalty: Optional[float] = None, - logit_bias: Optional[List[float]] = None, - logprobs: Optional[bool] = None, - max_tokens: Optional[int] = None, - n: Optional[int] = None, - presence_penalty: Optional[float] = None, - response_format: Optional[ChatCompletionInputGrammarType] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stream_options: Optional[ChatCompletionInputStreamOptions] = None, - temperature: Optional[float] = None, - tool_choice: Optional[Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"]] = None, - tool_prompt: Optional[str] = None, - tools: Optional[List[ChatCompletionInputTool]] = None, - top_logprobs: Optional[int] = None, - top_p: Optional[float] = None, - extra_body: Optional[Dict] = None, - ) -> Union[ChatCompletionOutput, Iterable[ChatCompletionStreamOutput]]: - """ - A method for completing conversations using a specified language model. - - > [!TIP] - > The `client.chat_completion` method is aliased as `client.chat.completions.create` for compatibility with OpenAI's client. - > Inputs and outputs are strictly the same and using either syntax will yield the same results. - > Check out the [Inference guide](https://huggingface.co/docs/huggingface_hub/guides/inference#openai-compatibility) - > for more details about OpenAI's compatibility. - - > [!TIP] - > You can pass provider-specific parameters to the model by using the `extra_body` argument. - - Args: - messages (List of [`ChatCompletionInputMessage`]): - Conversation history consisting of roles and content pairs. - model (`str`, *optional*): - The model to use for chat-completion. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended model for chat-based text-generation will be used. - See https://huggingface.co/tasks/text-generation for more details. - If `model` is a model ID, it is passed to the server as the `model` parameter. If you want to define a - custom URL while setting `model` in the request payload, you must set `base_url` when initializing [`InferenceClient`]. - frequency_penalty (`float`, *optional*): - Penalizes new tokens based on their existing frequency - in the text so far. Range: [-2.0, 2.0]. Defaults to 0.0. - logit_bias (`List[float]`, *optional*): - Adjusts the likelihood of specific tokens appearing in the generated output. - logprobs (`bool`, *optional*): - Whether to return log probabilities of the output tokens or not. If true, returns the log - probabilities of each output token returned in the content of message. - max_tokens (`int`, *optional*): - Maximum number of tokens allowed in the response. Defaults to 100. - n (`int`, *optional*): - The number of completions to generate for each prompt. - presence_penalty (`float`, *optional*): - Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the - text so far, increasing the model's likelihood to talk about new topics. - response_format ([`ChatCompletionInputGrammarType`], *optional*): - Grammar constraints. Can be either a JSONSchema or a regex. - seed (Optional[`int`], *optional*): - Seed for reproducible control flow. Defaults to None. - stop (`List[str]`, *optional*): - Up to four strings which trigger the end of the response. - Defaults to None. - stream (`bool`, *optional*): - Enable realtime streaming of responses. Defaults to False. - stream_options ([`ChatCompletionInputStreamOptions`], *optional*): - Options for streaming completions. - temperature (`float`, *optional*): - Controls randomness of the generations. Lower values ensure - less random completions. Range: [0, 2]. Defaults to 1.0. - top_logprobs (`int`, *optional*): - An integer between 0 and 5 specifying the number of most likely tokens to return at each token - position, each with an associated log probability. logprobs must be set to true if this parameter is - used. - top_p (`float`, *optional*): - Fraction of the most likely next words to sample from. - Must be between 0 and 1. Defaults to 1.0. - tool_choice ([`ChatCompletionInputToolChoiceClass`] or [`ChatCompletionInputToolChoiceEnum`], *optional*): - The tool to use for the completion. Defaults to "auto". - tool_prompt (`str`, *optional*): - A prompt to be appended before the tools. - tools (List of [`ChatCompletionInputTool`], *optional*): - A list of tools the model may call. Currently, only functions are supported as a tool. Use this to - provide a list of functions the model may generate JSON inputs for. - extra_body (`Dict`, *optional*): - Additional provider-specific parameters to pass to the model. Refer to the provider's documentation - for supported parameters. - Returns: - [`ChatCompletionOutput`] or Iterable of [`ChatCompletionStreamOutput`]: - Generated text returned from the server: - - if `stream=False`, the generated text is returned as a [`ChatCompletionOutput`] (default). - - if `stream=True`, the generated text is returned token by token as a sequence of [`ChatCompletionStreamOutput`]. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - - ```py - >>> from huggingface_hub import InferenceClient - >>> messages = [{"role": "user", "content": "What is the capital of France?"}] - >>> client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") - >>> client.chat_completion(messages, max_tokens=100) - ChatCompletionOutput( - choices=[ - ChatCompletionOutputComplete( - finish_reason='eos_token', - index=0, - message=ChatCompletionOutputMessage( - role='assistant', - content='The capital of France is Paris.', - name=None, - tool_calls=None - ), - logprobs=None - ) - ], - created=1719907176, - id='', - model='meta-llama/Meta-Llama-3-8B-Instruct', - object='text_completion', - system_fingerprint='2.0.4-sha-f426a33', - usage=ChatCompletionOutputUsage( - completion_tokens=8, - prompt_tokens=17, - total_tokens=25 - ) - ) - ``` - - Example using streaming: - ```py - >>> from huggingface_hub import InferenceClient - >>> messages = [{"role": "user", "content": "What is the capital of France?"}] - >>> client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") - >>> for token in client.chat_completion(messages, max_tokens=10, stream=True): - ... print(token) - ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content='The', role='assistant'), index=0, finish_reason=None)], created=1710498504) - ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content=' capital', role='assistant'), index=0, finish_reason=None)], created=1710498504) - (...) - ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content=' may', role='assistant'), index=0, finish_reason=None)], created=1710498504) - ``` - - Example using OpenAI's syntax: - ```py - # instead of `from openai import OpenAI` - from huggingface_hub import InferenceClient - - # instead of `client = OpenAI(...)` - client = InferenceClient( - base_url=..., - api_key=..., - ) - - output = client.chat.completions.create( - model="meta-llama/Meta-Llama-3-8B-Instruct", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Count to 10"}, - ], - stream=True, - max_tokens=1024, - ) - - for chunk in output: - print(chunk.choices[0].delta.content) - ``` - - Example using a third-party provider directly with extra (provider-specific) parameters. Usage will be billed on your Together AI account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="together", # Use Together AI provider - ... api_key="", # Pass your Together API key directly - ... ) - >>> client.chat_completion( - ... model="meta-llama/Meta-Llama-3-8B-Instruct", - ... messages=[{"role": "user", "content": "What is the capital of France?"}], - ... extra_body={"safety_model": "Meta-Llama/Llama-Guard-7b"}, - ... ) - ``` - - Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="sambanova", # Use Sambanova provider - ... api_key="hf_...", # Pass your HF token - ... ) - >>> client.chat_completion( - ... model="meta-llama/Meta-Llama-3-8B-Instruct", - ... messages=[{"role": "user", "content": "What is the capital of France?"}], - ... ) - ``` - - Example using Image + Text as input: - ```py - >>> from huggingface_hub import InferenceClient - - # provide a remote URL - >>> image_url ="https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" - # or a base64-encoded image - >>> image_path = "/path/to/image.jpeg" - >>> with open(image_path, "rb") as f: - ... base64_image = base64.b64encode(f.read()).decode("utf-8") - >>> image_url = f"data:image/jpeg;base64,{base64_image}" - - >>> client = InferenceClient("meta-llama/Llama-3.2-11B-Vision-Instruct") - >>> output = client.chat.completions.create( - ... messages=[ - ... { - ... "role": "user", - ... "content": [ - ... { - ... "type": "image_url", - ... "image_url": {"url": image_url}, - ... }, - ... { - ... "type": "text", - ... "text": "Describe this image in one sentence.", - ... }, - ... ], - ... }, - ... ], - ... ) - >>> output - The image depicts the iconic Statue of Liberty situated in New York Harbor, New York, on a clear day. - ``` - - Example using tools: - ```py - >>> client = InferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") - >>> messages = [ - ... { - ... "role": "system", - ... "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous.", - ... }, - ... { - ... "role": "user", - ... "content": "What's the weather like the next 3 days in San Francisco, CA?", - ... }, - ... ] - >>> tools = [ - ... { - ... "type": "function", - ... "function": { - ... "name": "get_current_weather", - ... "description": "Get the current weather", - ... "parameters": { - ... "type": "object", - ... "properties": { - ... "location": { - ... "type": "string", - ... "description": "The city and state, e.g. San Francisco, CA", - ... }, - ... "format": { - ... "type": "string", - ... "enum": ["celsius", "fahrenheit"], - ... "description": "The temperature unit to use. Infer this from the users location.", - ... }, - ... }, - ... "required": ["location", "format"], - ... }, - ... }, - ... }, - ... { - ... "type": "function", - ... "function": { - ... "name": "get_n_day_weather_forecast", - ... "description": "Get an N-day weather forecast", - ... "parameters": { - ... "type": "object", - ... "properties": { - ... "location": { - ... "type": "string", - ... "description": "The city and state, e.g. San Francisco, CA", - ... }, - ... "format": { - ... "type": "string", - ... "enum": ["celsius", "fahrenheit"], - ... "description": "The temperature unit to use. Infer this from the users location.", - ... }, - ... "num_days": { - ... "type": "integer", - ... "description": "The number of days to forecast", - ... }, - ... }, - ... "required": ["location", "format", "num_days"], - ... }, - ... }, - ... }, - ... ] - - >>> response = client.chat_completion( - ... model="meta-llama/Meta-Llama-3-70B-Instruct", - ... messages=messages, - ... tools=tools, - ... tool_choice="auto", - ... max_tokens=500, - ... ) - >>> response.choices[0].message.tool_calls[0].function - ChatCompletionOutputFunctionDefinition( - arguments={ - 'location': 'San Francisco, CA', - 'format': 'fahrenheit', - 'num_days': 3 - }, - name='get_n_day_weather_forecast', - description=None - ) - ``` - - Example using response_format: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") - >>> messages = [ - ... { - ... "role": "user", - ... "content": "I saw a puppy a cat and a raccoon during my bike ride in the park. What did I saw and when?", - ... }, - ... ] - >>> response_format = { - ... "type": "json", - ... "value": { - ... "properties": { - ... "location": {"type": "string"}, - ... "activity": {"type": "string"}, - ... "animals_seen": {"type": "integer", "minimum": 1, "maximum": 5}, - ... "animals": {"type": "array", "items": {"type": "string"}}, - ... }, - ... "required": ["location", "activity", "animals_seen", "animals"], - ... }, - ... } - >>> response = client.chat_completion( - ... messages=messages, - ... response_format=response_format, - ... max_tokens=500, - ... ) - >>> response.choices[0].message.content - '{\n\n"activity": "bike ride",\n"animals": ["puppy", "cat", "raccoon"],\n"animals_seen": 3,\n"location": "park"}' - ``` - """ - # Since `chat_completion(..., model=xxx)` is also a payload parameter for the server, we need to handle 'model' differently. - # `self.model` takes precedence over 'model' argument for building URL. - # `model` takes precedence for payload value. - model_id_or_url = self.model or model - payload_model = model or self.model - - # Get the provider helper - provider_helper = get_provider_helper( - self.provider, - task="conversational", - model=model_id_or_url - if model_id_or_url is not None and model_id_or_url.startswith(("http://", "https://")) - else payload_model, - ) - - # Prepare the payload - parameters = { - "model": payload_model, - "frequency_penalty": frequency_penalty, - "logit_bias": logit_bias, - "logprobs": logprobs, - "max_tokens": max_tokens, - "n": n, - "presence_penalty": presence_penalty, - "response_format": response_format, - "seed": seed, - "stop": stop, - "temperature": temperature, - "tool_choice": tool_choice, - "tool_prompt": tool_prompt, - "tools": tools, - "top_logprobs": top_logprobs, - "top_p": top_p, - "stream": stream, - "stream_options": stream_options, - **(extra_body or {}), - } - request_parameters = provider_helper.prepare_request( - inputs=messages, - parameters=parameters, - headers=self.headers, - model=model_id_or_url, - api_key=self.token, - ) - data = self._inner_post(request_parameters, stream=stream) - - if stream: - return _stream_chat_completion_response(data) # type: ignore[arg-type] - - return ChatCompletionOutput.parse_obj_as_instance(data) # type: ignore[arg-type] - - def document_question_answering( - self, - image: ContentT, - question: str, - *, - model: Optional[str] = None, - doc_stride: Optional[int] = None, - handle_impossible_answer: Optional[bool] = None, - lang: Optional[str] = None, - max_answer_len: Optional[int] = None, - max_question_len: Optional[int] = None, - max_seq_len: Optional[int] = None, - top_k: Optional[int] = None, - word_boxes: Optional[List[Union[List[float], str]]] = None, - ) -> List[DocumentQuestionAnsweringOutputElement]: - """ - Answer questions on document images. - - Args: - image (`Union[str, Path, bytes, BinaryIO]`): - The input image for the context. It can be raw bytes, an image file, or a URL to an online image. - question (`str`): - Question to be answered. - model (`str`, *optional*): - The model to use for the document question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended document question answering model will be used. - Defaults to None. - doc_stride (`int`, *optional*): - If the words in the document are too long to fit with the question for the model, it will be split in - several chunks with some overlap. This argument controls the size of that overlap. - handle_impossible_answer (`bool`, *optional*): - Whether to accept impossible as an answer - lang (`str`, *optional*): - Language to use while running OCR. Defaults to english. - max_answer_len (`int`, *optional*): - The maximum length of predicted answers (e.g., only answers with a shorter length are considered). - max_question_len (`int`, *optional*): - The maximum length of the question after tokenization. It will be truncated if needed. - max_seq_len (`int`, *optional*): - The maximum length of the total sentence (context + question) in tokens of each chunk passed to the - model. The context will be split in several chunks (using doc_stride as overlap) if needed. - top_k (`int`, *optional*): - The number of answers to return (will be chosen by order of likelihood). Can return less than top_k - answers if there are not enough options available within the context. - word_boxes (`List[Union[List[float], str`, *optional*): - A list of words and bounding boxes (normalized 0->1000). If provided, the inference will skip the OCR - step and use the provided bounding boxes instead. - Returns: - `List[DocumentQuestionAnsweringOutputElement]`: a list of [`DocumentQuestionAnsweringOutputElement`] items containing the predicted label, associated probability, word ids, and page number. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.document_question_answering(image="https://huggingface.co/spaces/impira/docquery/resolve/2359223c1837a7587402bda0f2643382a6eefeab/invoice.png", question="What is the invoice number?") - [DocumentQuestionAnsweringOutputElement(answer='us-001', end=16, score=0.9999666213989258, start=16)] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="document-question-answering", model=model_id) - inputs: Dict[str, Any] = {"question": question, "image": _b64_encode(image)} - request_parameters = provider_helper.prepare_request( - inputs=inputs, - parameters={ - "doc_stride": doc_stride, - "handle_impossible_answer": handle_impossible_answer, - "lang": lang, - "max_answer_len": max_answer_len, - "max_question_len": max_question_len, - "max_seq_len": max_seq_len, - "top_k": top_k, - "word_boxes": word_boxes, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return DocumentQuestionAnsweringOutputElement.parse_obj_as_list(response) - - def feature_extraction( - self, - text: str, - *, - normalize: Optional[bool] = None, - prompt_name: Optional[str] = None, - truncate: Optional[bool] = None, - truncation_direction: Optional[Literal["Left", "Right"]] = None, - model: Optional[str] = None, - ) -> "np.ndarray": - """ - Generate embeddings for a given text. - - Args: - text (`str`): - The text to embed. - model (`str`, *optional*): - The model to use for the feature extraction task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended feature extraction model will be used. - Defaults to None. - normalize (`bool`, *optional*): - Whether to normalize the embeddings or not. - Only available on server powered by Text-Embedding-Inference. - prompt_name (`str`, *optional*): - The name of the prompt that should be used by for encoding. If not set, no prompt will be applied. - Must be a key in the `Sentence Transformers` configuration `prompts` dictionary. - For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ",...}, - then the sentence "What is the capital of France?" will be encoded as "query: What is the capital of France?" - because the prompt text will be prepended before any text to encode. - truncate (`bool`, *optional*): - Whether to truncate the embeddings or not. - Only available on server powered by Text-Embedding-Inference. - truncation_direction (`Literal["Left", "Right"]`, *optional*): - Which side of the input should be truncated when `truncate=True` is passed. - - Returns: - `np.ndarray`: The embedding representing the input text as a float32 numpy array. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.feature_extraction("Hi, who are you?") - array([[ 2.424802 , 2.93384 , 1.1750331 , ..., 1.240499, -0.13776633, -0.7889173 ], - [-0.42943227, -0.6364878 , -1.693462 , ..., 0.41978157, -2.4336355 , 0.6162071 ], - ..., - [ 0.28552425, -0.928395 , -1.2077185 , ..., 0.76810825, -2.1069427 , 0.6236161 ]], dtype=float32) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="feature-extraction", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "normalize": normalize, - "prompt_name": prompt_name, - "truncate": truncate, - "truncation_direction": truncation_direction, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - np = _import_numpy() - return np.array(provider_helper.get_response(response), dtype="float32") - - def fill_mask( - self, - text: str, - *, - model: Optional[str] = None, - targets: Optional[List[str]] = None, - top_k: Optional[int] = None, - ) -> List[FillMaskOutputElement]: - """ - Fill in a hole with a missing word (token to be precise). - - Args: - text (`str`): - a string to be filled from, must contain the [MASK] token (check model card for exact name of the mask). - model (`str`, *optional*): - The model to use for the fill mask task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended fill mask model will be used. - targets (`List[str`, *optional*): - When passed, the model will limit the scores to the passed targets instead of looking up in the whole - vocabulary. If the provided targets are not in the model vocab, they will be tokenized and the first - resulting token will be used (with a warning, and that might be slower). - top_k (`int`, *optional*): - When passed, overrides the number of predictions to return. - Returns: - `List[FillMaskOutputElement]`: a list of [`FillMaskOutputElement`] items containing the predicted label, associated - probability, token reference, and completed text. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.fill_mask("The goal of life is .") - [ - FillMaskOutputElement(score=0.06897063553333282, token=11098, token_str=' happiness', sequence='The goal of life is happiness.'), - FillMaskOutputElement(score=0.06554922461509705, token=45075, token_str=' immortality', sequence='The goal of life is immortality.') - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="fill-mask", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={"targets": targets, "top_k": top_k}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return FillMaskOutputElement.parse_obj_as_list(response) - - def image_classification( - self, - image: ContentT, - *, - model: Optional[str] = None, - function_to_apply: Optional["ImageClassificationOutputTransform"] = None, - top_k: Optional[int] = None, - ) -> List[ImageClassificationOutputElement]: - """ - Perform image classification on the given image using the specified model. - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The image to classify. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - model (`str`, *optional*): - The model to use for image classification. Can be a model ID hosted on the Hugging Face Hub or a URL to a - deployed Inference Endpoint. If not provided, the default recommended model for image classification will be used. - function_to_apply (`"ImageClassificationOutputTransform"`, *optional*): - The function to apply to the model outputs in order to retrieve the scores. - top_k (`int`, *optional*): - When specified, limits the output to the top K most probable classes. - Returns: - `List[ImageClassificationOutputElement]`: a list of [`ImageClassificationOutputElement`] items containing the predicted label and associated probability. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.image_classification("https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg") - [ImageClassificationOutputElement(label='Blenheim spaniel', score=0.9779096841812134), ...] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="image-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={"function_to_apply": function_to_apply, "top_k": top_k}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return ImageClassificationOutputElement.parse_obj_as_list(response) - - def image_segmentation( - self, - image: ContentT, - *, - model: Optional[str] = None, - mask_threshold: Optional[float] = None, - overlap_mask_area_threshold: Optional[float] = None, - subtask: Optional["ImageSegmentationSubtask"] = None, - threshold: Optional[float] = None, - ) -> List[ImageSegmentationOutputElement]: - """ - Perform image segmentation on the given image using the specified model. - - > [!WARNING] - > You must have `PIL` installed if you want to work with images (`pip install Pillow`). - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The image to segment. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - model (`str`, *optional*): - The model to use for image segmentation. Can be a model ID hosted on the Hugging Face Hub or a URL to a - deployed Inference Endpoint. If not provided, the default recommended model for image segmentation will be used. - mask_threshold (`float`, *optional*): - Threshold to use when turning the predicted masks into binary values. - overlap_mask_area_threshold (`float`, *optional*): - Mask overlap threshold to eliminate small, disconnected segments. - subtask (`"ImageSegmentationSubtask"`, *optional*): - Segmentation task to be performed, depending on model capabilities. - threshold (`float`, *optional*): - Probability threshold to filter out predicted masks. - Returns: - `List[ImageSegmentationOutputElement]`: A list of [`ImageSegmentationOutputElement`] items containing the segmented masks and associated attributes. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.image_segmentation("cat.jpg") - [ImageSegmentationOutputElement(score=0.989008, label='LABEL_184', mask=), ...] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="image-segmentation", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={ - "mask_threshold": mask_threshold, - "overlap_mask_area_threshold": overlap_mask_area_threshold, - "subtask": subtask, - "threshold": threshold, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - output = ImageSegmentationOutputElement.parse_obj_as_list(response) - for item in output: - item.mask = _b64_to_image(item.mask) # type: ignore [assignment] - return output - - def image_to_image( - self, - image: ContentT, - prompt: Optional[str] = None, - *, - negative_prompt: Optional[str] = None, - num_inference_steps: Optional[int] = None, - guidance_scale: Optional[float] = None, - model: Optional[str] = None, - target_size: Optional[ImageToImageTargetSize] = None, - **kwargs, - ) -> "Image": - """ - Perform image-to-image translation using a specified model. - - > [!WARNING] - > You must have `PIL` installed if you want to work with images (`pip install Pillow`). - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The input image for translation. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - prompt (`str`, *optional*): - The text prompt to guide the image generation. - negative_prompt (`str`, *optional*): - One prompt to guide what NOT to include in image generation. - num_inference_steps (`int`, *optional*): - For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher - quality image at the expense of slower inference. - guidance_scale (`float`, *optional*): - For diffusion models. A higher guidance scale value encourages the model to generate images closely - linked to the text prompt at the expense of lower image quality. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - target_size (`ImageToImageTargetSize`, *optional*): - The size in pixels of the output image. This parameter is only supported by some providers and for - specific models. It will be ignored when unsupported. - - Returns: - `Image`: The translated image. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> image = client.image_to_image("cat.jpg", prompt="turn the cat into a tiger") - >>> image.save("tiger.jpg") - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="image-to-image", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={ - "prompt": prompt, - "negative_prompt": negative_prompt, - "target_size": target_size, - "num_inference_steps": num_inference_steps, - "guidance_scale": guidance_scale, - **kwargs, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - response = provider_helper.get_response(response, request_parameters) - return _bytes_to_image(response) - - def image_to_video( - self, - image: ContentT, - *, - model: Optional[str] = None, - prompt: Optional[str] = None, - negative_prompt: Optional[str] = None, - num_frames: Optional[float] = None, - num_inference_steps: Optional[int] = None, - guidance_scale: Optional[float] = None, - seed: Optional[int] = None, - target_size: Optional[ImageToVideoTargetSize] = None, - **kwargs, - ) -> bytes: - """ - Generate a video from an input image. - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The input image to generate a video from. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - prompt (`str`, *optional*): - The text prompt to guide the video generation. - negative_prompt (`str`, *optional*): - One prompt to guide what NOT to include in video generation. - num_frames (`float`, *optional*): - The num_frames parameter determines how many video frames are generated. - num_inference_steps (`int`, *optional*): - For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher - quality image at the expense of slower inference. - guidance_scale (`float`, *optional*): - For diffusion models. A higher guidance scale value encourages the model to generate videos closely - linked to the text prompt at the expense of lower image quality. - seed (`int`, *optional*): - The seed to use for the video generation. - target_size (`ImageToVideoTargetSize`, *optional*): - The size in pixel of the output video frames. - num_inference_steps (`int`, *optional*): - The number of denoising steps. More denoising steps usually lead to a higher quality video at the - expense of slower inference. - seed (`int`, *optional*): - Seed for the random number generator. - - Returns: - `bytes`: The generated video. - - Examples: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> video = client.image_to_video("cat.jpg", model="Wan-AI/Wan2.2-I2V-A14B", prompt="turn the cat into a tiger") - >>> with open("tiger.mp4", "wb") as f: - ... f.write(video) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="image-to-video", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={ - "prompt": prompt, - "negative_prompt": negative_prompt, - "num_frames": num_frames, - "num_inference_steps": num_inference_steps, - "guidance_scale": guidance_scale, - "seed": seed, - "target_size": target_size, - **kwargs, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - response = provider_helper.get_response(response, request_parameters) - return response - - def image_to_text(self, image: ContentT, *, model: Optional[str] = None) -> ImageToTextOutput: - """ - Takes an input image and return text. - - Models can have very different outputs depending on your use case (image captioning, optical character recognition - (OCR), Pix2Struct, etc). Please have a look to the model card to learn more about a model's specificities. - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The input image to caption. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - - Returns: - [`ImageToTextOutput`]: The generated text. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.image_to_text("cat.jpg") - 'a cat standing in a grassy field ' - >>> client.image_to_text("https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg") - 'a dog laying on the grass next to a flower pot ' - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="image-to-text", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - output_list: List[ImageToTextOutput] = ImageToTextOutput.parse_obj_as_list(response) - return output_list[0] - - def object_detection( - self, image: ContentT, *, model: Optional[str] = None, threshold: Optional[float] = None - ) -> List[ObjectDetectionOutputElement]: - """ - Perform object detection on the given image using the specified model. - - > [!WARNING] - > You must have `PIL` installed if you want to work with images (`pip install Pillow`). - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The image to detect objects on. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - model (`str`, *optional*): - The model to use for object detection. Can be a model ID hosted on the Hugging Face Hub or a URL to a - deployed Inference Endpoint. If not provided, the default recommended model for object detection (DETR) will be used. - threshold (`float`, *optional*): - The probability necessary to make a prediction. - Returns: - `List[ObjectDetectionOutputElement]`: A list of [`ObjectDetectionOutputElement`] items containing the bounding boxes and associated attributes. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - `ValueError`: - If the request output is not a List. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.object_detection("people.jpg") - [ObjectDetectionOutputElement(score=0.9486683011054993, label='person', box=ObjectDetectionBoundingBox(xmin=59, ymin=39, xmax=420, ymax=510)), ...] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="object-detection", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={"threshold": threshold}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return ObjectDetectionOutputElement.parse_obj_as_list(response) - - def question_answering( - self, - question: str, - context: str, - *, - model: Optional[str] = None, - align_to_words: Optional[bool] = None, - doc_stride: Optional[int] = None, - handle_impossible_answer: Optional[bool] = None, - max_answer_len: Optional[int] = None, - max_question_len: Optional[int] = None, - max_seq_len: Optional[int] = None, - top_k: Optional[int] = None, - ) -> Union[QuestionAnsweringOutputElement, List[QuestionAnsweringOutputElement]]: - """ - Retrieve the answer to a question from a given text. - - Args: - question (`str`): - Question to be answered. - context (`str`): - The context of the question. - model (`str`): - The model to use for the question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. - align_to_words (`bool`, *optional*): - Attempts to align the answer to real words. Improves quality on space separated languages. Might hurt - on non-space-separated languages (like Japanese or Chinese) - doc_stride (`int`, *optional*): - If the context is too long to fit with the question for the model, it will be split in several chunks - with some overlap. This argument controls the size of that overlap. - handle_impossible_answer (`bool`, *optional*): - Whether to accept impossible as an answer. - max_answer_len (`int`, *optional*): - The maximum length of predicted answers (e.g., only answers with a shorter length are considered). - max_question_len (`int`, *optional*): - The maximum length of the question after tokenization. It will be truncated if needed. - max_seq_len (`int`, *optional*): - The maximum length of the total sentence (context + question) in tokens of each chunk passed to the - model. The context will be split in several chunks (using docStride as overlap) if needed. - top_k (`int`, *optional*): - The number of answers to return (will be chosen by order of likelihood). Note that we return less than - topk answers if there are not enough options available within the context. - - Returns: - Union[`QuestionAnsweringOutputElement`, List[`QuestionAnsweringOutputElement`]]: - When top_k is 1 or not provided, it returns a single `QuestionAnsweringOutputElement`. - When top_k is greater than 1, it returns a list of `QuestionAnsweringOutputElement`. - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.question_answering(question="What's my name?", context="My name is Clara and I live in Berkeley.") - QuestionAnsweringOutputElement(answer='Clara', end=16, score=0.9326565265655518, start=11) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="question-answering", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs={"question": question, "context": context}, - parameters={ - "align_to_words": align_to_words, - "doc_stride": doc_stride, - "handle_impossible_answer": handle_impossible_answer, - "max_answer_len": max_answer_len, - "max_question_len": max_question_len, - "max_seq_len": max_seq_len, - "top_k": top_k, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - # Parse the response as a single `QuestionAnsweringOutputElement` when top_k is 1 or not provided, or a list of `QuestionAnsweringOutputElement` to ensure backward compatibility. - output = QuestionAnsweringOutputElement.parse_obj(response) - return output - - def sentence_similarity( - self, sentence: str, other_sentences: List[str], *, model: Optional[str] = None - ) -> List[float]: - """ - Compute the semantic similarity between a sentence and a list of other sentences by comparing their embeddings. - - Args: - sentence (`str`): - The main sentence to compare to others. - other_sentences (`List[str]`): - The list of sentences to compare to. - model (`str`, *optional*): - The model to use for the sentence similarity task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended sentence similarity model will be used. - Defaults to None. - - Returns: - `List[float]`: The similarity scores between the main sentence and the given comparison sentences. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.sentence_similarity( - ... "Machine learning is so easy.", - ... other_sentences=[ - ... "Deep learning is so straightforward.", - ... "This is so difficult, like rocket science.", - ... "I can't believe how much I struggled with this.", - ... ], - ... ) - [0.7785726189613342, 0.45876261591911316, 0.2906220555305481] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="sentence-similarity", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs={"source_sentence": sentence, "sentences": other_sentences}, - parameters={}, - extra_payload={}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return _bytes_to_list(response) - - def summarization( - self, - text: str, - *, - model: Optional[str] = None, - clean_up_tokenization_spaces: Optional[bool] = None, - generate_parameters: Optional[Dict[str, Any]] = None, - truncation: Optional["SummarizationTruncationStrategy"] = None, - ) -> SummarizationOutput: - """ - Generate a summary of a given text using a specified model. - - Args: - text (`str`): - The input text to summarize. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended model for summarization will be used. - clean_up_tokenization_spaces (`bool`, *optional*): - Whether to clean up the potential extra spaces in the text output. - generate_parameters (`Dict[str, Any]`, *optional*): - Additional parametrization of the text generation algorithm. - truncation (`"SummarizationTruncationStrategy"`, *optional*): - The truncation strategy to use. - Returns: - [`SummarizationOutput`]: The generated summary text. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.summarization("The Eiffel tower...") - SummarizationOutput(generated_text="The Eiffel tower is one of the most famous landmarks in the world....") - ``` - """ - parameters = { - "clean_up_tokenization_spaces": clean_up_tokenization_spaces, - "generate_parameters": generate_parameters, - "truncation": truncation, - } - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="summarization", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters=parameters, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return SummarizationOutput.parse_obj_as_list(response)[0] - - def table_question_answering( - self, - table: Dict[str, Any], - query: str, - *, - model: Optional[str] = None, - padding: Optional["Padding"] = None, - sequential: Optional[bool] = None, - truncation: Optional[bool] = None, - ) -> TableQuestionAnsweringOutputElement: - """ - Retrieve the answer to a question from information given in a table. - - Args: - table (`str`): - A table of data represented as a dict of lists where entries are headers and the lists are all the - values, all lists must have the same size. - query (`str`): - The query in plain text that you want to ask the table. - model (`str`): - The model to use for the table-question-answering task. Can be a model ID hosted on the Hugging Face - Hub or a URL to a deployed Inference Endpoint. - padding (`"Padding"`, *optional*): - Activates and controls padding. - sequential (`bool`, *optional*): - Whether to do inference sequentially or as a batch. Batching is faster, but models like SQA require the - inference to be done sequentially to extract relations within sequences, given their conversational - nature. - truncation (`bool`, *optional*): - Activates and controls truncation. - - Returns: - [`TableQuestionAnsweringOutputElement`]: a table question answering output containing the answer, coordinates, cells and the aggregator used. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> query = "How many stars does the transformers repository have?" - >>> table = {"Repository": ["Transformers", "Datasets", "Tokenizers"], "Stars": ["36542", "4512", "3934"]} - >>> client.table_question_answering(table, query, model="google/tapas-base-finetuned-wtq") - TableQuestionAnsweringOutputElement(answer='36542', coordinates=[[0, 1]], cells=['36542'], aggregator='AVERAGE') - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="table-question-answering", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs={"query": query, "table": table}, - parameters={"model": model, "padding": padding, "sequential": sequential, "truncation": truncation}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return TableQuestionAnsweringOutputElement.parse_obj_as_instance(response) - - def tabular_classification(self, table: Dict[str, Any], *, model: Optional[str] = None) -> List[str]: - """ - Classifying a target category (a group) based on a set of attributes. - - Args: - table (`Dict[str, Any]`): - Set of attributes to classify. - model (`str`, *optional*): - The model to use for the tabular classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended tabular classification model will be used. - Defaults to None. - - Returns: - `List`: a list of labels, one per row in the initial table. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> table = { - ... "fixed_acidity": ["7.4", "7.8", "10.3"], - ... "volatile_acidity": ["0.7", "0.88", "0.32"], - ... "citric_acid": ["0", "0", "0.45"], - ... "residual_sugar": ["1.9", "2.6", "6.4"], - ... "chlorides": ["0.076", "0.098", "0.073"], - ... "free_sulfur_dioxide": ["11", "25", "5"], - ... "total_sulfur_dioxide": ["34", "67", "13"], - ... "density": ["0.9978", "0.9968", "0.9976"], - ... "pH": ["3.51", "3.2", "3.23"], - ... "sulphates": ["0.56", "0.68", "0.82"], - ... "alcohol": ["9.4", "9.8", "12.6"], - ... } - >>> client.tabular_classification(table=table, model="julien-c/wine-quality") - ["5", "5", "5"] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="tabular-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=None, - extra_payload={"table": table}, - parameters={}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return _bytes_to_list(response) - - def tabular_regression(self, table: Dict[str, Any], *, model: Optional[str] = None) -> List[float]: - """ - Predicting a numerical target value given a set of attributes/features in a table. - - Args: - table (`Dict[str, Any]`): - Set of attributes stored in a table. The attributes used to predict the target can be both numerical and categorical. - model (`str`, *optional*): - The model to use for the tabular regression task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended tabular regression model will be used. - Defaults to None. - - Returns: - `List`: a list of predicted numerical target values. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> table = { - ... "Height": ["11.52", "12.48", "12.3778"], - ... "Length1": ["23.2", "24", "23.9"], - ... "Length2": ["25.4", "26.3", "26.5"], - ... "Length3": ["30", "31.2", "31.1"], - ... "Species": ["Bream", "Bream", "Bream"], - ... "Width": ["4.02", "4.3056", "4.6961"], - ... } - >>> client.tabular_regression(table, model="scikit-learn/Fish-Weight") - [110, 120, 130] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="tabular-regression", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=None, - parameters={}, - extra_payload={"table": table}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return _bytes_to_list(response) - - def text_classification( - self, - text: str, - *, - model: Optional[str] = None, - top_k: Optional[int] = None, - function_to_apply: Optional["TextClassificationOutputTransform"] = None, - ) -> List[TextClassificationOutputElement]: - """ - Perform text classification (e.g. sentiment-analysis) on the given text. - - Args: - text (`str`): - A string to be classified. - model (`str`, *optional*): - The model to use for the text classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended text classification model will be used. - Defaults to None. - top_k (`int`, *optional*): - When specified, limits the output to the top K most probable classes. - function_to_apply (`"TextClassificationOutputTransform"`, *optional*): - The function to apply to the model outputs in order to retrieve the scores. - - Returns: - `List[TextClassificationOutputElement]`: a list of [`TextClassificationOutputElement`] items containing the predicted label and associated probability. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.text_classification("I like you") - [ - TextClassificationOutputElement(label='POSITIVE', score=0.9998695850372314), - TextClassificationOutputElement(label='NEGATIVE', score=0.0001304351753788069), - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="text-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "function_to_apply": function_to_apply, - "top_k": top_k, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return TextClassificationOutputElement.parse_obj_as_list(response)[0] # type: ignore [return-value] - - @overload - def text_generation( - self, - prompt: str, - *, - details: Literal[True], - stream: Literal[True], - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> Iterable[TextGenerationStreamOutput]: ... - - @overload - def text_generation( - self, - prompt: str, - *, - details: Literal[True], - stream: Optional[Literal[False]] = None, - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> TextGenerationOutput: ... - - @overload - def text_generation( - self, - prompt: str, - *, - details: Optional[Literal[False]] = None, - stream: Literal[True], - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, # Manual default value - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> Iterable[str]: ... - - @overload - def text_generation( - self, - prompt: str, - *, - details: Optional[Literal[False]] = None, - stream: Optional[Literal[False]] = None, - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> str: ... - - @overload - def text_generation( - self, - prompt: str, - *, - details: Optional[bool] = None, - stream: Optional[bool] = None, - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> Union[str, TextGenerationOutput, Iterable[str], Iterable[TextGenerationStreamOutput]]: ... - - def text_generation( - self, - prompt: str, - *, - details: Optional[bool] = None, - stream: Optional[bool] = None, - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> Union[str, TextGenerationOutput, Iterable[str], Iterable[TextGenerationStreamOutput]]: - """ - Given a prompt, generate the following text. - - > [!TIP] - > If you want to generate a response from chat messages, you should use the [`InferenceClient.chat_completion`] method. - > It accepts a list of messages instead of a single text prompt and handles the chat templating for you. - - Args: - prompt (`str`): - Input text. - details (`bool`, *optional*): - By default, text_generation returns a string. Pass `details=True` if you want a detailed output (tokens, - probabilities, seed, finish reason, etc.). Only available for models running on with the - `text-generation-inference` backend. - stream (`bool`, *optional*): - By default, text_generation returns the full generated text. Pass `stream=True` if you want a stream of - tokens to be returned. Only available for models running on with the `text-generation-inference` - backend. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - adapter_id (`str`, *optional*): - Lora adapter id. - best_of (`int`, *optional*): - Generate best_of sequences and return the one if the highest token logprobs. - decoder_input_details (`bool`, *optional*): - Return the decoder input token logprobs and ids. You must set `details=True` as well for it to be taken - into account. Defaults to `False`. - do_sample (`bool`, *optional*): - Activate logits sampling - frequency_penalty (`float`, *optional*): - Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in - the text so far, decreasing the model's likelihood to repeat the same line verbatim. - grammar ([`TextGenerationInputGrammarType`], *optional*): - Grammar constraints. Can be either a JSONSchema or a regex. - max_new_tokens (`int`, *optional*): - Maximum number of generated tokens. Defaults to 100. - repetition_penalty (`float`, *optional*): - The parameter for repetition penalty. 1.0 means no penalty. See [this - paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. - return_full_text (`bool`, *optional*): - Whether to prepend the prompt to the generated text - seed (`int`, *optional*): - Random sampling seed - stop (`List[str]`, *optional*): - Stop generating tokens if a member of `stop` is generated. - stop_sequences (`List[str]`, *optional*): - Deprecated argument. Use `stop` instead. - temperature (`float`, *optional*): - The value used to module the logits distribution. - top_n_tokens (`int`, *optional*): - Return information about the `top_n_tokens` most likely tokens at each generation step, instead of - just the sampled token. - top_k (`int`, *optional`): - The number of highest probability vocabulary tokens to keep for top-k-filtering. - top_p (`float`, *optional`): - If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or - higher are kept for generation. - truncate (`int`, *optional`): - Truncate inputs tokens to the given size. - typical_p (`float`, *optional`): - Typical Decoding mass - See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information - watermark (`bool`, *optional*): - Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) - - Returns: - `Union[str, TextGenerationOutput, Iterable[str], Iterable[TextGenerationStreamOutput]]`: - Generated text returned from the server: - - if `stream=False` and `details=False`, the generated text is returned as a `str` (default) - - if `stream=True` and `details=False`, the generated text is returned token by token as a `Iterable[str]` - - if `stream=False` and `details=True`, the generated text is returned with more details as a [`~huggingface_hub.TextGenerationOutput`] - - if `details=True` and `stream=True`, the generated text is returned token by token as a iterable of [`~huggingface_hub.TextGenerationStreamOutput`] - - Raises: - `ValidationError`: - If input values are not valid. No HTTP call is made to the server. - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - - # Case 1: generate text - >>> client.text_generation("The huggingface_hub library is ", max_new_tokens=12) - '100% open source and built to be easy to use.' - - # Case 2: iterate over the generated tokens. Useful for large generation. - >>> for token in client.text_generation("The huggingface_hub library is ", max_new_tokens=12, stream=True): - ... print(token) - 100 - % - open - source - and - built - to - be - easy - to - use - . - - # Case 3: get more details about the generation process. - >>> client.text_generation("The huggingface_hub library is ", max_new_tokens=12, details=True) - TextGenerationOutput( - generated_text='100% open source and built to be easy to use.', - details=TextGenerationDetails( - finish_reason='length', - generated_tokens=12, - seed=None, - prefill=[ - TextGenerationPrefillOutputToken(id=487, text='The', logprob=None), - TextGenerationPrefillOutputToken(id=53789, text=' hugging', logprob=-13.171875), - (...) - TextGenerationPrefillOutputToken(id=204, text=' ', logprob=-7.0390625) - ], - tokens=[ - TokenElement(id=1425, text='100', logprob=-1.0175781, special=False), - TokenElement(id=16, text='%', logprob=-0.0463562, special=False), - (...) - TokenElement(id=25, text='.', logprob=-0.5703125, special=False) - ], - best_of_sequences=None - ) - ) - - # Case 4: iterate over the generated tokens with more details. - # Last object is more complete, containing the full generated text and the finish reason. - >>> for details in client.text_generation("The huggingface_hub library is ", max_new_tokens=12, details=True, stream=True): - ... print(details) - ... - TextGenerationStreamOutput(token=TokenElement(id=1425, text='100', logprob=-1.0175781, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=16, text='%', logprob=-0.0463562, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=1314, text=' open', logprob=-1.3359375, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=3178, text=' source', logprob=-0.28100586, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=273, text=' and', logprob=-0.5961914, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=3426, text=' built', logprob=-1.9423828, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=271, text=' to', logprob=-1.4121094, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=314, text=' be', logprob=-1.5224609, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=1833, text=' easy', logprob=-2.1132812, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=271, text=' to', logprob=-0.08520508, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=745, text=' use', logprob=-0.39453125, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement( - id=25, - text='.', - logprob=-0.5703125, - special=False), - generated_text='100% open source and built to be easy to use.', - details=TextGenerationStreamOutputStreamDetails(finish_reason='length', generated_tokens=12, seed=None) - ) - - # Case 5: generate constrained output using grammar - >>> response = client.text_generation( - ... prompt="I saw a puppy a cat and a raccoon during my bike ride in the park", - ... model="HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1", - ... max_new_tokens=100, - ... repetition_penalty=1.3, - ... grammar={ - ... "type": "json", - ... "value": { - ... "properties": { - ... "location": {"type": "string"}, - ... "activity": {"type": "string"}, - ... "animals_seen": {"type": "integer", "minimum": 1, "maximum": 5}, - ... "animals": {"type": "array", "items": {"type": "string"}}, - ... }, - ... "required": ["location", "activity", "animals_seen", "animals"], - ... }, - ... }, - ... ) - >>> json.loads(response) - { - "activity": "bike riding", - "animals": ["puppy", "cat", "raccoon"], - "animals_seen": 3, - "location": "park" - } - ``` - """ - if decoder_input_details and not details: - warnings.warn( - "`decoder_input_details=True` has been passed to the server but `details=False` is set meaning that" - " the output from the server will be truncated." - ) - decoder_input_details = False - - if stop_sequences is not None: - warnings.warn( - "`stop_sequences` is a deprecated argument for `text_generation` task" - " and will be removed in version '0.28.0'. Use `stop` instead.", - FutureWarning, - ) - if stop is None: - stop = stop_sequences # use deprecated arg if provided - - # Build payload - parameters = { - "adapter_id": adapter_id, - "best_of": best_of, - "decoder_input_details": decoder_input_details, - "details": details, - "do_sample": do_sample, - "frequency_penalty": frequency_penalty, - "grammar": grammar, - "max_new_tokens": max_new_tokens, - "repetition_penalty": repetition_penalty, - "return_full_text": return_full_text, - "seed": seed, - "stop": stop, - "temperature": temperature, - "top_k": top_k, - "top_n_tokens": top_n_tokens, - "top_p": top_p, - "truncate": truncate, - "typical_p": typical_p, - "watermark": watermark, - } - - # Remove some parameters if not a TGI server - unsupported_kwargs = _get_unsupported_text_generation_kwargs(model) - if len(unsupported_kwargs) > 0: - # The server does not support some parameters - # => means it is not a TGI server - # => remove unsupported parameters and warn the user - - ignored_parameters = [] - for key in unsupported_kwargs: - if parameters.get(key): - ignored_parameters.append(key) - parameters.pop(key, None) - if len(ignored_parameters) > 0: - warnings.warn( - "API endpoint/model for text-generation is not served via TGI. Ignoring following parameters:" - f" {', '.join(ignored_parameters)}.", - UserWarning, - ) - if details: - warnings.warn( - "API endpoint/model for text-generation is not served via TGI. Parameter `details=True` will" - " be ignored meaning only the generated text will be returned.", - UserWarning, - ) - details = False - if stream: - raise ValueError( - "API endpoint/model for text-generation is not served via TGI. Cannot return output as a stream." - " Please pass `stream=False` as input." - ) - - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="text-generation", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=prompt, - parameters=parameters, - extra_payload={"stream": stream}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - - # Handle errors separately for more precise error messages - try: - bytes_output = self._inner_post(request_parameters, stream=stream or False) - except HTTPError as e: - match = MODEL_KWARGS_NOT_USED_REGEX.search(str(e)) - if isinstance(e, BadRequestError) and match: - unused_params = [kwarg.strip("' ") for kwarg in match.group(1).split(",")] - _set_unsupported_text_generation_kwargs(model, unused_params) - return self.text_generation( # type: ignore - prompt=prompt, - details=details, - stream=stream, - model=model_id, - adapter_id=adapter_id, - best_of=best_of, - decoder_input_details=decoder_input_details, - do_sample=do_sample, - frequency_penalty=frequency_penalty, - grammar=grammar, - max_new_tokens=max_new_tokens, - repetition_penalty=repetition_penalty, - return_full_text=return_full_text, - seed=seed, - stop=stop, - temperature=temperature, - top_k=top_k, - top_n_tokens=top_n_tokens, - top_p=top_p, - truncate=truncate, - typical_p=typical_p, - watermark=watermark, - ) - raise_text_generation_error(e) - - # Parse output - if stream: - return _stream_text_generation_response(bytes_output, details) # type: ignore - - data = _bytes_to_dict(bytes_output) # type: ignore[arg-type] - - # Data can be a single element (dict) or an iterable of dicts where we select the first element of. - if isinstance(data, list): - data = data[0] - response = provider_helper.get_response(data, request_parameters) - return TextGenerationOutput.parse_obj_as_instance(response) if details else response["generated_text"] - - def text_to_image( - self, - prompt: str, - *, - negative_prompt: Optional[str] = None, - height: Optional[int] = None, - width: Optional[int] = None, - num_inference_steps: Optional[int] = None, - guidance_scale: Optional[float] = None, - model: Optional[str] = None, - scheduler: Optional[str] = None, - seed: Optional[int] = None, - extra_body: Optional[Dict[str, Any]] = None, - ) -> "Image": - """ - Generate an image based on a given text using a specified model. - - > [!WARNING] - > You must have `PIL` installed if you want to work with images (`pip install Pillow`). - - > [!TIP] - > You can pass provider-specific parameters to the model by using the `extra_body` argument. - - Args: - prompt (`str`): - The prompt to generate an image from. - negative_prompt (`str`, *optional*): - One prompt to guide what NOT to include in image generation. - height (`int`, *optional*): - The height in pixels of the output image - width (`int`, *optional*): - The width in pixels of the output image - num_inference_steps (`int`, *optional*): - The number of denoising steps. More denoising steps usually lead to a higher quality image at the - expense of slower inference. - guidance_scale (`float`, *optional*): - A higher guidance scale value encourages the model to generate images closely linked to the text - prompt, but values too high may cause saturation and other artifacts. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended text-to-image model will be used. - Defaults to None. - scheduler (`str`, *optional*): - Override the scheduler with a compatible one. - seed (`int`, *optional*): - Seed for the random number generator. - extra_body (`Dict[str, Any]`, *optional*): - Additional provider-specific parameters to pass to the model. Refer to the provider's documentation - for supported parameters. - - Returns: - `Image`: The generated image. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - - >>> image = client.text_to_image("An astronaut riding a horse on the moon.") - >>> image.save("astronaut.png") - - >>> image = client.text_to_image( - ... "An astronaut riding a horse on the moon.", - ... negative_prompt="low resolution, blurry", - ... model="stabilityai/stable-diffusion-2-1", - ... ) - >>> image.save("better_astronaut.png") - ``` - Example using a third-party provider directly. Usage will be billed on your fal.ai account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="fal-ai", # Use fal.ai provider - ... api_key="fal-ai-api-key", # Pass your fal.ai API key - ... ) - >>> image = client.text_to_image( - ... "A majestic lion in a fantasy forest", - ... model="black-forest-labs/FLUX.1-schnell", - ... ) - >>> image.save("lion.png") - ``` - - Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", # Use replicate provider - ... api_key="hf_...", # Pass your HF token - ... ) - >>> image = client.text_to_image( - ... "An astronaut riding a horse on the moon.", - ... model="black-forest-labs/FLUX.1-dev", - ... ) - >>> image.save("astronaut.png") - ``` - - Example using Replicate provider with extra parameters - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", # Use replicate provider - ... api_key="hf_...", # Pass your HF token - ... ) - >>> image = client.text_to_image( - ... "An astronaut riding a horse on the moon.", - ... model="black-forest-labs/FLUX.1-schnell", - ... extra_body={"output_quality": 100}, - ... ) - >>> image.save("astronaut.png") - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="text-to-image", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=prompt, - parameters={ - "negative_prompt": negative_prompt, - "height": height, - "width": width, - "num_inference_steps": num_inference_steps, - "guidance_scale": guidance_scale, - "scheduler": scheduler, - "seed": seed, - **(extra_body or {}), - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - response = provider_helper.get_response(response) - return _bytes_to_image(response) - - def text_to_video( - self, - prompt: str, - *, - model: Optional[str] = None, - guidance_scale: Optional[float] = None, - negative_prompt: Optional[List[str]] = None, - num_frames: Optional[float] = None, - num_inference_steps: Optional[int] = None, - seed: Optional[int] = None, - extra_body: Optional[Dict[str, Any]] = None, - ) -> bytes: - """ - Generate a video based on a given text. - - > [!TIP] - > You can pass provider-specific parameters to the model by using the `extra_body` argument. - - Args: - prompt (`str`): - The prompt to generate a video from. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended text-to-video model will be used. - Defaults to None. - guidance_scale (`float`, *optional*): - A higher guidance scale value encourages the model to generate videos closely linked to the text - prompt, but values too high may cause saturation and other artifacts. - negative_prompt (`List[str]`, *optional*): - One or several prompt to guide what NOT to include in video generation. - num_frames (`float`, *optional*): - The num_frames parameter determines how many video frames are generated. - num_inference_steps (`int`, *optional*): - The number of denoising steps. More denoising steps usually lead to a higher quality video at the - expense of slower inference. - seed (`int`, *optional*): - Seed for the random number generator. - extra_body (`Dict[str, Any]`, *optional*): - Additional provider-specific parameters to pass to the model. Refer to the provider's documentation - for supported parameters. - - Returns: - `bytes`: The generated video. - - Example: - - Example using a third-party provider directly. Usage will be billed on your fal.ai account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="fal-ai", # Using fal.ai provider - ... api_key="fal-ai-api-key", # Pass your fal.ai API key - ... ) - >>> video = client.text_to_video( - ... "A majestic lion running in a fantasy forest", - ... model="tencent/HunyuanVideo", - ... ) - >>> with open("lion.mp4", "wb") as file: - ... file.write(video) - ``` - - Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", # Using replicate provider - ... api_key="hf_...", # Pass your HF token - ... ) - >>> video = client.text_to_video( - ... "A cat running in a park", - ... model="genmo/mochi-1-preview", - ... ) - >>> with open("cat.mp4", "wb") as file: - ... file.write(video) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="text-to-video", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=prompt, - parameters={ - "guidance_scale": guidance_scale, - "negative_prompt": negative_prompt, - "num_frames": num_frames, - "num_inference_steps": num_inference_steps, - "seed": seed, - **(extra_body or {}), - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - response = provider_helper.get_response(response, request_parameters) - return response - - def text_to_speech( - self, - text: str, - *, - model: Optional[str] = None, - do_sample: Optional[bool] = None, - early_stopping: Optional[Union[bool, "TextToSpeechEarlyStoppingEnum"]] = None, - epsilon_cutoff: Optional[float] = None, - eta_cutoff: Optional[float] = None, - max_length: Optional[int] = None, - max_new_tokens: Optional[int] = None, - min_length: Optional[int] = None, - min_new_tokens: Optional[int] = None, - num_beam_groups: Optional[int] = None, - num_beams: Optional[int] = None, - penalty_alpha: Optional[float] = None, - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_p: Optional[float] = None, - typical_p: Optional[float] = None, - use_cache: Optional[bool] = None, - extra_body: Optional[Dict[str, Any]] = None, - ) -> bytes: - """ - Synthesize an audio of a voice pronouncing a given text. - - > [!TIP] - > You can pass provider-specific parameters to the model by using the `extra_body` argument. - - Args: - text (`str`): - The text to synthesize. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended text-to-speech model will be used. - Defaults to None. - do_sample (`bool`, *optional*): - Whether to use sampling instead of greedy decoding when generating new tokens. - early_stopping (`Union[bool, "TextToSpeechEarlyStoppingEnum"]`, *optional*): - Controls the stopping condition for beam-based methods. - epsilon_cutoff (`float`, *optional*): - If set to float strictly between 0 and 1, only tokens with a conditional probability greater than - epsilon_cutoff will be sampled. In the paper, suggested values range from 3e-4 to 9e-4, depending on - the size of the model. See [Truncation Sampling as Language Model - Desmoothing](https://hf.co/papers/2210.15191) for more details. - eta_cutoff (`float`, *optional*): - Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to float strictly - between 0 and 1, a token is only considered if it is greater than either eta_cutoff or sqrt(eta_cutoff) - * exp(-entropy(softmax(next_token_logits))). The latter term is intuitively the expected next token - probability, scaled by sqrt(eta_cutoff). In the paper, suggested values range from 3e-4 to 2e-3, - depending on the size of the model. See [Truncation Sampling as Language Model - Desmoothing](https://hf.co/papers/2210.15191) for more details. - max_length (`int`, *optional*): - The maximum length (in tokens) of the generated text, including the input. - max_new_tokens (`int`, *optional*): - The maximum number of tokens to generate. Takes precedence over max_length. - min_length (`int`, *optional*): - The minimum length (in tokens) of the generated text, including the input. - min_new_tokens (`int`, *optional*): - The minimum number of tokens to generate. Takes precedence over min_length. - num_beam_groups (`int`, *optional*): - Number of groups to divide num_beams into in order to ensure diversity among different groups of beams. - See [this paper](https://hf.co/papers/1610.02424) for more details. - num_beams (`int`, *optional*): - Number of beams to use for beam search. - penalty_alpha (`float`, *optional*): - The value balances the model confidence and the degeneration penalty in contrastive search decoding. - temperature (`float`, *optional*): - The value used to modulate the next token probabilities. - top_k (`int`, *optional*): - The number of highest probability vocabulary tokens to keep for top-k-filtering. - top_p (`float`, *optional*): - If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to - top_p or higher are kept for generation. - typical_p (`float`, *optional*): - Local typicality measures how similar the conditional probability of predicting a target token next is - to the expected conditional probability of predicting a random token next, given the partial text - already generated. If set to float < 1, the smallest set of the most locally typical tokens with - probabilities that add up to typical_p or higher are kept for generation. See [this - paper](https://hf.co/papers/2202.00666) for more details. - use_cache (`bool`, *optional*): - Whether the model should use the past last key/values attentions to speed up decoding - extra_body (`Dict[str, Any]`, *optional*): - Additional provider-specific parameters to pass to the model. Refer to the provider's documentation - for supported parameters. - Returns: - `bytes`: The generated audio. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from pathlib import Path - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - - >>> audio = client.text_to_speech("Hello world") - >>> Path("hello_world.flac").write_bytes(audio) - ``` - - Example using a third-party provider directly. Usage will be billed on your Replicate account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", - ... api_key="your-replicate-api-key", # Pass your Replicate API key directly - ... ) - >>> audio = client.text_to_speech( - ... text="Hello world", - ... model="OuteAI/OuteTTS-0.3-500M", - ... ) - >>> Path("hello_world.flac").write_bytes(audio) - ``` - - Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", - ... api_key="hf_...", # Pass your HF token - ... ) - >>> audio =client.text_to_speech( - ... text="Hello world", - ... model="OuteAI/OuteTTS-0.3-500M", - ... ) - >>> Path("hello_world.flac").write_bytes(audio) - ``` - Example using Replicate provider with extra parameters - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", # Use replicate provider - ... api_key="hf_...", # Pass your HF token - ... ) - >>> audio = client.text_to_speech( - ... "Hello, my name is Kororo, an awesome text-to-speech model.", - ... model="hexgrad/Kokoro-82M", - ... extra_body={"voice": "af_nicole"}, - ... ) - >>> Path("hello.flac").write_bytes(audio) - ``` - - Example music-gen using "YuE-s1-7B-anneal-en-cot" on fal.ai - ```py - >>> from huggingface_hub import InferenceClient - >>> lyrics = ''' - ... [verse] - ... In the town where I was born - ... Lived a man who sailed to sea - ... And he told us of his life - ... In the land of submarines - ... So we sailed on to the sun - ... 'Til we found a sea of green - ... And we lived beneath the waves - ... In our yellow submarine - - ... [chorus] - ... We all live in a yellow submarine - ... Yellow submarine, yellow submarine - ... We all live in a yellow submarine - ... Yellow submarine, yellow submarine - ... ''' - >>> genres = "pavarotti-style tenor voice" - >>> client = InferenceClient( - ... provider="fal-ai", - ... model="m-a-p/YuE-s1-7B-anneal-en-cot", - ... api_key=..., - ... ) - >>> audio = client.text_to_speech(lyrics, extra_body={"genres": genres}) - >>> with open("output.mp3", "wb") as f: - ... f.write(audio) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="text-to-speech", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "do_sample": do_sample, - "early_stopping": early_stopping, - "epsilon_cutoff": epsilon_cutoff, - "eta_cutoff": eta_cutoff, - "max_length": max_length, - "max_new_tokens": max_new_tokens, - "min_length": min_length, - "min_new_tokens": min_new_tokens, - "num_beam_groups": num_beam_groups, - "num_beams": num_beams, - "penalty_alpha": penalty_alpha, - "temperature": temperature, - "top_k": top_k, - "top_p": top_p, - "typical_p": typical_p, - "use_cache": use_cache, - **(extra_body or {}), - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - response = provider_helper.get_response(response) - return response - - def token_classification( - self, - text: str, - *, - model: Optional[str] = None, - aggregation_strategy: Optional["TokenClassificationAggregationStrategy"] = None, - ignore_labels: Optional[List[str]] = None, - stride: Optional[int] = None, - ) -> List[TokenClassificationOutputElement]: - """ - Perform token classification on the given text. - Usually used for sentence parsing, either grammatical, or Named Entity Recognition (NER) to understand keywords contained within text. - - Args: - text (`str`): - A string to be classified. - model (`str`, *optional*): - The model to use for the token classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended token classification model will be used. - Defaults to None. - aggregation_strategy (`"TokenClassificationAggregationStrategy"`, *optional*): - The strategy used to fuse tokens based on model predictions - ignore_labels (`List[str`, *optional*): - A list of labels to ignore - stride (`int`, *optional*): - The number of overlapping tokens between chunks when splitting the input text. - - Returns: - `List[TokenClassificationOutputElement]`: List of [`TokenClassificationOutputElement`] items containing the entity group, confidence score, word, start and end index. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.token_classification("My name is Sarah Jessica Parker but you can call me Jessica") - [ - TokenClassificationOutputElement( - entity_group='PER', - score=0.9971321225166321, - word='Sarah Jessica Parker', - start=11, - end=31, - ), - TokenClassificationOutputElement( - entity_group='PER', - score=0.9773476123809814, - word='Jessica', - start=52, - end=59, - ) - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="token-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "aggregation_strategy": aggregation_strategy, - "ignore_labels": ignore_labels, - "stride": stride, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return TokenClassificationOutputElement.parse_obj_as_list(response) - - def translation( - self, - text: str, - *, - model: Optional[str] = None, - src_lang: Optional[str] = None, - tgt_lang: Optional[str] = None, - clean_up_tokenization_spaces: Optional[bool] = None, - truncation: Optional["TranslationTruncationStrategy"] = None, - generate_parameters: Optional[Dict[str, Any]] = None, - ) -> TranslationOutput: - """ - Convert text from one language to another. - - Check out https://huggingface.co/tasks/translation for more information on how to choose the best model for - your specific use case. Source and target languages usually depend on the model. - However, it is possible to specify source and target languages for certain models. If you are working with one of these models, - you can use `src_lang` and `tgt_lang` arguments to pass the relevant information. - - Args: - text (`str`): - A string to be translated. - model (`str`, *optional*): - The model to use for the translation task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended translation model will be used. - Defaults to None. - src_lang (`str`, *optional*): - The source language of the text. Required for models that can translate from multiple languages. - tgt_lang (`str`, *optional*): - Target language to translate to. Required for models that can translate to multiple languages. - clean_up_tokenization_spaces (`bool`, *optional*): - Whether to clean up the potential extra spaces in the text output. - truncation (`"TranslationTruncationStrategy"`, *optional*): - The truncation strategy to use. - generate_parameters (`Dict[str, Any]`, *optional*): - Additional parametrization of the text generation algorithm. - - Returns: - [`TranslationOutput`]: The generated translated text. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - `ValueError`: - If only one of the `src_lang` and `tgt_lang` arguments are provided. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.translation("My name is Wolfgang and I live in Berlin") - 'Mein Name ist Wolfgang und ich lebe in Berlin.' - >>> client.translation("My name is Wolfgang and I live in Berlin", model="Helsinki-NLP/opus-mt-en-fr") - TranslationOutput(translation_text='Je m'appelle Wolfgang et je vis à Berlin.') - ``` - - Specifying languages: - ```py - >>> client.translation("My name is Sarah Jessica Parker but you can call me Jessica", model="facebook/mbart-large-50-many-to-many-mmt", src_lang="en_XX", tgt_lang="fr_XX") - "Mon nom est Sarah Jessica Parker mais vous pouvez m'appeler Jessica" - ``` - """ - # Throw error if only one of `src_lang` and `tgt_lang` was given - if src_lang is not None and tgt_lang is None: - raise ValueError("You cannot specify `src_lang` without specifying `tgt_lang`.") - - if src_lang is None and tgt_lang is not None: - raise ValueError("You cannot specify `tgt_lang` without specifying `src_lang`.") - - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="translation", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "src_lang": src_lang, - "tgt_lang": tgt_lang, - "clean_up_tokenization_spaces": clean_up_tokenization_spaces, - "truncation": truncation, - "generate_parameters": generate_parameters, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return TranslationOutput.parse_obj_as_list(response)[0] - - def visual_question_answering( - self, - image: ContentT, - question: str, - *, - model: Optional[str] = None, - top_k: Optional[int] = None, - ) -> List[VisualQuestionAnsweringOutputElement]: - """ - Answering open-ended questions based on an image. - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The input image for the context. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - question (`str`): - Question to be answered. - model (`str`, *optional*): - The model to use for the visual question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended visual question answering model will be used. - Defaults to None. - top_k (`int`, *optional*): - The number of answers to return (will be chosen by order of likelihood). Note that we return less than - topk answers if there are not enough options available within the context. - Returns: - `List[VisualQuestionAnsweringOutputElement]`: a list of [`VisualQuestionAnsweringOutputElement`] items containing the predicted label and associated probability. - - Raises: - `InferenceTimeoutError`: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.visual_question_answering( - ... image="https://huggingface.co/datasets/mishig/sample_images/resolve/main/tiger.jpg", - ... question="What is the animal doing?" - ... ) - [ - VisualQuestionAnsweringOutputElement(score=0.778609573841095, answer='laying down'), - VisualQuestionAnsweringOutputElement(score=0.6957435607910156, answer='sitting'), - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="visual-question-answering", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={"top_k": top_k}, - headers=self.headers, - model=model_id, - api_key=self.token, - extra_payload={"question": question, "image": _b64_encode(image)}, - ) - response = self._inner_post(request_parameters) - return VisualQuestionAnsweringOutputElement.parse_obj_as_list(response) - - def zero_shot_classification( - self, - text: str, - candidate_labels: List[str], - *, - multi_label: Optional[bool] = False, - hypothesis_template: Optional[str] = None, - model: Optional[str] = None, - ) -> List[ZeroShotClassificationOutputElement]: - """ - Provide as input a text and a set of candidate labels to classify the input text. - - Args: - text (`str`): - The input text to classify. - candidate_labels (`List[str]`): - The set of possible class labels to classify the text into. - labels (`List[str]`, *optional*): - (deprecated) List of strings. Each string is the verbalization of a possible label for the input text. - multi_label (`bool`, *optional*): - Whether multiple candidate labels can be true. If false, the scores are normalized such that the sum of - the label likelihoods for each sequence is 1. If true, the labels are considered independent and - probabilities are normalized for each candidate. - hypothesis_template (`str`, *optional*): - The sentence used in conjunction with `candidate_labels` to attempt the text classification by - replacing the placeholder with the candidate labels. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. If not provided, the default recommended zero-shot classification model will be used. - - - Returns: - `List[ZeroShotClassificationOutputElement]`: List of [`ZeroShotClassificationOutputElement`] items containing the predicted labels and their confidence. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example with `multi_label=False`: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> text = ( - ... "A new model offers an explanation for how the Galilean satellites formed around the solar system's" - ... "largest world. Konstantin Batygin did not set out to solve one of the solar system's most puzzling" - ... " mysteries when he went for a run up a hill in Nice, France." - ... ) - >>> labels = ["space & cosmos", "scientific discovery", "microbiology", "robots", "archeology"] - >>> client.zero_shot_classification(text, labels) - [ - ZeroShotClassificationOutputElement(label='scientific discovery', score=0.7961668968200684), - ZeroShotClassificationOutputElement(label='space & cosmos', score=0.18570658564567566), - ZeroShotClassificationOutputElement(label='microbiology', score=0.00730885099619627), - ZeroShotClassificationOutputElement(label='archeology', score=0.006258360575884581), - ZeroShotClassificationOutputElement(label='robots', score=0.004559356719255447), - ] - >>> client.zero_shot_classification(text, labels, multi_label=True) - [ - ZeroShotClassificationOutputElement(label='scientific discovery', score=0.9829297661781311), - ZeroShotClassificationOutputElement(label='space & cosmos', score=0.755190908908844), - ZeroShotClassificationOutputElement(label='microbiology', score=0.0005462635890580714), - ZeroShotClassificationOutputElement(label='archeology', score=0.00047131875180639327), - ZeroShotClassificationOutputElement(label='robots', score=0.00030448526376858354), - ] - ``` - - Example with `multi_label=True` and a custom `hypothesis_template`: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - >>> client.zero_shot_classification( - ... text="I really like our dinner and I'm very happy. I don't like the weather though.", - ... labels=["positive", "negative", "pessimistic", "optimistic"], - ... multi_label=True, - ... hypothesis_template="This text is {} towards the weather" - ... ) - [ - ZeroShotClassificationOutputElement(label='negative', score=0.9231801629066467), - ZeroShotClassificationOutputElement(label='pessimistic', score=0.8760990500450134), - ZeroShotClassificationOutputElement(label='optimistic', score=0.0008674879791215062), - ZeroShotClassificationOutputElement(label='positive', score=0.0005250611575320363) - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="zero-shot-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "candidate_labels": candidate_labels, - "multi_label": multi_label, - "hypothesis_template": hypothesis_template, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - output = _bytes_to_dict(response) - return [ - ZeroShotClassificationOutputElement.parse_obj_as_instance({"label": label, "score": score}) - for label, score in zip(output["labels"], output["scores"]) - ] - - def zero_shot_image_classification( - self, - image: ContentT, - candidate_labels: List[str], - *, - model: Optional[str] = None, - hypothesis_template: Optional[str] = None, - # deprecated argument - labels: List[str] = None, # type: ignore - ) -> List[ZeroShotImageClassificationOutputElement]: - """ - Provide input image and text labels to predict text labels for the image. - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The input image to caption. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - candidate_labels (`List[str]`): - The candidate labels for this image - labels (`List[str]`, *optional*): - (deprecated) List of string possible labels. There must be at least 2 labels. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. If not provided, the default recommended zero-shot image classification model will be used. - hypothesis_template (`str`, *optional*): - The sentence used in conjunction with `candidate_labels` to attempt the image classification by - replacing the placeholder with the candidate labels. - - Returns: - `List[ZeroShotImageClassificationOutputElement]`: List of [`ZeroShotImageClassificationOutputElement`] items containing the predicted labels and their confidence. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `HTTPError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient() - - >>> client.zero_shot_image_classification( - ... "https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg", - ... labels=["dog", "cat", "horse"], - ... ) - [ZeroShotImageClassificationOutputElement(label='dog', score=0.956),...] - ``` - """ - # Raise ValueError if input is less than 2 labels - if len(candidate_labels) < 2: - raise ValueError("You must specify at least 2 classes to compare.") - - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="zero-shot-image-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={ - "candidate_labels": candidate_labels, - "hypothesis_template": hypothesis_template, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = self._inner_post(request_parameters) - return ZeroShotImageClassificationOutputElement.parse_obj_as_list(response) - - def get_endpoint_info(self, *, model: Optional[str] = None) -> Dict[str, Any]: - """ - Get information about the deployed endpoint. - - This endpoint is only available on endpoints powered by Text-Generation-Inference (TGI) or Text-Embedding-Inference (TEI). - Endpoints powered by `transformers` return an empty payload. - - Args: - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - - Returns: - `Dict[str, Any]`: Information about the endpoint. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") - >>> client.get_endpoint_info() - { - 'model_id': 'meta-llama/Meta-Llama-3-70B-Instruct', - 'model_sha': None, - 'model_dtype': 'torch.float16', - 'model_device_type': 'cuda', - 'model_pipeline_tag': None, - 'max_concurrent_requests': 128, - 'max_best_of': 2, - 'max_stop_sequences': 4, - 'max_input_length': 8191, - 'max_total_tokens': 8192, - 'waiting_served_ratio': 0.3, - 'max_batch_total_tokens': 1259392, - 'max_waiting_tokens': 20, - 'max_batch_size': None, - 'validation_workers': 32, - 'max_client_batch_size': 4, - 'version': '2.0.2', - 'sha': 'dccab72549635c7eb5ddb17f43f0b7cdff07c214', - 'docker_label': 'sha-dccab72' - } - ``` - """ - if self.provider != "hf-inference": - raise ValueError(f"Getting endpoint info is not supported on '{self.provider}'.") - - model = model or self.model - if model is None: - raise ValueError("Model id not provided.") - if model.startswith(("http://", "https://")): - url = model.rstrip("/") + "/info" - else: - url = f"{constants.INFERENCE_ENDPOINT}/models/{model}/info" - - response = get_session().get(url, headers=build_hf_headers(token=self.token)) - hf_raise_for_status(response) - return response.json() - - def health_check(self, model: Optional[str] = None) -> bool: - """ - Check the health of the deployed endpoint. - - Health check is only available with Inference Endpoints powered by Text-Generation-Inference (TGI) or Text-Embedding-Inference (TEI). - - Args: - model (`str`, *optional*): - URL of the Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - - Returns: - `bool`: True if everything is working fine. - - Example: - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient("https://jzgu0buei5.us-east-1.aws.endpoints.huggingface.cloud") - >>> client.health_check() - True - ``` - """ - if self.provider != "hf-inference": - raise ValueError(f"Health check is not supported on '{self.provider}'.") - - model = model or self.model - if model is None: - raise ValueError("Model id not provided.") - if not model.startswith(("http://", "https://")): - raise ValueError("Model must be an Inference Endpoint URL.") - url = model.rstrip("/") + "/health" - - response = get_session().get(url, headers=build_hf_headers(token=self.token)) - return response.status_code == 200 - - @property - def chat(self) -> "ProxyClientChat": - return ProxyClientChat(self) - - -class _ProxyClient: - """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" - - def __init__(self, client: InferenceClient): - self._client = client - - -class ProxyClientChat(_ProxyClient): - """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" - - @property - def completions(self) -> "ProxyClientChatCompletions": - return ProxyClientChatCompletions(self._client) - - -class ProxyClientChatCompletions(_ProxyClient): - """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" - - @property - def create(self): - return self._client.chat_completion diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_common.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_common.py deleted file mode 100644 index c7803d14eee9161739f25f9fb5914a35469be0ff..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_common.py +++ /dev/null @@ -1,459 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains utilities used by both the sync and async inference clients.""" - -import base64 -import io -import json -import logging -import mimetypes -from dataclasses import dataclass -from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - BinaryIO, - Dict, - Iterable, - List, - Literal, - NoReturn, - Optional, - Union, - overload, -) - -from requests import HTTPError - -from huggingface_hub.errors import ( - GenerationError, - IncompleteGenerationError, - OverloadedError, - TextGenerationError, - UnknownError, - ValidationError, -) - -from ..utils import get_session, is_aiohttp_available, is_numpy_available, is_pillow_available -from ._generated.types import ChatCompletionStreamOutput, TextGenerationStreamOutput - - -if TYPE_CHECKING: - from aiohttp import ClientResponse, ClientSession - from PIL.Image import Image - -# TYPES -UrlT = str -PathT = Union[str, Path] -ContentT = Union[bytes, BinaryIO, PathT, UrlT, "Image", bytearray, memoryview] - -# Use to set a Accept: image/png header -TASKS_EXPECTING_IMAGES = {"text-to-image", "image-to-image"} - -logger = logging.getLogger(__name__) - - -@dataclass -class RequestParameters: - url: str - task: str - model: Optional[str] - json: Optional[Union[str, Dict, List]] - data: Optional[bytes] - headers: Dict[str, Any] - - -class MimeBytes(bytes): - """ - A bytes object with a mime type. - To be returned by `_prepare_payload_open_as_mime_bytes` in subclasses. - - Example: - ```python - >>> b = MimeBytes(b"hello", "text/plain") - >>> isinstance(b, bytes) - True - >>> b.mime_type - 'text/plain' - ``` - """ - - mime_type: Optional[str] - - def __new__(cls, data: bytes, mime_type: Optional[str] = None): - obj = super().__new__(cls, data) - obj.mime_type = mime_type - if isinstance(data, MimeBytes) and mime_type is None: - obj.mime_type = data.mime_type - return obj - - -## IMPORT UTILS - - -def _import_aiohttp(): - # Make sure `aiohttp` is installed on the machine. - if not is_aiohttp_available(): - raise ImportError("Please install aiohttp to use `AsyncInferenceClient` (`pip install aiohttp`).") - import aiohttp - - return aiohttp - - -def _import_numpy(): - """Make sure `numpy` is installed on the machine.""" - if not is_numpy_available(): - raise ImportError("Please install numpy to use deal with embeddings (`pip install numpy`).") - import numpy - - return numpy - - -def _import_pil_image(): - """Make sure `PIL` is installed on the machine.""" - if not is_pillow_available(): - raise ImportError( - "Please install Pillow to use deal with images (`pip install Pillow`). If you don't want the image to be" - " post-processed, use `client.post(...)` and get the raw response from the server." - ) - from PIL import Image - - return Image - - -## ENCODING / DECODING UTILS - - -@overload -def _open_as_mime_bytes(content: ContentT) -> MimeBytes: ... # means "if input is not None, output is not None" - - -@overload -def _open_as_mime_bytes(content: Literal[None]) -> Literal[None]: ... # means "if input is None, output is None" - - -def _open_as_mime_bytes(content: Optional[ContentT]) -> Optional[MimeBytes]: - """Open `content` as a binary file, either from a URL, a local path, raw bytes, or a PIL Image. - - Do nothing if `content` is None. - """ - # If content is None, yield None - if content is None: - return None - - # If content is bytes, return it - if isinstance(content, bytes): - return MimeBytes(content) - - # If content is raw binary data (bytearray, memoryview) - if isinstance(content, (bytearray, memoryview)): - return MimeBytes(bytes(content)) - - # If content is a binary file-like object - if hasattr(content, "read"): # duck-typing instead of isinstance(content, BinaryIO) - logger.debug("Reading content from BinaryIO") - data = content.read() - mime_type = mimetypes.guess_type(content.name)[0] if hasattr(content, "name") else None - if isinstance(data, str): - raise TypeError("Expected binary stream (bytes), but got text stream") - return MimeBytes(data, mime_type=mime_type) - - # If content is a string => must be either a URL or a path - if isinstance(content, str): - if content.startswith("https://") or content.startswith("http://"): - logger.debug(f"Downloading content from {content}") - response = get_session().get(content) - mime_type = response.headers.get("Content-Type") - if mime_type is None: - mime_type = mimetypes.guess_type(content)[0] - return MimeBytes(response.content, mime_type=mime_type) - - content = Path(content) - if not content.exists(): - raise FileNotFoundError( - f"File not found at {content}. If `data` is a string, it must either be a URL or a path to a local" - " file. To pass raw content, please encode it as bytes first." - ) - - # If content is a Path => open it - if isinstance(content, Path): - logger.debug(f"Opening content from {content}") - return MimeBytes(content.read_bytes(), mime_type=mimetypes.guess_type(content)[0]) - - # If content is a PIL Image => convert to bytes - if is_pillow_available(): - from PIL import Image - - if isinstance(content, Image.Image): - logger.debug("Converting PIL Image to bytes") - buffer = io.BytesIO() - format = content.format or "PNG" - content.save(buffer, format=format) - return MimeBytes(buffer.getvalue(), mime_type=f"image/{format.lower()}") - - # If nothing matched, raise error - raise TypeError( - f"Unsupported content type: {type(content)}. " - "Expected one of: bytes, bytearray, BinaryIO, memoryview, Path, str (URL or file path), or PIL.Image.Image." - ) - - -def _b64_encode(content: ContentT) -> str: - """Encode a raw file (image, audio) into base64. Can be bytes, an opened file, a path or a URL.""" - raw_bytes = _open_as_mime_bytes(content) - return base64.b64encode(raw_bytes).decode() - - -def _as_url(content: ContentT, default_mime_type: str) -> str: - if isinstance(content, str) and content.startswith(("http://", "https://", "data:")): - return content - - # Convert content to bytes - raw_bytes = _open_as_mime_bytes(content) - - # Get MIME type - mime_type = raw_bytes.mime_type or default_mime_type - - # Encode content to base64 - encoded_data = base64.b64encode(raw_bytes).decode() - - # Build data URL - return f"data:{mime_type};base64,{encoded_data}" - - -def _b64_to_image(encoded_image: str) -> "Image": - """Parse a base64-encoded string into a PIL Image.""" - Image = _import_pil_image() - return Image.open(io.BytesIO(base64.b64decode(encoded_image))) - - -def _bytes_to_list(content: bytes) -> List: - """Parse bytes from a Response object into a Python list. - - Expects the response body to be JSON-encoded data. - - NOTE: This is exactly the same implementation as `_bytes_to_dict` and will not complain if the returned data is a - dictionary. The only advantage of having both is to help the user (and mypy) understand what kind of data to expect. - """ - return json.loads(content.decode()) - - -def _bytes_to_dict(content: bytes) -> Dict: - """Parse bytes from a Response object into a Python dictionary. - - Expects the response body to be JSON-encoded data. - - NOTE: This is exactly the same implementation as `_bytes_to_list` and will not complain if the returned data is a - list. The only advantage of having both is to help the user (and mypy) understand what kind of data to expect. - """ - return json.loads(content.decode()) - - -def _bytes_to_image(content: bytes) -> "Image": - """Parse bytes from a Response object into a PIL Image. - - Expects the response body to be raw bytes. To deal with b64 encoded images, use `_b64_to_image` instead. - """ - Image = _import_pil_image() - return Image.open(io.BytesIO(content)) - - -def _as_dict(response: Union[bytes, Dict]) -> Dict: - return json.loads(response) if isinstance(response, bytes) else response - - -## STREAMING UTILS - - -def _stream_text_generation_response( - bytes_output_as_lines: Iterable[bytes], details: bool -) -> Union[Iterable[str], Iterable[TextGenerationStreamOutput]]: - """Used in `InferenceClient.text_generation`.""" - # Parse ServerSentEvents - for byte_payload in bytes_output_as_lines: - try: - output = _format_text_generation_stream_output(byte_payload, details) - except StopIteration: - break - if output is not None: - yield output - - -async def _async_stream_text_generation_response( - bytes_output_as_lines: AsyncIterable[bytes], details: bool -) -> Union[AsyncIterable[str], AsyncIterable[TextGenerationStreamOutput]]: - """Used in `AsyncInferenceClient.text_generation`.""" - # Parse ServerSentEvents - async for byte_payload in bytes_output_as_lines: - try: - output = _format_text_generation_stream_output(byte_payload, details) - except StopIteration: - break - if output is not None: - yield output - - -def _format_text_generation_stream_output( - byte_payload: bytes, details: bool -) -> Optional[Union[str, TextGenerationStreamOutput]]: - if not byte_payload.startswith(b"data:"): - return None # empty line - - if byte_payload.strip() == b"data: [DONE]": - raise StopIteration("[DONE] signal received.") - - # Decode payload - payload = byte_payload.decode("utf-8") - json_payload = json.loads(payload.lstrip("data:").rstrip("/n")) - - # Either an error as being returned - if json_payload.get("error") is not None: - raise _parse_text_generation_error(json_payload["error"], json_payload.get("error_type")) - - # Or parse token payload - output = TextGenerationStreamOutput.parse_obj_as_instance(json_payload) - return output.token.text if not details else output - - -def _stream_chat_completion_response( - bytes_lines: Iterable[bytes], -) -> Iterable[ChatCompletionStreamOutput]: - """Used in `InferenceClient.chat_completion` if model is served with TGI.""" - for item in bytes_lines: - try: - output = _format_chat_completion_stream_output(item) - except StopIteration: - break - if output is not None: - yield output - - -async def _async_stream_chat_completion_response( - bytes_lines: AsyncIterable[bytes], -) -> AsyncIterable[ChatCompletionStreamOutput]: - """Used in `AsyncInferenceClient.chat_completion`.""" - async for item in bytes_lines: - try: - output = _format_chat_completion_stream_output(item) - except StopIteration: - break - if output is not None: - yield output - - -def _format_chat_completion_stream_output( - byte_payload: bytes, -) -> Optional[ChatCompletionStreamOutput]: - if not byte_payload.startswith(b"data:"): - return None # empty line - - if byte_payload.strip() == b"data: [DONE]": - raise StopIteration("[DONE] signal received.") - - # Decode payload - payload = byte_payload.decode("utf-8") - json_payload = json.loads(payload.lstrip("data:").rstrip("/n")) - - # Either an error as being returned - if json_payload.get("error") is not None: - raise _parse_text_generation_error(json_payload["error"], json_payload.get("error_type")) - - # Or parse token payload - return ChatCompletionStreamOutput.parse_obj_as_instance(json_payload) - - -async def _async_yield_from(client: "ClientSession", response: "ClientResponse") -> AsyncIterable[bytes]: - try: - async for byte_payload in response.content: - yield byte_payload.strip() - finally: - # Always close the underlying HTTP session to avoid resource leaks - await client.close() - - -# "TGI servers" are servers running with the `text-generation-inference` backend. -# This backend is the go-to solution to run large language models at scale. However, -# for some smaller models (e.g. "gpt2") the default `transformers` + `api-inference` -# solution is still in use. -# -# Both approaches have very similar APIs, but not exactly the same. What we do first in -# the `text_generation` method is to assume the model is served via TGI. If we realize -# it's not the case (i.e. we receive an HTTP 400 Bad Request), we fallback to the -# default API with a warning message. When that's the case, We remember the unsupported -# attributes for this model in the `_UNSUPPORTED_TEXT_GENERATION_KWARGS` global variable. -# -# In addition, TGI servers have a built-in API route for chat-completion, which is not -# available on the default API. We use this route to provide a more consistent behavior -# when available. -# -# For more details, see https://github.com/huggingface/text-generation-inference and -# https://huggingface.co/docs/api-inference/detailed_parameters#text-generation-task. - -_UNSUPPORTED_TEXT_GENERATION_KWARGS: Dict[Optional[str], List[str]] = {} - - -def _set_unsupported_text_generation_kwargs(model: Optional[str], unsupported_kwargs: List[str]) -> None: - _UNSUPPORTED_TEXT_GENERATION_KWARGS.setdefault(model, []).extend(unsupported_kwargs) - - -def _get_unsupported_text_generation_kwargs(model: Optional[str]) -> List[str]: - return _UNSUPPORTED_TEXT_GENERATION_KWARGS.get(model, []) - - -# TEXT GENERATION ERRORS -# ---------------------- -# Text-generation errors are parsed separately to handle as much as possible the errors returned by the text generation -# inference project (https://github.com/huggingface/text-generation-inference). -# ---------------------- - - -def raise_text_generation_error(http_error: HTTPError) -> NoReturn: - """ - Try to parse text-generation-inference error message and raise HTTPError in any case. - - Args: - error (`HTTPError`): - The HTTPError that have been raised. - """ - # Try to parse a Text Generation Inference error - - try: - # Hacky way to retrieve payload in case of aiohttp error - payload = getattr(http_error, "response_error_payload", None) or http_error.response.json() - error = payload.get("error") - error_type = payload.get("error_type") - except Exception: # no payload - raise http_error - - # If error_type => more information than `hf_raise_for_status` - if error_type is not None: - exception = _parse_text_generation_error(error, error_type) - raise exception from http_error - - # Otherwise, fallback to default error - raise http_error - - -def _parse_text_generation_error(error: Optional[str], error_type: Optional[str]) -> TextGenerationError: - if error_type == "generation": - return GenerationError(error) # type: ignore - if error_type == "incomplete_generation": - return IncompleteGenerationError(error) # type: ignore - if error_type == "overloaded": - return OverloadedError(error) # type: ignore - if error_type == "validation": - return ValidationError(error) # type: ignore - return UnknownError(error) # type: ignore diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/__init__.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/_async_client.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/_async_client.py deleted file mode 100644 index 45285d8390cb0d8ab1a3b9cc6a0ce0d01f95b6c8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/_async_client.py +++ /dev/null @@ -1,3478 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# WARNING -# This entire file has been adapted from the sync-client code in `src/huggingface_hub/inference/_client.py`. -# Any change in InferenceClient will be automatically reflected in AsyncInferenceClient. -# To re-generate the code, run `make style` or `python ./utils/generate_async_inference_client.py --update`. -# WARNING -import asyncio -import base64 -import logging -import re -import warnings -from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Literal, Optional, Set, Union, overload - -from huggingface_hub import constants -from huggingface_hub.errors import InferenceTimeoutError -from huggingface_hub.inference._common import ( - TASKS_EXPECTING_IMAGES, - ContentT, - RequestParameters, - _async_stream_chat_completion_response, - _async_stream_text_generation_response, - _b64_encode, - _b64_to_image, - _bytes_to_dict, - _bytes_to_image, - _bytes_to_list, - _get_unsupported_text_generation_kwargs, - _import_numpy, - _set_unsupported_text_generation_kwargs, - raise_text_generation_error, -) -from huggingface_hub.inference._generated.types import ( - AudioClassificationOutputElement, - AudioClassificationOutputTransform, - AudioToAudioOutputElement, - AutomaticSpeechRecognitionOutput, - ChatCompletionInputGrammarType, - ChatCompletionInputMessage, - ChatCompletionInputStreamOptions, - ChatCompletionInputTool, - ChatCompletionInputToolChoiceClass, - ChatCompletionInputToolChoiceEnum, - ChatCompletionOutput, - ChatCompletionStreamOutput, - DocumentQuestionAnsweringOutputElement, - FillMaskOutputElement, - ImageClassificationOutputElement, - ImageClassificationOutputTransform, - ImageSegmentationOutputElement, - ImageSegmentationSubtask, - ImageToImageTargetSize, - ImageToTextOutput, - ImageToVideoTargetSize, - ObjectDetectionOutputElement, - Padding, - QuestionAnsweringOutputElement, - SummarizationOutput, - SummarizationTruncationStrategy, - TableQuestionAnsweringOutputElement, - TextClassificationOutputElement, - TextClassificationOutputTransform, - TextGenerationInputGrammarType, - TextGenerationOutput, - TextGenerationStreamOutput, - TextToSpeechEarlyStoppingEnum, - TokenClassificationAggregationStrategy, - TokenClassificationOutputElement, - TranslationOutput, - TranslationTruncationStrategy, - VisualQuestionAnsweringOutputElement, - ZeroShotClassificationOutputElement, - ZeroShotImageClassificationOutputElement, -) -from huggingface_hub.inference._providers import PROVIDER_OR_POLICY_T, get_provider_helper -from huggingface_hub.utils import build_hf_headers -from huggingface_hub.utils._auth import get_token - -from .._common import _async_yield_from, _import_aiohttp - - -if TYPE_CHECKING: - import numpy as np - from aiohttp import ClientResponse, ClientSession - from PIL.Image import Image - -logger = logging.getLogger(__name__) - - -MODEL_KWARGS_NOT_USED_REGEX = re.compile(r"The following `model_kwargs` are not used by the model: \[(.*?)\]") - - -class AsyncInferenceClient: - """ - Initialize a new Inference Client. - - [`InferenceClient`] aims to provide a unified experience to perform inference. The client can be used - seamlessly with either the (free) Inference API, self-hosted Inference Endpoints, or third-party Inference Providers. - - Args: - model (`str`, `optional`): - The model to run inference with. Can be a model id hosted on the Hugging Face Hub, e.g. `meta-llama/Meta-Llama-3-8B-Instruct` - or a URL to a deployed Inference Endpoint. Defaults to None, in which case a recommended model is - automatically selected for the task. - Note: for better compatibility with OpenAI's client, `model` has been aliased as `base_url`. Those 2 - arguments are mutually exclusive. If a URL is passed as `model` or `base_url` for chat completion, the `(/v1)/chat/completions` suffix path will be appended to the URL. - provider (`str`, *optional*): - Name of the provider to use for inference. Can be `"black-forest-labs"`, `"cerebras"`, `"clarifai"`, `"cohere"`, `"fal-ai"`, `"featherless-ai"`, `"fireworks-ai"`, `"groq"`, `"hf-inference"`, `"hyperbolic"`, `"nebius"`, `"novita"`, `"nscale"`, `"openai"`, `publicai`, `"replicate"`, `"sambanova"`, `"scaleway"`, `"together"` or `"zai-org"`. - Defaults to "auto" i.e. the first of the providers available for the model, sorted by the user's order in https://hf.co/settings/inference-providers. - If model is a URL or `base_url` is passed, then `provider` is not used. - token (`str`, *optional*): - Hugging Face token. Will default to the locally saved token if not provided. - Note: for better compatibility with OpenAI's client, `token` has been aliased as `api_key`. Those 2 - arguments are mutually exclusive and have the exact same behavior. - timeout (`float`, `optional`): - The maximum number of seconds to wait for a response from the server. Defaults to None, meaning it will loop until the server is available. - headers (`Dict[str, str]`, `optional`): - Additional headers to send to the server. By default only the authorization and user-agent headers are sent. - Values in this dictionary will override the default values. - bill_to (`str`, `optional`): - The billing account to use for the requests. By default the requests are billed on the user's account. - Requests can only be billed to an organization the user is a member of, and which has subscribed to Enterprise Hub. - cookies (`Dict[str, str]`, `optional`): - Additional cookies to send to the server. - trust_env ('bool', 'optional'): - Trust environment settings for proxy configuration if the parameter is `True` (`False` by default). - proxies (`Any`, `optional`): - Proxies to use for the request. - base_url (`str`, `optional`): - Base URL to run inference. This is a duplicated argument from `model` to make [`InferenceClient`] - follow the same pattern as `openai.OpenAI` client. Cannot be used if `model` is set. Defaults to None. - api_key (`str`, `optional`): - Token to use for authentication. This is a duplicated argument from `token` to make [`InferenceClient`] - follow the same pattern as `openai.OpenAI` client. Cannot be used if `token` is set. Defaults to None. - """ - - def __init__( - self, - model: Optional[str] = None, - *, - provider: Optional[PROVIDER_OR_POLICY_T] = None, - token: Optional[str] = None, - timeout: Optional[float] = None, - headers: Optional[Dict[str, str]] = None, - cookies: Optional[Dict[str, str]] = None, - trust_env: bool = False, - proxies: Optional[Any] = None, - bill_to: Optional[str] = None, - # OpenAI compatibility - base_url: Optional[str] = None, - api_key: Optional[str] = None, - ) -> None: - if model is not None and base_url is not None: - raise ValueError( - "Received both `model` and `base_url` arguments. Please provide only one of them." - " `base_url` is an alias for `model` to make the API compatible with OpenAI's client." - " If using `base_url` for chat completion, the `/chat/completions` suffix path will be appended to the base url." - " When passing a URL as `model`, the client will not append any suffix path to it." - ) - if token is not None and api_key is not None: - raise ValueError( - "Received both `token` and `api_key` arguments. Please provide only one of them." - " `api_key` is an alias for `token` to make the API compatible with OpenAI's client." - " It has the exact same behavior as `token`." - ) - token = token if token is not None else api_key - if isinstance(token, bool): - # Legacy behavior: previously is was possible to pass `token=False` to disable authentication. This is not - # supported anymore as authentication is required. Better to explicitly raise here rather than risking - # sending the locally saved token without the user knowing about it. - if token is False: - raise ValueError( - "Cannot use `token=False` to disable authentication as authentication is required to run Inference." - ) - warnings.warn( - "Using `token=True` to automatically use the locally saved token is deprecated and will be removed in a future release. " - "Please use `token=None` instead (default).", - DeprecationWarning, - ) - token = get_token() - - self.model: Optional[str] = base_url or model - self.token: Optional[str] = token - - self.headers = {**headers} if headers is not None else {} - if bill_to is not None: - if ( - constants.HUGGINGFACE_HEADER_X_BILL_TO in self.headers - and self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO] != bill_to - ): - warnings.warn( - f"Overriding existing '{self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO]}' value in headers with '{bill_to}'.", - UserWarning, - ) - self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO] = bill_to - - if token is not None and not token.startswith("hf_"): - warnings.warn( - "You've provided an external provider's API key, so requests will be billed directly by the provider. " - "The `bill_to` parameter is only applicable for Hugging Face billing and will be ignored.", - UserWarning, - ) - - # Configure provider - self.provider = provider - - self.cookies = cookies - self.timeout = timeout - self.trust_env = trust_env - self.proxies = proxies - - # Keep track of the sessions to close them properly - self._sessions: Dict["ClientSession", Set["ClientResponse"]] = dict() - - def __repr__(self): - return f"" - - @overload - async def _inner_post( # type: ignore[misc] - self, request_parameters: RequestParameters, *, stream: Literal[False] = ... - ) -> bytes: ... - - @overload - async def _inner_post( # type: ignore[misc] - self, request_parameters: RequestParameters, *, stream: Literal[True] = ... - ) -> AsyncIterable[bytes]: ... - - @overload - async def _inner_post( - self, request_parameters: RequestParameters, *, stream: bool = False - ) -> Union[bytes, AsyncIterable[bytes]]: ... - - async def _inner_post( - self, request_parameters: RequestParameters, *, stream: bool = False - ) -> Union[bytes, AsyncIterable[bytes]]: - """Make a request to the inference server.""" - - aiohttp = _import_aiohttp() - - # TODO: this should be handled in provider helpers directly - if request_parameters.task in TASKS_EXPECTING_IMAGES and "Accept" not in request_parameters.headers: - request_parameters.headers["Accept"] = "image/png" - - # Do not use context manager as we don't want to close the connection immediately when returning - # a stream - session = self._get_client_session(headers=request_parameters.headers) - - try: - response = await session.post( - request_parameters.url, json=request_parameters.json, data=request_parameters.data, proxy=self.proxies - ) - response_error_payload = None - if response.status != 200: - try: - response_error_payload = await response.json() # get payload before connection closed - except Exception: - pass - response.raise_for_status() - if stream: - return _async_yield_from(session, response) - else: - content = await response.read() - await session.close() - return content - except asyncio.TimeoutError as error: - await session.close() - # Convert any `TimeoutError` to a `InferenceTimeoutError` - raise InferenceTimeoutError(f"Inference call timed out: {request_parameters.url}") from error # type: ignore - except aiohttp.ClientResponseError as error: - error.response_error_payload = response_error_payload - await session.close() - raise error - except Exception: - await session.close() - raise - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc_value, traceback): - await self.close() - - def __del__(self): - if len(self._sessions) > 0: - warnings.warn( - "Deleting 'AsyncInferenceClient' client but some sessions are still open. " - "This can happen if you've stopped streaming data from the server before the stream was complete. " - "To close the client properly, you must call `await client.close()` " - "or use an async context (e.g. `async with AsyncInferenceClient(): ...`." - ) - - async def close(self): - """Close all open sessions. - - By default, 'aiohttp.ClientSession' objects are closed automatically when a call is completed. However, if you - are streaming data from the server and you stop before the stream is complete, you must call this method to - close the session properly. - - Another possibility is to use an async context (e.g. `async with AsyncInferenceClient(): ...`). - """ - await asyncio.gather(*[session.close() for session in self._sessions.keys()]) - - async def audio_classification( - self, - audio: ContentT, - *, - model: Optional[str] = None, - top_k: Optional[int] = None, - function_to_apply: Optional["AudioClassificationOutputTransform"] = None, - ) -> List[AudioClassificationOutputElement]: - """ - Perform audio classification on the provided audio content. - - Args: - audio (Union[str, Path, bytes, BinaryIO]): - The audio content to classify. It can be raw audio bytes, a local audio file, or a URL pointing to an - audio file. - model (`str`, *optional*): - The model to use for audio classification. Can be a model ID hosted on the Hugging Face Hub - or a URL to a deployed Inference Endpoint. If not provided, the default recommended model for - audio classification will be used. - top_k (`int`, *optional*): - When specified, limits the output to the top K most probable classes. - function_to_apply (`"AudioClassificationOutputTransform"`, *optional*): - The function to apply to the model outputs in order to retrieve the scores. - - Returns: - `List[AudioClassificationOutputElement]`: List of [`AudioClassificationOutputElement`] items containing the predicted labels and their confidence. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.audio_classification("audio.flac") - [ - AudioClassificationOutputElement(score=0.4976358711719513, label='hap'), - AudioClassificationOutputElement(score=0.3677836060523987, label='neu'), - ... - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="audio-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=audio, - parameters={"function_to_apply": function_to_apply, "top_k": top_k}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return AudioClassificationOutputElement.parse_obj_as_list(response) - - async def audio_to_audio( - self, - audio: ContentT, - *, - model: Optional[str] = None, - ) -> List[AudioToAudioOutputElement]: - """ - Performs multiple tasks related to audio-to-audio depending on the model (eg: speech enhancement, source separation). - - Args: - audio (Union[str, Path, bytes, BinaryIO]): - The audio content for the model. It can be raw audio bytes, a local audio file, or a URL pointing to an - audio file. - model (`str`, *optional*): - The model can be any model which takes an audio file and returns another audio file. Can be a model ID hosted on the Hugging Face Hub - or a URL to a deployed Inference Endpoint. If not provided, the default recommended model for - audio_to_audio will be used. - - Returns: - `List[AudioToAudioOutputElement]`: A list of [`AudioToAudioOutputElement`] items containing audios label, content-type, and audio content in blob. - - Raises: - `InferenceTimeoutError`: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> audio_output = await client.audio_to_audio("audio.flac") - >>> async for i, item in enumerate(audio_output): - >>> with open(f"output_{i}.flac", "wb") as f: - f.write(item.blob) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="audio-to-audio", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=audio, - parameters={}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - audio_output = AudioToAudioOutputElement.parse_obj_as_list(response) - for item in audio_output: - item.blob = base64.b64decode(item.blob) - return audio_output - - async def automatic_speech_recognition( - self, - audio: ContentT, - *, - model: Optional[str] = None, - extra_body: Optional[Dict] = None, - ) -> AutomaticSpeechRecognitionOutput: - """ - Perform automatic speech recognition (ASR or audio-to-text) on the given audio content. - - Args: - audio (Union[str, Path, bytes, BinaryIO]): - The content to transcribe. It can be raw audio bytes, local audio file, or a URL to an audio file. - model (`str`, *optional*): - The model to use for ASR. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended model for ASR will be used. - extra_body (`Dict`, *optional*): - Additional provider-specific parameters to pass to the model. Refer to the provider's documentation - for supported parameters. - Returns: - [`AutomaticSpeechRecognitionOutput`]: An item containing the transcribed text and optionally the timestamp chunks. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.automatic_speech_recognition("hello_world.flac").text - "hello world" - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="automatic-speech-recognition", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=audio, - parameters={**(extra_body or {})}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return AutomaticSpeechRecognitionOutput.parse_obj_as_instance(response) - - @overload - async def chat_completion( # type: ignore - self, - messages: List[Union[Dict, ChatCompletionInputMessage]], - *, - model: Optional[str] = None, - stream: Literal[False] = False, - frequency_penalty: Optional[float] = None, - logit_bias: Optional[List[float]] = None, - logprobs: Optional[bool] = None, - max_tokens: Optional[int] = None, - n: Optional[int] = None, - presence_penalty: Optional[float] = None, - response_format: Optional[ChatCompletionInputGrammarType] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stream_options: Optional[ChatCompletionInputStreamOptions] = None, - temperature: Optional[float] = None, - tool_choice: Optional[Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"]] = None, - tool_prompt: Optional[str] = None, - tools: Optional[List[ChatCompletionInputTool]] = None, - top_logprobs: Optional[int] = None, - top_p: Optional[float] = None, - extra_body: Optional[Dict] = None, - ) -> ChatCompletionOutput: ... - - @overload - async def chat_completion( # type: ignore - self, - messages: List[Union[Dict, ChatCompletionInputMessage]], - *, - model: Optional[str] = None, - stream: Literal[True] = True, - frequency_penalty: Optional[float] = None, - logit_bias: Optional[List[float]] = None, - logprobs: Optional[bool] = None, - max_tokens: Optional[int] = None, - n: Optional[int] = None, - presence_penalty: Optional[float] = None, - response_format: Optional[ChatCompletionInputGrammarType] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stream_options: Optional[ChatCompletionInputStreamOptions] = None, - temperature: Optional[float] = None, - tool_choice: Optional[Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"]] = None, - tool_prompt: Optional[str] = None, - tools: Optional[List[ChatCompletionInputTool]] = None, - top_logprobs: Optional[int] = None, - top_p: Optional[float] = None, - extra_body: Optional[Dict] = None, - ) -> AsyncIterable[ChatCompletionStreamOutput]: ... - - @overload - async def chat_completion( - self, - messages: List[Union[Dict, ChatCompletionInputMessage]], - *, - model: Optional[str] = None, - stream: bool = False, - frequency_penalty: Optional[float] = None, - logit_bias: Optional[List[float]] = None, - logprobs: Optional[bool] = None, - max_tokens: Optional[int] = None, - n: Optional[int] = None, - presence_penalty: Optional[float] = None, - response_format: Optional[ChatCompletionInputGrammarType] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stream_options: Optional[ChatCompletionInputStreamOptions] = None, - temperature: Optional[float] = None, - tool_choice: Optional[Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"]] = None, - tool_prompt: Optional[str] = None, - tools: Optional[List[ChatCompletionInputTool]] = None, - top_logprobs: Optional[int] = None, - top_p: Optional[float] = None, - extra_body: Optional[Dict] = None, - ) -> Union[ChatCompletionOutput, AsyncIterable[ChatCompletionStreamOutput]]: ... - - async def chat_completion( - self, - messages: List[Union[Dict, ChatCompletionInputMessage]], - *, - model: Optional[str] = None, - stream: bool = False, - # Parameters from ChatCompletionInput (handled manually) - frequency_penalty: Optional[float] = None, - logit_bias: Optional[List[float]] = None, - logprobs: Optional[bool] = None, - max_tokens: Optional[int] = None, - n: Optional[int] = None, - presence_penalty: Optional[float] = None, - response_format: Optional[ChatCompletionInputGrammarType] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stream_options: Optional[ChatCompletionInputStreamOptions] = None, - temperature: Optional[float] = None, - tool_choice: Optional[Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"]] = None, - tool_prompt: Optional[str] = None, - tools: Optional[List[ChatCompletionInputTool]] = None, - top_logprobs: Optional[int] = None, - top_p: Optional[float] = None, - extra_body: Optional[Dict] = None, - ) -> Union[ChatCompletionOutput, AsyncIterable[ChatCompletionStreamOutput]]: - """ - A method for completing conversations using a specified language model. - - > [!TIP] - > The `client.chat_completion` method is aliased as `client.chat.completions.create` for compatibility with OpenAI's client. - > Inputs and outputs are strictly the same and using either syntax will yield the same results. - > Check out the [Inference guide](https://huggingface.co/docs/huggingface_hub/guides/inference#openai-compatibility) - > for more details about OpenAI's compatibility. - - > [!TIP] - > You can pass provider-specific parameters to the model by using the `extra_body` argument. - - Args: - messages (List of [`ChatCompletionInputMessage`]): - Conversation history consisting of roles and content pairs. - model (`str`, *optional*): - The model to use for chat-completion. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended model for chat-based text-generation will be used. - See https://huggingface.co/tasks/text-generation for more details. - If `model` is a model ID, it is passed to the server as the `model` parameter. If you want to define a - custom URL while setting `model` in the request payload, you must set `base_url` when initializing [`InferenceClient`]. - frequency_penalty (`float`, *optional*): - Penalizes new tokens based on their existing frequency - in the text so far. Range: [-2.0, 2.0]. Defaults to 0.0. - logit_bias (`List[float]`, *optional*): - Adjusts the likelihood of specific tokens appearing in the generated output. - logprobs (`bool`, *optional*): - Whether to return log probabilities of the output tokens or not. If true, returns the log - probabilities of each output token returned in the content of message. - max_tokens (`int`, *optional*): - Maximum number of tokens allowed in the response. Defaults to 100. - n (`int`, *optional*): - The number of completions to generate for each prompt. - presence_penalty (`float`, *optional*): - Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the - text so far, increasing the model's likelihood to talk about new topics. - response_format ([`ChatCompletionInputGrammarType`], *optional*): - Grammar constraints. Can be either a JSONSchema or a regex. - seed (Optional[`int`], *optional*): - Seed for reproducible control flow. Defaults to None. - stop (`List[str]`, *optional*): - Up to four strings which trigger the end of the response. - Defaults to None. - stream (`bool`, *optional*): - Enable realtime streaming of responses. Defaults to False. - stream_options ([`ChatCompletionInputStreamOptions`], *optional*): - Options for streaming completions. - temperature (`float`, *optional*): - Controls randomness of the generations. Lower values ensure - less random completions. Range: [0, 2]. Defaults to 1.0. - top_logprobs (`int`, *optional*): - An integer between 0 and 5 specifying the number of most likely tokens to return at each token - position, each with an associated log probability. logprobs must be set to true if this parameter is - used. - top_p (`float`, *optional*): - Fraction of the most likely next words to sample from. - Must be between 0 and 1. Defaults to 1.0. - tool_choice ([`ChatCompletionInputToolChoiceClass`] or [`ChatCompletionInputToolChoiceEnum`], *optional*): - The tool to use for the completion. Defaults to "auto". - tool_prompt (`str`, *optional*): - A prompt to be appended before the tools. - tools (List of [`ChatCompletionInputTool`], *optional*): - A list of tools the model may call. Currently, only functions are supported as a tool. Use this to - provide a list of functions the model may generate JSON inputs for. - extra_body (`Dict`, *optional*): - Additional provider-specific parameters to pass to the model. Refer to the provider's documentation - for supported parameters. - Returns: - [`ChatCompletionOutput`] or Iterable of [`ChatCompletionStreamOutput`]: - Generated text returned from the server: - - if `stream=False`, the generated text is returned as a [`ChatCompletionOutput`] (default). - - if `stream=True`, the generated text is returned token by token as a sequence of [`ChatCompletionStreamOutput`]. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> messages = [{"role": "user", "content": "What is the capital of France?"}] - >>> client = AsyncInferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") - >>> await client.chat_completion(messages, max_tokens=100) - ChatCompletionOutput( - choices=[ - ChatCompletionOutputComplete( - finish_reason='eos_token', - index=0, - message=ChatCompletionOutputMessage( - role='assistant', - content='The capital of France is Paris.', - name=None, - tool_calls=None - ), - logprobs=None - ) - ], - created=1719907176, - id='', - model='meta-llama/Meta-Llama-3-8B-Instruct', - object='text_completion', - system_fingerprint='2.0.4-sha-f426a33', - usage=ChatCompletionOutputUsage( - completion_tokens=8, - prompt_tokens=17, - total_tokens=25 - ) - ) - ``` - - Example using streaming: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> messages = [{"role": "user", "content": "What is the capital of France?"}] - >>> client = AsyncInferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") - >>> async for token in await client.chat_completion(messages, max_tokens=10, stream=True): - ... print(token) - ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content='The', role='assistant'), index=0, finish_reason=None)], created=1710498504) - ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content=' capital', role='assistant'), index=0, finish_reason=None)], created=1710498504) - (...) - ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content=' may', role='assistant'), index=0, finish_reason=None)], created=1710498504) - ``` - - Example using OpenAI's syntax: - ```py - # Must be run in an async context - # instead of `from openai import OpenAI` - from huggingface_hub import AsyncInferenceClient - - # instead of `client = OpenAI(...)` - client = AsyncInferenceClient( - base_url=..., - api_key=..., - ) - - output = await client.chat.completions.create( - model="meta-llama/Meta-Llama-3-8B-Instruct", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Count to 10"}, - ], - stream=True, - max_tokens=1024, - ) - - for chunk in output: - print(chunk.choices[0].delta.content) - ``` - - Example using a third-party provider directly with extra (provider-specific) parameters. Usage will be billed on your Together AI account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="together", # Use Together AI provider - ... api_key="", # Pass your Together API key directly - ... ) - >>> client.chat_completion( - ... model="meta-llama/Meta-Llama-3-8B-Instruct", - ... messages=[{"role": "user", "content": "What is the capital of France?"}], - ... extra_body={"safety_model": "Meta-Llama/Llama-Guard-7b"}, - ... ) - ``` - - Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="sambanova", # Use Sambanova provider - ... api_key="hf_...", # Pass your HF token - ... ) - >>> client.chat_completion( - ... model="meta-llama/Meta-Llama-3-8B-Instruct", - ... messages=[{"role": "user", "content": "What is the capital of France?"}], - ... ) - ``` - - Example using Image + Text as input: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - - # provide a remote URL - >>> image_url ="https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" - # or a base64-encoded image - >>> image_path = "/path/to/image.jpeg" - >>> with open(image_path, "rb") as f: - ... base64_image = base64.b64encode(f.read()).decode("utf-8") - >>> image_url = f"data:image/jpeg;base64,{base64_image}" - - >>> client = AsyncInferenceClient("meta-llama/Llama-3.2-11B-Vision-Instruct") - >>> output = await client.chat.completions.create( - ... messages=[ - ... { - ... "role": "user", - ... "content": [ - ... { - ... "type": "image_url", - ... "image_url": {"url": image_url}, - ... }, - ... { - ... "type": "text", - ... "text": "Describe this image in one sentence.", - ... }, - ... ], - ... }, - ... ], - ... ) - >>> output - The image depicts the iconic Statue of Liberty situated in New York Harbor, New York, on a clear day. - ``` - - Example using tools: - ```py - # Must be run in an async context - >>> client = AsyncInferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") - >>> messages = [ - ... { - ... "role": "system", - ... "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous.", - ... }, - ... { - ... "role": "user", - ... "content": "What's the weather like the next 3 days in San Francisco, CA?", - ... }, - ... ] - >>> tools = [ - ... { - ... "type": "function", - ... "function": { - ... "name": "get_current_weather", - ... "description": "Get the current weather", - ... "parameters": { - ... "type": "object", - ... "properties": { - ... "location": { - ... "type": "string", - ... "description": "The city and state, e.g. San Francisco, CA", - ... }, - ... "format": { - ... "type": "string", - ... "enum": ["celsius", "fahrenheit"], - ... "description": "The temperature unit to use. Infer this from the users location.", - ... }, - ... }, - ... "required": ["location", "format"], - ... }, - ... }, - ... }, - ... { - ... "type": "function", - ... "function": { - ... "name": "get_n_day_weather_forecast", - ... "description": "Get an N-day weather forecast", - ... "parameters": { - ... "type": "object", - ... "properties": { - ... "location": { - ... "type": "string", - ... "description": "The city and state, e.g. San Francisco, CA", - ... }, - ... "format": { - ... "type": "string", - ... "enum": ["celsius", "fahrenheit"], - ... "description": "The temperature unit to use. Infer this from the users location.", - ... }, - ... "num_days": { - ... "type": "integer", - ... "description": "The number of days to forecast", - ... }, - ... }, - ... "required": ["location", "format", "num_days"], - ... }, - ... }, - ... }, - ... ] - - >>> response = await client.chat_completion( - ... model="meta-llama/Meta-Llama-3-70B-Instruct", - ... messages=messages, - ... tools=tools, - ... tool_choice="auto", - ... max_tokens=500, - ... ) - >>> response.choices[0].message.tool_calls[0].function - ChatCompletionOutputFunctionDefinition( - arguments={ - 'location': 'San Francisco, CA', - 'format': 'fahrenheit', - 'num_days': 3 - }, - name='get_n_day_weather_forecast', - description=None - ) - ``` - - Example using response_format: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") - >>> messages = [ - ... { - ... "role": "user", - ... "content": "I saw a puppy a cat and a raccoon during my bike ride in the park. What did I saw and when?", - ... }, - ... ] - >>> response_format = { - ... "type": "json", - ... "value": { - ... "properties": { - ... "location": {"type": "string"}, - ... "activity": {"type": "string"}, - ... "animals_seen": {"type": "integer", "minimum": 1, "maximum": 5}, - ... "animals": {"type": "array", "items": {"type": "string"}}, - ... }, - ... "required": ["location", "activity", "animals_seen", "animals"], - ... }, - ... } - >>> response = await client.chat_completion( - ... messages=messages, - ... response_format=response_format, - ... max_tokens=500, - ... ) - >>> response.choices[0].message.content - '{\n\n"activity": "bike ride",\n"animals": ["puppy", "cat", "raccoon"],\n"animals_seen": 3,\n"location": "park"}' - ``` - """ - # Since `chat_completion(..., model=xxx)` is also a payload parameter for the server, we need to handle 'model' differently. - # `self.model` takes precedence over 'model' argument for building URL. - # `model` takes precedence for payload value. - model_id_or_url = self.model or model - payload_model = model or self.model - - # Get the provider helper - provider_helper = get_provider_helper( - self.provider, - task="conversational", - model=model_id_or_url - if model_id_or_url is not None and model_id_or_url.startswith(("http://", "https://")) - else payload_model, - ) - - # Prepare the payload - parameters = { - "model": payload_model, - "frequency_penalty": frequency_penalty, - "logit_bias": logit_bias, - "logprobs": logprobs, - "max_tokens": max_tokens, - "n": n, - "presence_penalty": presence_penalty, - "response_format": response_format, - "seed": seed, - "stop": stop, - "temperature": temperature, - "tool_choice": tool_choice, - "tool_prompt": tool_prompt, - "tools": tools, - "top_logprobs": top_logprobs, - "top_p": top_p, - "stream": stream, - "stream_options": stream_options, - **(extra_body or {}), - } - request_parameters = provider_helper.prepare_request( - inputs=messages, - parameters=parameters, - headers=self.headers, - model=model_id_or_url, - api_key=self.token, - ) - data = await self._inner_post(request_parameters, stream=stream) - - if stream: - return _async_stream_chat_completion_response(data) # type: ignore[arg-type] - - return ChatCompletionOutput.parse_obj_as_instance(data) # type: ignore[arg-type] - - async def document_question_answering( - self, - image: ContentT, - question: str, - *, - model: Optional[str] = None, - doc_stride: Optional[int] = None, - handle_impossible_answer: Optional[bool] = None, - lang: Optional[str] = None, - max_answer_len: Optional[int] = None, - max_question_len: Optional[int] = None, - max_seq_len: Optional[int] = None, - top_k: Optional[int] = None, - word_boxes: Optional[List[Union[List[float], str]]] = None, - ) -> List[DocumentQuestionAnsweringOutputElement]: - """ - Answer questions on document images. - - Args: - image (`Union[str, Path, bytes, BinaryIO]`): - The input image for the context. It can be raw bytes, an image file, or a URL to an online image. - question (`str`): - Question to be answered. - model (`str`, *optional*): - The model to use for the document question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended document question answering model will be used. - Defaults to None. - doc_stride (`int`, *optional*): - If the words in the document are too long to fit with the question for the model, it will be split in - several chunks with some overlap. This argument controls the size of that overlap. - handle_impossible_answer (`bool`, *optional*): - Whether to accept impossible as an answer - lang (`str`, *optional*): - Language to use while running OCR. Defaults to english. - max_answer_len (`int`, *optional*): - The maximum length of predicted answers (e.g., only answers with a shorter length are considered). - max_question_len (`int`, *optional*): - The maximum length of the question after tokenization. It will be truncated if needed. - max_seq_len (`int`, *optional*): - The maximum length of the total sentence (context + question) in tokens of each chunk passed to the - model. The context will be split in several chunks (using doc_stride as overlap) if needed. - top_k (`int`, *optional*): - The number of answers to return (will be chosen by order of likelihood). Can return less than top_k - answers if there are not enough options available within the context. - word_boxes (`List[Union[List[float], str`, *optional*): - A list of words and bounding boxes (normalized 0->1000). If provided, the inference will skip the OCR - step and use the provided bounding boxes instead. - Returns: - `List[DocumentQuestionAnsweringOutputElement]`: a list of [`DocumentQuestionAnsweringOutputElement`] items containing the predicted label, associated probability, word ids, and page number. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.document_question_answering(image="https://huggingface.co/spaces/impira/docquery/resolve/2359223c1837a7587402bda0f2643382a6eefeab/invoice.png", question="What is the invoice number?") - [DocumentQuestionAnsweringOutputElement(answer='us-001', end=16, score=0.9999666213989258, start=16)] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="document-question-answering", model=model_id) - inputs: Dict[str, Any] = {"question": question, "image": _b64_encode(image)} - request_parameters = provider_helper.prepare_request( - inputs=inputs, - parameters={ - "doc_stride": doc_stride, - "handle_impossible_answer": handle_impossible_answer, - "lang": lang, - "max_answer_len": max_answer_len, - "max_question_len": max_question_len, - "max_seq_len": max_seq_len, - "top_k": top_k, - "word_boxes": word_boxes, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return DocumentQuestionAnsweringOutputElement.parse_obj_as_list(response) - - async def feature_extraction( - self, - text: str, - *, - normalize: Optional[bool] = None, - prompt_name: Optional[str] = None, - truncate: Optional[bool] = None, - truncation_direction: Optional[Literal["Left", "Right"]] = None, - model: Optional[str] = None, - ) -> "np.ndarray": - """ - Generate embeddings for a given text. - - Args: - text (`str`): - The text to embed. - model (`str`, *optional*): - The model to use for the feature extraction task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended feature extraction model will be used. - Defaults to None. - normalize (`bool`, *optional*): - Whether to normalize the embeddings or not. - Only available on server powered by Text-Embedding-Inference. - prompt_name (`str`, *optional*): - The name of the prompt that should be used by for encoding. If not set, no prompt will be applied. - Must be a key in the `Sentence Transformers` configuration `prompts` dictionary. - For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ",...}, - then the sentence "What is the capital of France?" will be encoded as "query: What is the capital of France?" - because the prompt text will be prepended before any text to encode. - truncate (`bool`, *optional*): - Whether to truncate the embeddings or not. - Only available on server powered by Text-Embedding-Inference. - truncation_direction (`Literal["Left", "Right"]`, *optional*): - Which side of the input should be truncated when `truncate=True` is passed. - - Returns: - `np.ndarray`: The embedding representing the input text as a float32 numpy array. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.feature_extraction("Hi, who are you?") - array([[ 2.424802 , 2.93384 , 1.1750331 , ..., 1.240499, -0.13776633, -0.7889173 ], - [-0.42943227, -0.6364878 , -1.693462 , ..., 0.41978157, -2.4336355 , 0.6162071 ], - ..., - [ 0.28552425, -0.928395 , -1.2077185 , ..., 0.76810825, -2.1069427 , 0.6236161 ]], dtype=float32) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="feature-extraction", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "normalize": normalize, - "prompt_name": prompt_name, - "truncate": truncate, - "truncation_direction": truncation_direction, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - np = _import_numpy() - return np.array(provider_helper.get_response(response), dtype="float32") - - async def fill_mask( - self, - text: str, - *, - model: Optional[str] = None, - targets: Optional[List[str]] = None, - top_k: Optional[int] = None, - ) -> List[FillMaskOutputElement]: - """ - Fill in a hole with a missing word (token to be precise). - - Args: - text (`str`): - a string to be filled from, must contain the [MASK] token (check model card for exact name of the mask). - model (`str`, *optional*): - The model to use for the fill mask task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended fill mask model will be used. - targets (`List[str`, *optional*): - When passed, the model will limit the scores to the passed targets instead of looking up in the whole - vocabulary. If the provided targets are not in the model vocab, they will be tokenized and the first - resulting token will be used (with a warning, and that might be slower). - top_k (`int`, *optional*): - When passed, overrides the number of predictions to return. - Returns: - `List[FillMaskOutputElement]`: a list of [`FillMaskOutputElement`] items containing the predicted label, associated - probability, token reference, and completed text. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.fill_mask("The goal of life is .") - [ - FillMaskOutputElement(score=0.06897063553333282, token=11098, token_str=' happiness', sequence='The goal of life is happiness.'), - FillMaskOutputElement(score=0.06554922461509705, token=45075, token_str=' immortality', sequence='The goal of life is immortality.') - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="fill-mask", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={"targets": targets, "top_k": top_k}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return FillMaskOutputElement.parse_obj_as_list(response) - - async def image_classification( - self, - image: ContentT, - *, - model: Optional[str] = None, - function_to_apply: Optional["ImageClassificationOutputTransform"] = None, - top_k: Optional[int] = None, - ) -> List[ImageClassificationOutputElement]: - """ - Perform image classification on the given image using the specified model. - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The image to classify. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - model (`str`, *optional*): - The model to use for image classification. Can be a model ID hosted on the Hugging Face Hub or a URL to a - deployed Inference Endpoint. If not provided, the default recommended model for image classification will be used. - function_to_apply (`"ImageClassificationOutputTransform"`, *optional*): - The function to apply to the model outputs in order to retrieve the scores. - top_k (`int`, *optional*): - When specified, limits the output to the top K most probable classes. - Returns: - `List[ImageClassificationOutputElement]`: a list of [`ImageClassificationOutputElement`] items containing the predicted label and associated probability. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.image_classification("https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg") - [ImageClassificationOutputElement(label='Blenheim spaniel', score=0.9779096841812134), ...] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="image-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={"function_to_apply": function_to_apply, "top_k": top_k}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return ImageClassificationOutputElement.parse_obj_as_list(response) - - async def image_segmentation( - self, - image: ContentT, - *, - model: Optional[str] = None, - mask_threshold: Optional[float] = None, - overlap_mask_area_threshold: Optional[float] = None, - subtask: Optional["ImageSegmentationSubtask"] = None, - threshold: Optional[float] = None, - ) -> List[ImageSegmentationOutputElement]: - """ - Perform image segmentation on the given image using the specified model. - - > [!WARNING] - > You must have `PIL` installed if you want to work with images (`pip install Pillow`). - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The image to segment. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - model (`str`, *optional*): - The model to use for image segmentation. Can be a model ID hosted on the Hugging Face Hub or a URL to a - deployed Inference Endpoint. If not provided, the default recommended model for image segmentation will be used. - mask_threshold (`float`, *optional*): - Threshold to use when turning the predicted masks into binary values. - overlap_mask_area_threshold (`float`, *optional*): - Mask overlap threshold to eliminate small, disconnected segments. - subtask (`"ImageSegmentationSubtask"`, *optional*): - Segmentation task to be performed, depending on model capabilities. - threshold (`float`, *optional*): - Probability threshold to filter out predicted masks. - Returns: - `List[ImageSegmentationOutputElement]`: A list of [`ImageSegmentationOutputElement`] items containing the segmented masks and associated attributes. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.image_segmentation("cat.jpg") - [ImageSegmentationOutputElement(score=0.989008, label='LABEL_184', mask=), ...] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="image-segmentation", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={ - "mask_threshold": mask_threshold, - "overlap_mask_area_threshold": overlap_mask_area_threshold, - "subtask": subtask, - "threshold": threshold, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - output = ImageSegmentationOutputElement.parse_obj_as_list(response) - for item in output: - item.mask = _b64_to_image(item.mask) # type: ignore [assignment] - return output - - async def image_to_image( - self, - image: ContentT, - prompt: Optional[str] = None, - *, - negative_prompt: Optional[str] = None, - num_inference_steps: Optional[int] = None, - guidance_scale: Optional[float] = None, - model: Optional[str] = None, - target_size: Optional[ImageToImageTargetSize] = None, - **kwargs, - ) -> "Image": - """ - Perform image-to-image translation using a specified model. - - > [!WARNING] - > You must have `PIL` installed if you want to work with images (`pip install Pillow`). - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The input image for translation. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - prompt (`str`, *optional*): - The text prompt to guide the image generation. - negative_prompt (`str`, *optional*): - One prompt to guide what NOT to include in image generation. - num_inference_steps (`int`, *optional*): - For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher - quality image at the expense of slower inference. - guidance_scale (`float`, *optional*): - For diffusion models. A higher guidance scale value encourages the model to generate images closely - linked to the text prompt at the expense of lower image quality. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - target_size (`ImageToImageTargetSize`, *optional*): - The size in pixels of the output image. This parameter is only supported by some providers and for - specific models. It will be ignored when unsupported. - - Returns: - `Image`: The translated image. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> image = await client.image_to_image("cat.jpg", prompt="turn the cat into a tiger") - >>> image.save("tiger.jpg") - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="image-to-image", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={ - "prompt": prompt, - "negative_prompt": negative_prompt, - "target_size": target_size, - "num_inference_steps": num_inference_steps, - "guidance_scale": guidance_scale, - **kwargs, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - response = provider_helper.get_response(response, request_parameters) - return _bytes_to_image(response) - - async def image_to_video( - self, - image: ContentT, - *, - model: Optional[str] = None, - prompt: Optional[str] = None, - negative_prompt: Optional[str] = None, - num_frames: Optional[float] = None, - num_inference_steps: Optional[int] = None, - guidance_scale: Optional[float] = None, - seed: Optional[int] = None, - target_size: Optional[ImageToVideoTargetSize] = None, - **kwargs, - ) -> bytes: - """ - Generate a video from an input image. - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The input image to generate a video from. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - prompt (`str`, *optional*): - The text prompt to guide the video generation. - negative_prompt (`str`, *optional*): - One prompt to guide what NOT to include in video generation. - num_frames (`float`, *optional*): - The num_frames parameter determines how many video frames are generated. - num_inference_steps (`int`, *optional*): - For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher - quality image at the expense of slower inference. - guidance_scale (`float`, *optional*): - For diffusion models. A higher guidance scale value encourages the model to generate videos closely - linked to the text prompt at the expense of lower image quality. - seed (`int`, *optional*): - The seed to use for the video generation. - target_size (`ImageToVideoTargetSize`, *optional*): - The size in pixel of the output video frames. - num_inference_steps (`int`, *optional*): - The number of denoising steps. More denoising steps usually lead to a higher quality video at the - expense of slower inference. - seed (`int`, *optional*): - Seed for the random number generator. - - Returns: - `bytes`: The generated video. - - Examples: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> video = await client.image_to_video("cat.jpg", model="Wan-AI/Wan2.2-I2V-A14B", prompt="turn the cat into a tiger") - >>> with open("tiger.mp4", "wb") as f: - ... f.write(video) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="image-to-video", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={ - "prompt": prompt, - "negative_prompt": negative_prompt, - "num_frames": num_frames, - "num_inference_steps": num_inference_steps, - "guidance_scale": guidance_scale, - "seed": seed, - "target_size": target_size, - **kwargs, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - response = provider_helper.get_response(response, request_parameters) - return response - - async def image_to_text(self, image: ContentT, *, model: Optional[str] = None) -> ImageToTextOutput: - """ - Takes an input image and return text. - - Models can have very different outputs depending on your use case (image captioning, optical character recognition - (OCR), Pix2Struct, etc). Please have a look to the model card to learn more about a model's specificities. - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The input image to caption. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - - Returns: - [`ImageToTextOutput`]: The generated text. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.image_to_text("cat.jpg") - 'a cat standing in a grassy field ' - >>> await client.image_to_text("https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg") - 'a dog laying on the grass next to a flower pot ' - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="image-to-text", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - output_list: List[ImageToTextOutput] = ImageToTextOutput.parse_obj_as_list(response) - return output_list[0] - - async def object_detection( - self, image: ContentT, *, model: Optional[str] = None, threshold: Optional[float] = None - ) -> List[ObjectDetectionOutputElement]: - """ - Perform object detection on the given image using the specified model. - - > [!WARNING] - > You must have `PIL` installed if you want to work with images (`pip install Pillow`). - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The image to detect objects on. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - model (`str`, *optional*): - The model to use for object detection. Can be a model ID hosted on the Hugging Face Hub or a URL to a - deployed Inference Endpoint. If not provided, the default recommended model for object detection (DETR) will be used. - threshold (`float`, *optional*): - The probability necessary to make a prediction. - Returns: - `List[ObjectDetectionOutputElement]`: A list of [`ObjectDetectionOutputElement`] items containing the bounding boxes and associated attributes. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - `ValueError`: - If the request output is not a List. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.object_detection("people.jpg") - [ObjectDetectionOutputElement(score=0.9486683011054993, label='person', box=ObjectDetectionBoundingBox(xmin=59, ymin=39, xmax=420, ymax=510)), ...] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="object-detection", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={"threshold": threshold}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return ObjectDetectionOutputElement.parse_obj_as_list(response) - - async def question_answering( - self, - question: str, - context: str, - *, - model: Optional[str] = None, - align_to_words: Optional[bool] = None, - doc_stride: Optional[int] = None, - handle_impossible_answer: Optional[bool] = None, - max_answer_len: Optional[int] = None, - max_question_len: Optional[int] = None, - max_seq_len: Optional[int] = None, - top_k: Optional[int] = None, - ) -> Union[QuestionAnsweringOutputElement, List[QuestionAnsweringOutputElement]]: - """ - Retrieve the answer to a question from a given text. - - Args: - question (`str`): - Question to be answered. - context (`str`): - The context of the question. - model (`str`): - The model to use for the question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. - align_to_words (`bool`, *optional*): - Attempts to align the answer to real words. Improves quality on space separated languages. Might hurt - on non-space-separated languages (like Japanese or Chinese) - doc_stride (`int`, *optional*): - If the context is too long to fit with the question for the model, it will be split in several chunks - with some overlap. This argument controls the size of that overlap. - handle_impossible_answer (`bool`, *optional*): - Whether to accept impossible as an answer. - max_answer_len (`int`, *optional*): - The maximum length of predicted answers (e.g., only answers with a shorter length are considered). - max_question_len (`int`, *optional*): - The maximum length of the question after tokenization. It will be truncated if needed. - max_seq_len (`int`, *optional*): - The maximum length of the total sentence (context + question) in tokens of each chunk passed to the - model. The context will be split in several chunks (using docStride as overlap) if needed. - top_k (`int`, *optional*): - The number of answers to return (will be chosen by order of likelihood). Note that we return less than - topk answers if there are not enough options available within the context. - - Returns: - Union[`QuestionAnsweringOutputElement`, List[`QuestionAnsweringOutputElement`]]: - When top_k is 1 or not provided, it returns a single `QuestionAnsweringOutputElement`. - When top_k is greater than 1, it returns a list of `QuestionAnsweringOutputElement`. - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.question_answering(question="What's my name?", context="My name is Clara and I live in Berkeley.") - QuestionAnsweringOutputElement(answer='Clara', end=16, score=0.9326565265655518, start=11) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="question-answering", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs={"question": question, "context": context}, - parameters={ - "align_to_words": align_to_words, - "doc_stride": doc_stride, - "handle_impossible_answer": handle_impossible_answer, - "max_answer_len": max_answer_len, - "max_question_len": max_question_len, - "max_seq_len": max_seq_len, - "top_k": top_k, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - # Parse the response as a single `QuestionAnsweringOutputElement` when top_k is 1 or not provided, or a list of `QuestionAnsweringOutputElement` to ensure backward compatibility. - output = QuestionAnsweringOutputElement.parse_obj(response) - return output - - async def sentence_similarity( - self, sentence: str, other_sentences: List[str], *, model: Optional[str] = None - ) -> List[float]: - """ - Compute the semantic similarity between a sentence and a list of other sentences by comparing their embeddings. - - Args: - sentence (`str`): - The main sentence to compare to others. - other_sentences (`List[str]`): - The list of sentences to compare to. - model (`str`, *optional*): - The model to use for the sentence similarity task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended sentence similarity model will be used. - Defaults to None. - - Returns: - `List[float]`: The similarity scores between the main sentence and the given comparison sentences. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.sentence_similarity( - ... "Machine learning is so easy.", - ... other_sentences=[ - ... "Deep learning is so straightforward.", - ... "This is so difficult, like rocket science.", - ... "I can't believe how much I struggled with this.", - ... ], - ... ) - [0.7785726189613342, 0.45876261591911316, 0.2906220555305481] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="sentence-similarity", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs={"source_sentence": sentence, "sentences": other_sentences}, - parameters={}, - extra_payload={}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return _bytes_to_list(response) - - async def summarization( - self, - text: str, - *, - model: Optional[str] = None, - clean_up_tokenization_spaces: Optional[bool] = None, - generate_parameters: Optional[Dict[str, Any]] = None, - truncation: Optional["SummarizationTruncationStrategy"] = None, - ) -> SummarizationOutput: - """ - Generate a summary of a given text using a specified model. - - Args: - text (`str`): - The input text to summarize. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended model for summarization will be used. - clean_up_tokenization_spaces (`bool`, *optional*): - Whether to clean up the potential extra spaces in the text output. - generate_parameters (`Dict[str, Any]`, *optional*): - Additional parametrization of the text generation algorithm. - truncation (`"SummarizationTruncationStrategy"`, *optional*): - The truncation strategy to use. - Returns: - [`SummarizationOutput`]: The generated summary text. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.summarization("The Eiffel tower...") - SummarizationOutput(generated_text="The Eiffel tower is one of the most famous landmarks in the world....") - ``` - """ - parameters = { - "clean_up_tokenization_spaces": clean_up_tokenization_spaces, - "generate_parameters": generate_parameters, - "truncation": truncation, - } - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="summarization", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters=parameters, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return SummarizationOutput.parse_obj_as_list(response)[0] - - async def table_question_answering( - self, - table: Dict[str, Any], - query: str, - *, - model: Optional[str] = None, - padding: Optional["Padding"] = None, - sequential: Optional[bool] = None, - truncation: Optional[bool] = None, - ) -> TableQuestionAnsweringOutputElement: - """ - Retrieve the answer to a question from information given in a table. - - Args: - table (`str`): - A table of data represented as a dict of lists where entries are headers and the lists are all the - values, all lists must have the same size. - query (`str`): - The query in plain text that you want to ask the table. - model (`str`): - The model to use for the table-question-answering task. Can be a model ID hosted on the Hugging Face - Hub or a URL to a deployed Inference Endpoint. - padding (`"Padding"`, *optional*): - Activates and controls padding. - sequential (`bool`, *optional*): - Whether to do inference sequentially or as a batch. Batching is faster, but models like SQA require the - inference to be done sequentially to extract relations within sequences, given their conversational - nature. - truncation (`bool`, *optional*): - Activates and controls truncation. - - Returns: - [`TableQuestionAnsweringOutputElement`]: a table question answering output containing the answer, coordinates, cells and the aggregator used. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> query = "How many stars does the transformers repository have?" - >>> table = {"Repository": ["Transformers", "Datasets", "Tokenizers"], "Stars": ["36542", "4512", "3934"]} - >>> await client.table_question_answering(table, query, model="google/tapas-base-finetuned-wtq") - TableQuestionAnsweringOutputElement(answer='36542', coordinates=[[0, 1]], cells=['36542'], aggregator='AVERAGE') - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="table-question-answering", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs={"query": query, "table": table}, - parameters={"model": model, "padding": padding, "sequential": sequential, "truncation": truncation}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return TableQuestionAnsweringOutputElement.parse_obj_as_instance(response) - - async def tabular_classification(self, table: Dict[str, Any], *, model: Optional[str] = None) -> List[str]: - """ - Classifying a target category (a group) based on a set of attributes. - - Args: - table (`Dict[str, Any]`): - Set of attributes to classify. - model (`str`, *optional*): - The model to use for the tabular classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended tabular classification model will be used. - Defaults to None. - - Returns: - `List`: a list of labels, one per row in the initial table. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> table = { - ... "fixed_acidity": ["7.4", "7.8", "10.3"], - ... "volatile_acidity": ["0.7", "0.88", "0.32"], - ... "citric_acid": ["0", "0", "0.45"], - ... "residual_sugar": ["1.9", "2.6", "6.4"], - ... "chlorides": ["0.076", "0.098", "0.073"], - ... "free_sulfur_dioxide": ["11", "25", "5"], - ... "total_sulfur_dioxide": ["34", "67", "13"], - ... "density": ["0.9978", "0.9968", "0.9976"], - ... "pH": ["3.51", "3.2", "3.23"], - ... "sulphates": ["0.56", "0.68", "0.82"], - ... "alcohol": ["9.4", "9.8", "12.6"], - ... } - >>> await client.tabular_classification(table=table, model="julien-c/wine-quality") - ["5", "5", "5"] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="tabular-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=None, - extra_payload={"table": table}, - parameters={}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return _bytes_to_list(response) - - async def tabular_regression(self, table: Dict[str, Any], *, model: Optional[str] = None) -> List[float]: - """ - Predicting a numerical target value given a set of attributes/features in a table. - - Args: - table (`Dict[str, Any]`): - Set of attributes stored in a table. The attributes used to predict the target can be both numerical and categorical. - model (`str`, *optional*): - The model to use for the tabular regression task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended tabular regression model will be used. - Defaults to None. - - Returns: - `List`: a list of predicted numerical target values. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> table = { - ... "Height": ["11.52", "12.48", "12.3778"], - ... "Length1": ["23.2", "24", "23.9"], - ... "Length2": ["25.4", "26.3", "26.5"], - ... "Length3": ["30", "31.2", "31.1"], - ... "Species": ["Bream", "Bream", "Bream"], - ... "Width": ["4.02", "4.3056", "4.6961"], - ... } - >>> await client.tabular_regression(table, model="scikit-learn/Fish-Weight") - [110, 120, 130] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="tabular-regression", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=None, - parameters={}, - extra_payload={"table": table}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return _bytes_to_list(response) - - async def text_classification( - self, - text: str, - *, - model: Optional[str] = None, - top_k: Optional[int] = None, - function_to_apply: Optional["TextClassificationOutputTransform"] = None, - ) -> List[TextClassificationOutputElement]: - """ - Perform text classification (e.g. sentiment-analysis) on the given text. - - Args: - text (`str`): - A string to be classified. - model (`str`, *optional*): - The model to use for the text classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended text classification model will be used. - Defaults to None. - top_k (`int`, *optional*): - When specified, limits the output to the top K most probable classes. - function_to_apply (`"TextClassificationOutputTransform"`, *optional*): - The function to apply to the model outputs in order to retrieve the scores. - - Returns: - `List[TextClassificationOutputElement]`: a list of [`TextClassificationOutputElement`] items containing the predicted label and associated probability. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.text_classification("I like you") - [ - TextClassificationOutputElement(label='POSITIVE', score=0.9998695850372314), - TextClassificationOutputElement(label='NEGATIVE', score=0.0001304351753788069), - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="text-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "function_to_apply": function_to_apply, - "top_k": top_k, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return TextClassificationOutputElement.parse_obj_as_list(response)[0] # type: ignore [return-value] - - @overload - async def text_generation( - self, - prompt: str, - *, - details: Literal[True], - stream: Literal[True], - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> AsyncIterable[TextGenerationStreamOutput]: ... - - @overload - async def text_generation( - self, - prompt: str, - *, - details: Literal[True], - stream: Optional[Literal[False]] = None, - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> TextGenerationOutput: ... - - @overload - async def text_generation( - self, - prompt: str, - *, - details: Optional[Literal[False]] = None, - stream: Literal[True], - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, # Manual default value - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> AsyncIterable[str]: ... - - @overload - async def text_generation( - self, - prompt: str, - *, - details: Optional[Literal[False]] = None, - stream: Optional[Literal[False]] = None, - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> str: ... - - @overload - async def text_generation( - self, - prompt: str, - *, - details: Optional[bool] = None, - stream: Optional[bool] = None, - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> Union[str, TextGenerationOutput, AsyncIterable[str], AsyncIterable[TextGenerationStreamOutput]]: ... - - async def text_generation( - self, - prompt: str, - *, - details: Optional[bool] = None, - stream: Optional[bool] = None, - model: Optional[str] = None, - # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) - adapter_id: Optional[str] = None, - best_of: Optional[int] = None, - decoder_input_details: Optional[bool] = None, - do_sample: Optional[bool] = None, - frequency_penalty: Optional[float] = None, - grammar: Optional[TextGenerationInputGrammarType] = None, - max_new_tokens: Optional[int] = None, - repetition_penalty: Optional[float] = None, - return_full_text: Optional[bool] = None, - seed: Optional[int] = None, - stop: Optional[List[str]] = None, - stop_sequences: Optional[List[str]] = None, # Deprecated, use `stop` instead - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_n_tokens: Optional[int] = None, - top_p: Optional[float] = None, - truncate: Optional[int] = None, - typical_p: Optional[float] = None, - watermark: Optional[bool] = None, - ) -> Union[str, TextGenerationOutput, AsyncIterable[str], AsyncIterable[TextGenerationStreamOutput]]: - """ - Given a prompt, generate the following text. - - > [!TIP] - > If you want to generate a response from chat messages, you should use the [`InferenceClient.chat_completion`] method. - > It accepts a list of messages instead of a single text prompt and handles the chat templating for you. - - Args: - prompt (`str`): - Input text. - details (`bool`, *optional*): - By default, text_generation returns a string. Pass `details=True` if you want a detailed output (tokens, - probabilities, seed, finish reason, etc.). Only available for models running on with the - `text-generation-inference` backend. - stream (`bool`, *optional*): - By default, text_generation returns the full generated text. Pass `stream=True` if you want a stream of - tokens to be returned. Only available for models running on with the `text-generation-inference` - backend. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - adapter_id (`str`, *optional*): - Lora adapter id. - best_of (`int`, *optional*): - Generate best_of sequences and return the one if the highest token logprobs. - decoder_input_details (`bool`, *optional*): - Return the decoder input token logprobs and ids. You must set `details=True` as well for it to be taken - into account. Defaults to `False`. - do_sample (`bool`, *optional*): - Activate logits sampling - frequency_penalty (`float`, *optional*): - Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in - the text so far, decreasing the model's likelihood to repeat the same line verbatim. - grammar ([`TextGenerationInputGrammarType`], *optional*): - Grammar constraints. Can be either a JSONSchema or a regex. - max_new_tokens (`int`, *optional*): - Maximum number of generated tokens. Defaults to 100. - repetition_penalty (`float`, *optional*): - The parameter for repetition penalty. 1.0 means no penalty. See [this - paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. - return_full_text (`bool`, *optional*): - Whether to prepend the prompt to the generated text - seed (`int`, *optional*): - Random sampling seed - stop (`List[str]`, *optional*): - Stop generating tokens if a member of `stop` is generated. - stop_sequences (`List[str]`, *optional*): - Deprecated argument. Use `stop` instead. - temperature (`float`, *optional*): - The value used to module the logits distribution. - top_n_tokens (`int`, *optional*): - Return information about the `top_n_tokens` most likely tokens at each generation step, instead of - just the sampled token. - top_k (`int`, *optional`): - The number of highest probability vocabulary tokens to keep for top-k-filtering. - top_p (`float`, *optional`): - If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or - higher are kept for generation. - truncate (`int`, *optional`): - Truncate inputs tokens to the given size. - typical_p (`float`, *optional`): - Typical Decoding mass - See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information - watermark (`bool`, *optional*): - Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) - - Returns: - `Union[str, TextGenerationOutput, Iterable[str], Iterable[TextGenerationStreamOutput]]`: - Generated text returned from the server: - - if `stream=False` and `details=False`, the generated text is returned as a `str` (default) - - if `stream=True` and `details=False`, the generated text is returned token by token as a `Iterable[str]` - - if `stream=False` and `details=True`, the generated text is returned with more details as a [`~huggingface_hub.TextGenerationOutput`] - - if `details=True` and `stream=True`, the generated text is returned token by token as a iterable of [`~huggingface_hub.TextGenerationStreamOutput`] - - Raises: - `ValidationError`: - If input values are not valid. No HTTP call is made to the server. - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - - # Case 1: generate text - >>> await client.text_generation("The huggingface_hub library is ", max_new_tokens=12) - '100% open source and built to be easy to use.' - - # Case 2: iterate over the generated tokens. Useful for large generation. - >>> async for token in await client.text_generation("The huggingface_hub library is ", max_new_tokens=12, stream=True): - ... print(token) - 100 - % - open - source - and - built - to - be - easy - to - use - . - - # Case 3: get more details about the generation process. - >>> await client.text_generation("The huggingface_hub library is ", max_new_tokens=12, details=True) - TextGenerationOutput( - generated_text='100% open source and built to be easy to use.', - details=TextGenerationDetails( - finish_reason='length', - generated_tokens=12, - seed=None, - prefill=[ - TextGenerationPrefillOutputToken(id=487, text='The', logprob=None), - TextGenerationPrefillOutputToken(id=53789, text=' hugging', logprob=-13.171875), - (...) - TextGenerationPrefillOutputToken(id=204, text=' ', logprob=-7.0390625) - ], - tokens=[ - TokenElement(id=1425, text='100', logprob=-1.0175781, special=False), - TokenElement(id=16, text='%', logprob=-0.0463562, special=False), - (...) - TokenElement(id=25, text='.', logprob=-0.5703125, special=False) - ], - best_of_sequences=None - ) - ) - - # Case 4: iterate over the generated tokens with more details. - # Last object is more complete, containing the full generated text and the finish reason. - >>> async for details in await client.text_generation("The huggingface_hub library is ", max_new_tokens=12, details=True, stream=True): - ... print(details) - ... - TextGenerationStreamOutput(token=TokenElement(id=1425, text='100', logprob=-1.0175781, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=16, text='%', logprob=-0.0463562, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=1314, text=' open', logprob=-1.3359375, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=3178, text=' source', logprob=-0.28100586, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=273, text=' and', logprob=-0.5961914, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=3426, text=' built', logprob=-1.9423828, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=271, text=' to', logprob=-1.4121094, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=314, text=' be', logprob=-1.5224609, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=1833, text=' easy', logprob=-2.1132812, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=271, text=' to', logprob=-0.08520508, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement(id=745, text=' use', logprob=-0.39453125, special=False), generated_text=None, details=None) - TextGenerationStreamOutput(token=TokenElement( - id=25, - text='.', - logprob=-0.5703125, - special=False), - generated_text='100% open source and built to be easy to use.', - details=TextGenerationStreamOutputStreamDetails(finish_reason='length', generated_tokens=12, seed=None) - ) - - # Case 5: generate constrained output using grammar - >>> response = await client.text_generation( - ... prompt="I saw a puppy a cat and a raccoon during my bike ride in the park", - ... model="HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1", - ... max_new_tokens=100, - ... repetition_penalty=1.3, - ... grammar={ - ... "type": "json", - ... "value": { - ... "properties": { - ... "location": {"type": "string"}, - ... "activity": {"type": "string"}, - ... "animals_seen": {"type": "integer", "minimum": 1, "maximum": 5}, - ... "animals": {"type": "array", "items": {"type": "string"}}, - ... }, - ... "required": ["location", "activity", "animals_seen", "animals"], - ... }, - ... }, - ... ) - >>> json.loads(response) - { - "activity": "bike riding", - "animals": ["puppy", "cat", "raccoon"], - "animals_seen": 3, - "location": "park" - } - ``` - """ - if decoder_input_details and not details: - warnings.warn( - "`decoder_input_details=True` has been passed to the server but `details=False` is set meaning that" - " the output from the server will be truncated." - ) - decoder_input_details = False - - if stop_sequences is not None: - warnings.warn( - "`stop_sequences` is a deprecated argument for `text_generation` task" - " and will be removed in version '0.28.0'. Use `stop` instead.", - FutureWarning, - ) - if stop is None: - stop = stop_sequences # use deprecated arg if provided - - # Build payload - parameters = { - "adapter_id": adapter_id, - "best_of": best_of, - "decoder_input_details": decoder_input_details, - "details": details, - "do_sample": do_sample, - "frequency_penalty": frequency_penalty, - "grammar": grammar, - "max_new_tokens": max_new_tokens, - "repetition_penalty": repetition_penalty, - "return_full_text": return_full_text, - "seed": seed, - "stop": stop, - "temperature": temperature, - "top_k": top_k, - "top_n_tokens": top_n_tokens, - "top_p": top_p, - "truncate": truncate, - "typical_p": typical_p, - "watermark": watermark, - } - - # Remove some parameters if not a TGI server - unsupported_kwargs = _get_unsupported_text_generation_kwargs(model) - if len(unsupported_kwargs) > 0: - # The server does not support some parameters - # => means it is not a TGI server - # => remove unsupported parameters and warn the user - - ignored_parameters = [] - for key in unsupported_kwargs: - if parameters.get(key): - ignored_parameters.append(key) - parameters.pop(key, None) - if len(ignored_parameters) > 0: - warnings.warn( - "API endpoint/model for text-generation is not served via TGI. Ignoring following parameters:" - f" {', '.join(ignored_parameters)}.", - UserWarning, - ) - if details: - warnings.warn( - "API endpoint/model for text-generation is not served via TGI. Parameter `details=True` will" - " be ignored meaning only the generated text will be returned.", - UserWarning, - ) - details = False - if stream: - raise ValueError( - "API endpoint/model for text-generation is not served via TGI. Cannot return output as a stream." - " Please pass `stream=False` as input." - ) - - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="text-generation", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=prompt, - parameters=parameters, - extra_payload={"stream": stream}, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - - # Handle errors separately for more precise error messages - try: - bytes_output = await self._inner_post(request_parameters, stream=stream or False) - except _import_aiohttp().ClientResponseError as e: - match = MODEL_KWARGS_NOT_USED_REGEX.search(e.response_error_payload["error"]) - if e.status == 400 and match: - unused_params = [kwarg.strip("' ") for kwarg in match.group(1).split(",")] - _set_unsupported_text_generation_kwargs(model, unused_params) - return await self.text_generation( # type: ignore - prompt=prompt, - details=details, - stream=stream, - model=model_id, - adapter_id=adapter_id, - best_of=best_of, - decoder_input_details=decoder_input_details, - do_sample=do_sample, - frequency_penalty=frequency_penalty, - grammar=grammar, - max_new_tokens=max_new_tokens, - repetition_penalty=repetition_penalty, - return_full_text=return_full_text, - seed=seed, - stop=stop, - temperature=temperature, - top_k=top_k, - top_n_tokens=top_n_tokens, - top_p=top_p, - truncate=truncate, - typical_p=typical_p, - watermark=watermark, - ) - raise_text_generation_error(e) - - # Parse output - if stream: - return _async_stream_text_generation_response(bytes_output, details) # type: ignore - - data = _bytes_to_dict(bytes_output) # type: ignore[arg-type] - - # Data can be a single element (dict) or an iterable of dicts where we select the first element of. - if isinstance(data, list): - data = data[0] - response = provider_helper.get_response(data, request_parameters) - return TextGenerationOutput.parse_obj_as_instance(response) if details else response["generated_text"] - - async def text_to_image( - self, - prompt: str, - *, - negative_prompt: Optional[str] = None, - height: Optional[int] = None, - width: Optional[int] = None, - num_inference_steps: Optional[int] = None, - guidance_scale: Optional[float] = None, - model: Optional[str] = None, - scheduler: Optional[str] = None, - seed: Optional[int] = None, - extra_body: Optional[Dict[str, Any]] = None, - ) -> "Image": - """ - Generate an image based on a given text using a specified model. - - > [!WARNING] - > You must have `PIL` installed if you want to work with images (`pip install Pillow`). - - > [!TIP] - > You can pass provider-specific parameters to the model by using the `extra_body` argument. - - Args: - prompt (`str`): - The prompt to generate an image from. - negative_prompt (`str`, *optional*): - One prompt to guide what NOT to include in image generation. - height (`int`, *optional*): - The height in pixels of the output image - width (`int`, *optional*): - The width in pixels of the output image - num_inference_steps (`int`, *optional*): - The number of denoising steps. More denoising steps usually lead to a higher quality image at the - expense of slower inference. - guidance_scale (`float`, *optional*): - A higher guidance scale value encourages the model to generate images closely linked to the text - prompt, but values too high may cause saturation and other artifacts. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended text-to-image model will be used. - Defaults to None. - scheduler (`str`, *optional*): - Override the scheduler with a compatible one. - seed (`int`, *optional*): - Seed for the random number generator. - extra_body (`Dict[str, Any]`, *optional*): - Additional provider-specific parameters to pass to the model. Refer to the provider's documentation - for supported parameters. - - Returns: - `Image`: The generated image. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - - >>> image = await client.text_to_image("An astronaut riding a horse on the moon.") - >>> image.save("astronaut.png") - - >>> image = await client.text_to_image( - ... "An astronaut riding a horse on the moon.", - ... negative_prompt="low resolution, blurry", - ... model="stabilityai/stable-diffusion-2-1", - ... ) - >>> image.save("better_astronaut.png") - ``` - Example using a third-party provider directly. Usage will be billed on your fal.ai account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="fal-ai", # Use fal.ai provider - ... api_key="fal-ai-api-key", # Pass your fal.ai API key - ... ) - >>> image = client.text_to_image( - ... "A majestic lion in a fantasy forest", - ... model="black-forest-labs/FLUX.1-schnell", - ... ) - >>> image.save("lion.png") - ``` - - Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", # Use replicate provider - ... api_key="hf_...", # Pass your HF token - ... ) - >>> image = client.text_to_image( - ... "An astronaut riding a horse on the moon.", - ... model="black-forest-labs/FLUX.1-dev", - ... ) - >>> image.save("astronaut.png") - ``` - - Example using Replicate provider with extra parameters - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", # Use replicate provider - ... api_key="hf_...", # Pass your HF token - ... ) - >>> image = client.text_to_image( - ... "An astronaut riding a horse on the moon.", - ... model="black-forest-labs/FLUX.1-schnell", - ... extra_body={"output_quality": 100}, - ... ) - >>> image.save("astronaut.png") - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="text-to-image", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=prompt, - parameters={ - "negative_prompt": negative_prompt, - "height": height, - "width": width, - "num_inference_steps": num_inference_steps, - "guidance_scale": guidance_scale, - "scheduler": scheduler, - "seed": seed, - **(extra_body or {}), - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - response = provider_helper.get_response(response) - return _bytes_to_image(response) - - async def text_to_video( - self, - prompt: str, - *, - model: Optional[str] = None, - guidance_scale: Optional[float] = None, - negative_prompt: Optional[List[str]] = None, - num_frames: Optional[float] = None, - num_inference_steps: Optional[int] = None, - seed: Optional[int] = None, - extra_body: Optional[Dict[str, Any]] = None, - ) -> bytes: - """ - Generate a video based on a given text. - - > [!TIP] - > You can pass provider-specific parameters to the model by using the `extra_body` argument. - - Args: - prompt (`str`): - The prompt to generate a video from. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended text-to-video model will be used. - Defaults to None. - guidance_scale (`float`, *optional*): - A higher guidance scale value encourages the model to generate videos closely linked to the text - prompt, but values too high may cause saturation and other artifacts. - negative_prompt (`List[str]`, *optional*): - One or several prompt to guide what NOT to include in video generation. - num_frames (`float`, *optional*): - The num_frames parameter determines how many video frames are generated. - num_inference_steps (`int`, *optional*): - The number of denoising steps. More denoising steps usually lead to a higher quality video at the - expense of slower inference. - seed (`int`, *optional*): - Seed for the random number generator. - extra_body (`Dict[str, Any]`, *optional*): - Additional provider-specific parameters to pass to the model. Refer to the provider's documentation - for supported parameters. - - Returns: - `bytes`: The generated video. - - Example: - - Example using a third-party provider directly. Usage will be billed on your fal.ai account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="fal-ai", # Using fal.ai provider - ... api_key="fal-ai-api-key", # Pass your fal.ai API key - ... ) - >>> video = client.text_to_video( - ... "A majestic lion running in a fantasy forest", - ... model="tencent/HunyuanVideo", - ... ) - >>> with open("lion.mp4", "wb") as file: - ... file.write(video) - ``` - - Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", # Using replicate provider - ... api_key="hf_...", # Pass your HF token - ... ) - >>> video = client.text_to_video( - ... "A cat running in a park", - ... model="genmo/mochi-1-preview", - ... ) - >>> with open("cat.mp4", "wb") as file: - ... file.write(video) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="text-to-video", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=prompt, - parameters={ - "guidance_scale": guidance_scale, - "negative_prompt": negative_prompt, - "num_frames": num_frames, - "num_inference_steps": num_inference_steps, - "seed": seed, - **(extra_body or {}), - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - response = provider_helper.get_response(response, request_parameters) - return response - - async def text_to_speech( - self, - text: str, - *, - model: Optional[str] = None, - do_sample: Optional[bool] = None, - early_stopping: Optional[Union[bool, "TextToSpeechEarlyStoppingEnum"]] = None, - epsilon_cutoff: Optional[float] = None, - eta_cutoff: Optional[float] = None, - max_length: Optional[int] = None, - max_new_tokens: Optional[int] = None, - min_length: Optional[int] = None, - min_new_tokens: Optional[int] = None, - num_beam_groups: Optional[int] = None, - num_beams: Optional[int] = None, - penalty_alpha: Optional[float] = None, - temperature: Optional[float] = None, - top_k: Optional[int] = None, - top_p: Optional[float] = None, - typical_p: Optional[float] = None, - use_cache: Optional[bool] = None, - extra_body: Optional[Dict[str, Any]] = None, - ) -> bytes: - """ - Synthesize an audio of a voice pronouncing a given text. - - > [!TIP] - > You can pass provider-specific parameters to the model by using the `extra_body` argument. - - Args: - text (`str`): - The text to synthesize. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. If not provided, the default recommended text-to-speech model will be used. - Defaults to None. - do_sample (`bool`, *optional*): - Whether to use sampling instead of greedy decoding when generating new tokens. - early_stopping (`Union[bool, "TextToSpeechEarlyStoppingEnum"]`, *optional*): - Controls the stopping condition for beam-based methods. - epsilon_cutoff (`float`, *optional*): - If set to float strictly between 0 and 1, only tokens with a conditional probability greater than - epsilon_cutoff will be sampled. In the paper, suggested values range from 3e-4 to 9e-4, depending on - the size of the model. See [Truncation Sampling as Language Model - Desmoothing](https://hf.co/papers/2210.15191) for more details. - eta_cutoff (`float`, *optional*): - Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to float strictly - between 0 and 1, a token is only considered if it is greater than either eta_cutoff or sqrt(eta_cutoff) - * exp(-entropy(softmax(next_token_logits))). The latter term is intuitively the expected next token - probability, scaled by sqrt(eta_cutoff). In the paper, suggested values range from 3e-4 to 2e-3, - depending on the size of the model. See [Truncation Sampling as Language Model - Desmoothing](https://hf.co/papers/2210.15191) for more details. - max_length (`int`, *optional*): - The maximum length (in tokens) of the generated text, including the input. - max_new_tokens (`int`, *optional*): - The maximum number of tokens to generate. Takes precedence over max_length. - min_length (`int`, *optional*): - The minimum length (in tokens) of the generated text, including the input. - min_new_tokens (`int`, *optional*): - The minimum number of tokens to generate. Takes precedence over min_length. - num_beam_groups (`int`, *optional*): - Number of groups to divide num_beams into in order to ensure diversity among different groups of beams. - See [this paper](https://hf.co/papers/1610.02424) for more details. - num_beams (`int`, *optional*): - Number of beams to use for beam search. - penalty_alpha (`float`, *optional*): - The value balances the model confidence and the degeneration penalty in contrastive search decoding. - temperature (`float`, *optional*): - The value used to modulate the next token probabilities. - top_k (`int`, *optional*): - The number of highest probability vocabulary tokens to keep for top-k-filtering. - top_p (`float`, *optional*): - If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to - top_p or higher are kept for generation. - typical_p (`float`, *optional*): - Local typicality measures how similar the conditional probability of predicting a target token next is - to the expected conditional probability of predicting a random token next, given the partial text - already generated. If set to float < 1, the smallest set of the most locally typical tokens with - probabilities that add up to typical_p or higher are kept for generation. See [this - paper](https://hf.co/papers/2202.00666) for more details. - use_cache (`bool`, *optional*): - Whether the model should use the past last key/values attentions to speed up decoding - extra_body (`Dict[str, Any]`, *optional*): - Additional provider-specific parameters to pass to the model. Refer to the provider's documentation - for supported parameters. - Returns: - `bytes`: The generated audio. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from pathlib import Path - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - - >>> audio = await client.text_to_speech("Hello world") - >>> Path("hello_world.flac").write_bytes(audio) - ``` - - Example using a third-party provider directly. Usage will be billed on your Replicate account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", - ... api_key="your-replicate-api-key", # Pass your Replicate API key directly - ... ) - >>> audio = client.text_to_speech( - ... text="Hello world", - ... model="OuteAI/OuteTTS-0.3-500M", - ... ) - >>> Path("hello_world.flac").write_bytes(audio) - ``` - - Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", - ... api_key="hf_...", # Pass your HF token - ... ) - >>> audio =client.text_to_speech( - ... text="Hello world", - ... model="OuteAI/OuteTTS-0.3-500M", - ... ) - >>> Path("hello_world.flac").write_bytes(audio) - ``` - Example using Replicate provider with extra parameters - ```py - >>> from huggingface_hub import InferenceClient - >>> client = InferenceClient( - ... provider="replicate", # Use replicate provider - ... api_key="hf_...", # Pass your HF token - ... ) - >>> audio = client.text_to_speech( - ... "Hello, my name is Kororo, an awesome text-to-speech model.", - ... model="hexgrad/Kokoro-82M", - ... extra_body={"voice": "af_nicole"}, - ... ) - >>> Path("hello.flac").write_bytes(audio) - ``` - - Example music-gen using "YuE-s1-7B-anneal-en-cot" on fal.ai - ```py - >>> from huggingface_hub import InferenceClient - >>> lyrics = ''' - ... [verse] - ... In the town where I was born - ... Lived a man who sailed to sea - ... And he told us of his life - ... In the land of submarines - ... So we sailed on to the sun - ... 'Til we found a sea of green - ... And we lived beneath the waves - ... In our yellow submarine - - ... [chorus] - ... We all live in a yellow submarine - ... Yellow submarine, yellow submarine - ... We all live in a yellow submarine - ... Yellow submarine, yellow submarine - ... ''' - >>> genres = "pavarotti-style tenor voice" - >>> client = InferenceClient( - ... provider="fal-ai", - ... model="m-a-p/YuE-s1-7B-anneal-en-cot", - ... api_key=..., - ... ) - >>> audio = client.text_to_speech(lyrics, extra_body={"genres": genres}) - >>> with open("output.mp3", "wb") as f: - ... f.write(audio) - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="text-to-speech", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "do_sample": do_sample, - "early_stopping": early_stopping, - "epsilon_cutoff": epsilon_cutoff, - "eta_cutoff": eta_cutoff, - "max_length": max_length, - "max_new_tokens": max_new_tokens, - "min_length": min_length, - "min_new_tokens": min_new_tokens, - "num_beam_groups": num_beam_groups, - "num_beams": num_beams, - "penalty_alpha": penalty_alpha, - "temperature": temperature, - "top_k": top_k, - "top_p": top_p, - "typical_p": typical_p, - "use_cache": use_cache, - **(extra_body or {}), - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - response = provider_helper.get_response(response) - return response - - async def token_classification( - self, - text: str, - *, - model: Optional[str] = None, - aggregation_strategy: Optional["TokenClassificationAggregationStrategy"] = None, - ignore_labels: Optional[List[str]] = None, - stride: Optional[int] = None, - ) -> List[TokenClassificationOutputElement]: - """ - Perform token classification on the given text. - Usually used for sentence parsing, either grammatical, or Named Entity Recognition (NER) to understand keywords contained within text. - - Args: - text (`str`): - A string to be classified. - model (`str`, *optional*): - The model to use for the token classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended token classification model will be used. - Defaults to None. - aggregation_strategy (`"TokenClassificationAggregationStrategy"`, *optional*): - The strategy used to fuse tokens based on model predictions - ignore_labels (`List[str`, *optional*): - A list of labels to ignore - stride (`int`, *optional*): - The number of overlapping tokens between chunks when splitting the input text. - - Returns: - `List[TokenClassificationOutputElement]`: List of [`TokenClassificationOutputElement`] items containing the entity group, confidence score, word, start and end index. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.token_classification("My name is Sarah Jessica Parker but you can call me Jessica") - [ - TokenClassificationOutputElement( - entity_group='PER', - score=0.9971321225166321, - word='Sarah Jessica Parker', - start=11, - end=31, - ), - TokenClassificationOutputElement( - entity_group='PER', - score=0.9773476123809814, - word='Jessica', - start=52, - end=59, - ) - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="token-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "aggregation_strategy": aggregation_strategy, - "ignore_labels": ignore_labels, - "stride": stride, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return TokenClassificationOutputElement.parse_obj_as_list(response) - - async def translation( - self, - text: str, - *, - model: Optional[str] = None, - src_lang: Optional[str] = None, - tgt_lang: Optional[str] = None, - clean_up_tokenization_spaces: Optional[bool] = None, - truncation: Optional["TranslationTruncationStrategy"] = None, - generate_parameters: Optional[Dict[str, Any]] = None, - ) -> TranslationOutput: - """ - Convert text from one language to another. - - Check out https://huggingface.co/tasks/translation for more information on how to choose the best model for - your specific use case. Source and target languages usually depend on the model. - However, it is possible to specify source and target languages for certain models. If you are working with one of these models, - you can use `src_lang` and `tgt_lang` arguments to pass the relevant information. - - Args: - text (`str`): - A string to be translated. - model (`str`, *optional*): - The model to use for the translation task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended translation model will be used. - Defaults to None. - src_lang (`str`, *optional*): - The source language of the text. Required for models that can translate from multiple languages. - tgt_lang (`str`, *optional*): - Target language to translate to. Required for models that can translate to multiple languages. - clean_up_tokenization_spaces (`bool`, *optional*): - Whether to clean up the potential extra spaces in the text output. - truncation (`"TranslationTruncationStrategy"`, *optional*): - The truncation strategy to use. - generate_parameters (`Dict[str, Any]`, *optional*): - Additional parametrization of the text generation algorithm. - - Returns: - [`TranslationOutput`]: The generated translated text. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - `ValueError`: - If only one of the `src_lang` and `tgt_lang` arguments are provided. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.translation("My name is Wolfgang and I live in Berlin") - 'Mein Name ist Wolfgang und ich lebe in Berlin.' - >>> await client.translation("My name is Wolfgang and I live in Berlin", model="Helsinki-NLP/opus-mt-en-fr") - TranslationOutput(translation_text='Je m'appelle Wolfgang et je vis à Berlin.') - ``` - - Specifying languages: - ```py - >>> client.translation("My name is Sarah Jessica Parker but you can call me Jessica", model="facebook/mbart-large-50-many-to-many-mmt", src_lang="en_XX", tgt_lang="fr_XX") - "Mon nom est Sarah Jessica Parker mais vous pouvez m'appeler Jessica" - ``` - """ - # Throw error if only one of `src_lang` and `tgt_lang` was given - if src_lang is not None and tgt_lang is None: - raise ValueError("You cannot specify `src_lang` without specifying `tgt_lang`.") - - if src_lang is None and tgt_lang is not None: - raise ValueError("You cannot specify `tgt_lang` without specifying `src_lang`.") - - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="translation", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "src_lang": src_lang, - "tgt_lang": tgt_lang, - "clean_up_tokenization_spaces": clean_up_tokenization_spaces, - "truncation": truncation, - "generate_parameters": generate_parameters, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return TranslationOutput.parse_obj_as_list(response)[0] - - async def visual_question_answering( - self, - image: ContentT, - question: str, - *, - model: Optional[str] = None, - top_k: Optional[int] = None, - ) -> List[VisualQuestionAnsweringOutputElement]: - """ - Answering open-ended questions based on an image. - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The input image for the context. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - question (`str`): - Question to be answered. - model (`str`, *optional*): - The model to use for the visual question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to - a deployed Inference Endpoint. If not provided, the default recommended visual question answering model will be used. - Defaults to None. - top_k (`int`, *optional*): - The number of answers to return (will be chosen by order of likelihood). Note that we return less than - topk answers if there are not enough options available within the context. - Returns: - `List[VisualQuestionAnsweringOutputElement]`: a list of [`VisualQuestionAnsweringOutputElement`] items containing the predicted label and associated probability. - - Raises: - `InferenceTimeoutError`: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.visual_question_answering( - ... image="https://huggingface.co/datasets/mishig/sample_images/resolve/main/tiger.jpg", - ... question="What is the animal doing?" - ... ) - [ - VisualQuestionAnsweringOutputElement(score=0.778609573841095, answer='laying down'), - VisualQuestionAnsweringOutputElement(score=0.6957435607910156, answer='sitting'), - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="visual-question-answering", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={"top_k": top_k}, - headers=self.headers, - model=model_id, - api_key=self.token, - extra_payload={"question": question, "image": _b64_encode(image)}, - ) - response = await self._inner_post(request_parameters) - return VisualQuestionAnsweringOutputElement.parse_obj_as_list(response) - - async def zero_shot_classification( - self, - text: str, - candidate_labels: List[str], - *, - multi_label: Optional[bool] = False, - hypothesis_template: Optional[str] = None, - model: Optional[str] = None, - ) -> List[ZeroShotClassificationOutputElement]: - """ - Provide as input a text and a set of candidate labels to classify the input text. - - Args: - text (`str`): - The input text to classify. - candidate_labels (`List[str]`): - The set of possible class labels to classify the text into. - labels (`List[str]`, *optional*): - (deprecated) List of strings. Each string is the verbalization of a possible label for the input text. - multi_label (`bool`, *optional*): - Whether multiple candidate labels can be true. If false, the scores are normalized such that the sum of - the label likelihoods for each sequence is 1. If true, the labels are considered independent and - probabilities are normalized for each candidate. - hypothesis_template (`str`, *optional*): - The sentence used in conjunction with `candidate_labels` to attempt the text classification by - replacing the placeholder with the candidate labels. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. If not provided, the default recommended zero-shot classification model will be used. - - - Returns: - `List[ZeroShotClassificationOutputElement]`: List of [`ZeroShotClassificationOutputElement`] items containing the predicted labels and their confidence. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example with `multi_label=False`: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> text = ( - ... "A new model offers an explanation for how the Galilean satellites formed around the solar system's" - ... "largest world. Konstantin Batygin did not set out to solve one of the solar system's most puzzling" - ... " mysteries when he went for a run up a hill in Nice, France." - ... ) - >>> labels = ["space & cosmos", "scientific discovery", "microbiology", "robots", "archeology"] - >>> await client.zero_shot_classification(text, labels) - [ - ZeroShotClassificationOutputElement(label='scientific discovery', score=0.7961668968200684), - ZeroShotClassificationOutputElement(label='space & cosmos', score=0.18570658564567566), - ZeroShotClassificationOutputElement(label='microbiology', score=0.00730885099619627), - ZeroShotClassificationOutputElement(label='archeology', score=0.006258360575884581), - ZeroShotClassificationOutputElement(label='robots', score=0.004559356719255447), - ] - >>> await client.zero_shot_classification(text, labels, multi_label=True) - [ - ZeroShotClassificationOutputElement(label='scientific discovery', score=0.9829297661781311), - ZeroShotClassificationOutputElement(label='space & cosmos', score=0.755190908908844), - ZeroShotClassificationOutputElement(label='microbiology', score=0.0005462635890580714), - ZeroShotClassificationOutputElement(label='archeology', score=0.00047131875180639327), - ZeroShotClassificationOutputElement(label='robots', score=0.00030448526376858354), - ] - ``` - - Example with `multi_label=True` and a custom `hypothesis_template`: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - >>> await client.zero_shot_classification( - ... text="I really like our dinner and I'm very happy. I don't like the weather though.", - ... labels=["positive", "negative", "pessimistic", "optimistic"], - ... multi_label=True, - ... hypothesis_template="This text is {} towards the weather" - ... ) - [ - ZeroShotClassificationOutputElement(label='negative', score=0.9231801629066467), - ZeroShotClassificationOutputElement(label='pessimistic', score=0.8760990500450134), - ZeroShotClassificationOutputElement(label='optimistic', score=0.0008674879791215062), - ZeroShotClassificationOutputElement(label='positive', score=0.0005250611575320363) - ] - ``` - """ - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="zero-shot-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=text, - parameters={ - "candidate_labels": candidate_labels, - "multi_label": multi_label, - "hypothesis_template": hypothesis_template, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - output = _bytes_to_dict(response) - return [ - ZeroShotClassificationOutputElement.parse_obj_as_instance({"label": label, "score": score}) - for label, score in zip(output["labels"], output["scores"]) - ] - - async def zero_shot_image_classification( - self, - image: ContentT, - candidate_labels: List[str], - *, - model: Optional[str] = None, - hypothesis_template: Optional[str] = None, - # deprecated argument - labels: List[str] = None, # type: ignore - ) -> List[ZeroShotImageClassificationOutputElement]: - """ - Provide input image and text labels to predict text labels for the image. - - Args: - image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): - The input image to caption. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. - candidate_labels (`List[str]`): - The candidate labels for this image - labels (`List[str]`, *optional*): - (deprecated) List of string possible labels. There must be at least 2 labels. - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. If not provided, the default recommended zero-shot image classification model will be used. - hypothesis_template (`str`, *optional*): - The sentence used in conjunction with `candidate_labels` to attempt the image classification by - replacing the placeholder with the candidate labels. - - Returns: - `List[ZeroShotImageClassificationOutputElement]`: List of [`ZeroShotImageClassificationOutputElement`] items containing the predicted labels and their confidence. - - Raises: - [`InferenceTimeoutError`]: - If the model is unavailable or the request times out. - `aiohttp.ClientResponseError`: - If the request fails with an HTTP error status code other than HTTP 503. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient() - - >>> await client.zero_shot_image_classification( - ... "https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg", - ... labels=["dog", "cat", "horse"], - ... ) - [ZeroShotImageClassificationOutputElement(label='dog', score=0.956),...] - ``` - """ - # Raise ValueError if input is less than 2 labels - if len(candidate_labels) < 2: - raise ValueError("You must specify at least 2 classes to compare.") - - model_id = model or self.model - provider_helper = get_provider_helper(self.provider, task="zero-shot-image-classification", model=model_id) - request_parameters = provider_helper.prepare_request( - inputs=image, - parameters={ - "candidate_labels": candidate_labels, - "hypothesis_template": hypothesis_template, - }, - headers=self.headers, - model=model_id, - api_key=self.token, - ) - response = await self._inner_post(request_parameters) - return ZeroShotImageClassificationOutputElement.parse_obj_as_list(response) - - def _get_client_session(self, headers: Optional[Dict] = None) -> "ClientSession": - aiohttp = _import_aiohttp() - client_headers = self.headers.copy() - if headers is not None: - client_headers.update(headers) - - # Return a new aiohttp ClientSession with correct settings. - session = aiohttp.ClientSession( - headers=client_headers, - cookies=self.cookies, - timeout=aiohttp.ClientTimeout(self.timeout), - trust_env=self.trust_env, - ) - - # Keep track of sessions to close them later - self._sessions[session] = set() - - # Override the `._request` method to register responses to be closed - session._wrapped_request = session._request - - async def _request(method, url, **kwargs): - response = await session._wrapped_request(method, url, **kwargs) - self._sessions[session].add(response) - return response - - session._request = _request - - # Override the 'close' method to - # 1. close ongoing responses - # 2. deregister the session when closed - session._close = session.close - - async def close_session(): - for response in self._sessions[session]: - response.close() - await session._close() - self._sessions.pop(session, None) - - session.close = close_session - return session - - async def get_endpoint_info(self, *, model: Optional[str] = None) -> Dict[str, Any]: - """ - Get information about the deployed endpoint. - - This endpoint is only available on endpoints powered by Text-Generation-Inference (TGI) or Text-Embedding-Inference (TEI). - Endpoints powered by `transformers` return an empty payload. - - Args: - model (`str`, *optional*): - The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed - Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - - Returns: - `Dict[str, Any]`: Information about the endpoint. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") - >>> await client.get_endpoint_info() - { - 'model_id': 'meta-llama/Meta-Llama-3-70B-Instruct', - 'model_sha': None, - 'model_dtype': 'torch.float16', - 'model_device_type': 'cuda', - 'model_pipeline_tag': None, - 'max_concurrent_requests': 128, - 'max_best_of': 2, - 'max_stop_sequences': 4, - 'max_input_length': 8191, - 'max_total_tokens': 8192, - 'waiting_served_ratio': 0.3, - 'max_batch_total_tokens': 1259392, - 'max_waiting_tokens': 20, - 'max_batch_size': None, - 'validation_workers': 32, - 'max_client_batch_size': 4, - 'version': '2.0.2', - 'sha': 'dccab72549635c7eb5ddb17f43f0b7cdff07c214', - 'docker_label': 'sha-dccab72' - } - ``` - """ - if self.provider != "hf-inference": - raise ValueError(f"Getting endpoint info is not supported on '{self.provider}'.") - - model = model or self.model - if model is None: - raise ValueError("Model id not provided.") - if model.startswith(("http://", "https://")): - url = model.rstrip("/") + "/info" - else: - url = f"{constants.INFERENCE_ENDPOINT}/models/{model}/info" - - async with self._get_client_session(headers=build_hf_headers(token=self.token)) as client: - response = await client.get(url, proxy=self.proxies) - response.raise_for_status() - return await response.json() - - async def health_check(self, model: Optional[str] = None) -> bool: - """ - Check the health of the deployed endpoint. - - Health check is only available with Inference Endpoints powered by Text-Generation-Inference (TGI) or Text-Embedding-Inference (TEI). - - Args: - model (`str`, *optional*): - URL of the Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. - - Returns: - `bool`: True if everything is working fine. - - Example: - ```py - # Must be run in an async context - >>> from huggingface_hub import AsyncInferenceClient - >>> client = AsyncInferenceClient("https://jzgu0buei5.us-east-1.aws.endpoints.huggingface.cloud") - >>> await client.health_check() - True - ``` - """ - if self.provider != "hf-inference": - raise ValueError(f"Health check is not supported on '{self.provider}'.") - - model = model or self.model - if model is None: - raise ValueError("Model id not provided.") - if not model.startswith(("http://", "https://")): - raise ValueError("Model must be an Inference Endpoint URL.") - url = model.rstrip("/") + "/health" - - async with self._get_client_session(headers=build_hf_headers(token=self.token)) as client: - response = await client.get(url, proxy=self.proxies) - return response.status == 200 - - @property - def chat(self) -> "ProxyClientChat": - return ProxyClientChat(self) - - -class _ProxyClient: - """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" - - def __init__(self, client: AsyncInferenceClient): - self._client = client - - -class ProxyClientChat(_ProxyClient): - """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" - - @property - def completions(self) -> "ProxyClientChatCompletions": - return ProxyClientChatCompletions(self._client) - - -class ProxyClientChatCompletions(_ProxyClient): - """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" - - @property - def create(self): - return self._client.chat_completion diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/__init__.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/__init__.py deleted file mode 100644 index bfffc0ae3bce71532382ee87d03c40dc376cfae7..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/__init__.py +++ /dev/null @@ -1,192 +0,0 @@ -# This file is auto-generated by `utils/generate_inference_types.py`. -# Do not modify it manually. -# -# ruff: noqa: F401 - -from .audio_classification import ( - AudioClassificationInput, - AudioClassificationOutputElement, - AudioClassificationOutputTransform, - AudioClassificationParameters, -) -from .audio_to_audio import AudioToAudioInput, AudioToAudioOutputElement -from .automatic_speech_recognition import ( - AutomaticSpeechRecognitionEarlyStoppingEnum, - AutomaticSpeechRecognitionGenerationParameters, - AutomaticSpeechRecognitionInput, - AutomaticSpeechRecognitionOutput, - AutomaticSpeechRecognitionOutputChunk, - AutomaticSpeechRecognitionParameters, -) -from .base import BaseInferenceType -from .chat_completion import ( - ChatCompletionInput, - ChatCompletionInputFunctionDefinition, - ChatCompletionInputFunctionName, - ChatCompletionInputGrammarType, - ChatCompletionInputJSONSchema, - ChatCompletionInputMessage, - ChatCompletionInputMessageChunk, - ChatCompletionInputMessageChunkType, - ChatCompletionInputResponseFormatJSONObject, - ChatCompletionInputResponseFormatJSONSchema, - ChatCompletionInputResponseFormatText, - ChatCompletionInputStreamOptions, - ChatCompletionInputTool, - ChatCompletionInputToolCall, - ChatCompletionInputToolChoiceClass, - ChatCompletionInputToolChoiceEnum, - ChatCompletionInputURL, - ChatCompletionOutput, - ChatCompletionOutputComplete, - ChatCompletionOutputFunctionDefinition, - ChatCompletionOutputLogprob, - ChatCompletionOutputLogprobs, - ChatCompletionOutputMessage, - ChatCompletionOutputToolCall, - ChatCompletionOutputTopLogprob, - ChatCompletionOutputUsage, - ChatCompletionStreamOutput, - ChatCompletionStreamOutputChoice, - ChatCompletionStreamOutputDelta, - ChatCompletionStreamOutputDeltaToolCall, - ChatCompletionStreamOutputFunction, - ChatCompletionStreamOutputLogprob, - ChatCompletionStreamOutputLogprobs, - ChatCompletionStreamOutputTopLogprob, - ChatCompletionStreamOutputUsage, -) -from .depth_estimation import DepthEstimationInput, DepthEstimationOutput -from .document_question_answering import ( - DocumentQuestionAnsweringInput, - DocumentQuestionAnsweringInputData, - DocumentQuestionAnsweringOutputElement, - DocumentQuestionAnsweringParameters, -) -from .feature_extraction import FeatureExtractionInput, FeatureExtractionInputTruncationDirection -from .fill_mask import FillMaskInput, FillMaskOutputElement, FillMaskParameters -from .image_classification import ( - ImageClassificationInput, - ImageClassificationOutputElement, - ImageClassificationOutputTransform, - ImageClassificationParameters, -) -from .image_segmentation import ( - ImageSegmentationInput, - ImageSegmentationOutputElement, - ImageSegmentationParameters, - ImageSegmentationSubtask, -) -from .image_to_image import ImageToImageInput, ImageToImageOutput, ImageToImageParameters, ImageToImageTargetSize -from .image_to_text import ( - ImageToTextEarlyStoppingEnum, - ImageToTextGenerationParameters, - ImageToTextInput, - ImageToTextOutput, - ImageToTextParameters, -) -from .image_to_video import ImageToVideoInput, ImageToVideoOutput, ImageToVideoParameters, ImageToVideoTargetSize -from .object_detection import ( - ObjectDetectionBoundingBox, - ObjectDetectionInput, - ObjectDetectionOutputElement, - ObjectDetectionParameters, -) -from .question_answering import ( - QuestionAnsweringInput, - QuestionAnsweringInputData, - QuestionAnsweringOutputElement, - QuestionAnsweringParameters, -) -from .sentence_similarity import SentenceSimilarityInput, SentenceSimilarityInputData -from .summarization import ( - SummarizationInput, - SummarizationOutput, - SummarizationParameters, - SummarizationTruncationStrategy, -) -from .table_question_answering import ( - Padding, - TableQuestionAnsweringInput, - TableQuestionAnsweringInputData, - TableQuestionAnsweringOutputElement, - TableQuestionAnsweringParameters, -) -from .text2text_generation import ( - Text2TextGenerationInput, - Text2TextGenerationOutput, - Text2TextGenerationParameters, - Text2TextGenerationTruncationStrategy, -) -from .text_classification import ( - TextClassificationInput, - TextClassificationOutputElement, - TextClassificationOutputTransform, - TextClassificationParameters, -) -from .text_generation import ( - TextGenerationInput, - TextGenerationInputGenerateParameters, - TextGenerationInputGrammarType, - TextGenerationOutput, - TextGenerationOutputBestOfSequence, - TextGenerationOutputDetails, - TextGenerationOutputFinishReason, - TextGenerationOutputPrefillToken, - TextGenerationOutputToken, - TextGenerationStreamOutput, - TextGenerationStreamOutputStreamDetails, - TextGenerationStreamOutputToken, - TypeEnum, -) -from .text_to_audio import ( - TextToAudioEarlyStoppingEnum, - TextToAudioGenerationParameters, - TextToAudioInput, - TextToAudioOutput, - TextToAudioParameters, -) -from .text_to_image import TextToImageInput, TextToImageOutput, TextToImageParameters -from .text_to_speech import ( - TextToSpeechEarlyStoppingEnum, - TextToSpeechGenerationParameters, - TextToSpeechInput, - TextToSpeechOutput, - TextToSpeechParameters, -) -from .text_to_video import TextToVideoInput, TextToVideoOutput, TextToVideoParameters -from .token_classification import ( - TokenClassificationAggregationStrategy, - TokenClassificationInput, - TokenClassificationOutputElement, - TokenClassificationParameters, -) -from .translation import TranslationInput, TranslationOutput, TranslationParameters, TranslationTruncationStrategy -from .video_classification import ( - VideoClassificationInput, - VideoClassificationOutputElement, - VideoClassificationOutputTransform, - VideoClassificationParameters, -) -from .visual_question_answering import ( - VisualQuestionAnsweringInput, - VisualQuestionAnsweringInputData, - VisualQuestionAnsweringOutputElement, - VisualQuestionAnsweringParameters, -) -from .zero_shot_classification import ( - ZeroShotClassificationInput, - ZeroShotClassificationOutputElement, - ZeroShotClassificationParameters, -) -from .zero_shot_image_classification import ( - ZeroShotImageClassificationInput, - ZeroShotImageClassificationOutputElement, - ZeroShotImageClassificationParameters, -) -from .zero_shot_object_detection import ( - ZeroShotObjectDetectionBoundingBox, - ZeroShotObjectDetectionInput, - ZeroShotObjectDetectionOutputElement, - ZeroShotObjectDetectionParameters, -) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/audio_classification.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/audio_classification.py deleted file mode 100644 index 053055787bce933e1fbd393cfbc00d81c43c8c2d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/audio_classification.py +++ /dev/null @@ -1,43 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Literal, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -AudioClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] - - -@dataclass_with_extra -class AudioClassificationParameters(BaseInferenceType): - """Additional inference parameters for Audio Classification""" - - function_to_apply: Optional["AudioClassificationOutputTransform"] = None - """The function to apply to the model outputs in order to retrieve the scores.""" - top_k: Optional[int] = None - """When specified, limits the output to the top K most probable classes.""" - - -@dataclass_with_extra -class AudioClassificationInput(BaseInferenceType): - """Inputs for Audio Classification inference""" - - inputs: str - """The input audio data as a base64-encoded string. If no `parameters` are provided, you can - also provide the audio data as a raw bytes payload. - """ - parameters: Optional[AudioClassificationParameters] = None - """Additional inference parameters for Audio Classification""" - - -@dataclass_with_extra -class AudioClassificationOutputElement(BaseInferenceType): - """Outputs for Audio Classification inference""" - - label: str - """The predicted class label.""" - score: float - """The corresponding probability.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/audio_to_audio.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/audio_to_audio.py deleted file mode 100644 index 43f376b5345fab6b854b028d1c17416c020d7bc1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/audio_to_audio.py +++ /dev/null @@ -1,30 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class AudioToAudioInput(BaseInferenceType): - """Inputs for Audio to Audio inference""" - - inputs: Any - """The input audio data""" - - -@dataclass_with_extra -class AudioToAudioOutputElement(BaseInferenceType): - """Outputs of inference for the Audio To Audio task - A generated audio file with its label. - """ - - blob: Any - """The generated audio file.""" - content_type: str - """The content type of audio file.""" - label: str - """The label of the audio file.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/automatic_speech_recognition.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/automatic_speech_recognition.py deleted file mode 100644 index f6bfd28256c82309b160f337aba5a54e2dd11872..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/automatic_speech_recognition.py +++ /dev/null @@ -1,113 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import List, Literal, Optional, Union - -from .base import BaseInferenceType, dataclass_with_extra - - -AutomaticSpeechRecognitionEarlyStoppingEnum = Literal["never"] - - -@dataclass_with_extra -class AutomaticSpeechRecognitionGenerationParameters(BaseInferenceType): - """Parametrization of the text generation process""" - - do_sample: Optional[bool] = None - """Whether to use sampling instead of greedy decoding when generating new tokens.""" - early_stopping: Optional[Union[bool, "AutomaticSpeechRecognitionEarlyStoppingEnum"]] = None - """Controls the stopping condition for beam-based methods.""" - epsilon_cutoff: Optional[float] = None - """If set to float strictly between 0 and 1, only tokens with a conditional probability - greater than epsilon_cutoff will be sampled. In the paper, suggested values range from - 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language - Model Desmoothing](https://hf.co/papers/2210.15191) for more details. - """ - eta_cutoff: Optional[float] = None - """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to - float strictly between 0 and 1, a token is only considered if it is greater than either - eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter - term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In - the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. - See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) - for more details. - """ - max_length: Optional[int] = None - """The maximum length (in tokens) of the generated text, including the input.""" - max_new_tokens: Optional[int] = None - """The maximum number of tokens to generate. Takes precedence over max_length.""" - min_length: Optional[int] = None - """The minimum length (in tokens) of the generated text, including the input.""" - min_new_tokens: Optional[int] = None - """The minimum number of tokens to generate. Takes precedence over min_length.""" - num_beam_groups: Optional[int] = None - """Number of groups to divide num_beams into in order to ensure diversity among different - groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. - """ - num_beams: Optional[int] = None - """Number of beams to use for beam search.""" - penalty_alpha: Optional[float] = None - """The value balances the model confidence and the degeneration penalty in contrastive - search decoding. - """ - temperature: Optional[float] = None - """The value used to modulate the next token probabilities.""" - top_k: Optional[int] = None - """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" - top_p: Optional[float] = None - """If set to float < 1, only the smallest set of most probable tokens with probabilities - that add up to top_p or higher are kept for generation. - """ - typical_p: Optional[float] = None - """Local typicality measures how similar the conditional probability of predicting a target - token next is to the expected conditional probability of predicting a random token next, - given the partial text already generated. If set to float < 1, the smallest set of the - most locally typical tokens with probabilities that add up to typical_p or higher are - kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. - """ - use_cache: Optional[bool] = None - """Whether the model should use the past last key/values attentions to speed up decoding""" - - -@dataclass_with_extra -class AutomaticSpeechRecognitionParameters(BaseInferenceType): - """Additional inference parameters for Automatic Speech Recognition""" - - generation_parameters: Optional[AutomaticSpeechRecognitionGenerationParameters] = None - """Parametrization of the text generation process""" - return_timestamps: Optional[bool] = None - """Whether to output corresponding timestamps with the generated text""" - - -@dataclass_with_extra -class AutomaticSpeechRecognitionInput(BaseInferenceType): - """Inputs for Automatic Speech Recognition inference""" - - inputs: str - """The input audio data as a base64-encoded string. If no `parameters` are provided, you can - also provide the audio data as a raw bytes payload. - """ - parameters: Optional[AutomaticSpeechRecognitionParameters] = None - """Additional inference parameters for Automatic Speech Recognition""" - - -@dataclass_with_extra -class AutomaticSpeechRecognitionOutputChunk(BaseInferenceType): - text: str - """A chunk of text identified by the model""" - timestamp: List[float] - """The start and end timestamps corresponding with the text""" - - -@dataclass_with_extra -class AutomaticSpeechRecognitionOutput(BaseInferenceType): - """Outputs of inference for the Automatic Speech Recognition task""" - - text: str - """The recognized text.""" - chunks: Optional[List[AutomaticSpeechRecognitionOutputChunk]] = None - """When returnTimestamps is enabled, chunks contains a list of audio chunks identified by - the model. - """ diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/base.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/base.py deleted file mode 100644 index 1f0c4687ceccbfb738da3f38c583c2516d065a01..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/base.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright 2024 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains a base class for all inference types.""" - -import inspect -import json -from dataclasses import asdict, dataclass -from typing import Any, Dict, List, Type, TypeVar, Union, get_args - - -T = TypeVar("T", bound="BaseInferenceType") - - -def _repr_with_extra(self): - fields = list(self.__dataclass_fields__.keys()) - other_fields = list(k for k in self.__dict__ if k not in fields) - return f"{self.__class__.__name__}({', '.join(f'{k}={self.__dict__[k]!r}' for k in fields + other_fields)})" - - -def dataclass_with_extra(cls: Type[T]) -> Type[T]: - """Decorator to add a custom __repr__ method to a dataclass, showing all fields, including extra ones. - - This decorator only works with dataclasses that inherit from `BaseInferenceType`. - """ - cls = dataclass(cls) - cls.__repr__ = _repr_with_extra # type: ignore[method-assign] - return cls - - -@dataclass -class BaseInferenceType(dict): - """Base class for all inference types. - - Object is a dataclass and a dict for backward compatibility but plan is to remove the dict part in the future. - - Handle parsing from dict, list and json strings in a permissive way to ensure future-compatibility (e.g. all fields - are made optional, and non-expected fields are added as dict attributes). - """ - - @classmethod - def parse_obj_as_list(cls: Type[T], data: Union[bytes, str, List, Dict]) -> List[T]: - """Alias to parse server response and return a single instance. - - See `parse_obj` for more details. - """ - output = cls.parse_obj(data) - if not isinstance(output, list): - raise ValueError(f"Invalid input data for {cls}. Expected a list, but got {type(output)}.") - return output - - @classmethod - def parse_obj_as_instance(cls: Type[T], data: Union[bytes, str, List, Dict]) -> T: - """Alias to parse server response and return a single instance. - - See `parse_obj` for more details. - """ - output = cls.parse_obj(data) - if isinstance(output, list): - raise ValueError(f"Invalid input data for {cls}. Expected a single instance, but got a list.") - return output - - @classmethod - def parse_obj(cls: Type[T], data: Union[bytes, str, List, Dict]) -> Union[List[T], T]: - """Parse server response as a dataclass or list of dataclasses. - - To enable future-compatibility, we want to handle cases where the server return more fields than expected. - In such cases, we don't want to raise an error but still create the dataclass object. Remaining fields are - added as dict attributes. - """ - # Parse server response (from bytes) - if isinstance(data, bytes): - data = data.decode() - if isinstance(data, str): - data = json.loads(data) - - # If a list, parse each item individually - if isinstance(data, List): - return [cls.parse_obj(d) for d in data] # type: ignore [misc] - - # At this point, we expect a dict - if not isinstance(data, dict): - raise ValueError(f"Invalid data type: {type(data)}") - - init_values = {} - other_values = {} - for key, value in data.items(): - key = normalize_key(key) - if key in cls.__dataclass_fields__ and cls.__dataclass_fields__[key].init: - if isinstance(value, dict) or isinstance(value, list): - field_type = cls.__dataclass_fields__[key].type - - # if `field_type` is a `BaseInferenceType`, parse it - if inspect.isclass(field_type) and issubclass(field_type, BaseInferenceType): - value = field_type.parse_obj(value) - - # otherwise, recursively parse nested dataclasses (if possible) - # `get_args` returns handle Union and Optional for us - else: - expected_types = get_args(field_type) - for expected_type in expected_types: - if getattr(expected_type, "_name", None) == "List": - expected_type = get_args(expected_type)[ - 0 - ] # assume same type for all items in the list - if inspect.isclass(expected_type) and issubclass(expected_type, BaseInferenceType): - value = expected_type.parse_obj(value) - break - init_values[key] = value - else: - other_values[key] = value - - # Make all missing fields default to None - # => ensure that dataclass initialization will never fail even if the server does not return all fields. - for key in cls.__dataclass_fields__: - if key not in init_values: - init_values[key] = None - - # Initialize dataclass with expected values - item = cls(**init_values) - - # Add remaining fields as dict attributes - item.update(other_values) - - # Add remaining fields as extra dataclass fields. - # They won't be part of the dataclass fields but will be accessible as attributes. - # Use @dataclass_with_extra to show them in __repr__. - item.__dict__.update(other_values) - return item - - def __post_init__(self): - self.update(asdict(self)) - - def __setitem__(self, __key: Any, __value: Any) -> None: - # Hacky way to keep dataclass values in sync when dict is updated - super().__setitem__(__key, __value) - if __key in self.__dataclass_fields__ and getattr(self, __key, None) != __value: - self.__setattr__(__key, __value) - return - - def __setattr__(self, __name: str, __value: Any) -> None: - # Hacky way to keep dict values is sync when dataclass is updated - super().__setattr__(__name, __value) - if self.get(__name) != __value: - self[__name] = __value - return - - -def normalize_key(key: str) -> str: - # e.g "content-type" -> "content_type", "Accept" -> "accept" - return key.replace("-", "_").replace(" ", "_").lower() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/chat_completion.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/chat_completion.py deleted file mode 100644 index ba708a7009bf14cd182a999ccf95f07ee2a002b8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/chat_completion.py +++ /dev/null @@ -1,347 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Dict, List, Literal, Optional, Union - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class ChatCompletionInputURL(BaseInferenceType): - url: str - - -ChatCompletionInputMessageChunkType = Literal["text", "image_url"] - - -@dataclass_with_extra -class ChatCompletionInputMessageChunk(BaseInferenceType): - type: "ChatCompletionInputMessageChunkType" - image_url: Optional[ChatCompletionInputURL] = None - text: Optional[str] = None - - -@dataclass_with_extra -class ChatCompletionInputFunctionDefinition(BaseInferenceType): - name: str - parameters: Any - description: Optional[str] = None - - -@dataclass_with_extra -class ChatCompletionInputToolCall(BaseInferenceType): - function: ChatCompletionInputFunctionDefinition - id: str - type: str - - -@dataclass_with_extra -class ChatCompletionInputMessage(BaseInferenceType): - role: str - content: Optional[Union[List[ChatCompletionInputMessageChunk], str]] = None - name: Optional[str] = None - tool_calls: Optional[List[ChatCompletionInputToolCall]] = None - - -@dataclass_with_extra -class ChatCompletionInputJSONSchema(BaseInferenceType): - name: str - """ - The name of the response format. - """ - description: Optional[str] = None - """ - A description of what the response format is for, used by the model to determine - how to respond in the format. - """ - schema: Optional[Dict[str, object]] = None - """ - The schema for the response format, described as a JSON Schema object. Learn how - to build JSON schemas [here](https://json-schema.org/). - """ - strict: Optional[bool] = None - """ - Whether to enable strict schema adherence when generating the output. If set to - true, the model will always follow the exact schema defined in the `schema` - field. - """ - - -@dataclass_with_extra -class ChatCompletionInputResponseFormatText(BaseInferenceType): - type: Literal["text"] - - -@dataclass_with_extra -class ChatCompletionInputResponseFormatJSONSchema(BaseInferenceType): - type: Literal["json_schema"] - json_schema: ChatCompletionInputJSONSchema - - -@dataclass_with_extra -class ChatCompletionInputResponseFormatJSONObject(BaseInferenceType): - type: Literal["json_object"] - - -ChatCompletionInputGrammarType = Union[ - ChatCompletionInputResponseFormatText, - ChatCompletionInputResponseFormatJSONSchema, - ChatCompletionInputResponseFormatJSONObject, -] - - -@dataclass_with_extra -class ChatCompletionInputStreamOptions(BaseInferenceType): - include_usage: Optional[bool] = None - """If set, an additional chunk will be streamed before the data: [DONE] message. The usage - field on this chunk shows the token usage statistics for the entire request, and the - choices field will always be an empty array. All other chunks will also include a usage - field, but with a null value. - """ - - -@dataclass_with_extra -class ChatCompletionInputFunctionName(BaseInferenceType): - name: str - - -@dataclass_with_extra -class ChatCompletionInputToolChoiceClass(BaseInferenceType): - function: ChatCompletionInputFunctionName - - -ChatCompletionInputToolChoiceEnum = Literal["auto", "none", "required"] - - -@dataclass_with_extra -class ChatCompletionInputTool(BaseInferenceType): - function: ChatCompletionInputFunctionDefinition - type: str - - -@dataclass_with_extra -class ChatCompletionInput(BaseInferenceType): - """Chat Completion Input. - Auto-generated from TGI specs. - For more details, check out - https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. - """ - - messages: List[ChatCompletionInputMessage] - """A list of messages comprising the conversation so far.""" - frequency_penalty: Optional[float] = None - """Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing - frequency in the text so far, - decreasing the model's likelihood to repeat the same line verbatim. - """ - logit_bias: Optional[List[float]] = None - """UNUSED - Modify the likelihood of specified tokens appearing in the completion. Accepts a JSON - object that maps tokens - (specified by their token ID in the tokenizer) to an associated bias value from -100 to - 100. Mathematically, - the bias is added to the logits generated by the model prior to sampling. The exact - effect will vary per model, - but values between -1 and 1 should decrease or increase likelihood of selection; values - like -100 or 100 should - result in a ban or exclusive selection of the relevant token. - """ - logprobs: Optional[bool] = None - """Whether to return log probabilities of the output tokens or not. If true, returns the log - probabilities of each - output token returned in the content of message. - """ - max_tokens: Optional[int] = None - """The maximum number of tokens that can be generated in the chat completion.""" - model: Optional[str] = None - """[UNUSED] ID of the model to use. See the model endpoint compatibility table for details - on which models work with the Chat API. - """ - n: Optional[int] = None - """UNUSED - How many chat completion choices to generate for each input message. Note that you will - be charged based on the - number of generated tokens across all of the choices. Keep n as 1 to minimize costs. - """ - presence_penalty: Optional[float] = None - """Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they - appear in the text so far, - increasing the model's likelihood to talk about new topics - """ - response_format: Optional[ChatCompletionInputGrammarType] = None - seed: Optional[int] = None - stop: Optional[List[str]] = None - """Up to 4 sequences where the API will stop generating further tokens.""" - stream: Optional[bool] = None - stream_options: Optional[ChatCompletionInputStreamOptions] = None - temperature: Optional[float] = None - """What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the - output more random, while - lower values like 0.2 will make it more focused and deterministic. - We generally recommend altering this or `top_p` but not both. - """ - tool_choice: Optional[Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"]] = None - tool_prompt: Optional[str] = None - """A prompt to be appended before the tools""" - tools: Optional[List[ChatCompletionInputTool]] = None - """A list of tools the model may call. Currently, only functions are supported as a tool. - Use this to provide a list of - functions the model may generate JSON inputs for. - """ - top_logprobs: Optional[int] = None - """An integer between 0 and 5 specifying the number of most likely tokens to return at each - token position, each with - an associated log probability. logprobs must be set to true if this parameter is used. - """ - top_p: Optional[float] = None - """An alternative to sampling with temperature, called nucleus sampling, where the model - considers the results of the - tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% - probability mass are considered. - """ - - -@dataclass_with_extra -class ChatCompletionOutputTopLogprob(BaseInferenceType): - logprob: float - token: str - - -@dataclass_with_extra -class ChatCompletionOutputLogprob(BaseInferenceType): - logprob: float - token: str - top_logprobs: List[ChatCompletionOutputTopLogprob] - - -@dataclass_with_extra -class ChatCompletionOutputLogprobs(BaseInferenceType): - content: List[ChatCompletionOutputLogprob] - - -@dataclass_with_extra -class ChatCompletionOutputFunctionDefinition(BaseInferenceType): - arguments: str - name: str - description: Optional[str] = None - - -@dataclass_with_extra -class ChatCompletionOutputToolCall(BaseInferenceType): - function: ChatCompletionOutputFunctionDefinition - id: str - type: str - - -@dataclass_with_extra -class ChatCompletionOutputMessage(BaseInferenceType): - role: str - content: Optional[str] = None - reasoning: Optional[str] = None - tool_call_id: Optional[str] = None - tool_calls: Optional[List[ChatCompletionOutputToolCall]] = None - - -@dataclass_with_extra -class ChatCompletionOutputComplete(BaseInferenceType): - finish_reason: str - index: int - message: ChatCompletionOutputMessage - logprobs: Optional[ChatCompletionOutputLogprobs] = None - - -@dataclass_with_extra -class ChatCompletionOutputUsage(BaseInferenceType): - completion_tokens: int - prompt_tokens: int - total_tokens: int - - -@dataclass_with_extra -class ChatCompletionOutput(BaseInferenceType): - """Chat Completion Output. - Auto-generated from TGI specs. - For more details, check out - https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. - """ - - choices: List[ChatCompletionOutputComplete] - created: int - id: str - model: str - system_fingerprint: str - usage: ChatCompletionOutputUsage - - -@dataclass_with_extra -class ChatCompletionStreamOutputFunction(BaseInferenceType): - arguments: str - name: Optional[str] = None - - -@dataclass_with_extra -class ChatCompletionStreamOutputDeltaToolCall(BaseInferenceType): - function: ChatCompletionStreamOutputFunction - id: str - index: int - type: str - - -@dataclass_with_extra -class ChatCompletionStreamOutputDelta(BaseInferenceType): - role: str - content: Optional[str] = None - reasoning: Optional[str] = None - tool_call_id: Optional[str] = None - tool_calls: Optional[List[ChatCompletionStreamOutputDeltaToolCall]] = None - - -@dataclass_with_extra -class ChatCompletionStreamOutputTopLogprob(BaseInferenceType): - logprob: float - token: str - - -@dataclass_with_extra -class ChatCompletionStreamOutputLogprob(BaseInferenceType): - logprob: float - token: str - top_logprobs: List[ChatCompletionStreamOutputTopLogprob] - - -@dataclass_with_extra -class ChatCompletionStreamOutputLogprobs(BaseInferenceType): - content: List[ChatCompletionStreamOutputLogprob] - - -@dataclass_with_extra -class ChatCompletionStreamOutputChoice(BaseInferenceType): - delta: ChatCompletionStreamOutputDelta - index: int - finish_reason: Optional[str] = None - logprobs: Optional[ChatCompletionStreamOutputLogprobs] = None - - -@dataclass_with_extra -class ChatCompletionStreamOutputUsage(BaseInferenceType): - completion_tokens: int - prompt_tokens: int - total_tokens: int - - -@dataclass_with_extra -class ChatCompletionStreamOutput(BaseInferenceType): - """Chat Completion Stream Output. - Auto-generated from TGI specs. - For more details, check out - https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. - """ - - choices: List[ChatCompletionStreamOutputChoice] - created: int - id: str - model: str - system_fingerprint: str - usage: Optional[ChatCompletionStreamOutputUsage] = None diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/depth_estimation.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/depth_estimation.py deleted file mode 100644 index 1e09bdffa194f97444e484de6e930f67ac030207..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/depth_estimation.py +++ /dev/null @@ -1,28 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Dict, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class DepthEstimationInput(BaseInferenceType): - """Inputs for Depth Estimation inference""" - - inputs: Any - """The input image data""" - parameters: Optional[Dict[str, Any]] = None - """Additional inference parameters for Depth Estimation""" - - -@dataclass_with_extra -class DepthEstimationOutput(BaseInferenceType): - """Outputs of inference for the Depth Estimation task""" - - depth: Any - """The predicted depth as an image""" - predicted_depth: Any - """The predicted depth as a tensor""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/document_question_answering.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/document_question_answering.py deleted file mode 100644 index 2457d2c8c237f055f660e0e8291d846bb036949d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/document_question_answering.py +++ /dev/null @@ -1,80 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, List, Optional, Union - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class DocumentQuestionAnsweringInputData(BaseInferenceType): - """One (document, question) pair to answer""" - - image: Any - """The image on which the question is asked""" - question: str - """A question to ask of the document""" - - -@dataclass_with_extra -class DocumentQuestionAnsweringParameters(BaseInferenceType): - """Additional inference parameters for Document Question Answering""" - - doc_stride: Optional[int] = None - """If the words in the document are too long to fit with the question for the model, it will - be split in several chunks with some overlap. This argument controls the size of that - overlap. - """ - handle_impossible_answer: Optional[bool] = None - """Whether to accept impossible as an answer""" - lang: Optional[str] = None - """Language to use while running OCR. Defaults to english.""" - max_answer_len: Optional[int] = None - """The maximum length of predicted answers (e.g., only answers with a shorter length are - considered). - """ - max_question_len: Optional[int] = None - """The maximum length of the question after tokenization. It will be truncated if needed.""" - max_seq_len: Optional[int] = None - """The maximum length of the total sentence (context + question) in tokens of each chunk - passed to the model. The context will be split in several chunks (using doc_stride as - overlap) if needed. - """ - top_k: Optional[int] = None - """The number of answers to return (will be chosen by order of likelihood). Can return less - than top_k answers if there are not enough options available within the context. - """ - word_boxes: Optional[List[Union[List[float], str]]] = None - """A list of words and bounding boxes (normalized 0->1000). If provided, the inference will - skip the OCR step and use the provided bounding boxes instead. - """ - - -@dataclass_with_extra -class DocumentQuestionAnsweringInput(BaseInferenceType): - """Inputs for Document Question Answering inference""" - - inputs: DocumentQuestionAnsweringInputData - """One (document, question) pair to answer""" - parameters: Optional[DocumentQuestionAnsweringParameters] = None - """Additional inference parameters for Document Question Answering""" - - -@dataclass_with_extra -class DocumentQuestionAnsweringOutputElement(BaseInferenceType): - """Outputs of inference for the Document Question Answering task""" - - answer: str - """The answer to the question.""" - end: int - """The end word index of the answer (in the OCR’d version of the input or provided word - boxes). - """ - score: float - """The probability associated to the answer.""" - start: int - """The start word index of the answer (in the OCR’d version of the input or provided word - boxes). - """ diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/feature_extraction.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/feature_extraction.py deleted file mode 100644 index e965ddbac2af0a5bf73e662a7c18c847611d18a1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/feature_extraction.py +++ /dev/null @@ -1,36 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import List, Literal, Optional, Union - -from .base import BaseInferenceType, dataclass_with_extra - - -FeatureExtractionInputTruncationDirection = Literal["Left", "Right"] - - -@dataclass_with_extra -class FeatureExtractionInput(BaseInferenceType): - """Feature Extraction Input. - Auto-generated from TEI specs. - For more details, check out - https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts. - """ - - inputs: Union[List[str], str] - """The text or list of texts to embed.""" - normalize: Optional[bool] = None - prompt_name: Optional[str] = None - """The name of the prompt that should be used by for encoding. If not set, no prompt - will be applied. - Must be a key in the `sentence-transformers` configuration `prompts` dictionary. - For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ", - ...}, - then the sentence "What is the capital of France?" will be encoded as - "query: What is the capital of France?" because the prompt text will be prepended before - any text to encode. - """ - truncate: Optional[bool] = None - truncation_direction: Optional["FeatureExtractionInputTruncationDirection"] = None diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/fill_mask.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/fill_mask.py deleted file mode 100644 index dfcdc56bc507e50280d38e0f63b024ada6a7ea94..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/fill_mask.py +++ /dev/null @@ -1,47 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, List, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class FillMaskParameters(BaseInferenceType): - """Additional inference parameters for Fill Mask""" - - targets: Optional[List[str]] = None - """When passed, the model will limit the scores to the passed targets instead of looking up - in the whole vocabulary. If the provided targets are not in the model vocab, they will be - tokenized and the first resulting token will be used (with a warning, and that might be - slower). - """ - top_k: Optional[int] = None - """When passed, overrides the number of predictions to return.""" - - -@dataclass_with_extra -class FillMaskInput(BaseInferenceType): - """Inputs for Fill Mask inference""" - - inputs: str - """The text with masked tokens""" - parameters: Optional[FillMaskParameters] = None - """Additional inference parameters for Fill Mask""" - - -@dataclass_with_extra -class FillMaskOutputElement(BaseInferenceType): - """Outputs of inference for the Fill Mask task""" - - score: float - """The corresponding probability""" - sequence: str - """The corresponding input with the mask token prediction.""" - token: int - """The predicted token id (to replace the masked one).""" - token_str: Any - fill_mask_output_token_str: Optional[str] = None - """The predicted token (to replace the masked one).""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_classification.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_classification.py deleted file mode 100644 index 0fdda6c83ff4c7aee5dc7794f0530e89d6b43047..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_classification.py +++ /dev/null @@ -1,43 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Literal, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -ImageClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] - - -@dataclass_with_extra -class ImageClassificationParameters(BaseInferenceType): - """Additional inference parameters for Image Classification""" - - function_to_apply: Optional["ImageClassificationOutputTransform"] = None - """The function to apply to the model outputs in order to retrieve the scores.""" - top_k: Optional[int] = None - """When specified, limits the output to the top K most probable classes.""" - - -@dataclass_with_extra -class ImageClassificationInput(BaseInferenceType): - """Inputs for Image Classification inference""" - - inputs: str - """The input image data as a base64-encoded string. If no `parameters` are provided, you can - also provide the image data as a raw bytes payload. - """ - parameters: Optional[ImageClassificationParameters] = None - """Additional inference parameters for Image Classification""" - - -@dataclass_with_extra -class ImageClassificationOutputElement(BaseInferenceType): - """Outputs of inference for the Image Classification task""" - - label: str - """The predicted class label.""" - score: float - """The corresponding probability.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_segmentation.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_segmentation.py deleted file mode 100644 index 3dbf61db83ec2ae6ceafd901c4425567cd2e5b03..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_segmentation.py +++ /dev/null @@ -1,51 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Literal, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -ImageSegmentationSubtask = Literal["instance", "panoptic", "semantic"] - - -@dataclass_with_extra -class ImageSegmentationParameters(BaseInferenceType): - """Additional inference parameters for Image Segmentation""" - - mask_threshold: Optional[float] = None - """Threshold to use when turning the predicted masks into binary values.""" - overlap_mask_area_threshold: Optional[float] = None - """Mask overlap threshold to eliminate small, disconnected segments.""" - subtask: Optional["ImageSegmentationSubtask"] = None - """Segmentation task to be performed, depending on model capabilities.""" - threshold: Optional[float] = None - """Probability threshold to filter out predicted masks.""" - - -@dataclass_with_extra -class ImageSegmentationInput(BaseInferenceType): - """Inputs for Image Segmentation inference""" - - inputs: str - """The input image data as a base64-encoded string. If no `parameters` are provided, you can - also provide the image data as a raw bytes payload. - """ - parameters: Optional[ImageSegmentationParameters] = None - """Additional inference parameters for Image Segmentation""" - - -@dataclass_with_extra -class ImageSegmentationOutputElement(BaseInferenceType): - """Outputs of inference for the Image Segmentation task - A predicted mask / segment - """ - - label: str - """The label of the predicted segment.""" - mask: str - """The corresponding mask as a black-and-white image (base64-encoded).""" - score: Optional[float] = None - """The score or confidence degree the model has.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_to_image.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_to_image.py deleted file mode 100644 index b14c79fedf228bb66fa88327c6d2601e77b8d6c6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_to_image.py +++ /dev/null @@ -1,60 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class ImageToImageTargetSize(BaseInferenceType): - """The size in pixels of the output image. This parameter is only supported by some - providers and for specific models. It will be ignored when unsupported. - """ - - height: int - width: int - - -@dataclass_with_extra -class ImageToImageParameters(BaseInferenceType): - """Additional inference parameters for Image To Image""" - - guidance_scale: Optional[float] = None - """For diffusion models. A higher guidance scale value encourages the model to generate - images closely linked to the text prompt at the expense of lower image quality. - """ - negative_prompt: Optional[str] = None - """One prompt to guide what NOT to include in image generation.""" - num_inference_steps: Optional[int] = None - """For diffusion models. The number of denoising steps. More denoising steps usually lead to - a higher quality image at the expense of slower inference. - """ - prompt: Optional[str] = None - """The text prompt to guide the image generation.""" - target_size: Optional[ImageToImageTargetSize] = None - """The size in pixels of the output image. This parameter is only supported by some - providers and for specific models. It will be ignored when unsupported. - """ - - -@dataclass_with_extra -class ImageToImageInput(BaseInferenceType): - """Inputs for Image To Image inference""" - - inputs: str - """The input image data as a base64-encoded string. If no `parameters` are provided, you can - also provide the image data as a raw bytes payload. - """ - parameters: Optional[ImageToImageParameters] = None - """Additional inference parameters for Image To Image""" - - -@dataclass_with_extra -class ImageToImageOutput(BaseInferenceType): - """Outputs of inference for the Image To Image task""" - - image: Any - """The output image returned as raw bytes in the payload.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_to_text.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_to_text.py deleted file mode 100644 index b65e0e0068e80dbcab5a4706fb5d49be2538c4ca..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_to_text.py +++ /dev/null @@ -1,100 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Literal, Optional, Union - -from .base import BaseInferenceType, dataclass_with_extra - - -ImageToTextEarlyStoppingEnum = Literal["never"] - - -@dataclass_with_extra -class ImageToTextGenerationParameters(BaseInferenceType): - """Parametrization of the text generation process""" - - do_sample: Optional[bool] = None - """Whether to use sampling instead of greedy decoding when generating new tokens.""" - early_stopping: Optional[Union[bool, "ImageToTextEarlyStoppingEnum"]] = None - """Controls the stopping condition for beam-based methods.""" - epsilon_cutoff: Optional[float] = None - """If set to float strictly between 0 and 1, only tokens with a conditional probability - greater than epsilon_cutoff will be sampled. In the paper, suggested values range from - 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language - Model Desmoothing](https://hf.co/papers/2210.15191) for more details. - """ - eta_cutoff: Optional[float] = None - """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to - float strictly between 0 and 1, a token is only considered if it is greater than either - eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter - term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In - the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. - See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) - for more details. - """ - max_length: Optional[int] = None - """The maximum length (in tokens) of the generated text, including the input.""" - max_new_tokens: Optional[int] = None - """The maximum number of tokens to generate. Takes precedence over max_length.""" - min_length: Optional[int] = None - """The minimum length (in tokens) of the generated text, including the input.""" - min_new_tokens: Optional[int] = None - """The minimum number of tokens to generate. Takes precedence over min_length.""" - num_beam_groups: Optional[int] = None - """Number of groups to divide num_beams into in order to ensure diversity among different - groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. - """ - num_beams: Optional[int] = None - """Number of beams to use for beam search.""" - penalty_alpha: Optional[float] = None - """The value balances the model confidence and the degeneration penalty in contrastive - search decoding. - """ - temperature: Optional[float] = None - """The value used to modulate the next token probabilities.""" - top_k: Optional[int] = None - """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" - top_p: Optional[float] = None - """If set to float < 1, only the smallest set of most probable tokens with probabilities - that add up to top_p or higher are kept for generation. - """ - typical_p: Optional[float] = None - """Local typicality measures how similar the conditional probability of predicting a target - token next is to the expected conditional probability of predicting a random token next, - given the partial text already generated. If set to float < 1, the smallest set of the - most locally typical tokens with probabilities that add up to typical_p or higher are - kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. - """ - use_cache: Optional[bool] = None - """Whether the model should use the past last key/values attentions to speed up decoding""" - - -@dataclass_with_extra -class ImageToTextParameters(BaseInferenceType): - """Additional inference parameters for Image To Text""" - - generation_parameters: Optional[ImageToTextGenerationParameters] = None - """Parametrization of the text generation process""" - max_new_tokens: Optional[int] = None - """The amount of maximum tokens to generate.""" - - -@dataclass_with_extra -class ImageToTextInput(BaseInferenceType): - """Inputs for Image To Text inference""" - - inputs: Any - """The input image data""" - parameters: Optional[ImageToTextParameters] = None - """Additional inference parameters for Image To Text""" - - -@dataclass_with_extra -class ImageToTextOutput(BaseInferenceType): - """Outputs of inference for the Image To Text task""" - - generated_text: Any - image_to_text_output_generated_text: Optional[str] = None - """The generated text.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_to_video.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_to_video.py deleted file mode 100644 index 92192a2a05b7a825c6dd55e96702fece0f3b3316..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/image_to_video.py +++ /dev/null @@ -1,60 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class ImageToVideoTargetSize(BaseInferenceType): - """The size in pixel of the output video frames.""" - - height: int - width: int - - -@dataclass_with_extra -class ImageToVideoParameters(BaseInferenceType): - """Additional inference parameters for Image To Video""" - - guidance_scale: Optional[float] = None - """For diffusion models. A higher guidance scale value encourages the model to generate - videos closely linked to the text prompt at the expense of lower image quality. - """ - negative_prompt: Optional[str] = None - """One prompt to guide what NOT to include in video generation.""" - num_frames: Optional[float] = None - """The num_frames parameter determines how many video frames are generated.""" - num_inference_steps: Optional[int] = None - """The number of denoising steps. More denoising steps usually lead to a higher quality - video at the expense of slower inference. - """ - prompt: Optional[str] = None - """The text prompt to guide the video generation.""" - seed: Optional[int] = None - """Seed for the random number generator.""" - target_size: Optional[ImageToVideoTargetSize] = None - """The size in pixel of the output video frames.""" - - -@dataclass_with_extra -class ImageToVideoInput(BaseInferenceType): - """Inputs for Image To Video inference""" - - inputs: str - """The input image data as a base64-encoded string. If no `parameters` are provided, you can - also provide the image data as a raw bytes payload. - """ - parameters: Optional[ImageToVideoParameters] = None - """Additional inference parameters for Image To Video""" - - -@dataclass_with_extra -class ImageToVideoOutput(BaseInferenceType): - """Outputs of inference for the Image To Video task""" - - video: Any - """The generated video returned as raw bytes in the payload.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/object_detection.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/object_detection.py deleted file mode 100644 index 75f3ebcfe1199462d0df60879b5ba6e517f7001e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/object_detection.py +++ /dev/null @@ -1,58 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class ObjectDetectionParameters(BaseInferenceType): - """Additional inference parameters for Object Detection""" - - threshold: Optional[float] = None - """The probability necessary to make a prediction.""" - - -@dataclass_with_extra -class ObjectDetectionInput(BaseInferenceType): - """Inputs for Object Detection inference""" - - inputs: str - """The input image data as a base64-encoded string. If no `parameters` are provided, you can - also provide the image data as a raw bytes payload. - """ - parameters: Optional[ObjectDetectionParameters] = None - """Additional inference parameters for Object Detection""" - - -@dataclass_with_extra -class ObjectDetectionBoundingBox(BaseInferenceType): - """The predicted bounding box. Coordinates are relative to the top left corner of the input - image. - """ - - xmax: int - """The x-coordinate of the bottom-right corner of the bounding box.""" - xmin: int - """The x-coordinate of the top-left corner of the bounding box.""" - ymax: int - """The y-coordinate of the bottom-right corner of the bounding box.""" - ymin: int - """The y-coordinate of the top-left corner of the bounding box.""" - - -@dataclass_with_extra -class ObjectDetectionOutputElement(BaseInferenceType): - """Outputs of inference for the Object Detection task""" - - box: ObjectDetectionBoundingBox - """The predicted bounding box. Coordinates are relative to the top left corner of the input - image. - """ - label: str - """The predicted label for the bounding box.""" - score: float - """The associated score / probability.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/question_answering.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/question_answering.py deleted file mode 100644 index 014ab41893c560a2c266bc04a1d60bc933be31c7..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/question_answering.py +++ /dev/null @@ -1,74 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class QuestionAnsweringInputData(BaseInferenceType): - """One (context, question) pair to answer""" - - context: str - """The context to be used for answering the question""" - question: str - """The question to be answered""" - - -@dataclass_with_extra -class QuestionAnsweringParameters(BaseInferenceType): - """Additional inference parameters for Question Answering""" - - align_to_words: Optional[bool] = None - """Attempts to align the answer to real words. Improves quality on space separated - languages. Might hurt on non-space-separated languages (like Japanese or Chinese) - """ - doc_stride: Optional[int] = None - """If the context is too long to fit with the question for the model, it will be split in - several chunks with some overlap. This argument controls the size of that overlap. - """ - handle_impossible_answer: Optional[bool] = None - """Whether to accept impossible as an answer.""" - max_answer_len: Optional[int] = None - """The maximum length of predicted answers (e.g., only answers with a shorter length are - considered). - """ - max_question_len: Optional[int] = None - """The maximum length of the question after tokenization. It will be truncated if needed.""" - max_seq_len: Optional[int] = None - """The maximum length of the total sentence (context + question) in tokens of each chunk - passed to the model. The context will be split in several chunks (using docStride as - overlap) if needed. - """ - top_k: Optional[int] = None - """The number of answers to return (will be chosen by order of likelihood). Note that we - return less than topk answers if there are not enough options available within the - context. - """ - - -@dataclass_with_extra -class QuestionAnsweringInput(BaseInferenceType): - """Inputs for Question Answering inference""" - - inputs: QuestionAnsweringInputData - """One (context, question) pair to answer""" - parameters: Optional[QuestionAnsweringParameters] = None - """Additional inference parameters for Question Answering""" - - -@dataclass_with_extra -class QuestionAnsweringOutputElement(BaseInferenceType): - """Outputs of inference for the Question Answering task""" - - answer: str - """The answer to the question.""" - end: int - """The character position in the input where the answer ends.""" - score: float - """The probability associated to the answer.""" - start: int - """The character position in the input where the answer begins.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/sentence_similarity.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/sentence_similarity.py deleted file mode 100644 index 66e8bb4d9322d4847556b7a17dc17bd208a37d0c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/sentence_similarity.py +++ /dev/null @@ -1,27 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Dict, List, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class SentenceSimilarityInputData(BaseInferenceType): - sentences: List[str] - """A list of strings which will be compared against the source_sentence.""" - source_sentence: str - """The string that you wish to compare the other strings with. This can be a phrase, - sentence, or longer passage, depending on the model being used. - """ - - -@dataclass_with_extra -class SentenceSimilarityInput(BaseInferenceType): - """Inputs for Sentence similarity inference""" - - inputs: SentenceSimilarityInputData - parameters: Optional[Dict[str, Any]] = None - """Additional inference parameters for Sentence Similarity""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/summarization.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/summarization.py deleted file mode 100644 index 33eae6fcba0e8724babf145f93be005868429c33..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/summarization.py +++ /dev/null @@ -1,41 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Dict, Literal, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -SummarizationTruncationStrategy = Literal["do_not_truncate", "longest_first", "only_first", "only_second"] - - -@dataclass_with_extra -class SummarizationParameters(BaseInferenceType): - """Additional inference parameters for summarization.""" - - clean_up_tokenization_spaces: Optional[bool] = None - """Whether to clean up the potential extra spaces in the text output.""" - generate_parameters: Optional[Dict[str, Any]] = None - """Additional parametrization of the text generation algorithm.""" - truncation: Optional["SummarizationTruncationStrategy"] = None - """The truncation strategy to use.""" - - -@dataclass_with_extra -class SummarizationInput(BaseInferenceType): - """Inputs for Summarization inference""" - - inputs: str - """The input text to summarize.""" - parameters: Optional[SummarizationParameters] = None - """Additional inference parameters for summarization.""" - - -@dataclass_with_extra -class SummarizationOutput(BaseInferenceType): - """Outputs of inference for the Summarization task""" - - summary_text: str - """The summarized text.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/table_question_answering.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/table_question_answering.py deleted file mode 100644 index 10e208eeeb50a689d2826a160432a2b005ec006c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/table_question_answering.py +++ /dev/null @@ -1,62 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Dict, List, Literal, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class TableQuestionAnsweringInputData(BaseInferenceType): - """One (table, question) pair to answer""" - - question: str - """The question to be answered about the table""" - table: Dict[str, List[str]] - """The table to serve as context for the questions""" - - -Padding = Literal["do_not_pad", "longest", "max_length"] - - -@dataclass_with_extra -class TableQuestionAnsweringParameters(BaseInferenceType): - """Additional inference parameters for Table Question Answering""" - - padding: Optional["Padding"] = None - """Activates and controls padding.""" - sequential: Optional[bool] = None - """Whether to do inference sequentially or as a batch. Batching is faster, but models like - SQA require the inference to be done sequentially to extract relations within sequences, - given their conversational nature. - """ - truncation: Optional[bool] = None - """Activates and controls truncation.""" - - -@dataclass_with_extra -class TableQuestionAnsweringInput(BaseInferenceType): - """Inputs for Table Question Answering inference""" - - inputs: TableQuestionAnsweringInputData - """One (table, question) pair to answer""" - parameters: Optional[TableQuestionAnsweringParameters] = None - """Additional inference parameters for Table Question Answering""" - - -@dataclass_with_extra -class TableQuestionAnsweringOutputElement(BaseInferenceType): - """Outputs of inference for the Table Question Answering task""" - - answer: str - """The answer of the question given the table. If there is an aggregator, the answer will be - preceded by `AGGREGATOR >`. - """ - cells: List[str] - """List of strings made up of the answer cell values.""" - coordinates: List[List[int]] - """Coordinates of the cells of the answers.""" - aggregator: Optional[str] = None - """If the model has an aggregator, this returns the aggregator.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text2text_generation.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text2text_generation.py deleted file mode 100644 index 34ac74e21e8a30d889f1a251f648d4c365325be6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text2text_generation.py +++ /dev/null @@ -1,42 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Dict, Literal, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -Text2TextGenerationTruncationStrategy = Literal["do_not_truncate", "longest_first", "only_first", "only_second"] - - -@dataclass_with_extra -class Text2TextGenerationParameters(BaseInferenceType): - """Additional inference parameters for Text2text Generation""" - - clean_up_tokenization_spaces: Optional[bool] = None - """Whether to clean up the potential extra spaces in the text output.""" - generate_parameters: Optional[Dict[str, Any]] = None - """Additional parametrization of the text generation algorithm""" - truncation: Optional["Text2TextGenerationTruncationStrategy"] = None - """The truncation strategy to use""" - - -@dataclass_with_extra -class Text2TextGenerationInput(BaseInferenceType): - """Inputs for Text2text Generation inference""" - - inputs: str - """The input text data""" - parameters: Optional[Text2TextGenerationParameters] = None - """Additional inference parameters for Text2text Generation""" - - -@dataclass_with_extra -class Text2TextGenerationOutput(BaseInferenceType): - """Outputs of inference for the Text2text Generation task""" - - generated_text: Any - text2_text_generation_output_generated_text: Optional[str] = None - """The generated text.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_classification.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_classification.py deleted file mode 100644 index 9a172b23f844fa58f757a644d52138a18e7b6ddb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_classification.py +++ /dev/null @@ -1,41 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Literal, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -TextClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] - - -@dataclass_with_extra -class TextClassificationParameters(BaseInferenceType): - """Additional inference parameters for Text Classification""" - - function_to_apply: Optional["TextClassificationOutputTransform"] = None - """The function to apply to the model outputs in order to retrieve the scores.""" - top_k: Optional[int] = None - """When specified, limits the output to the top K most probable classes.""" - - -@dataclass_with_extra -class TextClassificationInput(BaseInferenceType): - """Inputs for Text Classification inference""" - - inputs: str - """The text to classify""" - parameters: Optional[TextClassificationParameters] = None - """Additional inference parameters for Text Classification""" - - -@dataclass_with_extra -class TextClassificationOutputElement(BaseInferenceType): - """Outputs of inference for the Text Classification task""" - - label: str - """The predicted class label.""" - score: float - """The corresponding probability.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_generation.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_generation.py deleted file mode 100644 index 9b79cc691dce3a6d42aef716d4a93a719f2d600c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_generation.py +++ /dev/null @@ -1,168 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, List, Literal, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -TypeEnum = Literal["json", "regex", "json_schema"] - - -@dataclass_with_extra -class TextGenerationInputGrammarType(BaseInferenceType): - type: "TypeEnum" - value: Any - """A string that represents a [JSON Schema](https://json-schema.org/). - JSON Schema is a declarative language that allows to annotate JSON documents - with types and descriptions. - """ - - -@dataclass_with_extra -class TextGenerationInputGenerateParameters(BaseInferenceType): - adapter_id: Optional[str] = None - """Lora adapter id""" - best_of: Optional[int] = None - """Generate best_of sequences and return the one if the highest token logprobs.""" - decoder_input_details: Optional[bool] = None - """Whether to return decoder input token logprobs and ids.""" - details: Optional[bool] = None - """Whether to return generation details.""" - do_sample: Optional[bool] = None - """Activate logits sampling.""" - frequency_penalty: Optional[float] = None - """The parameter for frequency penalty. 1.0 means no penalty - Penalize new tokens based on their existing frequency in the text so far, - decreasing the model's likelihood to repeat the same line verbatim. - """ - grammar: Optional[TextGenerationInputGrammarType] = None - max_new_tokens: Optional[int] = None - """Maximum number of tokens to generate.""" - repetition_penalty: Optional[float] = None - """The parameter for repetition penalty. 1.0 means no penalty. - See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. - """ - return_full_text: Optional[bool] = None - """Whether to prepend the prompt to the generated text""" - seed: Optional[int] = None - """Random sampling seed.""" - stop: Optional[List[str]] = None - """Stop generating tokens if a member of `stop` is generated.""" - temperature: Optional[float] = None - """The value used to module the logits distribution.""" - top_k: Optional[int] = None - """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" - top_n_tokens: Optional[int] = None - """The number of highest probability vocabulary tokens to keep for top-n-filtering.""" - top_p: Optional[float] = None - """Top-p value for nucleus sampling.""" - truncate: Optional[int] = None - """Truncate inputs tokens to the given size.""" - typical_p: Optional[float] = None - """Typical Decoding mass - See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) - for more information. - """ - watermark: Optional[bool] = None - """Watermarking with [A Watermark for Large Language - Models](https://arxiv.org/abs/2301.10226). - """ - - -@dataclass_with_extra -class TextGenerationInput(BaseInferenceType): - """Text Generation Input. - Auto-generated from TGI specs. - For more details, check out - https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. - """ - - inputs: str - parameters: Optional[TextGenerationInputGenerateParameters] = None - stream: Optional[bool] = None - - -TextGenerationOutputFinishReason = Literal["length", "eos_token", "stop_sequence"] - - -@dataclass_with_extra -class TextGenerationOutputPrefillToken(BaseInferenceType): - id: int - logprob: float - text: str - - -@dataclass_with_extra -class TextGenerationOutputToken(BaseInferenceType): - id: int - logprob: float - special: bool - text: str - - -@dataclass_with_extra -class TextGenerationOutputBestOfSequence(BaseInferenceType): - finish_reason: "TextGenerationOutputFinishReason" - generated_text: str - generated_tokens: int - prefill: List[TextGenerationOutputPrefillToken] - tokens: List[TextGenerationOutputToken] - seed: Optional[int] = None - top_tokens: Optional[List[List[TextGenerationOutputToken]]] = None - - -@dataclass_with_extra -class TextGenerationOutputDetails(BaseInferenceType): - finish_reason: "TextGenerationOutputFinishReason" - generated_tokens: int - prefill: List[TextGenerationOutputPrefillToken] - tokens: List[TextGenerationOutputToken] - best_of_sequences: Optional[List[TextGenerationOutputBestOfSequence]] = None - seed: Optional[int] = None - top_tokens: Optional[List[List[TextGenerationOutputToken]]] = None - - -@dataclass_with_extra -class TextGenerationOutput(BaseInferenceType): - """Text Generation Output. - Auto-generated from TGI specs. - For more details, check out - https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. - """ - - generated_text: str - details: Optional[TextGenerationOutputDetails] = None - - -@dataclass_with_extra -class TextGenerationStreamOutputStreamDetails(BaseInferenceType): - finish_reason: "TextGenerationOutputFinishReason" - generated_tokens: int - input_length: int - seed: Optional[int] = None - - -@dataclass_with_extra -class TextGenerationStreamOutputToken(BaseInferenceType): - id: int - logprob: float - special: bool - text: str - - -@dataclass_with_extra -class TextGenerationStreamOutput(BaseInferenceType): - """Text Generation Stream Output. - Auto-generated from TGI specs. - For more details, check out - https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. - """ - - index: int - token: TextGenerationStreamOutputToken - details: Optional[TextGenerationStreamOutputStreamDetails] = None - generated_text: Optional[str] = None - top_tokens: Optional[List[TextGenerationStreamOutputToken]] = None diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_audio.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_audio.py deleted file mode 100644 index 87af80a598af70800b8386f034c65de0b397479e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_audio.py +++ /dev/null @@ -1,99 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Literal, Optional, Union - -from .base import BaseInferenceType, dataclass_with_extra - - -TextToAudioEarlyStoppingEnum = Literal["never"] - - -@dataclass_with_extra -class TextToAudioGenerationParameters(BaseInferenceType): - """Parametrization of the text generation process""" - - do_sample: Optional[bool] = None - """Whether to use sampling instead of greedy decoding when generating new tokens.""" - early_stopping: Optional[Union[bool, "TextToAudioEarlyStoppingEnum"]] = None - """Controls the stopping condition for beam-based methods.""" - epsilon_cutoff: Optional[float] = None - """If set to float strictly between 0 and 1, only tokens with a conditional probability - greater than epsilon_cutoff will be sampled. In the paper, suggested values range from - 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language - Model Desmoothing](https://hf.co/papers/2210.15191) for more details. - """ - eta_cutoff: Optional[float] = None - """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to - float strictly between 0 and 1, a token is only considered if it is greater than either - eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter - term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In - the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. - See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) - for more details. - """ - max_length: Optional[int] = None - """The maximum length (in tokens) of the generated text, including the input.""" - max_new_tokens: Optional[int] = None - """The maximum number of tokens to generate. Takes precedence over max_length.""" - min_length: Optional[int] = None - """The minimum length (in tokens) of the generated text, including the input.""" - min_new_tokens: Optional[int] = None - """The minimum number of tokens to generate. Takes precedence over min_length.""" - num_beam_groups: Optional[int] = None - """Number of groups to divide num_beams into in order to ensure diversity among different - groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. - """ - num_beams: Optional[int] = None - """Number of beams to use for beam search.""" - penalty_alpha: Optional[float] = None - """The value balances the model confidence and the degeneration penalty in contrastive - search decoding. - """ - temperature: Optional[float] = None - """The value used to modulate the next token probabilities.""" - top_k: Optional[int] = None - """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" - top_p: Optional[float] = None - """If set to float < 1, only the smallest set of most probable tokens with probabilities - that add up to top_p or higher are kept for generation. - """ - typical_p: Optional[float] = None - """Local typicality measures how similar the conditional probability of predicting a target - token next is to the expected conditional probability of predicting a random token next, - given the partial text already generated. If set to float < 1, the smallest set of the - most locally typical tokens with probabilities that add up to typical_p or higher are - kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. - """ - use_cache: Optional[bool] = None - """Whether the model should use the past last key/values attentions to speed up decoding""" - - -@dataclass_with_extra -class TextToAudioParameters(BaseInferenceType): - """Additional inference parameters for Text To Audio""" - - generation_parameters: Optional[TextToAudioGenerationParameters] = None - """Parametrization of the text generation process""" - - -@dataclass_with_extra -class TextToAudioInput(BaseInferenceType): - """Inputs for Text To Audio inference""" - - inputs: str - """The input text data""" - parameters: Optional[TextToAudioParameters] = None - """Additional inference parameters for Text To Audio""" - - -@dataclass_with_extra -class TextToAudioOutput(BaseInferenceType): - """Outputs of inference for the Text To Audio task""" - - audio: Any - """The generated audio waveform.""" - sampling_rate: float - """The sampling rate of the generated audio waveform.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_image.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_image.py deleted file mode 100644 index 20c963731371339975019ca5d40c95303d79209b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_image.py +++ /dev/null @@ -1,50 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class TextToImageParameters(BaseInferenceType): - """Additional inference parameters for Text To Image""" - - guidance_scale: Optional[float] = None - """A higher guidance scale value encourages the model to generate images closely linked to - the text prompt, but values too high may cause saturation and other artifacts. - """ - height: Optional[int] = None - """The height in pixels of the output image""" - negative_prompt: Optional[str] = None - """One prompt to guide what NOT to include in image generation.""" - num_inference_steps: Optional[int] = None - """The number of denoising steps. More denoising steps usually lead to a higher quality - image at the expense of slower inference. - """ - scheduler: Optional[str] = None - """Override the scheduler with a compatible one.""" - seed: Optional[int] = None - """Seed for the random number generator.""" - width: Optional[int] = None - """The width in pixels of the output image""" - - -@dataclass_with_extra -class TextToImageInput(BaseInferenceType): - """Inputs for Text To Image inference""" - - inputs: str - """The input text data (sometimes called "prompt")""" - parameters: Optional[TextToImageParameters] = None - """Additional inference parameters for Text To Image""" - - -@dataclass_with_extra -class TextToImageOutput(BaseInferenceType): - """Outputs of inference for the Text To Image task""" - - image: Any - """The generated image returned as raw bytes in the payload.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_speech.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_speech.py deleted file mode 100644 index ce2db8f3f901cc99b5d2fcbb362c4b07b2a718e0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_speech.py +++ /dev/null @@ -1,99 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Literal, Optional, Union - -from .base import BaseInferenceType, dataclass_with_extra - - -TextToSpeechEarlyStoppingEnum = Literal["never"] - - -@dataclass_with_extra -class TextToSpeechGenerationParameters(BaseInferenceType): - """Parametrization of the text generation process""" - - do_sample: Optional[bool] = None - """Whether to use sampling instead of greedy decoding when generating new tokens.""" - early_stopping: Optional[Union[bool, "TextToSpeechEarlyStoppingEnum"]] = None - """Controls the stopping condition for beam-based methods.""" - epsilon_cutoff: Optional[float] = None - """If set to float strictly between 0 and 1, only tokens with a conditional probability - greater than epsilon_cutoff will be sampled. In the paper, suggested values range from - 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language - Model Desmoothing](https://hf.co/papers/2210.15191) for more details. - """ - eta_cutoff: Optional[float] = None - """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to - float strictly between 0 and 1, a token is only considered if it is greater than either - eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter - term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In - the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. - See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) - for more details. - """ - max_length: Optional[int] = None - """The maximum length (in tokens) of the generated text, including the input.""" - max_new_tokens: Optional[int] = None - """The maximum number of tokens to generate. Takes precedence over max_length.""" - min_length: Optional[int] = None - """The minimum length (in tokens) of the generated text, including the input.""" - min_new_tokens: Optional[int] = None - """The minimum number of tokens to generate. Takes precedence over min_length.""" - num_beam_groups: Optional[int] = None - """Number of groups to divide num_beams into in order to ensure diversity among different - groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. - """ - num_beams: Optional[int] = None - """Number of beams to use for beam search.""" - penalty_alpha: Optional[float] = None - """The value balances the model confidence and the degeneration penalty in contrastive - search decoding. - """ - temperature: Optional[float] = None - """The value used to modulate the next token probabilities.""" - top_k: Optional[int] = None - """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" - top_p: Optional[float] = None - """If set to float < 1, only the smallest set of most probable tokens with probabilities - that add up to top_p or higher are kept for generation. - """ - typical_p: Optional[float] = None - """Local typicality measures how similar the conditional probability of predicting a target - token next is to the expected conditional probability of predicting a random token next, - given the partial text already generated. If set to float < 1, the smallest set of the - most locally typical tokens with probabilities that add up to typical_p or higher are - kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. - """ - use_cache: Optional[bool] = None - """Whether the model should use the past last key/values attentions to speed up decoding""" - - -@dataclass_with_extra -class TextToSpeechParameters(BaseInferenceType): - """Additional inference parameters for Text To Speech""" - - generation_parameters: Optional[TextToSpeechGenerationParameters] = None - """Parametrization of the text generation process""" - - -@dataclass_with_extra -class TextToSpeechInput(BaseInferenceType): - """Inputs for Text To Speech inference""" - - inputs: str - """The input text data""" - parameters: Optional[TextToSpeechParameters] = None - """Additional inference parameters for Text To Speech""" - - -@dataclass_with_extra -class TextToSpeechOutput(BaseInferenceType): - """Outputs of inference for the Text To Speech task""" - - audio: Any - """The generated audio""" - sampling_rate: Optional[float] = None - """The sampling rate of the generated audio waveform.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_video.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_video.py deleted file mode 100644 index e54a1bc094e4aaf7132e502aa268bc052ab34f0a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/text_to_video.py +++ /dev/null @@ -1,46 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, List, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class TextToVideoParameters(BaseInferenceType): - """Additional inference parameters for Text To Video""" - - guidance_scale: Optional[float] = None - """A higher guidance scale value encourages the model to generate videos closely linked to - the text prompt, but values too high may cause saturation and other artifacts. - """ - negative_prompt: Optional[List[str]] = None - """One or several prompt to guide what NOT to include in video generation.""" - num_frames: Optional[float] = None - """The num_frames parameter determines how many video frames are generated.""" - num_inference_steps: Optional[int] = None - """The number of denoising steps. More denoising steps usually lead to a higher quality - video at the expense of slower inference. - """ - seed: Optional[int] = None - """Seed for the random number generator.""" - - -@dataclass_with_extra -class TextToVideoInput(BaseInferenceType): - """Inputs for Text To Video inference""" - - inputs: str - """The input text data (sometimes called "prompt")""" - parameters: Optional[TextToVideoParameters] = None - """Additional inference parameters for Text To Video""" - - -@dataclass_with_extra -class TextToVideoOutput(BaseInferenceType): - """Outputs of inference for the Text To Video task""" - - video: Any - """The generated video returned as raw bytes in the payload.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/token_classification.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/token_classification.py deleted file mode 100644 index e039b6a1db7dcd54dbc9434d3254da0770c6799e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/token_classification.py +++ /dev/null @@ -1,51 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import List, Literal, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -TokenClassificationAggregationStrategy = Literal["none", "simple", "first", "average", "max"] - - -@dataclass_with_extra -class TokenClassificationParameters(BaseInferenceType): - """Additional inference parameters for Token Classification""" - - aggregation_strategy: Optional["TokenClassificationAggregationStrategy"] = None - """The strategy used to fuse tokens based on model predictions""" - ignore_labels: Optional[List[str]] = None - """A list of labels to ignore""" - stride: Optional[int] = None - """The number of overlapping tokens between chunks when splitting the input text.""" - - -@dataclass_with_extra -class TokenClassificationInput(BaseInferenceType): - """Inputs for Token Classification inference""" - - inputs: str - """The input text data""" - parameters: Optional[TokenClassificationParameters] = None - """Additional inference parameters for Token Classification""" - - -@dataclass_with_extra -class TokenClassificationOutputElement(BaseInferenceType): - """Outputs of inference for the Token Classification task""" - - end: int - """The character position in the input where this group ends.""" - score: float - """The associated score / probability""" - start: int - """The character position in the input where this group begins.""" - word: str - """The corresponding text""" - entity: Optional[str] = None - """The predicted label for a single token""" - entity_group: Optional[str] = None - """The predicted label for a group of one or more tokens""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/translation.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/translation.py deleted file mode 100644 index df95b7dbb1f4ce5b80cec034e004bb6e71387be8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/translation.py +++ /dev/null @@ -1,49 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Dict, Literal, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -TranslationTruncationStrategy = Literal["do_not_truncate", "longest_first", "only_first", "only_second"] - - -@dataclass_with_extra -class TranslationParameters(BaseInferenceType): - """Additional inference parameters for Translation""" - - clean_up_tokenization_spaces: Optional[bool] = None - """Whether to clean up the potential extra spaces in the text output.""" - generate_parameters: Optional[Dict[str, Any]] = None - """Additional parametrization of the text generation algorithm.""" - src_lang: Optional[str] = None - """The source language of the text. Required for models that can translate from multiple - languages. - """ - tgt_lang: Optional[str] = None - """Target language to translate to. Required for models that can translate to multiple - languages. - """ - truncation: Optional["TranslationTruncationStrategy"] = None - """The truncation strategy to use.""" - - -@dataclass_with_extra -class TranslationInput(BaseInferenceType): - """Inputs for Translation inference""" - - inputs: str - """The text to translate.""" - parameters: Optional[TranslationParameters] = None - """Additional inference parameters for Translation""" - - -@dataclass_with_extra -class TranslationOutput(BaseInferenceType): - """Outputs of inference for the Translation task""" - - translation_text: str - """The translated text.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/video_classification.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/video_classification.py deleted file mode 100644 index e1d7a15bb4ee5fa63aa6ebc3750191bd38549212..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/video_classification.py +++ /dev/null @@ -1,45 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Literal, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -VideoClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] - - -@dataclass_with_extra -class VideoClassificationParameters(BaseInferenceType): - """Additional inference parameters for Video Classification""" - - frame_sampling_rate: Optional[int] = None - """The sampling rate used to select frames from the video.""" - function_to_apply: Optional["VideoClassificationOutputTransform"] = None - """The function to apply to the model outputs in order to retrieve the scores.""" - num_frames: Optional[int] = None - """The number of sampled frames to consider for classification.""" - top_k: Optional[int] = None - """When specified, limits the output to the top K most probable classes.""" - - -@dataclass_with_extra -class VideoClassificationInput(BaseInferenceType): - """Inputs for Video Classification inference""" - - inputs: Any - """The input video data""" - parameters: Optional[VideoClassificationParameters] = None - """Additional inference parameters for Video Classification""" - - -@dataclass_with_extra -class VideoClassificationOutputElement(BaseInferenceType): - """Outputs of inference for the Video Classification task""" - - label: str - """The predicted class label.""" - score: float - """The corresponding probability.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/visual_question_answering.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/visual_question_answering.py deleted file mode 100644 index d368f1621289bc11a17be3e590cf8a040019d455..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/visual_question_answering.py +++ /dev/null @@ -1,49 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import Any, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class VisualQuestionAnsweringInputData(BaseInferenceType): - """One (image, question) pair to answer""" - - image: Any - """The image.""" - question: str - """The question to answer based on the image.""" - - -@dataclass_with_extra -class VisualQuestionAnsweringParameters(BaseInferenceType): - """Additional inference parameters for Visual Question Answering""" - - top_k: Optional[int] = None - """The number of answers to return (will be chosen by order of likelihood). Note that we - return less than topk answers if there are not enough options available within the - context. - """ - - -@dataclass_with_extra -class VisualQuestionAnsweringInput(BaseInferenceType): - """Inputs for Visual Question Answering inference""" - - inputs: VisualQuestionAnsweringInputData - """One (image, question) pair to answer""" - parameters: Optional[VisualQuestionAnsweringParameters] = None - """Additional inference parameters for Visual Question Answering""" - - -@dataclass_with_extra -class VisualQuestionAnsweringOutputElement(BaseInferenceType): - """Outputs of inference for the Visual Question Answering task""" - - score: float - """The associated score / probability""" - answer: Optional[str] = None - """The answer to the question""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/zero_shot_classification.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/zero_shot_classification.py deleted file mode 100644 index 47b32492e358edcc0de6aa09d53635b0a8156b25..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/zero_shot_classification.py +++ /dev/null @@ -1,45 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import List, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class ZeroShotClassificationParameters(BaseInferenceType): - """Additional inference parameters for Zero Shot Classification""" - - candidate_labels: List[str] - """The set of possible class labels to classify the text into.""" - hypothesis_template: Optional[str] = None - """The sentence used in conjunction with `candidate_labels` to attempt the text - classification by replacing the placeholder with the candidate labels. - """ - multi_label: Optional[bool] = None - """Whether multiple candidate labels can be true. If false, the scores are normalized such - that the sum of the label likelihoods for each sequence is 1. If true, the labels are - considered independent and probabilities are normalized for each candidate. - """ - - -@dataclass_with_extra -class ZeroShotClassificationInput(BaseInferenceType): - """Inputs for Zero Shot Classification inference""" - - inputs: str - """The text to classify""" - parameters: ZeroShotClassificationParameters - """Additional inference parameters for Zero Shot Classification""" - - -@dataclass_with_extra -class ZeroShotClassificationOutputElement(BaseInferenceType): - """Outputs of inference for the Zero Shot Classification task""" - - label: str - """The predicted class label.""" - score: float - """The corresponding probability.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/zero_shot_image_classification.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/zero_shot_image_classification.py deleted file mode 100644 index 998d66b6b4e3356f0f09a0ad25ebdaf2e76cd03f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/zero_shot_image_classification.py +++ /dev/null @@ -1,40 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import List, Optional - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class ZeroShotImageClassificationParameters(BaseInferenceType): - """Additional inference parameters for Zero Shot Image Classification""" - - candidate_labels: List[str] - """The candidate labels for this image""" - hypothesis_template: Optional[str] = None - """The sentence used in conjunction with `candidate_labels` to attempt the image - classification by replacing the placeholder with the candidate labels. - """ - - -@dataclass_with_extra -class ZeroShotImageClassificationInput(BaseInferenceType): - """Inputs for Zero Shot Image Classification inference""" - - inputs: str - """The input image data to classify as a base64-encoded string.""" - parameters: ZeroShotImageClassificationParameters - """Additional inference parameters for Zero Shot Image Classification""" - - -@dataclass_with_extra -class ZeroShotImageClassificationOutputElement(BaseInferenceType): - """Outputs of inference for the Zero Shot Image Classification task""" - - label: str - """The predicted class label.""" - score: float - """The corresponding probability.""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/zero_shot_object_detection.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/zero_shot_object_detection.py deleted file mode 100644 index 8ef76b5fcb93e8126266e4b1464934d01024b1b7..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_generated/types/zero_shot_object_detection.py +++ /dev/null @@ -1,52 +0,0 @@ -# Inference code generated from the JSON schema spec in @huggingface/tasks. -# -# See: -# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts -# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. -from typing import List - -from .base import BaseInferenceType, dataclass_with_extra - - -@dataclass_with_extra -class ZeroShotObjectDetectionParameters(BaseInferenceType): - """Additional inference parameters for Zero Shot Object Detection""" - - candidate_labels: List[str] - """The candidate labels for this image""" - - -@dataclass_with_extra -class ZeroShotObjectDetectionInput(BaseInferenceType): - """Inputs for Zero Shot Object Detection inference""" - - inputs: str - """The input image data as a base64-encoded string.""" - parameters: ZeroShotObjectDetectionParameters - """Additional inference parameters for Zero Shot Object Detection""" - - -@dataclass_with_extra -class ZeroShotObjectDetectionBoundingBox(BaseInferenceType): - """The predicted bounding box. Coordinates are relative to the top left corner of the input - image. - """ - - xmax: int - xmin: int - ymax: int - ymin: int - - -@dataclass_with_extra -class ZeroShotObjectDetectionOutputElement(BaseInferenceType): - """Outputs of inference for the Zero Shot Object Detection task""" - - box: ZeroShotObjectDetectionBoundingBox - """The predicted bounding box. Coordinates are relative to the top left corner of the input - image. - """ - label: str - """A candidate label""" - score: float - """The associated score / probability""" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/__init__.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/_cli_hacks.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/_cli_hacks.py deleted file mode 100644 index 64251bbb745dc3b4b561f0eb249be65108b20d82..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/_cli_hacks.py +++ /dev/null @@ -1,88 +0,0 @@ -import asyncio -import sys -from functools import partial - -import typer - - -def _patch_anyio_open_process(): - """ - Patch anyio.open_process to allow detached processes on Windows and Unix-like systems. - - This is necessary to prevent the MCP client from being interrupted by Ctrl+C when running in the CLI. - """ - import subprocess - - import anyio - - if getattr(anyio, "_tiny_agents_patched", False): - return - anyio._tiny_agents_patched = True # ty: ignore[invalid-assignment] - - original_open_process = anyio.open_process - - if sys.platform == "win32": - # On Windows, we need to set the creation flags to create a new process group - - async def open_process_in_new_group(*args, **kwargs): - """ - Wrapper for open_process to handle Windows-specific process creation flags. - """ - # Ensure we pass the creation flags for Windows - kwargs.setdefault("creationflags", subprocess.CREATE_NEW_PROCESS_GROUP) - return await original_open_process(*args, **kwargs) - - anyio.open_process = open_process_in_new_group # ty: ignore[invalid-assignment] - else: - # For Unix-like systems, we can use setsid to create a new session - async def open_process_in_new_group(*args, **kwargs): - """ - Wrapper for open_process to handle Unix-like systems with start_new_session=True. - """ - kwargs.setdefault("start_new_session", True) - return await original_open_process(*args, **kwargs) - - anyio.open_process = open_process_in_new_group # ty: ignore[invalid-assignment] - - -async def _async_prompt(exit_event: asyncio.Event, prompt: str = "» ") -> str: - """ - Asynchronous prompt function that reads input from stdin without blocking. - - This function is designed to work in an asynchronous context, allowing the event loop to gracefully stop it (e.g. on Ctrl+C). - - Alternatively, we could use https://github.com/vxgmichel/aioconsole but that would be an additional dependency. - """ - loop = asyncio.get_event_loop() - - if sys.platform == "win32": - # Windows: Use run_in_executor to avoid blocking the event loop - # Degraded solution: this is not ideal as user will have to CTRL+C once more to stop the prompt (and it'll not be graceful) - return await loop.run_in_executor(None, partial(typer.prompt, prompt, prompt_suffix=" ")) - else: - # UNIX-like: Use loop.add_reader for non-blocking stdin read - future = loop.create_future() - - def on_input(): - line = sys.stdin.readline() - loop.remove_reader(sys.stdin) - future.set_result(line) - - print(prompt, end=" ", flush=True) - loop.add_reader(sys.stdin, on_input) # not supported on Windows - - # Wait for user input or exit event - # Wait until either the user hits enter or exit_event is set - exit_task = asyncio.create_task(exit_event.wait()) - await asyncio.wait( - [future, exit_task], - return_when=asyncio.FIRST_COMPLETED, - ) - - # Check which one has been triggered - if exit_event.is_set(): - future.cancel() - return "" - - line = await future - return line.strip() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/agent.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/agent.py deleted file mode 100644 index b9eb347ed60a7178caecc8d54d4b6b2593d80884..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/agent.py +++ /dev/null @@ -1,100 +0,0 @@ -from __future__ import annotations - -import asyncio -from typing import AsyncGenerator, Dict, Iterable, List, Optional, Union - -from huggingface_hub import ChatCompletionInputMessage, ChatCompletionStreamOutput, MCPClient - -from .._providers import PROVIDER_OR_POLICY_T -from .constants import DEFAULT_SYSTEM_PROMPT, EXIT_LOOP_TOOLS, MAX_NUM_TURNS -from .types import ServerConfig - - -class Agent(MCPClient): - """ - Implementation of a Simple Agent, which is a simple while loop built right on top of an [`MCPClient`]. - - > [!WARNING] - > This class is experimental and might be subject to breaking changes in the future without prior notice. - - Args: - model (`str`, *optional*): - The model to run inference with. Can be a model id hosted on the Hugging Face Hub, e.g. `meta-llama/Meta-Llama-3-8B-Instruct` - or a URL to a deployed Inference Endpoint or other local or remote endpoint. - servers (`Iterable[Dict]`): - MCP servers to connect to. Each server is a dictionary containing a `type` key and a `config` key. The `type` key can be `"stdio"` or `"sse"`, and the `config` key is a dictionary of arguments for the server. - provider (`str`, *optional*): - Name of the provider to use for inference. Defaults to "auto" i.e. the first of the providers available for the model, sorted by the user's order in https://hf.co/settings/inference-providers. - If model is a URL or `base_url` is passed, then `provider` is not used. - base_url (`str`, *optional*): - The base URL to run inference. Defaults to None. - api_key (`str`, *optional*): - Token to use for authentication. Will default to the locally Hugging Face saved token if not provided. You can also use your own provider API key to interact directly with the provider's service. - prompt (`str`, *optional*): - The system prompt to use for the agent. Defaults to the default system prompt in `constants.py`. - """ - - def __init__( - self, - *, - model: Optional[str] = None, - servers: Iterable[ServerConfig], - provider: Optional[PROVIDER_OR_POLICY_T] = None, - base_url: Optional[str] = None, - api_key: Optional[str] = None, - prompt: Optional[str] = None, - ): - super().__init__(model=model, provider=provider, base_url=base_url, api_key=api_key) - self._servers_cfg = list(servers) - self.messages: List[Union[Dict, ChatCompletionInputMessage]] = [ - {"role": "system", "content": prompt or DEFAULT_SYSTEM_PROMPT} - ] - - async def load_tools(self) -> None: - for cfg in self._servers_cfg: - await self.add_mcp_server(**cfg) - - async def run( - self, - user_input: str, - *, - abort_event: Optional[asyncio.Event] = None, - ) -> AsyncGenerator[Union[ChatCompletionStreamOutput, ChatCompletionInputMessage], None]: - """ - Run the agent with the given user input. - - Args: - user_input (`str`): - The user input to run the agent with. - abort_event (`asyncio.Event`, *optional*): - An event that can be used to abort the agent. If the event is set, the agent will stop running. - """ - self.messages.append({"role": "user", "content": user_input}) - - num_turns: int = 0 - next_turn_should_call_tools = True - - while True: - if abort_event and abort_event.is_set(): - return - - async for item in self.process_single_turn_with_tools( - self.messages, - exit_loop_tools=EXIT_LOOP_TOOLS, - exit_if_first_chunk_no_tool=(num_turns > 0 and next_turn_should_call_tools), - ): - yield item - - num_turns += 1 - last = self.messages[-1] - - if last.get("role") == "tool" and last.get("name") in {t.function.name for t in EXIT_LOOP_TOOLS}: - return - - if last.get("role") != "tool" and num_turns > MAX_NUM_TURNS: - return - - if last.get("role") != "tool" and next_turn_should_call_tools: - return - - next_turn_should_call_tools = last.get("role") != "tool" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/cli.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/cli.py deleted file mode 100644 index a8aaea687a2b372e5379f09dffc219e5ea5b38b8..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/cli.py +++ /dev/null @@ -1,247 +0,0 @@ -import asyncio -import os -import signal -import traceback -from typing import Optional - -import typer -from rich import print - -from ._cli_hacks import _async_prompt, _patch_anyio_open_process -from .agent import Agent -from .utils import _load_agent_config - - -app = typer.Typer( - rich_markup_mode="rich", - help="A squad of lightweight composable AI applications built on Hugging Face's Inference Client and MCP stack.", -) - -run_cli = typer.Typer( - name="run", - help="Run the Agent in the CLI", - invoke_without_command=True, -) -app.add_typer(run_cli, name="run") - - -async def run_agent( - agent_path: Optional[str], -) -> None: - """ - Tiny Agent loop. - - Args: - agent_path (`str`, *optional*): - Path to a local folder containing an `agent.json` and optionally a custom `PROMPT.md` or `AGENTS.md` file or a built-in agent stored in a Hugging Face dataset. - - """ - _patch_anyio_open_process() # Hacky way to prevent stdio connections to be stopped by Ctrl+C - - config, prompt = _load_agent_config(agent_path) - - inputs = config.get("inputs", []) - servers = config.get("servers", []) - - abort_event = asyncio.Event() - exit_event = asyncio.Event() - first_sigint = True - - loop = asyncio.get_running_loop() - original_sigint_handler = signal.getsignal(signal.SIGINT) - - def _sigint_handler() -> None: - nonlocal first_sigint - if first_sigint: - first_sigint = False - abort_event.set() - print("\n[red]Interrupted. Press Ctrl+C again to quit.[/red]", flush=True) - return - - print("\n[red]Exiting...[/red]", flush=True) - exit_event.set() - - try: - sigint_registered_in_loop = False - try: - loop.add_signal_handler(signal.SIGINT, _sigint_handler) - sigint_registered_in_loop = True - except (AttributeError, NotImplementedError): - # Windows (or any loop that doesn't support it) : fall back to sync - signal.signal(signal.SIGINT, lambda *_: _sigint_handler()) - - # Handle inputs (i.e. env variables injection) - resolved_inputs: dict[str, str] = {} - - if len(inputs) > 0: - print( - "[bold blue]Some initial inputs are required by the agent. " - "Please provide a value or leave empty to load from env.[/bold blue]" - ) - for input_item in inputs: - input_id = input_item["id"] - description = input_item["description"] - env_special_value = f"${{input:{input_id}}}" - - # Check if the input is used by any server or as an apiKey - input_usages = set() - for server in servers: - # Check stdio's "env" and http/sse's "headers" mappings - env_or_headers = server.get("env", {}) if server["type"] == "stdio" else server.get("headers", {}) - for key, value in env_or_headers.items(): - if env_special_value in value: - input_usages.add(key) - - raw_api_key = config.get("apiKey") - if isinstance(raw_api_key, str) and env_special_value in raw_api_key: - input_usages.add("apiKey") - - if not input_usages: - print( - f"[yellow]Input '{input_id}' defined in config but not used by any server or as an API key." - " Skipping.[/yellow]" - ) - continue - - # Prompt user for input - env_variable_key = input_id.replace("-", "_").upper() - print( - f"[blue] • {input_id}[/blue]: {description}. (default: load from {env_variable_key}).", - end=" ", - ) - user_input = (await _async_prompt(exit_event=exit_event)).strip() - if exit_event.is_set(): - return - - # Fallback to environment variable when user left blank - final_value = user_input - if not final_value: - final_value = os.getenv(env_variable_key, "") - if final_value: - print(f"[green]Value successfully loaded from '{env_variable_key}'[/green]") - else: - print( - f"[yellow]No value found for '{env_variable_key}' in environment variables. Continuing.[/yellow]" - ) - resolved_inputs[input_id] = final_value - - # Inject resolved value (can be empty) into stdio's env or http/sse's headers - for server in servers: - env_or_headers = server.get("env", {}) if server["type"] == "stdio" else server.get("headers", {}) - for key, value in env_or_headers.items(): - if env_special_value in value: - env_or_headers[key] = env_or_headers[key].replace(env_special_value, final_value) - - print() - - raw_api_key = config.get("apiKey") - if isinstance(raw_api_key, str): - substituted_api_key = raw_api_key - for input_id, val in resolved_inputs.items(): - substituted_api_key = substituted_api_key.replace(f"${{input:{input_id}}}", val) - config["apiKey"] = substituted_api_key - # Main agent loop - async with Agent( - provider=config.get("provider"), # type: ignore[arg-type] - model=config.get("model"), - base_url=config.get("endpointUrl"), # type: ignore[arg-type] - api_key=config.get("apiKey"), - servers=servers, # type: ignore[arg-type] - prompt=prompt, - ) as agent: - await agent.load_tools() - print(f"[bold blue]Agent loaded with {len(agent.available_tools)} tools:[/bold blue]") - for t in agent.available_tools: - print(f"[blue] • {t.function.name}[/blue]") - - while True: - abort_event.clear() - - # Check if we should exit - if exit_event.is_set(): - return - - try: - user_input = await _async_prompt(exit_event=exit_event) - first_sigint = True - except EOFError: - print("\n[red]EOF received, exiting.[/red]", flush=True) - break - except KeyboardInterrupt: - if not first_sigint and abort_event.is_set(): - continue - else: - print("\n[red]Keyboard interrupt during input processing.[/red]", flush=True) - break - - try: - async for chunk in agent.run(user_input, abort_event=abort_event): - if abort_event.is_set() and not first_sigint: - break - if exit_event.is_set(): - return - - if hasattr(chunk, "choices"): - delta = chunk.choices[0].delta - if delta.content: - print(delta.content, end="", flush=True) - if delta.tool_calls: - for call in delta.tool_calls: - if call.id: - print(f"", end="") - if call.function.name: - print(f"{call.function.name}", end=" ") - if call.function.arguments: - print(f"{call.function.arguments}", end="") - else: - print( - f"\n\n[green]Tool[{chunk.name}] {chunk.tool_call_id}\n{chunk.content}[/green]\n", - flush=True, - ) - - print() - - except Exception as e: - tb_str = traceback.format_exc() - print(f"\n[bold red]Error during agent run: {e}\n{tb_str}[/bold red]", flush=True) - first_sigint = True # Allow graceful interrupt for the next command - - except Exception as e: - tb_str = traceback.format_exc() - print(f"\n[bold red]An unexpected error occurred: {e}\n{tb_str}[/bold red]", flush=True) - raise e - - finally: - if sigint_registered_in_loop: - try: - loop.remove_signal_handler(signal.SIGINT) - except (AttributeError, NotImplementedError): - pass - else: - signal.signal(signal.SIGINT, original_sigint_handler) - - -@run_cli.callback() -def run( - path: Optional[str] = typer.Argument( - None, - help=( - "Path to a local folder containing an agent.json file or a built-in agent " - "stored in the 'tiny-agents/tiny-agents' Hugging Face dataset " - "(https://huggingface.co/datasets/tiny-agents/tiny-agents)" - ), - show_default=False, - ), -): - try: - asyncio.run(run_agent(path)) - except KeyboardInterrupt: - print("\n[red]Application terminated by KeyboardInterrupt.[/red]", flush=True) - raise typer.Exit(code=130) - except Exception as e: - print(f"\n[bold red]An unexpected error occurred: {e}[/bold red]", flush=True) - raise e - - -if __name__ == "__main__": - app() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/constants.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/constants.py deleted file mode 100644 index 1ccade43b151cc9650bfd8cb43d7e907c92447ef..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/constants.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -import sys -from pathlib import Path -from typing import List - -from huggingface_hub import ChatCompletionInputTool - - -FILENAME_CONFIG = "agent.json" -PROMPT_FILENAMES = ("PROMPT.md", "AGENTS.md") - -DEFAULT_AGENT = { - "model": "Qwen/Qwen2.5-72B-Instruct", - "provider": "nebius", - "servers": [ - { - "type": "stdio", - "command": "npx", - "args": [ - "-y", - "@modelcontextprotocol/server-filesystem", - str(Path.home() / ("Desktop" if sys.platform == "darwin" else "")), - ], - }, - { - "type": "stdio", - "command": "npx", - "args": ["@playwright/mcp@latest"], - }, - ], -} - - -DEFAULT_SYSTEM_PROMPT = """ -You are an agent - please keep going until the user’s query is completely -resolved, before ending your turn and yielding back to the user. Only terminate -your turn when you are sure that the problem is solved, or if you need more -info from the user to solve the problem. -If you are not sure about anything pertaining to the user’s request, use your -tools to read files and gather the relevant information: do NOT guess or make -up an answer. -You MUST plan extensively before each function call, and reflect extensively -on the outcomes of the previous function calls. DO NOT do this entire process -by making function calls only, as this can impair your ability to solve the -problem and think insightfully. -""".strip() - -MAX_NUM_TURNS = 10 - -TASK_COMPLETE_TOOL: ChatCompletionInputTool = ChatCompletionInputTool.parse_obj( # type: ignore[assignment] - { - "type": "function", - "function": { - "name": "task_complete", - "description": "Call this tool when the task given by the user is complete", - "parameters": { - "type": "object", - "properties": {}, - }, - }, - } -) - -ASK_QUESTION_TOOL: ChatCompletionInputTool = ChatCompletionInputTool.parse_obj( # type: ignore[assignment] - { - "type": "function", - "function": { - "name": "ask_question", - "description": "Ask the user for more info required to solve or clarify their problem.", - "parameters": { - "type": "object", - "properties": {}, - }, - }, - } -) - -EXIT_LOOP_TOOLS: List[ChatCompletionInputTool] = [TASK_COMPLETE_TOOL, ASK_QUESTION_TOOL] - - -DEFAULT_REPO_ID = "tiny-agents/tiny-agents" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/mcp_client.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/mcp_client.py deleted file mode 100644 index 67d1fc5d15c898a4130f341e62e60d32c7663d28..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/mcp_client.py +++ /dev/null @@ -1,384 +0,0 @@ -import json -import logging -from contextlib import AsyncExitStack -from datetime import timedelta -from pathlib import Path -from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Literal, Optional, Union, overload - -from typing_extensions import NotRequired, TypeAlias, TypedDict, Unpack - -from ...utils._runtime import get_hf_hub_version -from .._generated._async_client import AsyncInferenceClient -from .._generated.types import ( - ChatCompletionInputMessage, - ChatCompletionInputTool, - ChatCompletionStreamOutput, - ChatCompletionStreamOutputDeltaToolCall, -) -from .._providers import PROVIDER_OR_POLICY_T -from .utils import format_result - - -if TYPE_CHECKING: - from mcp import ClientSession - -logger = logging.getLogger(__name__) - -# Type alias for tool names -ToolName: TypeAlias = str - -ServerType: TypeAlias = Literal["stdio", "sse", "http"] - - -class StdioServerParameters_T(TypedDict): - command: str - args: NotRequired[List[str]] - env: NotRequired[Dict[str, str]] - cwd: NotRequired[Union[str, Path, None]] - - -class SSEServerParameters_T(TypedDict): - url: str - headers: NotRequired[Dict[str, Any]] - timeout: NotRequired[float] - sse_read_timeout: NotRequired[float] - - -class StreamableHTTPParameters_T(TypedDict): - url: str - headers: NotRequired[dict[str, Any]] - timeout: NotRequired[timedelta] - sse_read_timeout: NotRequired[timedelta] - terminate_on_close: NotRequired[bool] - - -class MCPClient: - """ - Client for connecting to one or more MCP servers and processing chat completions with tools. - - > [!WARNING] - > This class is experimental and might be subject to breaking changes in the future without prior notice. - - Args: - model (`str`, `optional`): - The model to run inference with. Can be a model id hosted on the Hugging Face Hub, e.g. `meta-llama/Meta-Llama-3-8B-Instruct` - or a URL to a deployed Inference Endpoint or other local or remote endpoint. - provider (`str`, *optional*): - Name of the provider to use for inference. Defaults to "auto" i.e. the first of the providers available for the model, sorted by the user's order in https://hf.co/settings/inference-providers. - If model is a URL or `base_url` is passed, then `provider` is not used. - base_url (`str`, *optional*): - The base URL to run inference. Defaults to None. - api_key (`str`, `optional`): - Token to use for authentication. Will default to the locally Hugging Face saved token if not provided. You can also use your own provider API key to interact directly with the provider's service. - """ - - def __init__( - self, - *, - model: Optional[str] = None, - provider: Optional[PROVIDER_OR_POLICY_T] = None, - base_url: Optional[str] = None, - api_key: Optional[str] = None, - ): - # Initialize MCP sessions as a dictionary of ClientSession objects - self.sessions: Dict[ToolName, "ClientSession"] = {} - self.exit_stack = AsyncExitStack() - self.available_tools: List[ChatCompletionInputTool] = [] - # To be able to send the model in the payload if `base_url` is provided - if model is None and base_url is None: - raise ValueError("At least one of `model` or `base_url` should be set in `MCPClient`.") - self.payload_model = model - self.client = AsyncInferenceClient( - model=None if base_url is not None else model, - provider=provider, - api_key=api_key, - base_url=base_url, - ) - - async def __aenter__(self): - """Enter the context manager""" - await self.client.__aenter__() - await self.exit_stack.__aenter__() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Exit the context manager""" - await self.client.__aexit__(exc_type, exc_val, exc_tb) - await self.cleanup() - - async def cleanup(self): - """Clean up resources""" - await self.client.close() - await self.exit_stack.aclose() - - @overload - async def add_mcp_server(self, type: Literal["stdio"], **params: Unpack[StdioServerParameters_T]): ... - - @overload - async def add_mcp_server(self, type: Literal["sse"], **params: Unpack[SSEServerParameters_T]): ... - - @overload - async def add_mcp_server(self, type: Literal["http"], **params: Unpack[StreamableHTTPParameters_T]): ... - - async def add_mcp_server(self, type: ServerType, **params: Any): - """Connect to an MCP server - - Args: - type (`str`): - Type of the server to connect to. Can be one of: - - "stdio": Standard input/output server (local) - - "sse": Server-sent events (SSE) server - - "http": StreamableHTTP server - **params (`Dict[str, Any]`): - Server parameters that can be either: - - For stdio servers: - - command (str): The command to run the MCP server - - args (List[str], optional): Arguments for the command - - env (Dict[str, str], optional): Environment variables for the command - - cwd (Union[str, Path, None], optional): Working directory for the command - - allowed_tools (List[str], optional): List of tool names to allow from this server - - For SSE servers: - - url (str): The URL of the SSE server - - headers (Dict[str, Any], optional): Headers for the SSE connection - - timeout (float, optional): Connection timeout - - sse_read_timeout (float, optional): SSE read timeout - - allowed_tools (List[str], optional): List of tool names to allow from this server - - For StreamableHTTP servers: - - url (str): The URL of the StreamableHTTP server - - headers (Dict[str, Any], optional): Headers for the StreamableHTTP connection - - timeout (timedelta, optional): Connection timeout - - sse_read_timeout (timedelta, optional): SSE read timeout - - terminate_on_close (bool, optional): Whether to terminate on close - - allowed_tools (List[str], optional): List of tool names to allow from this server - """ - from mcp import ClientSession, StdioServerParameters - from mcp import types as mcp_types - - # Extract allowed_tools configuration if provided - allowed_tools = params.pop("allowed_tools", None) - - # Determine server type and create appropriate parameters - if type == "stdio": - # Handle stdio server - from mcp.client.stdio import stdio_client - - logger.info(f"Connecting to stdio MCP server with command: {params['command']} {params.get('args', [])}") - - client_kwargs = {"command": params["command"]} - for key in ["args", "env", "cwd"]: - if params.get(key) is not None: - client_kwargs[key] = params[key] - server_params = StdioServerParameters(**client_kwargs) - read, write = await self.exit_stack.enter_async_context(stdio_client(server_params)) - elif type == "sse": - # Handle SSE server - from mcp.client.sse import sse_client - - logger.info(f"Connecting to SSE MCP server at: {params['url']}") - - client_kwargs = {"url": params["url"]} - for key in ["headers", "timeout", "sse_read_timeout"]: - if params.get(key) is not None: - client_kwargs[key] = params[key] - read, write = await self.exit_stack.enter_async_context(sse_client(**client_kwargs)) - elif type == "http": - # Handle StreamableHTTP server - from mcp.client.streamable_http import streamablehttp_client - - logger.info(f"Connecting to StreamableHTTP MCP server at: {params['url']}") - - client_kwargs = {"url": params["url"]} - for key in ["headers", "timeout", "sse_read_timeout", "terminate_on_close"]: - if params.get(key) is not None: - client_kwargs[key] = params[key] - read, write, _ = await self.exit_stack.enter_async_context(streamablehttp_client(**client_kwargs)) - # ^ TODO: should be handle `get_session_id_callback`? (function to retrieve the current session ID) - else: - raise ValueError(f"Unsupported server type: {type}") - - session = await self.exit_stack.enter_async_context( - ClientSession( - read_stream=read, - write_stream=write, - client_info=mcp_types.Implementation( - name="huggingface_hub.MCPClient", - version=get_hf_hub_version(), - ), - ) - ) - - logger.debug("Initializing session...") - await session.initialize() - - # List available tools - response = await session.list_tools() - logger.debug("Connected to server with tools:", [tool.name for tool in response.tools]) - - # Filter tools based on allowed_tools configuration - filtered_tools = response.tools - - if allowed_tools is not None: - filtered_tools = [tool for tool in response.tools if tool.name in allowed_tools] - logger.debug( - f"Tool filtering applied. Using {len(filtered_tools)} of {len(response.tools)} available tools: {[tool.name for tool in filtered_tools]}" - ) - - for tool in filtered_tools: - if tool.name in self.sessions: - logger.warning(f"Tool '{tool.name}' already defined by another server. Skipping.") - continue - - # Map tool names to their server for later lookup - self.sessions[tool.name] = session - - # Add tool to the list of available tools (for use in chat completions) - self.available_tools.append( - ChatCompletionInputTool.parse_obj_as_instance( - { - "type": "function", - "function": { - "name": tool.name, - "description": tool.description, - "parameters": tool.inputSchema, - }, - } - ) - ) - - async def process_single_turn_with_tools( - self, - messages: List[Union[Dict, ChatCompletionInputMessage]], - exit_loop_tools: Optional[List[ChatCompletionInputTool]] = None, - exit_if_first_chunk_no_tool: bool = False, - ) -> AsyncIterable[Union[ChatCompletionStreamOutput, ChatCompletionInputMessage]]: - """Process a query using `self.model` and available tools, yielding chunks and tool outputs. - - Args: - messages (`List[Dict]`): - List of message objects representing the conversation history - exit_loop_tools (`List[ChatCompletionInputTool]`, *optional*): - List of tools that should exit the generator when called - exit_if_first_chunk_no_tool (`bool`, *optional*): - Exit if no tool is present in the first chunks. Default to False. - - Yields: - [`ChatCompletionStreamOutput`] chunks or [`ChatCompletionInputMessage`] objects - """ - # Prepare tools list based on options - tools = self.available_tools - if exit_loop_tools is not None: - tools = [*exit_loop_tools, *self.available_tools] - - # Create the streaming request - response = await self.client.chat.completions.create( - model=self.payload_model, - messages=messages, - tools=tools, - tool_choice="auto", - stream=True, - ) - - message: Dict[str, Any] = {"role": "unknown", "content": ""} - final_tool_calls: Dict[int, ChatCompletionStreamOutputDeltaToolCall] = {} - num_of_chunks = 0 - - # Read from stream - async for chunk in response: - num_of_chunks += 1 - delta = chunk.choices[0].delta if chunk.choices and len(chunk.choices) > 0 else None - if not delta: - continue - - # Process message - if delta.role: - message["role"] = delta.role - if delta.content: - message["content"] += delta.content - - # Process tool calls - if delta.tool_calls: - for tool_call in delta.tool_calls: - idx = tool_call.index - # first chunk for this tool call - if idx not in final_tool_calls: - final_tool_calls[idx] = tool_call - if final_tool_calls[idx].function.arguments is None: - final_tool_calls[idx].function.arguments = "" - continue - # safety before concatenating text to .function.arguments - if final_tool_calls[idx].function.arguments is None: - final_tool_calls[idx].function.arguments = "" - - if tool_call.function.arguments: - final_tool_calls[idx].function.arguments += tool_call.function.arguments - - # Optionally exit early if no tools in first chunks - if exit_if_first_chunk_no_tool and num_of_chunks <= 2 and len(final_tool_calls) == 0: - return - - # Yield each chunk to caller - yield chunk - - # Add the assistant message with tool calls (if any) to messages - if message["content"] or final_tool_calls: - # if the role is unknown, set it to assistant - if message.get("role") == "unknown": - message["role"] = "assistant" - # Convert final_tool_calls to the format expected by OpenAI - if final_tool_calls: - tool_calls_list: List[Dict[str, Any]] = [] - for tc in final_tool_calls.values(): - tool_calls_list.append( - { - "id": tc.id, - "type": "function", - "function": { - "name": tc.function.name, - "arguments": tc.function.arguments or "{}", - }, - } - ) - message["tool_calls"] = tool_calls_list - messages.append(message) - - # Process tool calls one by one - for tool_call in final_tool_calls.values(): - function_name = tool_call.function.name - try: - function_args = json.loads(tool_call.function.arguments or "{}") - except json.JSONDecodeError as err: - tool_message = { - "role": "tool", - "tool_call_id": tool_call.id, - "name": function_name, - "content": f"Invalid JSON generated by the model: {err}", - } - tool_message_as_obj = ChatCompletionInputMessage.parse_obj_as_instance(tool_message) - messages.append(tool_message_as_obj) - yield tool_message_as_obj - continue # move to next tool call - - tool_message = {"role": "tool", "tool_call_id": tool_call.id, "content": "", "name": function_name} - - # Check if this is an exit loop tool - if exit_loop_tools and function_name in [t.function.name for t in exit_loop_tools]: - tool_message_as_obj = ChatCompletionInputMessage.parse_obj_as_instance(tool_message) - messages.append(tool_message_as_obj) - yield tool_message_as_obj - return - - # Execute tool call with the appropriate session - session = self.sessions.get(function_name) - if session is not None: - try: - result = await session.call_tool(function_name, function_args) - tool_message["content"] = format_result(result) - except Exception as err: - tool_message["content"] = f"Error: MCP tool call failed with error message: {err}" - else: - tool_message["content"] = f"Error: No session found for tool: {function_name}" - - # Yield tool message - tool_message_as_obj = ChatCompletionInputMessage.parse_obj_as_instance(tool_message) - messages.append(tool_message_as_obj) - yield tool_message_as_obj diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/types.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/types.py deleted file mode 100644 index 100f67832ea02d7d5b6886d117536e97efe1c6ff..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/types.py +++ /dev/null @@ -1,45 +0,0 @@ -from typing import Dict, List, Literal, TypedDict, Union - -from typing_extensions import NotRequired - - -class InputConfig(TypedDict, total=False): - id: str - description: str - type: str - password: bool - - -class StdioServerConfig(TypedDict): - type: Literal["stdio"] - command: str - args: List[str] - env: Dict[str, str] - cwd: str - allowed_tools: NotRequired[List[str]] - - -class HTTPServerConfig(TypedDict): - type: Literal["http"] - url: str - headers: Dict[str, str] - allowed_tools: NotRequired[List[str]] - - -class SSEServerConfig(TypedDict): - type: Literal["sse"] - url: str - headers: Dict[str, str] - allowed_tools: NotRequired[List[str]] - - -ServerConfig = Union[StdioServerConfig, HTTPServerConfig, SSEServerConfig] - - -# AgentConfig root object -class AgentConfig(TypedDict): - model: str - provider: str - apiKey: NotRequired[str] - inputs: List[InputConfig] - servers: List[ServerConfig] diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/utils.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/utils.py deleted file mode 100644 index ddab10d6770397e4b1ad20ef4470679f3bfd60bb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_mcp/utils.py +++ /dev/null @@ -1,128 +0,0 @@ -""" -Utility functions for MCPClient and Tiny Agents. - -Formatting utilities taken from the JS SDK: https://github.com/huggingface/huggingface.js/blob/main/packages/mcp-client/src/ResultFormatter.ts. -""" - -import json -from pathlib import Path -from typing import TYPE_CHECKING, List, Optional, Tuple - -from huggingface_hub import snapshot_download -from huggingface_hub.errors import EntryNotFoundError - -from .constants import DEFAULT_AGENT, DEFAULT_REPO_ID, FILENAME_CONFIG, PROMPT_FILENAMES -from .types import AgentConfig - - -if TYPE_CHECKING: - from mcp import types as mcp_types - - -def format_result(result: "mcp_types.CallToolResult") -> str: - """ - Formats a mcp.types.CallToolResult content into a human-readable string. - - Args: - result (CallToolResult) - Object returned by mcp.ClientSession.call_tool. - - Returns: - str - A formatted string representing the content of the result. - """ - content = result.content - - if len(content) == 0: - return "[No content]" - - formatted_parts: List[str] = [] - - for item in content: - if item.type == "text": - formatted_parts.append(item.text) - - elif item.type == "image": - formatted_parts.append( - f"[Binary Content: Image {item.mimeType}, {_get_base64_size(item.data)} bytes]\n" - f"The task is complete and the content accessible to the User" - ) - - elif item.type == "audio": - formatted_parts.append( - f"[Binary Content: Audio {item.mimeType}, {_get_base64_size(item.data)} bytes]\n" - f"The task is complete and the content accessible to the User" - ) - - elif item.type == "resource": - resource = item.resource - - if hasattr(resource, "text"): - formatted_parts.append(resource.text) - - elif hasattr(resource, "blob"): - formatted_parts.append( - f"[Binary Content ({resource.uri}): {resource.mimeType}, {_get_base64_size(resource.blob)} bytes]\n" - f"The task is complete and the content accessible to the User" - ) - - return "\n".join(formatted_parts) - - -def _get_base64_size(base64_str: str) -> int: - """Estimate the byte size of a base64-encoded string.""" - # Remove any prefix like "data:image/png;base64," - if "," in base64_str: - base64_str = base64_str.split(",")[1] - - padding = 0 - if base64_str.endswith("=="): - padding = 2 - elif base64_str.endswith("="): - padding = 1 - - return (len(base64_str) * 3) // 4 - padding - - -def _load_agent_config(agent_path: Optional[str]) -> Tuple[AgentConfig, Optional[str]]: - """Load server config and prompt.""" - - def _read_dir(directory: Path) -> Tuple[AgentConfig, Optional[str]]: - cfg_file = directory / FILENAME_CONFIG - if not cfg_file.exists(): - raise FileNotFoundError(f" Config file not found in {directory}! Please make sure it exists locally") - - config: AgentConfig = json.loads(cfg_file.read_text(encoding="utf-8")) - prompt: Optional[str] = None - for filename in PROMPT_FILENAMES: - prompt_file = directory / filename - if prompt_file.exists(): - prompt = prompt_file.read_text(encoding="utf-8") - break - return config, prompt - - if agent_path is None: - return DEFAULT_AGENT, None # type: ignore[return-value] - - path = Path(agent_path).expanduser() - - if path.is_file(): - return json.loads(path.read_text(encoding="utf-8")), None - - if path.is_dir(): - return _read_dir(path) - - # fetch from the Hub - try: - repo_dir = Path( - snapshot_download( - repo_id=DEFAULT_REPO_ID, - allow_patterns=f"{agent_path}/*", - repo_type="dataset", - ) - ) - return _read_dir(repo_dir / agent_path) - except Exception as err: - raise EntryNotFoundError( - f" Agent {agent_path} not found in tiny-agents/tiny-agents! Please make sure it exists in https://huggingface.co/datasets/tiny-agents/tiny-agents." - ) from err diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/__init__.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/__init__.py deleted file mode 100644 index 79d2bd75c8329f73bf466cd6b14467579595d180..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/__init__.py +++ /dev/null @@ -1,231 +0,0 @@ -from typing import Dict, Literal, Optional, Union - -from huggingface_hub.inference._providers.featherless_ai import ( - FeatherlessConversationalTask, - FeatherlessTextGenerationTask, -) -from huggingface_hub.utils import logging - -from ._common import TaskProviderHelper, _fetch_inference_provider_mapping -from .black_forest_labs import BlackForestLabsTextToImageTask -from .cerebras import CerebrasConversationalTask -from .clarifai import ClarifaiConversationalTask -from .cohere import CohereConversationalTask -from .fal_ai import ( - FalAIAutomaticSpeechRecognitionTask, - FalAIImageToImageTask, - FalAIImageToVideoTask, - FalAITextToImageTask, - FalAITextToSpeechTask, - FalAITextToVideoTask, -) -from .fireworks_ai import FireworksAIConversationalTask -from .groq import GroqConversationalTask -from .hf_inference import ( - HFInferenceBinaryInputTask, - HFInferenceConversational, - HFInferenceFeatureExtractionTask, - HFInferenceTask, -) -from .hyperbolic import HyperbolicTextGenerationTask, HyperbolicTextToImageTask -from .nebius import ( - NebiusConversationalTask, - NebiusFeatureExtractionTask, - NebiusTextGenerationTask, - NebiusTextToImageTask, -) -from .novita import NovitaConversationalTask, NovitaTextGenerationTask, NovitaTextToVideoTask -from .nscale import NscaleConversationalTask, NscaleTextToImageTask -from .openai import OpenAIConversationalTask -from .publicai import PublicAIConversationalTask -from .replicate import ReplicateImageToImageTask, ReplicateTask, ReplicateTextToImageTask, ReplicateTextToSpeechTask -from .sambanova import SambanovaConversationalTask, SambanovaFeatureExtractionTask -from .scaleway import ScalewayConversationalTask, ScalewayFeatureExtractionTask -from .together import TogetherConversationalTask, TogetherTextGenerationTask, TogetherTextToImageTask -from .zai_org import ZaiConversationalTask - - -logger = logging.get_logger(__name__) - - -PROVIDER_T = Literal[ - "black-forest-labs", - "cerebras", - "clarifai", - "cohere", - "fal-ai", - "featherless-ai", - "fireworks-ai", - "groq", - "hf-inference", - "hyperbolic", - "nebius", - "novita", - "nscale", - "openai", - "publicai", - "replicate", - "sambanova", - "scaleway", - "together", - "zai-org", -] - -PROVIDER_OR_POLICY_T = Union[PROVIDER_T, Literal["auto"]] - -PROVIDERS: Dict[PROVIDER_T, Dict[str, TaskProviderHelper]] = { - "black-forest-labs": { - "text-to-image": BlackForestLabsTextToImageTask(), - }, - "cerebras": { - "conversational": CerebrasConversationalTask(), - }, - "clarifai": { - "conversational": ClarifaiConversationalTask(), - }, - "cohere": { - "conversational": CohereConversationalTask(), - }, - "fal-ai": { - "automatic-speech-recognition": FalAIAutomaticSpeechRecognitionTask(), - "text-to-image": FalAITextToImageTask(), - "text-to-speech": FalAITextToSpeechTask(), - "text-to-video": FalAITextToVideoTask(), - "image-to-video": FalAIImageToVideoTask(), - "image-to-image": FalAIImageToImageTask(), - }, - "featherless-ai": { - "conversational": FeatherlessConversationalTask(), - "text-generation": FeatherlessTextGenerationTask(), - }, - "fireworks-ai": { - "conversational": FireworksAIConversationalTask(), - }, - "groq": { - "conversational": GroqConversationalTask(), - }, - "hf-inference": { - "text-to-image": HFInferenceTask("text-to-image"), - "conversational": HFInferenceConversational(), - "text-generation": HFInferenceTask("text-generation"), - "text-classification": HFInferenceTask("text-classification"), - "question-answering": HFInferenceTask("question-answering"), - "audio-classification": HFInferenceBinaryInputTask("audio-classification"), - "automatic-speech-recognition": HFInferenceBinaryInputTask("automatic-speech-recognition"), - "fill-mask": HFInferenceTask("fill-mask"), - "feature-extraction": HFInferenceFeatureExtractionTask(), - "image-classification": HFInferenceBinaryInputTask("image-classification"), - "image-segmentation": HFInferenceBinaryInputTask("image-segmentation"), - "document-question-answering": HFInferenceTask("document-question-answering"), - "image-to-text": HFInferenceBinaryInputTask("image-to-text"), - "object-detection": HFInferenceBinaryInputTask("object-detection"), - "audio-to-audio": HFInferenceBinaryInputTask("audio-to-audio"), - "zero-shot-image-classification": HFInferenceBinaryInputTask("zero-shot-image-classification"), - "zero-shot-classification": HFInferenceTask("zero-shot-classification"), - "image-to-image": HFInferenceBinaryInputTask("image-to-image"), - "sentence-similarity": HFInferenceTask("sentence-similarity"), - "table-question-answering": HFInferenceTask("table-question-answering"), - "tabular-classification": HFInferenceTask("tabular-classification"), - "text-to-speech": HFInferenceTask("text-to-speech"), - "token-classification": HFInferenceTask("token-classification"), - "translation": HFInferenceTask("translation"), - "summarization": HFInferenceTask("summarization"), - "visual-question-answering": HFInferenceBinaryInputTask("visual-question-answering"), - }, - "hyperbolic": { - "text-to-image": HyperbolicTextToImageTask(), - "conversational": HyperbolicTextGenerationTask("conversational"), - "text-generation": HyperbolicTextGenerationTask("text-generation"), - }, - "nebius": { - "text-to-image": NebiusTextToImageTask(), - "conversational": NebiusConversationalTask(), - "text-generation": NebiusTextGenerationTask(), - "feature-extraction": NebiusFeatureExtractionTask(), - }, - "novita": { - "text-generation": NovitaTextGenerationTask(), - "conversational": NovitaConversationalTask(), - "text-to-video": NovitaTextToVideoTask(), - }, - "nscale": { - "conversational": NscaleConversationalTask(), - "text-to-image": NscaleTextToImageTask(), - }, - "openai": { - "conversational": OpenAIConversationalTask(), - }, - "publicai": { - "conversational": PublicAIConversationalTask(), - }, - "replicate": { - "image-to-image": ReplicateImageToImageTask(), - "text-to-image": ReplicateTextToImageTask(), - "text-to-speech": ReplicateTextToSpeechTask(), - "text-to-video": ReplicateTask("text-to-video"), - }, - "sambanova": { - "conversational": SambanovaConversationalTask(), - "feature-extraction": SambanovaFeatureExtractionTask(), - }, - "scaleway": { - "conversational": ScalewayConversationalTask(), - "feature-extraction": ScalewayFeatureExtractionTask(), - }, - "together": { - "text-to-image": TogetherTextToImageTask(), - "conversational": TogetherConversationalTask(), - "text-generation": TogetherTextGenerationTask(), - }, - "zai-org": { - "conversational": ZaiConversationalTask(), - }, -} - - -def get_provider_helper( - provider: Optional[PROVIDER_OR_POLICY_T], task: str, model: Optional[str] -) -> TaskProviderHelper: - """Get provider helper instance by name and task. - - Args: - provider (`str`, *optional*): name of the provider, or "auto" to automatically select the provider for the model. - task (`str`): Name of the task - model (`str`, *optional*): Name of the model - Returns: - TaskProviderHelper: Helper instance for the specified provider and task - - Raises: - ValueError: If provider or task is not supported - """ - - if (model is None and provider in (None, "auto")) or ( - model is not None and model.startswith(("http://", "https://")) - ): - provider = "hf-inference" - - if provider is None: - logger.info( - "Defaulting to 'auto' which will select the first provider available for the model, sorted by the user's order in https://hf.co/settings/inference-providers." - ) - provider = "auto" - - if provider == "auto": - if model is None: - raise ValueError("Specifying a model is required when provider is 'auto'") - provider_mapping = _fetch_inference_provider_mapping(model) - provider = next(iter(provider_mapping)).provider - - provider_tasks = PROVIDERS.get(provider) # type: ignore - if provider_tasks is None: - raise ValueError( - f"Provider '{provider}' not supported. Available values: 'auto' or any provider from {list(PROVIDERS.keys())}." - "Passing 'auto' (default value) will automatically select the first provider available for the model, sorted " - "by the user's order in https://hf.co/settings/inference-providers." - ) - - if task not in provider_tasks: - raise ValueError( - f"Task '{task}' not supported for provider '{provider}'. Available tasks: {list(provider_tasks.keys())}" - ) - return provider_tasks[task] diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/_common.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/_common.py deleted file mode 100644 index 366fc3f45d6760e21c748e0ead7e4b3510efbc72..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/_common.py +++ /dev/null @@ -1,323 +0,0 @@ -from functools import lru_cache -from typing import Any, Dict, List, Optional, Union, overload - -from huggingface_hub import constants -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import MimeBytes, RequestParameters -from huggingface_hub.inference._generated.types.chat_completion import ChatCompletionInputMessage -from huggingface_hub.utils import build_hf_headers, get_token, logging - - -logger = logging.get_logger(__name__) - - -# Dev purposes only. -# If you want to try to run inference for a new model locally before it's registered on huggingface.co -# for a given Inference Provider, you can add it to the following dictionary. -HARDCODED_MODEL_INFERENCE_MAPPING: Dict[str, Dict[str, InferenceProviderMapping]] = { - # "HF model ID" => InferenceProviderMapping object initialized with "Model ID on Inference Provider's side" - # - # Example: - # "Qwen/Qwen2.5-Coder-32B-Instruct": InferenceProviderMapping(hf_model_id="Qwen/Qwen2.5-Coder-32B-Instruct", - # provider_id="Qwen2.5-Coder-32B-Instruct", - # task="conversational", - # status="live") - "cerebras": {}, - "cohere": {}, - "clarifai": {}, - "fal-ai": {}, - "fireworks-ai": {}, - "groq": {}, - "hf-inference": {}, - "hyperbolic": {}, - "nebius": {}, - "nscale": {}, - "replicate": {}, - "sambanova": {}, - "scaleway": {}, - "together": {}, - "zai-org": {}, -} - - -@overload -def filter_none(obj: Dict[str, Any]) -> Dict[str, Any]: ... -@overload -def filter_none(obj: List[Any]) -> List[Any]: ... - - -def filter_none(obj: Union[Dict[str, Any], List[Any]]) -> Union[Dict[str, Any], List[Any]]: - if isinstance(obj, dict): - cleaned: Dict[str, Any] = {} - for k, v in obj.items(): - if v is None: - continue - if isinstance(v, (dict, list)): - v = filter_none(v) - cleaned[k] = v - return cleaned - - if isinstance(obj, list): - return [filter_none(v) if isinstance(v, (dict, list)) else v for v in obj] - - raise ValueError(f"Expected dict or list, got {type(obj)}") - - -class TaskProviderHelper: - """Base class for task-specific provider helpers.""" - - def __init__(self, provider: str, base_url: str, task: str) -> None: - self.provider = provider - self.task = task - self.base_url = base_url - - def prepare_request( - self, - *, - inputs: Any, - parameters: Dict[str, Any], - headers: Dict, - model: Optional[str], - api_key: Optional[str], - extra_payload: Optional[Dict[str, Any]] = None, - ) -> RequestParameters: - """ - Prepare the request to be sent to the provider. - - Each step (api_key, model, headers, url, payload) can be customized in subclasses. - """ - # api_key from user, or local token, or raise error - api_key = self._prepare_api_key(api_key) - - # mapped model from HF model ID - provider_mapping_info = self._prepare_mapping_info(model) - - # default HF headers + user headers (to customize in subclasses) - headers = self._prepare_headers(headers, api_key) - - # routed URL if HF token, or direct URL (to customize in '_prepare_route' in subclasses) - url = self._prepare_url(api_key, provider_mapping_info.provider_id) - - # prepare payload (to customize in subclasses) - payload = self._prepare_payload_as_dict(inputs, parameters, provider_mapping_info=provider_mapping_info) - if payload is not None: - payload = recursive_merge(payload, filter_none(extra_payload or {})) - - # body data (to customize in subclasses) - data = self._prepare_payload_as_bytes(inputs, parameters, provider_mapping_info, extra_payload) - - # check if both payload and data are set and return - if payload is not None and data is not None: - raise ValueError("Both payload and data cannot be set in the same request.") - if payload is None and data is None: - raise ValueError("Either payload or data must be set in the request.") - - # normalize headers to lowercase and add content-type if not present - normalized_headers = self._normalize_headers(headers, payload, data) - - return RequestParameters( - url=url, - task=self.task, - model=provider_mapping_info.provider_id, - json=payload, - data=data, - headers=normalized_headers, - ) - - def get_response( - self, - response: Union[bytes, Dict], - request_params: Optional[RequestParameters] = None, - ) -> Any: - """ - Return the response in the expected format. - - Override this method in subclasses for customized response handling.""" - return response - - def _prepare_api_key(self, api_key: Optional[str]) -> str: - """Return the API key to use for the request. - - Usually not overwritten in subclasses.""" - if api_key is None: - api_key = get_token() - if api_key is None: - raise ValueError( - f"You must provide an api_key to work with {self.provider} API or log in with `hf auth login`." - ) - return api_key - - def _prepare_mapping_info(self, model: Optional[str]) -> InferenceProviderMapping: - """Return the mapped model ID to use for the request. - - Usually not overwritten in subclasses.""" - if model is None: - raise ValueError(f"Please provide an HF model ID supported by {self.provider}.") - - # hardcoded mapping for local testing - if HARDCODED_MODEL_INFERENCE_MAPPING.get(self.provider, {}).get(model): - return HARDCODED_MODEL_INFERENCE_MAPPING[self.provider][model] - - provider_mapping = None - for mapping in _fetch_inference_provider_mapping(model): - if mapping.provider == self.provider: - provider_mapping = mapping - break - - if provider_mapping is None: - raise ValueError(f"Model {model} is not supported by provider {self.provider}.") - - if provider_mapping.task != self.task: - raise ValueError( - f"Model {model} is not supported for task {self.task} and provider {self.provider}. " - f"Supported task: {provider_mapping.task}." - ) - - if provider_mapping.status == "staging": - logger.warning( - f"Model {model} is in staging mode for provider {self.provider}. Meant for test purposes only." - ) - if provider_mapping.status == "error": - logger.warning( - f"Our latest automated health check on model '{model}' for provider '{self.provider}' did not complete successfully. " - "Inference call might fail." - ) - return provider_mapping - - def _normalize_headers( - self, headers: Dict[str, Any], payload: Optional[Dict[str, Any]], data: Optional[MimeBytes] - ) -> Dict[str, Any]: - """Normalize the headers to use for the request. - - Override this method in subclasses for customized headers. - """ - normalized_headers = {key.lower(): value for key, value in headers.items() if value is not None} - if normalized_headers.get("content-type") is None: - if data is not None and data.mime_type is not None: - normalized_headers["content-type"] = data.mime_type - elif payload is not None: - normalized_headers["content-type"] = "application/json" - return normalized_headers - - def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]: - """Return the headers to use for the request. - - Override this method in subclasses for customized headers. - """ - return {**build_hf_headers(token=api_key), **headers} - - def _prepare_url(self, api_key: str, mapped_model: str) -> str: - """Return the URL to use for the request. - - Usually not overwritten in subclasses.""" - base_url = self._prepare_base_url(api_key) - route = self._prepare_route(mapped_model, api_key) - return f"{base_url.rstrip('/')}/{route.lstrip('/')}" - - def _prepare_base_url(self, api_key: str) -> str: - """Return the base URL to use for the request. - - Usually not overwritten in subclasses.""" - # Route to the proxy if the api_key is a HF TOKEN - if api_key.startswith("hf_"): - logger.info(f"Calling '{self.provider}' provider through Hugging Face router.") - return constants.INFERENCE_PROXY_TEMPLATE.format(provider=self.provider) - else: - logger.info(f"Calling '{self.provider}' provider directly.") - return self.base_url - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - """Return the route to use for the request. - - Override this method in subclasses for customized routes. - """ - return "" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - """Return the payload to use for the request, as a dict. - - Override this method in subclasses for customized payloads. - Only one of `_prepare_payload_as_dict` and `_prepare_payload_as_bytes` should return a value. - """ - return None - - def _prepare_payload_as_bytes( - self, - inputs: Any, - parameters: Dict, - provider_mapping_info: InferenceProviderMapping, - extra_payload: Optional[Dict], - ) -> Optional[MimeBytes]: - """Return the body to use for the request, as bytes. - - Override this method in subclasses for customized body data. - Only one of `_prepare_payload_as_dict` and `_prepare_payload_as_bytes` should return a value. - """ - return None - - -class BaseConversationalTask(TaskProviderHelper): - """ - Base class for conversational (chat completion) tasks. - The schema follows the OpenAI API format defined here: https://platform.openai.com/docs/api-reference/chat - """ - - def __init__(self, provider: str, base_url: str): - super().__init__(provider=provider, base_url=base_url, task="conversational") - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/v1/chat/completions" - - def _prepare_payload_as_dict( - self, - inputs: List[Union[Dict, ChatCompletionInputMessage]], - parameters: Dict, - provider_mapping_info: InferenceProviderMapping, - ) -> Optional[Dict]: - return filter_none({"messages": inputs, **parameters, "model": provider_mapping_info.provider_id}) - - -class BaseTextGenerationTask(TaskProviderHelper): - """ - Base class for text-generation (completion) tasks. - The schema follows the OpenAI API format defined here: https://platform.openai.com/docs/api-reference/completions - """ - - def __init__(self, provider: str, base_url: str): - super().__init__(provider=provider, base_url=base_url, task="text-generation") - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/v1/completions" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - return filter_none({"prompt": inputs, **parameters, "model": provider_mapping_info.provider_id}) - - -@lru_cache(maxsize=None) -def _fetch_inference_provider_mapping(model: str) -> List["InferenceProviderMapping"]: - """ - Fetch provider mappings for a model from the Hub. - """ - from huggingface_hub.hf_api import HfApi - - info = HfApi().model_info(model, expand=["inferenceProviderMapping"]) - provider_mapping = info.inference_provider_mapping - if provider_mapping is None: - raise ValueError(f"No provider mapping found for model {model}") - return provider_mapping - - -def recursive_merge(dict1: Dict, dict2: Dict) -> Dict: - return { - **dict1, - **{ - key: recursive_merge(dict1[key], value) - if (key in dict1 and isinstance(dict1[key], dict) and isinstance(value, dict)) - else value - for key, value in dict2.items() - }, - } diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/black_forest_labs.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/black_forest_labs.py deleted file mode 100644 index a5d96832256e3505d503a7d23bbcee76e485561a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/black_forest_labs.py +++ /dev/null @@ -1,69 +0,0 @@ -import time -from typing import Any, Dict, Optional, Union - -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import RequestParameters, _as_dict -from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none -from huggingface_hub.utils import logging -from huggingface_hub.utils._http import get_session - - -logger = logging.get_logger(__name__) - -MAX_POLLING_ATTEMPTS = 6 -POLLING_INTERVAL = 1.0 - - -class BlackForestLabsTextToImageTask(TaskProviderHelper): - def __init__(self): - super().__init__(provider="black-forest-labs", base_url="https://api.us1.bfl.ai", task="text-to-image") - - def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]: - headers = super()._prepare_headers(headers, api_key) - if not api_key.startswith("hf_"): - _ = headers.pop("authorization") - headers["X-Key"] = api_key - return headers - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return f"/v1/{mapped_model}" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - parameters = filter_none(parameters) - if "num_inference_steps" in parameters: - parameters["steps"] = parameters.pop("num_inference_steps") - if "guidance_scale" in parameters: - parameters["guidance"] = parameters.pop("guidance_scale") - - return {"prompt": inputs, **parameters} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - """ - Polling mechanism for Black Forest Labs since the API is asynchronous. - """ - url = _as_dict(response).get("polling_url") - session = get_session() - for _ in range(MAX_POLLING_ATTEMPTS): - time.sleep(POLLING_INTERVAL) - - response = session.get(url, headers={"Content-Type": "application/json"}) # type: ignore - response.raise_for_status() # type: ignore - response_json: Dict = response.json() # type: ignore - status = response_json.get("status") - logger.info( - f"Polling generation result from {url}. Current status: {status}. " - f"Will retry after {POLLING_INTERVAL} seconds if not ready." - ) - - if ( - status == "Ready" - and isinstance(response_json.get("result"), dict) - and (sample_url := response_json["result"].get("sample")) - ): - image_resp = session.get(sample_url) - image_resp.raise_for_status() - return image_resp.content - - raise TimeoutError(f"Failed to get the image URL after {MAX_POLLING_ATTEMPTS} attempts.") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/cerebras.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/cerebras.py deleted file mode 100644 index a9b9c3aacb3e134a8e755297c15ece198ffe633d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/cerebras.py +++ /dev/null @@ -1,6 +0,0 @@ -from ._common import BaseConversationalTask - - -class CerebrasConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider="cerebras", base_url="https://api.cerebras.ai") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/clarifai.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/clarifai.py deleted file mode 100644 index 5f118b7fc9a8dafb01305758791191ccef045a5d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/clarifai.py +++ /dev/null @@ -1,13 +0,0 @@ -from ._common import BaseConversationalTask - - -_PROVIDER = "clarifai" -_BASE_URL = "https://api.clarifai.com" - - -class ClarifaiConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider=_PROVIDER, base_url=_BASE_URL) - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/v2/ext/openai/v1/chat/completions" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/cohere.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/cohere.py deleted file mode 100644 index a5e9191caec50b0e659dddceba3e817a4ac28307..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/cohere.py +++ /dev/null @@ -1,32 +0,0 @@ -from typing import Any, Dict, Optional - -from huggingface_hub.hf_api import InferenceProviderMapping - -from ._common import BaseConversationalTask - - -_PROVIDER = "cohere" -_BASE_URL = "https://api.cohere.com" - - -class CohereConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider=_PROVIDER, base_url=_BASE_URL) - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/compatibility/v1/chat/completions" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) - response_format = parameters.get("response_format") - if isinstance(response_format, dict) and response_format.get("type") == "json_schema": - json_schema_details = response_format.get("json_schema") - if isinstance(json_schema_details, dict) and "schema" in json_schema_details: - payload["response_format"] = { # type: ignore [index] - "type": "json_object", - "schema": json_schema_details["schema"], - } - - return payload diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/fal_ai.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/fal_ai.py deleted file mode 100644 index bc2c41d04f811f6a6508ea6abf84593add31ef42..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/fal_ai.py +++ /dev/null @@ -1,248 +0,0 @@ -import base64 -import time -from abc import ABC -from typing import Any, Dict, Optional, Union -from urllib.parse import urlparse - -from huggingface_hub import constants -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import RequestParameters, _as_dict, _as_url -from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none -from huggingface_hub.utils import get_session, hf_raise_for_status -from huggingface_hub.utils.logging import get_logger - - -logger = get_logger(__name__) - -# Arbitrary polling interval -_POLLING_INTERVAL = 0.5 - - -class FalAITask(TaskProviderHelper, ABC): - def __init__(self, task: str): - super().__init__(provider="fal-ai", base_url="https://fal.run", task=task) - - def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]: - headers = super()._prepare_headers(headers, api_key) - if not api_key.startswith("hf_"): - headers["authorization"] = f"Key {api_key}" - return headers - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return f"/{mapped_model}" - - -class FalAIQueueTask(TaskProviderHelper, ABC): - def __init__(self, task: str): - super().__init__(provider="fal-ai", base_url="https://queue.fal.run", task=task) - - def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]: - headers = super()._prepare_headers(headers, api_key) - if not api_key.startswith("hf_"): - headers["authorization"] = f"Key {api_key}" - return headers - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - if api_key.startswith("hf_"): - # Use the queue subdomain for HF routing - return f"/{mapped_model}?_subdomain=queue" - return f"/{mapped_model}" - - def get_response( - self, - response: Union[bytes, Dict], - request_params: Optional[RequestParameters] = None, - ) -> Any: - response_dict = _as_dict(response) - - request_id = response_dict.get("request_id") - if not request_id: - raise ValueError("No request ID found in the response") - if request_params is None: - raise ValueError( - f"A `RequestParameters` object should be provided to get {self.task} responses with Fal AI." - ) - - # extract the base url and query params - parsed_url = urlparse(request_params.url) - # a bit hacky way to concatenate the provider name without parsing `parsed_url.path` - base_url = f"{parsed_url.scheme}://{parsed_url.netloc}{'/fal-ai' if parsed_url.netloc == 'router.huggingface.co' else ''}" - query_param = f"?{parsed_url.query}" if parsed_url.query else "" - - # extracting the provider model id for status and result urls - # from the response as it might be different from the mapped model in `request_params.url` - model_id = urlparse(response_dict.get("response_url")).path - status_url = f"{base_url}{str(model_id)}/status{query_param}" - result_url = f"{base_url}{str(model_id)}{query_param}" - - status = response_dict.get("status") - logger.info("Generating the output.. this can take several minutes.") - while status != "COMPLETED": - time.sleep(_POLLING_INTERVAL) - status_response = get_session().get(status_url, headers=request_params.headers) - hf_raise_for_status(status_response) - status = status_response.json().get("status") - - return get_session().get(result_url, headers=request_params.headers).json() - - -class FalAIAutomaticSpeechRecognitionTask(FalAITask): - def __init__(self): - super().__init__("automatic-speech-recognition") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - if isinstance(inputs, str) and inputs.startswith(("http://", "https://")): - # If input is a URL, pass it directly - audio_url = inputs - else: - # If input is a file path, read it first - if isinstance(inputs, str): - with open(inputs, "rb") as f: - inputs = f.read() - - audio_b64 = base64.b64encode(inputs).decode() - content_type = "audio/mpeg" - audio_url = f"data:{content_type};base64,{audio_b64}" - - return {"audio_url": audio_url, **filter_none(parameters)} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - text = _as_dict(response)["text"] - if not isinstance(text, str): - raise ValueError(f"Unexpected output format from FalAI API. Expected string, got {type(text)}.") - return text - - -class FalAITextToImageTask(FalAITask): - def __init__(self): - super().__init__("text-to-image") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - payload: Dict[str, Any] = { - "prompt": inputs, - **filter_none(parameters), - } - if "width" in payload and "height" in payload: - payload["image_size"] = { - "width": payload.pop("width"), - "height": payload.pop("height"), - } - if provider_mapping_info.adapter_weights_path is not None: - lora_path = constants.HUGGINGFACE_CO_URL_TEMPLATE.format( - repo_id=provider_mapping_info.hf_model_id, - revision="main", - filename=provider_mapping_info.adapter_weights_path, - ) - payload["loras"] = [{"path": lora_path, "scale": 1}] - if provider_mapping_info.provider_id == "fal-ai/lora": - # little hack: fal requires the base model for stable-diffusion-based loras but not for flux-based - # See payloads in https://fal.ai/models/fal-ai/lora/api vs https://fal.ai/models/fal-ai/flux-lora/api - payload["model_name"] = "stabilityai/stable-diffusion-xl-base-1.0" - - return payload - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - url = _as_dict(response)["images"][0]["url"] - return get_session().get(url).content - - -class FalAITextToSpeechTask(FalAITask): - def __init__(self): - super().__init__("text-to-speech") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - return {"text": inputs, **filter_none(parameters)} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - url = _as_dict(response)["audio"]["url"] - return get_session().get(url).content - - -class FalAITextToVideoTask(FalAIQueueTask): - def __init__(self): - super().__init__("text-to-video") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - return {"prompt": inputs, **filter_none(parameters)} - - def get_response( - self, - response: Union[bytes, Dict], - request_params: Optional[RequestParameters] = None, - ) -> Any: - output = super().get_response(response, request_params) - url = _as_dict(output)["video"]["url"] - return get_session().get(url).content - - -class FalAIImageToImageTask(FalAIQueueTask): - def __init__(self): - super().__init__("image-to-image") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - image_url = _as_url(inputs, default_mime_type="image/jpeg") - if "target_size" in parameters: - parameters["image_size"] = parameters.pop("target_size") - payload: Dict[str, Any] = { - "image_url": image_url, - **filter_none(parameters), - } - if provider_mapping_info.adapter_weights_path is not None: - lora_path = constants.HUGGINGFACE_CO_URL_TEMPLATE.format( - repo_id=provider_mapping_info.hf_model_id, - revision="main", - filename=provider_mapping_info.adapter_weights_path, - ) - payload["loras"] = [{"path": lora_path, "scale": 1}] - - return payload - - def get_response( - self, - response: Union[bytes, Dict], - request_params: Optional[RequestParameters] = None, - ) -> Any: - output = super().get_response(response, request_params) - url = _as_dict(output)["images"][0]["url"] - return get_session().get(url).content - - -class FalAIImageToVideoTask(FalAIQueueTask): - def __init__(self): - super().__init__("image-to-video") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - image_url = _as_url(inputs, default_mime_type="image/jpeg") - payload: Dict[str, Any] = { - "image_url": image_url, - **filter_none(parameters), - } - if provider_mapping_info.adapter_weights_path is not None: - lora_path = constants.HUGGINGFACE_CO_URL_TEMPLATE.format( - repo_id=provider_mapping_info.hf_model_id, - revision="main", - filename=provider_mapping_info.adapter_weights_path, - ) - payload["loras"] = [{"path": lora_path, "scale": 1}] - return payload - - def get_response( - self, - response: Union[bytes, Dict], - request_params: Optional[RequestParameters] = None, - ) -> Any: - output = super().get_response(response, request_params) - url = _as_dict(output)["video"]["url"] - return get_session().get(url).content diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/featherless_ai.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/featherless_ai.py deleted file mode 100644 index 6ad1c48134f5c990b6ac4fca5ff919f4cc0d2373..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/featherless_ai.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Any, Dict, Optional, Union - -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import RequestParameters, _as_dict - -from ._common import BaseConversationalTask, BaseTextGenerationTask, filter_none - - -_PROVIDER = "featherless-ai" -_BASE_URL = "https://api.featherless.ai" - - -class FeatherlessTextGenerationTask(BaseTextGenerationTask): - def __init__(self): - super().__init__(provider=_PROVIDER, base_url=_BASE_URL) - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - params = filter_none(parameters.copy()) - params["max_tokens"] = params.pop("max_new_tokens", None) - - return {"prompt": inputs, **params, "model": provider_mapping_info.provider_id} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - output = _as_dict(response)["choices"][0] - return { - "generated_text": output["text"], - "details": { - "finish_reason": output.get("finish_reason"), - "seed": output.get("seed"), - }, - } - - -class FeatherlessConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider=_PROVIDER, base_url=_BASE_URL) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/fireworks_ai.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/fireworks_ai.py deleted file mode 100644 index b4cc19a5700047f6516b2784d9785a99d7e32451..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/fireworks_ai.py +++ /dev/null @@ -1,27 +0,0 @@ -from typing import Any, Dict, Optional - -from huggingface_hub.hf_api import InferenceProviderMapping - -from ._common import BaseConversationalTask - - -class FireworksAIConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider="fireworks-ai", base_url="https://api.fireworks.ai") - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/inference/v1/chat/completions" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) - response_format = parameters.get("response_format") - if isinstance(response_format, dict) and response_format.get("type") == "json_schema": - json_schema_details = response_format.get("json_schema") - if isinstance(json_schema_details, dict) and "schema" in json_schema_details: - payload["response_format"] = { # type: ignore [index] - "type": "json_object", - "schema": json_schema_details["schema"], - } - return payload diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/groq.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/groq.py deleted file mode 100644 index 11e677504e89bc02b966e7d37d9e11f1b94b297f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/groq.py +++ /dev/null @@ -1,9 +0,0 @@ -from ._common import BaseConversationalTask - - -class GroqConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider="groq", base_url="https://api.groq.com") - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/openai/v1/chat/completions" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/hf_inference.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/hf_inference.py deleted file mode 100644 index d90d00c4f3e5b93029ed979df6e310635a639d93..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/hf_inference.py +++ /dev/null @@ -1,228 +0,0 @@ -import json -from functools import lru_cache -from pathlib import Path -from typing import Any, Dict, Optional, Union -from urllib.parse import urlparse, urlunparse - -from huggingface_hub import constants -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import ( - MimeBytes, - RequestParameters, - _b64_encode, - _bytes_to_dict, - _open_as_mime_bytes, -) -from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none -from huggingface_hub.utils import build_hf_headers, get_session, get_token, hf_raise_for_status - - -class HFInferenceTask(TaskProviderHelper): - """Base class for HF Inference API tasks.""" - - def __init__(self, task: str): - super().__init__( - provider="hf-inference", - base_url=constants.INFERENCE_PROXY_TEMPLATE.format(provider="hf-inference"), - task=task, - ) - - def _prepare_api_key(self, api_key: Optional[str]) -> str: - # special case: for HF Inference we allow not providing an API key - return api_key or get_token() # type: ignore[return-value] - - def _prepare_mapping_info(self, model: Optional[str]) -> InferenceProviderMapping: - if model is not None and model.startswith(("http://", "https://")): - return InferenceProviderMapping( - provider="hf-inference", providerId=model, hf_model_id=model, task=self.task, status="live" - ) - model_id = model if model is not None else _fetch_recommended_models().get(self.task) - if model_id is None: - raise ValueError( - f"Task {self.task} has no recommended model for HF Inference. Please specify a model" - " explicitly. Visit https://huggingface.co/tasks for more info." - ) - _check_supported_task(model_id, self.task) - return InferenceProviderMapping( - provider="hf-inference", providerId=model_id, hf_model_id=model_id, task=self.task, status="live" - ) - - def _prepare_url(self, api_key: str, mapped_model: str) -> str: - # hf-inference provider can handle URLs (e.g. Inference Endpoints or TGI deployment) - if mapped_model.startswith(("http://", "https://")): - return mapped_model - return ( - # Feature-extraction and sentence-similarity are the only cases where we handle models with several tasks. - f"{self.base_url}/models/{mapped_model}/pipeline/{self.task}" - if self.task in ("feature-extraction", "sentence-similarity") - # Otherwise, we use the default endpoint - else f"{self.base_url}/models/{mapped_model}" - ) - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - if isinstance(inputs, bytes): - raise ValueError(f"Unexpected binary input for task {self.task}.") - if isinstance(inputs, Path): - raise ValueError(f"Unexpected path input for task {self.task} (got {inputs})") - return filter_none({"inputs": inputs, "parameters": parameters}) - - -class HFInferenceBinaryInputTask(HFInferenceTask): - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - return None - - def _prepare_payload_as_bytes( - self, - inputs: Any, - parameters: Dict, - provider_mapping_info: InferenceProviderMapping, - extra_payload: Optional[Dict], - ) -> Optional[MimeBytes]: - parameters = filter_none(parameters) - extra_payload = extra_payload or {} - has_parameters = len(parameters) > 0 or len(extra_payload) > 0 - - # Raise if not a binary object or a local path or a URL. - if not isinstance(inputs, (bytes, Path)) and not isinstance(inputs, str): - raise ValueError(f"Expected binary inputs or a local path or a URL. Got {inputs}") - - # Send inputs as raw content when no parameters are provided - if not has_parameters: - return _open_as_mime_bytes(inputs) - - # Otherwise encode as b64 - return MimeBytes( - json.dumps({"inputs": _b64_encode(inputs), "parameters": parameters, **extra_payload}).encode("utf-8"), - mime_type="application/json", - ) - - -class HFInferenceConversational(HFInferenceTask): - def __init__(self): - super().__init__("conversational") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - payload = filter_none(parameters) - mapped_model = provider_mapping_info.provider_id - payload_model = parameters.get("model") or mapped_model - - if payload_model is None or payload_model.startswith(("http://", "https://")): - payload_model = "dummy" - - response_format = parameters.get("response_format") - if isinstance(response_format, dict) and response_format.get("type") == "json_schema": - payload["response_format"] = { - "type": "json_object", - "value": response_format["json_schema"]["schema"], - } - return {**payload, "model": payload_model, "messages": inputs} - - def _prepare_url(self, api_key: str, mapped_model: str) -> str: - base_url = ( - mapped_model - if mapped_model.startswith(("http://", "https://")) - else f"{constants.INFERENCE_PROXY_TEMPLATE.format(provider='hf-inference')}/models/{mapped_model}" - ) - return _build_chat_completion_url(base_url) - - -def _build_chat_completion_url(model_url: str) -> str: - parsed = urlparse(model_url) - path = parsed.path.rstrip("/") - - # If the path already ends with /chat/completions, we're done! - if path.endswith("/chat/completions"): - return model_url - - # Append /chat/completions if not already present - if path.endswith("/v1"): - new_path = path + "/chat/completions" - # If path was empty or just "/", set the full path - elif not path: - new_path = "/v1/chat/completions" - # Append /v1/chat/completions if not already present - else: - new_path = path + "/v1/chat/completions" - - # Reconstruct the URL with the new path and original query parameters. - new_parsed = parsed._replace(path=new_path) - return str(urlunparse(new_parsed)) - - -@lru_cache(maxsize=1) -def _fetch_recommended_models() -> Dict[str, Optional[str]]: - response = get_session().get(f"{constants.ENDPOINT}/api/tasks", headers=build_hf_headers()) - hf_raise_for_status(response) - return {task: next(iter(details["widgetModels"]), None) for task, details in response.json().items()} - - -@lru_cache(maxsize=None) -def _check_supported_task(model: str, task: str) -> None: - from huggingface_hub.hf_api import HfApi - - model_info = HfApi().model_info(model) - pipeline_tag = model_info.pipeline_tag - tags = model_info.tags or [] - is_conversational = "conversational" in tags - if task in ("text-generation", "conversational"): - if pipeline_tag == "text-generation": - # text-generation + conversational tag -> both tasks allowed - if is_conversational: - return - # text-generation without conversational tag -> only text-generation allowed - if task == "text-generation": - return - raise ValueError(f"Model '{model}' doesn't support task '{task}'.") - - if pipeline_tag == "text2text-generation": - if task == "text-generation": - return - raise ValueError(f"Model '{model}' doesn't support task '{task}'.") - - if pipeline_tag == "image-text-to-text": - if is_conversational and task == "conversational": - return # Only conversational allowed if tagged as conversational - raise ValueError("Non-conversational image-text-to-text task is not supported.") - - if ( - task in ("feature-extraction", "sentence-similarity") - and pipeline_tag in ("feature-extraction", "sentence-similarity") - and task in tags - ): - # feature-extraction and sentence-similarity are interchangeable for HF Inference - return - - # For all other tasks, just check pipeline tag - if pipeline_tag != task: - raise ValueError( - f"Model '{model}' doesn't support task '{task}'. Supported tasks: '{pipeline_tag}', got: '{task}'" - ) - return - - -class HFInferenceFeatureExtractionTask(HFInferenceTask): - def __init__(self): - super().__init__("feature-extraction") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - if isinstance(inputs, bytes): - raise ValueError(f"Unexpected binary input for task {self.task}.") - if isinstance(inputs, Path): - raise ValueError(f"Unexpected path input for task {self.task} (got {inputs})") - - # Parameters are sent at root-level for feature-extraction task - # See specs: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/tasks/feature-extraction/spec/input.json - return {"inputs": inputs, **filter_none(parameters)} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - if isinstance(response, bytes): - return _bytes_to_dict(response) - return response diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/hyperbolic.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/hyperbolic.py deleted file mode 100644 index 6dcb14cc275f6b80db5643361b9dfd3cbf8d91a2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/hyperbolic.py +++ /dev/null @@ -1,47 +0,0 @@ -import base64 -from typing import Any, Dict, Optional, Union - -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import RequestParameters, _as_dict -from huggingface_hub.inference._providers._common import BaseConversationalTask, TaskProviderHelper, filter_none - - -class HyperbolicTextToImageTask(TaskProviderHelper): - def __init__(self): - super().__init__(provider="hyperbolic", base_url="https://api.hyperbolic.xyz", task="text-to-image") - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/v1/images/generations" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - mapped_model = provider_mapping_info.provider_id - parameters = filter_none(parameters) - if "num_inference_steps" in parameters: - parameters["steps"] = parameters.pop("num_inference_steps") - if "guidance_scale" in parameters: - parameters["cfg_scale"] = parameters.pop("guidance_scale") - # For Hyperbolic, the width and height are required parameters - if "width" not in parameters: - parameters["width"] = 512 - if "height" not in parameters: - parameters["height"] = 512 - return {"prompt": inputs, "model_name": mapped_model, **parameters} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - response_dict = _as_dict(response) - return base64.b64decode(response_dict["images"][0]["image"]) - - -class HyperbolicTextGenerationTask(BaseConversationalTask): - """ - Special case for Hyperbolic, where text-generation task is handled as a conversational task. - """ - - def __init__(self, task: str): - super().__init__( - provider="hyperbolic", - base_url="https://api.hyperbolic.xyz", - ) - self.task = task diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/nebius.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/nebius.py deleted file mode 100644 index 85ad67c4c8835d7fb8bfe5f36e426614174a66ba..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/nebius.py +++ /dev/null @@ -1,83 +0,0 @@ -import base64 -from typing import Any, Dict, Optional, Union - -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import RequestParameters, _as_dict -from huggingface_hub.inference._providers._common import ( - BaseConversationalTask, - BaseTextGenerationTask, - TaskProviderHelper, - filter_none, -) - - -class NebiusTextGenerationTask(BaseTextGenerationTask): - def __init__(self): - super().__init__(provider="nebius", base_url="https://api.studio.nebius.ai") - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - output = _as_dict(response)["choices"][0] - return { - "generated_text": output["text"], - "details": { - "finish_reason": output.get("finish_reason"), - "seed": output.get("seed"), - }, - } - - -class NebiusConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider="nebius", base_url="https://api.studio.nebius.ai") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) - response_format = parameters.get("response_format") - if isinstance(response_format, dict) and response_format.get("type") == "json_schema": - json_schema_details = response_format.get("json_schema") - if isinstance(json_schema_details, dict) and "schema" in json_schema_details: - payload["guided_json"] = json_schema_details["schema"] # type: ignore [index] - return payload - - -class NebiusTextToImageTask(TaskProviderHelper): - def __init__(self): - super().__init__(task="text-to-image", provider="nebius", base_url="https://api.studio.nebius.ai") - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/v1/images/generations" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - mapped_model = provider_mapping_info.provider_id - parameters = filter_none(parameters) - if "guidance_scale" in parameters: - parameters.pop("guidance_scale") - if parameters.get("response_format") not in ("b64_json", "url"): - parameters["response_format"] = "b64_json" - - return {"prompt": inputs, **parameters, "model": mapped_model} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - response_dict = _as_dict(response) - return base64.b64decode(response_dict["data"][0]["b64_json"]) - - -class NebiusFeatureExtractionTask(TaskProviderHelper): - def __init__(self): - super().__init__(task="feature-extraction", provider="nebius", base_url="https://api.studio.nebius.ai") - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/v1/embeddings" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - return {"input": inputs, "model": provider_mapping_info.provider_id} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - embeddings = _as_dict(response)["data"] - return [embedding["embedding"] for embedding in embeddings] diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/novita.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/novita.py deleted file mode 100644 index 44adc9017b456f487513cde251086075d84b69f0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/novita.py +++ /dev/null @@ -1,69 +0,0 @@ -from typing import Any, Dict, Optional, Union - -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import RequestParameters, _as_dict -from huggingface_hub.inference._providers._common import ( - BaseConversationalTask, - BaseTextGenerationTask, - TaskProviderHelper, - filter_none, -) -from huggingface_hub.utils import get_session - - -_PROVIDER = "novita" -_BASE_URL = "https://api.novita.ai" - - -class NovitaTextGenerationTask(BaseTextGenerationTask): - def __init__(self): - super().__init__(provider=_PROVIDER, base_url=_BASE_URL) - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - # there is no v1/ route for novita - return "/v3/openai/completions" - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - output = _as_dict(response)["choices"][0] - return { - "generated_text": output["text"], - "details": { - "finish_reason": output.get("finish_reason"), - "seed": output.get("seed"), - }, - } - - -class NovitaConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider=_PROVIDER, base_url=_BASE_URL) - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - # there is no v1/ route for novita - return "/v3/openai/chat/completions" - - -class NovitaTextToVideoTask(TaskProviderHelper): - def __init__(self): - super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task="text-to-video") - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return f"/v3/hf/{mapped_model}" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - return {"prompt": inputs, **filter_none(parameters)} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - response_dict = _as_dict(response) - if not ( - isinstance(response_dict, dict) - and "video" in response_dict - and isinstance(response_dict["video"], dict) - and "video_url" in response_dict["video"] - ): - raise ValueError("Expected response format: { 'video': { 'video_url': string } }") - - video_url = response_dict["video"]["video_url"] - return get_session().get(video_url).content diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/nscale.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/nscale.py deleted file mode 100644 index ce5b20e354e246e93a7dd9831e4acf69ebcfad63..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/nscale.py +++ /dev/null @@ -1,44 +0,0 @@ -import base64 -from typing import Any, Dict, Optional, Union - -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import RequestParameters, _as_dict - -from ._common import BaseConversationalTask, TaskProviderHelper, filter_none - - -class NscaleConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider="nscale", base_url="https://inference.api.nscale.com") - - -class NscaleTextToImageTask(TaskProviderHelper): - def __init__(self): - super().__init__(provider="nscale", base_url="https://inference.api.nscale.com", task="text-to-image") - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/v1/images/generations" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - mapped_model = provider_mapping_info.provider_id - # Combine all parameters except inputs and parameters - parameters = filter_none(parameters) - if "width" in parameters and "height" in parameters: - parameters["size"] = f"{parameters.pop('width')}x{parameters.pop('height')}" - if "num_inference_steps" in parameters: - parameters.pop("num_inference_steps") - if "cfg_scale" in parameters: - parameters.pop("cfg_scale") - payload = { - "response_format": "b64_json", - "prompt": inputs, - "model": mapped_model, - **parameters, - } - return payload - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - response_dict = _as_dict(response) - return base64.b64decode(response_dict["data"][0]["b64_json"]) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/openai.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/openai.py deleted file mode 100644 index 7a554093c173ea8f664cb7fbd9616ce3a08ce78c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/openai.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Optional - -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._providers._common import BaseConversationalTask - - -class OpenAIConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider="openai", base_url="https://api.openai.com") - - def _prepare_api_key(self, api_key: Optional[str]) -> str: - if api_key is None: - raise ValueError("You must provide an api_key to work with OpenAI API.") - if api_key.startswith("hf_"): - raise ValueError( - "OpenAI provider is not available through Hugging Face routing, please use your own OpenAI API key." - ) - return api_key - - def _prepare_mapping_info(self, model: Optional[str]) -> InferenceProviderMapping: - if model is None: - raise ValueError("Please provide an OpenAI model ID, e.g. `gpt-4o` or `o1`.") - return InferenceProviderMapping( - provider="openai", providerId=model, task="conversational", status="live", hf_model_id=model - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/publicai.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/publicai.py deleted file mode 100644 index 4c88528e4f1e2eefaf6be9315c490db19ff5ca1e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/publicai.py +++ /dev/null @@ -1,6 +0,0 @@ -from ._common import BaseConversationalTask - - -class PublicAIConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider="publicai", base_url="https://api.publicai.co") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/replicate.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/replicate.py deleted file mode 100644 index 139582cc801eaf0bdd93e006df404432f2375fb3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/replicate.py +++ /dev/null @@ -1,90 +0,0 @@ -from typing import Any, Dict, Optional, Union - -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import RequestParameters, _as_dict, _as_url -from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none -from huggingface_hub.utils import get_session - - -_PROVIDER = "replicate" -_BASE_URL = "https://api.replicate.com" - - -class ReplicateTask(TaskProviderHelper): - def __init__(self, task: str): - super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task) - - def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]: - headers = super()._prepare_headers(headers, api_key) - headers["Prefer"] = "wait" - return headers - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - if ":" in mapped_model: - return "/v1/predictions" - return f"/v1/models/{mapped_model}/predictions" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - mapped_model = provider_mapping_info.provider_id - payload: Dict[str, Any] = {"input": {"prompt": inputs, **filter_none(parameters)}} - if ":" in mapped_model: - version = mapped_model.split(":", 1)[1] - payload["version"] = version - return payload - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - response_dict = _as_dict(response) - if response_dict.get("output") is None: - raise TimeoutError( - f"Inference request timed out after 60 seconds. No output generated for model {response_dict.get('model')}" - "The model might be in cold state or starting up. Please try again later." - ) - output_url = ( - response_dict["output"] if isinstance(response_dict["output"], str) else response_dict["output"][0] - ) - return get_session().get(output_url).content - - -class ReplicateTextToImageTask(ReplicateTask): - def __init__(self): - super().__init__("text-to-image") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - payload: Dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore[assignment] - if provider_mapping_info.adapter_weights_path is not None: - payload["input"]["lora_weights"] = f"https://huggingface.co/{provider_mapping_info.hf_model_id}" - return payload - - -class ReplicateTextToSpeechTask(ReplicateTask): - def __init__(self): - super().__init__("text-to-speech") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - payload: Dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore[assignment] - payload["input"]["text"] = payload["input"].pop("prompt") # rename "prompt" to "text" for TTS - return payload - - -class ReplicateImageToImageTask(ReplicateTask): - def __init__(self): - super().__init__("image-to-image") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - image_url = _as_url(inputs, default_mime_type="image/jpeg") - - payload: Dict[str, Any] = {"input": {"input_image": image_url, **filter_none(parameters)}} - - mapped_model = provider_mapping_info.provider_id - if ":" in mapped_model: - version = mapped_model.split(":", 1)[1] - payload["version"] = version - return payload diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/sambanova.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/sambanova.py deleted file mode 100644 index ed96fb766ce49003b605bda8ef8ee34da0ebe2f4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/sambanova.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import Any, Dict, Optional, Union - -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import RequestParameters, _as_dict -from huggingface_hub.inference._providers._common import BaseConversationalTask, TaskProviderHelper, filter_none - - -class SambanovaConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider="sambanova", base_url="https://api.sambanova.ai") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - response_format_config = parameters.get("response_format") - if isinstance(response_format_config, dict): - if response_format_config.get("type") == "json_schema": - json_schema_config = response_format_config.get("json_schema", {}) - strict = json_schema_config.get("strict") - if isinstance(json_schema_config, dict) and (strict is True or strict is None): - json_schema_config["strict"] = False - - payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) - return payload - - -class SambanovaFeatureExtractionTask(TaskProviderHelper): - def __init__(self): - super().__init__(provider="sambanova", base_url="https://api.sambanova.ai", task="feature-extraction") - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/v1/embeddings" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - parameters = filter_none(parameters) - return {"input": inputs, "model": provider_mapping_info.provider_id, **parameters} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - embeddings = _as_dict(response)["data"] - return [embedding["embedding"] for embedding in embeddings] diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/scaleway.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/scaleway.py deleted file mode 100644 index cfdd75416f1a11f3f4908d1c29541920cba76d79..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/scaleway.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import Any, Dict, Optional, Union - -from huggingface_hub.inference._common import RequestParameters, _as_dict - -from ._common import BaseConversationalTask, InferenceProviderMapping, TaskProviderHelper, filter_none - - -class ScalewayConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider="scaleway", base_url="https://api.scaleway.ai") - - -class ScalewayFeatureExtractionTask(TaskProviderHelper): - def __init__(self): - super().__init__(provider="scaleway", base_url="https://api.scaleway.ai", task="feature-extraction") - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/v1/embeddings" - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - parameters = filter_none(parameters) - return {"input": inputs, "model": provider_mapping_info.provider_id, **parameters} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - embeddings = _as_dict(response)["data"] - return [embedding["embedding"] for embedding in embeddings] diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/together.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/together.py deleted file mode 100644 index de166b7baf8d50b255f29cf8cc9b9d3fa639646e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/together.py +++ /dev/null @@ -1,88 +0,0 @@ -import base64 -from abc import ABC -from typing import Any, Dict, Optional, Union - -from huggingface_hub.hf_api import InferenceProviderMapping -from huggingface_hub.inference._common import RequestParameters, _as_dict -from huggingface_hub.inference._providers._common import ( - BaseConversationalTask, - BaseTextGenerationTask, - TaskProviderHelper, - filter_none, -) - - -_PROVIDER = "together" -_BASE_URL = "https://api.together.xyz" - - -class TogetherTask(TaskProviderHelper, ABC): - """Base class for Together API tasks.""" - - def __init__(self, task: str): - super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task) - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - if self.task == "text-to-image": - return "/v1/images/generations" - elif self.task == "conversational": - return "/v1/chat/completions" - elif self.task == "text-generation": - return "/v1/completions" - raise ValueError(f"Unsupported task '{self.task}' for Together API.") - - -class TogetherTextGenerationTask(BaseTextGenerationTask): - def __init__(self): - super().__init__(provider=_PROVIDER, base_url=_BASE_URL) - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - output = _as_dict(response)["choices"][0] - return { - "generated_text": output["text"], - "details": { - "finish_reason": output.get("finish_reason"), - "seed": output.get("seed"), - }, - } - - -class TogetherConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider=_PROVIDER, base_url=_BASE_URL) - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) - response_format = parameters.get("response_format") - if isinstance(response_format, dict) and response_format.get("type") == "json_schema": - json_schema_details = response_format.get("json_schema") - if isinstance(json_schema_details, dict) and "schema" in json_schema_details: - payload["response_format"] = { # type: ignore [index] - "type": "json_object", - "schema": json_schema_details["schema"], - } - - return payload - - -class TogetherTextToImageTask(TogetherTask): - def __init__(self): - super().__init__("text-to-image") - - def _prepare_payload_as_dict( - self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping - ) -> Optional[Dict]: - mapped_model = provider_mapping_info.provider_id - parameters = filter_none(parameters) - if "num_inference_steps" in parameters: - parameters["steps"] = parameters.pop("num_inference_steps") - if "guidance_scale" in parameters: - parameters["guidance"] = parameters.pop("guidance_scale") - - return {"prompt": inputs, "response_format": "base64", **parameters, "model": mapped_model} - - def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: - response_dict = _as_dict(response) - return base64.b64decode(response_dict["data"][0]["b64_json"]) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/zai_org.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/zai_org.py deleted file mode 100644 index d6f4c42b5abc78a98474b2f8899d6b30a4a58f8d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference/_providers/zai_org.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Any, Dict - -from huggingface_hub.inference._providers._common import BaseConversationalTask - - -class ZaiConversationalTask(BaseConversationalTask): - def __init__(self): - super().__init__(provider="zai-org", base_url="https://api.z.ai") - - def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]: - headers = super()._prepare_headers(headers, api_key) - headers["Accept-Language"] = "en-US,en" - headers["x-source-channel"] = "hugging_face" - return headers - - def _prepare_route(self, mapped_model: str, api_key: str) -> str: - return "/api/paas/v4/chat/completions" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference_api.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference_api.py deleted file mode 100644 index f895fcc61c3867838b013ecd3f6789cbc010b5b3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/inference_api.py +++ /dev/null @@ -1,217 +0,0 @@ -import io -from typing import Any, Dict, List, Optional, Union - -from . import constants -from .hf_api import HfApi -from .utils import build_hf_headers, get_session, is_pillow_available, logging, validate_hf_hub_args -from .utils._deprecation import _deprecate_method - - -logger = logging.get_logger(__name__) - - -ALL_TASKS = [ - # NLP - "text-classification", - "token-classification", - "table-question-answering", - "question-answering", - "zero-shot-classification", - "translation", - "summarization", - "conversational", - "feature-extraction", - "text-generation", - "text2text-generation", - "fill-mask", - "sentence-similarity", - # Audio - "text-to-speech", - "automatic-speech-recognition", - "audio-to-audio", - "audio-classification", - "voice-activity-detection", - # Computer vision - "image-classification", - "object-detection", - "image-segmentation", - "text-to-image", - "image-to-image", - # Others - "tabular-classification", - "tabular-regression", -] - - -class InferenceApi: - """Client to configure requests and make calls to the HuggingFace Inference API. - - Example: - - ```python - >>> from huggingface_hub.inference_api import InferenceApi - - >>> # Mask-fill example - >>> inference = InferenceApi("bert-base-uncased") - >>> inference(inputs="The goal of life is [MASK].") - [{'sequence': 'the goal of life is life.', 'score': 0.10933292657136917, 'token': 2166, 'token_str': 'life'}] - - >>> # Question Answering example - >>> inference = InferenceApi("deepset/roberta-base-squad2") - >>> inputs = { - ... "question": "What's my name?", - ... "context": "My name is Clara and I live in Berkeley.", - ... } - >>> inference(inputs) - {'score': 0.9326569437980652, 'start': 11, 'end': 16, 'answer': 'Clara'} - - >>> # Zero-shot example - >>> inference = InferenceApi("typeform/distilbert-base-uncased-mnli") - >>> inputs = "Hi, I recently bought a device from your company but it is not working as advertised and I would like to get reimbursed!" - >>> params = {"candidate_labels": ["refund", "legal", "faq"]} - >>> inference(inputs, params) - {'sequence': 'Hi, I recently bought a device from your company but it is not working as advertised and I would like to get reimbursed!', 'labels': ['refund', 'faq', 'legal'], 'scores': [0.9378499388694763, 0.04914155602455139, 0.013008488342165947]} - - >>> # Overriding configured task - >>> inference = InferenceApi("bert-base-uncased", task="feature-extraction") - - >>> # Text-to-image - >>> inference = InferenceApi("stabilityai/stable-diffusion-2-1") - >>> inference("cat") - - - >>> # Return as raw response to parse the output yourself - >>> inference = InferenceApi("mio/amadeus") - >>> response = inference("hello world", raw_response=True) - >>> response.headers - {"Content-Type": "audio/flac", ...} - >>> response.content # raw bytes from server - b'(...)' - ``` - """ - - @validate_hf_hub_args - @_deprecate_method( - version="1.0", - message=( - "`InferenceApi` client is deprecated in favor of the more feature-complete `InferenceClient`. Check out" - " this guide to learn how to convert your script to use it:" - " https://huggingface.co/docs/huggingface_hub/guides/inference#legacy-inferenceapi-client." - ), - ) - def __init__( - self, - repo_id: str, - task: Optional[str] = None, - token: Optional[str] = None, - gpu: bool = False, - ): - """Inits headers and API call information. - - Args: - repo_id (``str``): - Id of repository (e.g. `user/bert-base-uncased`). - task (``str``, `optional`, defaults ``None``): - Whether to force a task instead of using task specified in the - repository. - token (`str`, `optional`): - The API token to use as HTTP bearer authorization. This is not - the authentication token. You can find the token in - https://huggingface.co/settings/token. Alternatively, you can - find both your organizations and personal API tokens using - `HfApi().whoami(token)`. - gpu (`bool`, `optional`, defaults `False`): - Whether to use GPU instead of CPU for inference(requires Startup - plan at least). - """ - self.options = {"wait_for_model": True, "use_gpu": gpu} - self.headers = build_hf_headers(token=token) - - # Configure task - model_info = HfApi(token=token).model_info(repo_id=repo_id) - if not model_info.pipeline_tag and not task: - raise ValueError( - "Task not specified in the repository. Please add it to the model card" - " using pipeline_tag" - " (https://huggingface.co/docs#how-is-a-models-type-of-inference-api-and-widget-determined)" - ) - - if task and task != model_info.pipeline_tag: - if task not in ALL_TASKS: - raise ValueError(f"Invalid task {task}. Make sure it's valid.") - - logger.warning( - "You're using a different task than the one specified in the" - " repository. Be sure to know what you're doing :)" - ) - self.task = task - else: - assert model_info.pipeline_tag is not None, "Pipeline tag cannot be None" - self.task = model_info.pipeline_tag - - self.api_url = f"{constants.INFERENCE_ENDPOINT}/pipeline/{self.task}/{repo_id}" - - def __repr__(self): - # Do not add headers to repr to avoid leaking token. - return f"InferenceAPI(api_url='{self.api_url}', task='{self.task}', options={self.options})" - - def __call__( - self, - inputs: Optional[Union[str, Dict, List[str], List[List[str]]]] = None, - params: Optional[Dict] = None, - data: Optional[bytes] = None, - raw_response: bool = False, - ) -> Any: - """Make a call to the Inference API. - - Args: - inputs (`str` or `Dict` or `List[str]` or `List[List[str]]`, *optional*): - Inputs for the prediction. - params (`Dict`, *optional*): - Additional parameters for the models. Will be sent as `parameters` in the - payload. - data (`bytes`, *optional*): - Bytes content of the request. In this case, leave `inputs` and `params` empty. - raw_response (`bool`, defaults to `False`): - If `True`, the raw `Response` object is returned. You can parse its content - as preferred. By default, the content is parsed into a more practical format - (json dictionary or PIL Image for example). - """ - # Build payload - payload: Dict[str, Any] = { - "options": self.options, - } - if inputs: - payload["inputs"] = inputs - if params: - payload["parameters"] = params - - # Make API call - response = get_session().post(self.api_url, headers=self.headers, json=payload, data=data) - - # Let the user handle the response - if raw_response: - return response - - # By default, parse the response for the user. - content_type = response.headers.get("Content-Type") or "" - if content_type.startswith("image"): - if not is_pillow_available(): - raise ImportError( - f"Task '{self.task}' returned as image but Pillow is not installed." - " Please install it (`pip install Pillow`) or pass" - " `raw_response=True` to get the raw `Response` object and parse" - " the image by yourself." - ) - - from PIL import Image - - return Image.open(io.BytesIO(response.content)) - elif content_type == "application/json": - return response.json() - else: - raise NotImplementedError( - f"{content_type} output type is not implemented yet. You can pass" - " `raw_response=True` to get the raw `Response` object and parse the" - " output by yourself." - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/keras_mixin.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/keras_mixin.py deleted file mode 100644 index c284947c1d3c25da421b90e902683054830788d3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/keras_mixin.py +++ /dev/null @@ -1,497 +0,0 @@ -import collections.abc as collections -import json -import os -import warnings -from functools import wraps -from pathlib import Path -from shutil import copytree -from typing import Any, Dict, List, Optional, Union - -from huggingface_hub import ModelHubMixin, snapshot_download -from huggingface_hub.utils import ( - get_tf_version, - is_graphviz_available, - is_pydot_available, - is_tf_available, - yaml_dump, -) - -from . import constants -from .hf_api import HfApi -from .utils import SoftTemporaryDirectory, logging, validate_hf_hub_args -from .utils._typing import CallableT - - -logger = logging.get_logger(__name__) - -keras = None -if is_tf_available(): - # Depending on which version of TensorFlow is installed, we need to import - # keras from the correct location. - # See https://github.com/tensorflow/tensorflow/releases/tag/v2.16.1. - # Note: saving a keras model only works with Keras<3.0. - try: - import tf_keras as keras # type: ignore - except ImportError: - import tensorflow as tf # type: ignore - - keras = tf.keras - - -def _requires_keras_2_model(fn: CallableT) -> CallableT: - # Wrapper to raise if user tries to save a Keras 3.x model - @wraps(fn) - def _inner(model, *args, **kwargs): - if not hasattr(model, "history"): # hacky way to check if model is Keras 2.x - raise NotImplementedError( - f"Cannot use '{fn.__name__}': Keras 3.x is not supported." - " Please save models manually and upload them using `upload_folder` or `hf upload`." - ) - return fn(model, *args, **kwargs) - - return _inner # type: ignore [return-value] - - -def _flatten_dict(dictionary, parent_key=""): - """Flatten a nested dictionary. - Reference: https://stackoverflow.com/a/6027615/10319735 - - Args: - dictionary (`dict`): - The nested dictionary to be flattened. - parent_key (`str`): - The parent key to be prefixed to the children keys. - Necessary for recursing over the nested dictionary. - - Returns: - The flattened dictionary. - """ - items = [] - for key, value in dictionary.items(): - new_key = f"{parent_key}.{key}" if parent_key else key - if isinstance(value, collections.MutableMapping): - items.extend( - _flatten_dict( - value, - new_key, - ).items() - ) - else: - items.append((new_key, value)) - return dict(items) - - -def _create_hyperparameter_table(model): - """Parse hyperparameter dictionary into a markdown table.""" - table = None - if model.optimizer is not None: - optimizer_params = model.optimizer.get_config() - # flatten the configuration - optimizer_params = _flatten_dict(optimizer_params) - optimizer_params["training_precision"] = keras.mixed_precision.global_policy().name - table = "| Hyperparameters | Value |\n| :-- | :-- |\n" - for key, value in optimizer_params.items(): - table += f"| {key} | {value} |\n" - return table - - -def _plot_network(model, save_directory): - keras.utils.plot_model( - model, - to_file=f"{save_directory}/model.png", - show_shapes=False, - show_dtype=False, - show_layer_names=True, - rankdir="TB", - expand_nested=False, - dpi=96, - layer_range=None, - ) - - -def _create_model_card( - model, - repo_dir: Path, - plot_model: bool = True, - metadata: Optional[dict] = None, -): - """ - Creates a model card for the repository. - - Do not overwrite an existing README.md file. - """ - readme_path = repo_dir / "README.md" - if readme_path.exists(): - return - - hyperparameters = _create_hyperparameter_table(model) - if plot_model and is_graphviz_available() and is_pydot_available(): - _plot_network(model, repo_dir) - if metadata is None: - metadata = {} - metadata["library_name"] = "keras" - model_card: str = "---\n" - model_card += yaml_dump(metadata, default_flow_style=False) - model_card += "---\n" - model_card += "\n## Model description\n\nMore information needed\n" - model_card += "\n## Intended uses & limitations\n\nMore information needed\n" - model_card += "\n## Training and evaluation data\n\nMore information needed\n" - if hyperparameters is not None: - model_card += "\n## Training procedure\n" - model_card += "\n### Training hyperparameters\n" - model_card += "\nThe following hyperparameters were used during training:\n\n" - model_card += hyperparameters - model_card += "\n" - if plot_model and os.path.exists(f"{repo_dir}/model.png"): - model_card += "\n ## Model Plot\n" - model_card += "\n
" - model_card += "\nView Model Plot\n" - path_to_plot = "./model.png" - model_card += f"\n![Model Image]({path_to_plot})\n" - model_card += "\n
" - - readme_path.write_text(model_card) - - -@_requires_keras_2_model -def save_pretrained_keras( - model, - save_directory: Union[str, Path], - config: Optional[Dict[str, Any]] = None, - include_optimizer: bool = False, - plot_model: bool = True, - tags: Optional[Union[list, str]] = None, - **model_save_kwargs, -): - """ - Saves a Keras model to save_directory in SavedModel format. Use this if - you're using the Functional or Sequential APIs. - - Args: - model (`Keras.Model`): - The [Keras - model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) - you'd like to save. The model must be compiled and built. - save_directory (`str` or `Path`): - Specify directory in which you want to save the Keras model. - config (`dict`, *optional*): - Configuration object to be saved alongside the model weights. - include_optimizer(`bool`, *optional*, defaults to `False`): - Whether or not to include optimizer in serialization. - plot_model (`bool`, *optional*, defaults to `True`): - Setting this to `True` will plot the model and put it in the model - card. Requires graphviz and pydot to be installed. - tags (Union[`str`,`list`], *optional*): - List of tags that are related to model or string of a single tag. See example tags - [here](https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1). - model_save_kwargs(`dict`, *optional*): - model_save_kwargs will be passed to - [`tf.keras.models.save_model()`](https://www.tensorflow.org/api_docs/python/tf/keras/models/save_model). - """ - if keras is None: - raise ImportError("Called a Tensorflow-specific function but could not import it.") - - if not model.built: - raise ValueError("Model should be built before trying to save") - - save_directory = Path(save_directory) - save_directory.mkdir(parents=True, exist_ok=True) - - # saving config - if config: - if not isinstance(config, dict): - raise RuntimeError(f"Provided config to save_pretrained_keras should be a dict. Got: '{type(config)}'") - - with (save_directory / constants.CONFIG_NAME).open("w") as f: - json.dump(config, f) - - metadata = {} - if isinstance(tags, list): - metadata["tags"] = tags - elif isinstance(tags, str): - metadata["tags"] = [tags] - - task_name = model_save_kwargs.pop("task_name", None) - if task_name is not None: - warnings.warn( - "`task_name` input argument is deprecated. Pass `tags` instead.", - FutureWarning, - ) - if "tags" in metadata: - metadata["tags"].append(task_name) - else: - metadata["tags"] = [task_name] - - if model.history is not None: - if model.history.history != {}: - path = save_directory / "history.json" - if path.exists(): - warnings.warn( - "`history.json` file already exists, it will be overwritten by the history of this version.", - UserWarning, - ) - with path.open("w", encoding="utf-8") as f: - json.dump(model.history.history, f, indent=2, sort_keys=True) - - _create_model_card(model, save_directory, plot_model, metadata) - keras.models.save_model(model, save_directory, include_optimizer=include_optimizer, **model_save_kwargs) - - -def from_pretrained_keras(*args, **kwargs) -> "KerasModelHubMixin": - r""" - Instantiate a pretrained Keras model from a pre-trained model from the Hub. - The model is expected to be in `SavedModel` format. - - Args: - pretrained_model_name_or_path (`str` or `os.PathLike`): - Can be either: - - A string, the `model id` of a pretrained model hosted inside a - model repo on huggingface.co. Valid model ids can be located - at the root-level, like `bert-base-uncased`, or namespaced - under a user or organization name, like - `dbmdz/bert-base-german-cased`. - - You can add `revision` by appending `@` at the end of model_id - simply like this: `dbmdz/bert-base-german-cased@main` Revision - is the specific model version to use. It can be a branch name, - a tag name, or a commit id, since we use a git-based system - for storing models and other artifacts on huggingface.co, so - `revision` can be any identifier allowed by git. - - A path to a `directory` containing model weights saved using - [`~transformers.PreTrainedModel.save_pretrained`], e.g., - `./my_model_directory/`. - - `None` if you are both providing the configuration and state - dictionary (resp. with keyword arguments `config` and - `state_dict`). - force_download (`bool`, *optional*, defaults to `False`): - Whether to force the (re-)download of the model weights and - configuration files, overriding the cached versions if they exist. - proxies (`Dict[str, str]`, *optional*): - A dictionary of proxy servers to use by protocol or endpoint, e.g., - `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The - proxies are used on each request. - token (`str` or `bool`, *optional*): - The token to use as HTTP bearer authorization for remote files. If - `True`, will use the token generated when running `transformers-cli - login` (stored in `~/.huggingface`). - cache_dir (`Union[str, os.PathLike]`, *optional*): - Path to a directory in which a downloaded pretrained model - configuration should be cached if the standard cache should not be - used. - local_files_only(`bool`, *optional*, defaults to `False`): - Whether to only look at local files (i.e., do not try to download - the model). - model_kwargs (`Dict`, *optional*): - model_kwargs will be passed to the model during initialization - - > [!TIP] - > Passing `token=True` is required when you want to use a private - > model. - """ - return KerasModelHubMixin.from_pretrained(*args, **kwargs) - - -@validate_hf_hub_args -@_requires_keras_2_model -def push_to_hub_keras( - model, - repo_id: str, - *, - config: Optional[dict] = None, - commit_message: str = "Push Keras model using huggingface_hub.", - private: Optional[bool] = None, - api_endpoint: Optional[str] = None, - token: Optional[str] = None, - branch: Optional[str] = None, - create_pr: Optional[bool] = None, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - delete_patterns: Optional[Union[List[str], str]] = None, - log_dir: Optional[str] = None, - include_optimizer: bool = False, - tags: Optional[Union[list, str]] = None, - plot_model: bool = True, - **model_save_kwargs, -): - """ - Upload model checkpoint to the Hub. - - Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be pushed to the hub. Use - `delete_patterns` to delete existing remote files in the same commit. See [`upload_folder`] reference for more - details. - - Args: - model (`Keras.Model`): - The [Keras model](`https://www.tensorflow.org/api_docs/python/tf/keras/Model`) you'd like to push to the - Hub. The model must be compiled and built. - repo_id (`str`): - ID of the repository to push to (example: `"username/my-model"`). - commit_message (`str`, *optional*, defaults to "Add Keras model"): - Message to commit while pushing. - private (`bool`, *optional*): - Whether the repository created should be private. - If `None` (default), the repo will be public unless the organization's default is private. - api_endpoint (`str`, *optional*): - The API endpoint to use when pushing the model to the hub. - token (`str`, *optional*): - The token to use as HTTP bearer authorization for remote files. If - not set, will use the token set when logging in with - `hf auth login` (stored in `~/.huggingface`). - branch (`str`, *optional*): - The git branch on which to push the model. This defaults to - the default branch as specified in your repository, which - defaults to `"main"`. - create_pr (`boolean`, *optional*): - Whether or not to create a Pull Request from `branch` with that commit. - Defaults to `False`. - config (`dict`, *optional*): - Configuration object to be saved alongside the model weights. - allow_patterns (`List[str]` or `str`, *optional*): - If provided, only files matching at least one pattern are pushed. - ignore_patterns (`List[str]` or `str`, *optional*): - If provided, files matching any of the patterns are not pushed. - delete_patterns (`List[str]` or `str`, *optional*): - If provided, remote files matching any of the patterns will be deleted from the repo. - log_dir (`str`, *optional*): - TensorBoard logging directory to be pushed. The Hub automatically - hosts and displays a TensorBoard instance if log files are included - in the repository. - include_optimizer (`bool`, *optional*, defaults to `False`): - Whether or not to include optimizer during serialization. - tags (Union[`list`, `str`], *optional*): - List of tags that are related to model or string of a single tag. See example tags - [here](https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1). - plot_model (`bool`, *optional*, defaults to `True`): - Setting this to `True` will plot the model and put it in the model - card. Requires graphviz and pydot to be installed. - model_save_kwargs(`dict`, *optional*): - model_save_kwargs will be passed to - [`tf.keras.models.save_model()`](https://www.tensorflow.org/api_docs/python/tf/keras/models/save_model). - - Returns: - The url of the commit of your model in the given repository. - """ - api = HfApi(endpoint=api_endpoint) - repo_id = api.create_repo(repo_id=repo_id, token=token, private=private, exist_ok=True).repo_id - - # Push the files to the repo in a single commit - with SoftTemporaryDirectory() as tmp: - saved_path = Path(tmp) / repo_id - save_pretrained_keras( - model, - saved_path, - config=config, - include_optimizer=include_optimizer, - tags=tags, - plot_model=plot_model, - **model_save_kwargs, - ) - - # If `log_dir` provided, delete remote logs and upload new ones - if log_dir is not None: - delete_patterns = ( - [] - if delete_patterns is None - else ( - [delete_patterns] # convert `delete_patterns` to a list - if isinstance(delete_patterns, str) - else delete_patterns - ) - ) - delete_patterns.append("logs/*") - copytree(log_dir, saved_path / "logs") - - return api.upload_folder( - repo_type="model", - repo_id=repo_id, - folder_path=saved_path, - commit_message=commit_message, - token=token, - revision=branch, - create_pr=create_pr, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - delete_patterns=delete_patterns, - ) - - -class KerasModelHubMixin(ModelHubMixin): - """ - Implementation of [`ModelHubMixin`] to provide model Hub upload/download - capabilities to Keras models. - - - ```python - >>> import tensorflow as tf - >>> from huggingface_hub import KerasModelHubMixin - - - >>> class MyModel(tf.keras.Model, KerasModelHubMixin): - ... def __init__(self, **kwargs): - ... super().__init__() - ... self.config = kwargs.pop("config", None) - ... self.dummy_inputs = ... - ... self.layer = ... - - ... def call(self, *args): - ... return ... - - - >>> # Initialize and compile the model as you normally would - >>> model = MyModel() - >>> model.compile(...) - >>> # Build the graph by training it or passing dummy inputs - >>> _ = model(model.dummy_inputs) - >>> # Save model weights to local directory - >>> model.save_pretrained("my-awesome-model") - >>> # Push model weights to the Hub - >>> model.push_to_hub("my-awesome-model") - >>> # Download and initialize weights from the Hub - >>> model = MyModel.from_pretrained("username/super-cool-model") - ``` - """ - - def _save_pretrained(self, save_directory): - save_pretrained_keras(self, save_directory) - - @classmethod - def _from_pretrained( - cls, - model_id, - revision, - cache_dir, - force_download, - proxies, - resume_download, - local_files_only, - token, - config: Optional[Dict[str, Any]] = None, - **model_kwargs, - ): - """Here we just call [`from_pretrained_keras`] function so both the mixin and - functional APIs stay in sync. - - TODO - Some args above aren't used since we are calling - snapshot_download instead of hf_hub_download. - """ - if keras is None: - raise ImportError("Called a TensorFlow-specific function but could not import it.") - - # Root is either a local filepath matching model_id or a cached snapshot - if not os.path.isdir(model_id): - storage_folder = snapshot_download( - repo_id=model_id, - revision=revision, - cache_dir=cache_dir, - library_name="keras", - library_version=get_tf_version(), - ) - else: - storage_folder = model_id - - # TODO: change this in a future PR. We are not returning a KerasModelHubMixin instance here... - model = keras.models.load_model(storage_folder) - - # For now, we add a new attribute, config, to store the config loaded from the hub/a local dir. - model.config = config - - return model diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/lfs.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/lfs.py deleted file mode 100644 index 40b6ad087ca6bd33874433439a2c4f5b23d100c5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/lfs.py +++ /dev/null @@ -1,466 +0,0 @@ -# coding=utf-8 -# Copyright 2019-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Git LFS related type definitions and utilities""" - -import inspect -import io -import re -import warnings -from dataclasses import dataclass -from math import ceil -from os.path import getsize -from pathlib import Path -from typing import TYPE_CHECKING, BinaryIO, Dict, Iterable, List, Optional, Tuple, TypedDict -from urllib.parse import unquote - -from huggingface_hub import constants - -from .utils import ( - build_hf_headers, - fix_hf_endpoint_in_url, - get_session, - hf_raise_for_status, - http_backoff, - logging, - tqdm, - validate_hf_hub_args, -) -from .utils._lfs import SliceFileObj -from .utils.sha import sha256, sha_fileobj -from .utils.tqdm import is_tqdm_disabled - - -if TYPE_CHECKING: - from ._commit_api import CommitOperationAdd - -logger = logging.get_logger(__name__) - -OID_REGEX = re.compile(r"^[0-9a-f]{40}$") - -LFS_MULTIPART_UPLOAD_COMMAND = "lfs-multipart-upload" - -LFS_HEADERS = { - "Accept": "application/vnd.git-lfs+json", - "Content-Type": "application/vnd.git-lfs+json", -} - - -@dataclass -class UploadInfo: - """ - Dataclass holding required information to determine whether a blob - should be uploaded to the hub using the LFS protocol or the regular protocol - - Args: - sha256 (`bytes`): - SHA256 hash of the blob - size (`int`): - Size in bytes of the blob - sample (`bytes`): - First 512 bytes of the blob - """ - - sha256: bytes - size: int - sample: bytes - - @classmethod - def from_path(cls, path: str): - size = getsize(path) - with io.open(path, "rb") as file: - sample = file.peek(512)[:512] - sha = sha_fileobj(file) - return cls(size=size, sha256=sha, sample=sample) - - @classmethod - def from_bytes(cls, data: bytes): - sha = sha256(data).digest() - return cls(size=len(data), sample=data[:512], sha256=sha) - - @classmethod - def from_fileobj(cls, fileobj: BinaryIO): - sample = fileobj.read(512) - fileobj.seek(0, io.SEEK_SET) - sha = sha_fileobj(fileobj) - size = fileobj.tell() - fileobj.seek(0, io.SEEK_SET) - return cls(size=size, sha256=sha, sample=sample) - - -@validate_hf_hub_args -def post_lfs_batch_info( - upload_infos: Iterable[UploadInfo], - token: Optional[str], - repo_type: str, - repo_id: str, - revision: Optional[str] = None, - endpoint: Optional[str] = None, - headers: Optional[Dict[str, str]] = None, - transfers: Optional[List[str]] = None, -) -> Tuple[List[dict], List[dict], Optional[str]]: - """ - Requests the LFS batch endpoint to retrieve upload instructions - - Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md - - Args: - upload_infos (`Iterable` of `UploadInfo`): - `UploadInfo` for the files that are being uploaded, typically obtained - from `CommitOperationAdd.upload_info` - repo_type (`str`): - Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. - repo_id (`str`): - A namespace (user or an organization) and a repo name separated - by a `/`. - revision (`str`, *optional*): - The git revision to upload to. - headers (`dict`, *optional*): - Additional headers to include in the request - transfers (`list`, *optional*): - List of transfer methods to use. Defaults to ["basic", "multipart"]. - - Returns: - `LfsBatchInfo`: 3-tuple: - - First element is the list of upload instructions from the server - - Second element is a list of errors, if any - - Third element is the chosen transfer adapter if provided by the server (e.g. "basic", "multipart", "xet") - - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If an argument is invalid or the server response is malformed. - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - If the server returned an error. - """ - endpoint = endpoint if endpoint is not None else constants.ENDPOINT - url_prefix = "" - if repo_type in constants.REPO_TYPES_URL_PREFIXES: - url_prefix = constants.REPO_TYPES_URL_PREFIXES[repo_type] - batch_url = f"{endpoint}/{url_prefix}{repo_id}.git/info/lfs/objects/batch" - payload: Dict = { - "operation": "upload", - "transfers": transfers if transfers is not None else ["basic", "multipart"], - "objects": [ - { - "oid": upload.sha256.hex(), - "size": upload.size, - } - for upload in upload_infos - ], - "hash_algo": "sha256", - } - if revision is not None: - payload["ref"] = {"name": unquote(revision)} # revision has been previously 'quoted' - - headers = { - **LFS_HEADERS, - **build_hf_headers(token=token), - **(headers or {}), - } - resp = get_session().post(batch_url, headers=headers, json=payload) - hf_raise_for_status(resp) - batch_info = resp.json() - - objects = batch_info.get("objects", None) - if not isinstance(objects, list): - raise ValueError("Malformed response from server") - - chosen_transfer = batch_info.get("transfer") - chosen_transfer = chosen_transfer if isinstance(chosen_transfer, str) else None - - return ( - [_validate_batch_actions(obj) for obj in objects if "error" not in obj], - [_validate_batch_error(obj) for obj in objects if "error" in obj], - chosen_transfer, - ) - - -class PayloadPartT(TypedDict): - partNumber: int - etag: str - - -class CompletionPayloadT(TypedDict): - """Payload that will be sent to the Hub when uploading multi-part.""" - - oid: str - parts: List[PayloadPartT] - - -def lfs_upload( - operation: "CommitOperationAdd", - lfs_batch_action: Dict, - token: Optional[str] = None, - headers: Optional[Dict[str, str]] = None, - endpoint: Optional[str] = None, -) -> None: - """ - Handles uploading a given object to the Hub with the LFS protocol. - - Can be a No-op if the content of the file is already present on the hub large file storage. - - Args: - operation (`CommitOperationAdd`): - The add operation triggering this upload. - lfs_batch_action (`dict`): - Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for - more details. - headers (`dict`, *optional*): - Headers to include in the request, including authentication and user agent headers. - - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If `lfs_batch_action` is improperly formatted - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - If the upload resulted in an error - """ - # 0. If LFS file is already present, skip upload - _validate_batch_actions(lfs_batch_action) - actions = lfs_batch_action.get("actions") - if actions is None: - # The file was already uploaded - logger.debug(f"Content of file {operation.path_in_repo} is already present upstream - skipping upload") - return - - # 1. Validate server response (check required keys in dict) - upload_action = lfs_batch_action["actions"]["upload"] - _validate_lfs_action(upload_action) - verify_action = lfs_batch_action["actions"].get("verify") - if verify_action is not None: - _validate_lfs_action(verify_action) - - # 2. Upload file (either single part or multi-part) - header = upload_action.get("header", {}) - chunk_size = header.get("chunk_size") - upload_url = fix_hf_endpoint_in_url(upload_action["href"], endpoint=endpoint) - if chunk_size is not None: - try: - chunk_size = int(chunk_size) - except (ValueError, TypeError): - raise ValueError( - f"Malformed response from LFS batch endpoint: `chunk_size` should be an integer. Got '{chunk_size}'." - ) - _upload_multi_part(operation=operation, header=header, chunk_size=chunk_size, upload_url=upload_url) - else: - _upload_single_part(operation=operation, upload_url=upload_url) - - # 3. Verify upload went well - if verify_action is not None: - _validate_lfs_action(verify_action) - verify_url = fix_hf_endpoint_in_url(verify_action["href"], endpoint) - verify_resp = get_session().post( - verify_url, - headers=build_hf_headers(token=token, headers=headers), - json={"oid": operation.upload_info.sha256.hex(), "size": operation.upload_info.size}, - ) - hf_raise_for_status(verify_resp) - logger.debug(f"{operation.path_in_repo}: Upload successful") - - -def _validate_lfs_action(lfs_action: dict): - """validates response from the LFS batch endpoint""" - if not ( - isinstance(lfs_action.get("href"), str) - and (lfs_action.get("header") is None or isinstance(lfs_action.get("header"), dict)) - ): - raise ValueError("lfs_action is improperly formatted") - return lfs_action - - -def _validate_batch_actions(lfs_batch_actions: dict): - """validates response from the LFS batch endpoint""" - if not (isinstance(lfs_batch_actions.get("oid"), str) and isinstance(lfs_batch_actions.get("size"), int)): - raise ValueError("lfs_batch_actions is improperly formatted") - - upload_action = lfs_batch_actions.get("actions", {}).get("upload") - verify_action = lfs_batch_actions.get("actions", {}).get("verify") - if upload_action is not None: - _validate_lfs_action(upload_action) - if verify_action is not None: - _validate_lfs_action(verify_action) - return lfs_batch_actions - - -def _validate_batch_error(lfs_batch_error: dict): - """validates response from the LFS batch endpoint""" - if not (isinstance(lfs_batch_error.get("oid"), str) and isinstance(lfs_batch_error.get("size"), int)): - raise ValueError("lfs_batch_error is improperly formatted") - error_info = lfs_batch_error.get("error") - if not ( - isinstance(error_info, dict) - and isinstance(error_info.get("message"), str) - and isinstance(error_info.get("code"), int) - ): - raise ValueError("lfs_batch_error is improperly formatted") - return lfs_batch_error - - -def _upload_single_part(operation: "CommitOperationAdd", upload_url: str) -> None: - """ - Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol) - - Args: - upload_url (`str`): - The URL to PUT the file to. - fileobj: - The file-like object holding the data to upload. - - Returns: `requests.Response` - - Raises: - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - If the upload resulted in an error. - """ - with operation.as_file(with_tqdm=True) as fileobj: - # S3 might raise a transient 500 error -> let's retry if that happens - response = http_backoff("PUT", upload_url, data=fileobj) - hf_raise_for_status(response) - - -def _upload_multi_part(operation: "CommitOperationAdd", header: Dict, chunk_size: int, upload_url: str) -> None: - """ - Uploads file using HF multipart LFS transfer protocol. - """ - # 1. Get upload URLs for each part - sorted_parts_urls = _get_sorted_parts_urls(header=header, upload_info=operation.upload_info, chunk_size=chunk_size) - - # 2. Upload parts (either with hf_transfer or in pure Python) - use_hf_transfer = constants.HF_HUB_ENABLE_HF_TRANSFER - if ( - constants.HF_HUB_ENABLE_HF_TRANSFER - and not isinstance(operation.path_or_fileobj, str) - and not isinstance(operation.path_or_fileobj, Path) - ): - warnings.warn( - "hf_transfer is enabled but does not support uploading from bytes or BinaryIO, falling back to regular" - " upload" - ) - use_hf_transfer = False - - response_headers = ( - _upload_parts_hf_transfer(operation=operation, sorted_parts_urls=sorted_parts_urls, chunk_size=chunk_size) - if use_hf_transfer - else _upload_parts_iteratively(operation=operation, sorted_parts_urls=sorted_parts_urls, chunk_size=chunk_size) - ) - - # 3. Send completion request - completion_res = get_session().post( - upload_url, - json=_get_completion_payload(response_headers, operation.upload_info.sha256.hex()), - headers=LFS_HEADERS, - ) - hf_raise_for_status(completion_res) - - -def _get_sorted_parts_urls(header: Dict, upload_info: UploadInfo, chunk_size: int) -> List[str]: - sorted_part_upload_urls = [ - upload_url - for _, upload_url in sorted( - [ - (int(part_num, 10), upload_url) - for part_num, upload_url in header.items() - if part_num.isdigit() and len(part_num) > 0 - ], - key=lambda t: t[0], - ) - ] - num_parts = len(sorted_part_upload_urls) - if num_parts != ceil(upload_info.size / chunk_size): - raise ValueError("Invalid server response to upload large LFS file") - return sorted_part_upload_urls - - -def _get_completion_payload(response_headers: List[Dict], oid: str) -> CompletionPayloadT: - parts: List[PayloadPartT] = [] - for part_number, header in enumerate(response_headers): - etag = header.get("etag") - if etag is None or etag == "": - raise ValueError(f"Invalid etag (`{etag}`) returned for part {part_number + 1}") - parts.append( - { - "partNumber": part_number + 1, - "etag": etag, - } - ) - return {"oid": oid, "parts": parts} - - -def _upload_parts_iteratively( - operation: "CommitOperationAdd", sorted_parts_urls: List[str], chunk_size: int -) -> List[Dict]: - headers = [] - with operation.as_file(with_tqdm=True) as fileobj: - for part_idx, part_upload_url in enumerate(sorted_parts_urls): - with SliceFileObj( - fileobj, - seek_from=chunk_size * part_idx, - read_limit=chunk_size, - ) as fileobj_slice: - # S3 might raise a transient 500 error -> let's retry if that happens - part_upload_res = http_backoff("PUT", part_upload_url, data=fileobj_slice) - hf_raise_for_status(part_upload_res) - headers.append(part_upload_res.headers) - return headers # type: ignore - - -def _upload_parts_hf_transfer( - operation: "CommitOperationAdd", sorted_parts_urls: List[str], chunk_size: int -) -> List[Dict]: - # Upload file using an external Rust-based package. Upload is faster but support less features (no progress bars). - try: - from hf_transfer import multipart_upload - except ImportError: - raise ValueError( - "Fast uploading using 'hf_transfer' is enabled (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is" - " not available in your environment. Try `pip install hf_transfer`." - ) - - supports_callback = "callback" in inspect.signature(multipart_upload).parameters - if not supports_callback: - warnings.warn( - "You are using an outdated version of `hf_transfer`. Consider upgrading to latest version to enable progress bars using `pip install -U hf_transfer`." - ) - - total = operation.upload_info.size - desc = operation.path_in_repo - if len(desc) > 40: - desc = f"(…){desc[-40:]}" - - with tqdm( - unit="B", - unit_scale=True, - total=total, - initial=0, - desc=desc, - disable=is_tqdm_disabled(logger.getEffectiveLevel()), - name="huggingface_hub.lfs_upload", - ) as progress: - try: - output = multipart_upload( - file_path=operation.path_or_fileobj, - parts_urls=sorted_parts_urls, - chunk_size=chunk_size, - max_files=128, - parallel_failures=127, # could be removed - max_retries=5, - **({"callback": progress.update} if supports_callback else {}), - ) - except Exception as e: - raise RuntimeError( - "An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for" - " better error handling." - ) from e - if not supports_callback: - progress.update(total) - return output diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/py.typed b/bundle/python-cpu/Lib/site-packages/huggingface_hub/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/repocard.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/repocard.py deleted file mode 100644 index 357935c3f1831df2afc86a30f82f10fa8039a225..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/repocard.py +++ /dev/null @@ -1,827 +0,0 @@ -import os -import re -from pathlib import Path -from typing import Any, Dict, Literal, Optional, Type, Union - -import requests -import yaml - -from huggingface_hub.file_download import hf_hub_download -from huggingface_hub.hf_api import upload_file -from huggingface_hub.repocard_data import ( - CardData, - DatasetCardData, - EvalResult, - ModelCardData, - SpaceCardData, - eval_results_to_model_index, - model_index_to_eval_results, -) -from huggingface_hub.utils import get_session, is_jinja_available, yaml_dump - -from . import constants -from .errors import EntryNotFoundError -from .utils import SoftTemporaryDirectory, logging, validate_hf_hub_args - - -logger = logging.get_logger(__name__) - - -TEMPLATE_MODELCARD_PATH = Path(__file__).parent / "templates" / "modelcard_template.md" -TEMPLATE_DATASETCARD_PATH = Path(__file__).parent / "templates" / "datasetcard_template.md" - -# exact same regex as in the Hub server. Please keep in sync. -# See https://github.com/huggingface/moon-landing/blob/main/server/lib/ViewMarkdown.ts#L18 -REGEX_YAML_BLOCK = re.compile(r"^(\s*---[\r\n]+)([\S\s]*?)([\r\n]+---(\r\n|\n|$))") - - -class RepoCard: - card_data_class = CardData - default_template_path = TEMPLATE_MODELCARD_PATH - repo_type = "model" - - def __init__(self, content: str, ignore_metadata_errors: bool = False): - """Initialize a RepoCard from string content. The content should be a - Markdown file with a YAML block at the beginning and a Markdown body. - - Args: - content (`str`): The content of the Markdown file. - - Example: - ```python - >>> from huggingface_hub.repocard import RepoCard - >>> text = ''' - ... --- - ... language: en - ... license: mit - ... --- - ... - ... # My repo - ... ''' - >>> card = RepoCard(text) - >>> card.data.to_dict() - {'language': 'en', 'license': 'mit'} - >>> card.text - '\\n# My repo\\n' - - ``` - > [!TIP] - > Raises the following error: - > - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > when the content of the repo card metadata is not a dictionary. - """ - - # Set the content of the RepoCard, as well as underlying .data and .text attributes. - # See the `content` property setter for more details. - self.ignore_metadata_errors = ignore_metadata_errors - self.content = content - - @property - def content(self): - """The content of the RepoCard, including the YAML block and the Markdown body.""" - line_break = _detect_line_ending(self._content) or "\n" - return f"---{line_break}{self.data.to_yaml(line_break=line_break, original_order=self._original_order)}{line_break}---{line_break}{self.text}" - - @content.setter - def content(self, content: str): - """Set the content of the RepoCard.""" - self._content = content - - match = REGEX_YAML_BLOCK.search(content) - if match: - # Metadata found in the YAML block - yaml_block = match.group(2) - self.text = content[match.end() :] - data_dict = yaml.safe_load(yaml_block) - - if data_dict is None: - data_dict = {} - - # The YAML block's data should be a dictionary - if not isinstance(data_dict, dict): - raise ValueError("repo card metadata block should be a dict") - else: - # Model card without metadata... create empty metadata - logger.warning("Repo card metadata block was not found. Setting CardData to empty.") - data_dict = {} - self.text = content - - self.data = self.card_data_class(**data_dict, ignore_metadata_errors=self.ignore_metadata_errors) - self._original_order = list(data_dict.keys()) - - def __str__(self): - return self.content - - def save(self, filepath: Union[Path, str]): - r"""Save a RepoCard to a file. - - Args: - filepath (`Union[Path, str]`): Filepath to the markdown file to save. - - Example: - ```python - >>> from huggingface_hub.repocard import RepoCard - >>> card = RepoCard("---\nlanguage: en\n---\n# This is a test repo card") - >>> card.save("/tmp/test.md") - - ``` - """ - filepath = Path(filepath) - filepath.parent.mkdir(parents=True, exist_ok=True) - # Preserve newlines as in the existing file. - with open(filepath, mode="w", newline="", encoding="utf-8") as f: - f.write(str(self)) - - @classmethod - def load( - cls, - repo_id_or_path: Union[str, Path], - repo_type: Optional[str] = None, - token: Optional[str] = None, - ignore_metadata_errors: bool = False, - ): - """Initialize a RepoCard from a Hugging Face Hub repo's README.md or a local filepath. - - Args: - repo_id_or_path (`Union[str, Path]`): - The repo ID associated with a Hugging Face Hub repo or a local filepath. - repo_type (`str`, *optional*): - The type of Hugging Face repo to push to. Defaults to None, which will use use "model". Other options - are "dataset" and "space". Not used when loading from a local filepath. If this is called from a child - class, the default value will be the child class's `repo_type`. - token (`str`, *optional*): - Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to the stored token. - ignore_metadata_errors (`str`): - If True, errors while parsing the metadata section will be ignored. Some information might be lost during - the process. Use it at your own risk. - - Returns: - [`huggingface_hub.repocard.RepoCard`]: The RepoCard (or subclass) initialized from the repo's - README.md file or filepath. - - Example: - ```python - >>> from huggingface_hub.repocard import RepoCard - >>> card = RepoCard.load("nateraw/food") - >>> assert card.data.tags == ["generated_from_trainer", "image-classification", "pytorch"] - - ``` - """ - - if Path(repo_id_or_path).is_file(): - card_path = Path(repo_id_or_path) - elif isinstance(repo_id_or_path, str): - card_path = Path( - hf_hub_download( - repo_id_or_path, - constants.REPOCARD_NAME, - repo_type=repo_type or cls.repo_type, - token=token, - ) - ) - else: - raise ValueError(f"Cannot load RepoCard: path not found on disk ({repo_id_or_path}).") - - # Preserve newlines in the existing file. - with card_path.open(mode="r", newline="", encoding="utf-8") as f: - return cls(f.read(), ignore_metadata_errors=ignore_metadata_errors) - - def validate(self, repo_type: Optional[str] = None): - """Validates card against Hugging Face Hub's card validation logic. - Using this function requires access to the internet, so it is only called - internally by [`huggingface_hub.repocard.RepoCard.push_to_hub`]. - - Args: - repo_type (`str`, *optional*, defaults to "model"): - The type of Hugging Face repo to push to. Options are "model", "dataset", and "space". - If this function is called from a child class, the default will be the child class's `repo_type`. - - > [!TIP] - > Raises the following errors: - > - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if the card fails validation checks. - > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) - > if the request to the Hub API fails for any other reason. - """ - - # If repo type is provided, otherwise, use the repo type of the card. - repo_type = repo_type or self.repo_type - - body = { - "repoType": repo_type, - "content": str(self), - } - headers = {"Accept": "text/plain"} - - try: - r = get_session().post("https://huggingface.co/api/validate-yaml", body, headers=headers) - r.raise_for_status() - except requests.exceptions.HTTPError as exc: - if r.status_code == 400: - raise ValueError(r.text) - else: - raise exc - - def push_to_hub( - self, - repo_id: str, - token: Optional[str] = None, - repo_type: Optional[str] = None, - commit_message: Optional[str] = None, - commit_description: Optional[str] = None, - revision: Optional[str] = None, - create_pr: Optional[bool] = None, - parent_commit: Optional[str] = None, - ): - """Push a RepoCard to a Hugging Face Hub repo. - - Args: - repo_id (`str`): - The repo ID of the Hugging Face Hub repo to push to. Example: "nateraw/food". - token (`str`, *optional*): - Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to - the stored token. - repo_type (`str`, *optional*, defaults to "model"): - The type of Hugging Face repo to push to. Options are "model", "dataset", and "space". If this - function is called by a child class, it will default to the child class's `repo_type`. - commit_message (`str`, *optional*): - The summary / title / first line of the generated commit. - commit_description (`str`, *optional*) - The description of the generated commit. - revision (`str`, *optional*): - The git revision to commit from. Defaults to the head of the `"main"` branch. - create_pr (`bool`, *optional*): - Whether or not to create a Pull Request with this commit. Defaults to `False`. - parent_commit (`str`, *optional*): - The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. - If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. - If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. - Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be - especially useful if the repo is updated / committed to concurrently. - Returns: - `str`: URL of the commit which updated the card metadata. - """ - - # If repo type is provided, otherwise, use the repo type of the card. - repo_type = repo_type or self.repo_type - - # Validate card before pushing to hub - self.validate(repo_type=repo_type) - - with SoftTemporaryDirectory() as tmpdir: - tmp_path = Path(tmpdir) / constants.REPOCARD_NAME - tmp_path.write_text(str(self), encoding="utf-8") - url = upload_file( - path_or_fileobj=str(tmp_path), - path_in_repo=constants.REPOCARD_NAME, - repo_id=repo_id, - token=token, - repo_type=repo_type, - commit_message=commit_message, - commit_description=commit_description, - create_pr=create_pr, - revision=revision, - parent_commit=parent_commit, - ) - return url - - @classmethod - def from_template( - cls, - card_data: CardData, - template_path: Optional[str] = None, - template_str: Optional[str] = None, - **template_kwargs, - ): - """Initialize a RepoCard from a template. By default, it uses the default template. - - Templates are Jinja2 templates that can be customized by passing keyword arguments. - - Args: - card_data (`huggingface_hub.CardData`): - A huggingface_hub.CardData instance containing the metadata you want to include in the YAML - header of the repo card on the Hugging Face Hub. - template_path (`str`, *optional*): - A path to a markdown file with optional Jinja template variables that can be filled - in with `template_kwargs`. Defaults to the default template. - - Returns: - [`huggingface_hub.repocard.RepoCard`]: A RepoCard instance with the specified card data and content from the - template. - """ - if is_jinja_available(): - import jinja2 - else: - raise ImportError( - "Using RepoCard.from_template requires Jinja2 to be installed. Please" - " install it with `pip install Jinja2`." - ) - - kwargs = card_data.to_dict().copy() - kwargs.update(template_kwargs) # Template_kwargs have priority - - if template_path is not None: - template_str = Path(template_path).read_text() - if template_str is None: - template_str = Path(cls.default_template_path).read_text() - template = jinja2.Template(template_str) - content = template.render(card_data=card_data.to_yaml(), **kwargs) - return cls(content) - - -class ModelCard(RepoCard): - card_data_class = ModelCardData - default_template_path = TEMPLATE_MODELCARD_PATH - repo_type = "model" - - @classmethod - def from_template( # type: ignore # violates Liskov property but easier to use - cls, - card_data: ModelCardData, - template_path: Optional[str] = None, - template_str: Optional[str] = None, - **template_kwargs, - ): - """Initialize a ModelCard from a template. By default, it uses the default template, which can be found here: - https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/modelcard_template.md - - Templates are Jinja2 templates that can be customized by passing keyword arguments. - - Args: - card_data (`huggingface_hub.ModelCardData`): - A huggingface_hub.ModelCardData instance containing the metadata you want to include in the YAML - header of the model card on the Hugging Face Hub. - template_path (`str`, *optional*): - A path to a markdown file with optional Jinja template variables that can be filled - in with `template_kwargs`. Defaults to the default template. - - Returns: - [`huggingface_hub.ModelCard`]: A ModelCard instance with the specified card data and content from the - template. - - Example: - ```python - >>> from huggingface_hub import ModelCard, ModelCardData, EvalResult - - >>> # Using the Default Template - >>> card_data = ModelCardData( - ... language='en', - ... license='mit', - ... library_name='timm', - ... tags=['image-classification', 'resnet'], - ... datasets=['beans'], - ... metrics=['accuracy'], - ... ) - >>> card = ModelCard.from_template( - ... card_data, - ... model_description='This model does x + y...' - ... ) - - >>> # Including Evaluation Results - >>> card_data = ModelCardData( - ... language='en', - ... tags=['image-classification', 'resnet'], - ... eval_results=[ - ... EvalResult( - ... task_type='image-classification', - ... dataset_type='beans', - ... dataset_name='Beans', - ... metric_type='accuracy', - ... metric_value=0.9, - ... ), - ... ], - ... model_name='my-cool-model', - ... ) - >>> card = ModelCard.from_template(card_data) - - >>> # Using a Custom Template - >>> card_data = ModelCardData( - ... language='en', - ... tags=['image-classification', 'resnet'] - ... ) - >>> card = ModelCard.from_template( - ... card_data=card_data, - ... template_path='./src/huggingface_hub/templates/modelcard_template.md', - ... custom_template_var='custom value', # will be replaced in template if it exists - ... ) - - ``` - """ - return super().from_template(card_data, template_path, template_str, **template_kwargs) - - -class DatasetCard(RepoCard): - card_data_class = DatasetCardData - default_template_path = TEMPLATE_DATASETCARD_PATH - repo_type = "dataset" - - @classmethod - def from_template( # type: ignore # violates Liskov property but easier to use - cls, - card_data: DatasetCardData, - template_path: Optional[str] = None, - template_str: Optional[str] = None, - **template_kwargs, - ): - """Initialize a DatasetCard from a template. By default, it uses the default template, which can be found here: - https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/datasetcard_template.md - - Templates are Jinja2 templates that can be customized by passing keyword arguments. - - Args: - card_data (`huggingface_hub.DatasetCardData`): - A huggingface_hub.DatasetCardData instance containing the metadata you want to include in the YAML - header of the dataset card on the Hugging Face Hub. - template_path (`str`, *optional*): - A path to a markdown file with optional Jinja template variables that can be filled - in with `template_kwargs`. Defaults to the default template. - - Returns: - [`huggingface_hub.DatasetCard`]: A DatasetCard instance with the specified card data and content from the - template. - - Example: - ```python - >>> from huggingface_hub import DatasetCard, DatasetCardData - - >>> # Using the Default Template - >>> card_data = DatasetCardData( - ... language='en', - ... license='mit', - ... annotations_creators='crowdsourced', - ... task_categories=['text-classification'], - ... task_ids=['sentiment-classification', 'text-scoring'], - ... multilinguality='monolingual', - ... pretty_name='My Text Classification Dataset', - ... ) - >>> card = DatasetCard.from_template( - ... card_data, - ... pretty_name=card_data.pretty_name, - ... ) - - >>> # Using a Custom Template - >>> card_data = DatasetCardData( - ... language='en', - ... license='mit', - ... ) - >>> card = DatasetCard.from_template( - ... card_data=card_data, - ... template_path='./src/huggingface_hub/templates/datasetcard_template.md', - ... custom_template_var='custom value', # will be replaced in template if it exists - ... ) - - ``` - """ - return super().from_template(card_data, template_path, template_str, **template_kwargs) - - -class SpaceCard(RepoCard): - card_data_class = SpaceCardData - default_template_path = TEMPLATE_MODELCARD_PATH - repo_type = "space" - - -def _detect_line_ending(content: str) -> Literal["\r", "\n", "\r\n", None]: # noqa: F722 - """Detect the line ending of a string. Used by RepoCard to avoid making huge diff on newlines. - - Uses same implementation as in Hub server, keep it in sync. - - Returns: - str: The detected line ending of the string. - """ - cr = content.count("\r") - lf = content.count("\n") - crlf = content.count("\r\n") - if cr + lf == 0: - return None - if crlf == cr and crlf == lf: - return "\r\n" - if cr > lf: - return "\r" - else: - return "\n" - - -def metadata_load(local_path: Union[str, Path]) -> Optional[Dict]: - content = Path(local_path).read_text() - match = REGEX_YAML_BLOCK.search(content) - if match: - yaml_block = match.group(2) - data = yaml.safe_load(yaml_block) - if data is None or isinstance(data, dict): - return data - raise ValueError("repo card metadata block should be a dict") - else: - return None - - -def metadata_save(local_path: Union[str, Path], data: Dict) -> None: - """ - Save the metadata dict in the upper YAML part Trying to preserve newlines as - in the existing file. Docs about open() with newline="" parameter: - https://docs.python.org/3/library/functions.html?highlight=open#open Does - not work with "^M" linebreaks, which are replaced by \n - """ - line_break = "\n" - content = "" - # try to detect existing newline character - if os.path.exists(local_path): - with open(local_path, "r", newline="", encoding="utf8") as readme: - content = readme.read() - if isinstance(readme.newlines, tuple): - line_break = readme.newlines[0] - elif isinstance(readme.newlines, str): - line_break = readme.newlines - - # creates a new file if it not - with open(local_path, "w", newline="", encoding="utf8") as readme: - data_yaml = yaml_dump(data, sort_keys=False, line_break=line_break) - # sort_keys: keep dict order - match = REGEX_YAML_BLOCK.search(content) - if match: - output = content[: match.start()] + f"---{line_break}{data_yaml}---{line_break}" + content[match.end() :] - else: - output = f"---{line_break}{data_yaml}---{line_break}{content}" - - readme.write(output) - readme.close() - - -def metadata_eval_result( - *, - model_pretty_name: str, - task_pretty_name: str, - task_id: str, - metrics_pretty_name: str, - metrics_id: str, - metrics_value: Any, - dataset_pretty_name: str, - dataset_id: str, - metrics_config: Optional[str] = None, - metrics_verified: bool = False, - dataset_config: Optional[str] = None, - dataset_split: Optional[str] = None, - dataset_revision: Optional[str] = None, - metrics_verification_token: Optional[str] = None, -) -> Dict: - """ - Creates a metadata dict with the result from a model evaluated on a dataset. - - Args: - model_pretty_name (`str`): - The name of the model in natural language. - task_pretty_name (`str`): - The name of a task in natural language. - task_id (`str`): - Example: automatic-speech-recognition. A task id. - metrics_pretty_name (`str`): - A name for the metric in natural language. Example: Test WER. - metrics_id (`str`): - Example: wer. A metric id from https://hf.co/metrics. - metrics_value (`Any`): - The value from the metric. Example: 20.0 or "20.0 ± 1.2". - dataset_pretty_name (`str`): - The name of the dataset in natural language. - dataset_id (`str`): - Example: common_voice. A dataset id from https://hf.co/datasets. - metrics_config (`str`, *optional*): - The name of the metric configuration used in `load_metric()`. - Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`. - metrics_verified (`bool`, *optional*, defaults to `False`): - Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set. - dataset_config (`str`, *optional*): - Example: fr. The name of the dataset configuration used in `load_dataset()`. - dataset_split (`str`, *optional*): - Example: test. The name of the dataset split used in `load_dataset()`. - dataset_revision (`str`, *optional*): - Example: 5503434ddd753f426f4b38109466949a1217c2bb. The name of the dataset dataset revision - used in `load_dataset()`. - metrics_verification_token (`bool`, *optional*): - A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. - - Returns: - `dict`: a metadata dict with the result from a model evaluated on a dataset. - - Example: - ```python - >>> from huggingface_hub import metadata_eval_result - >>> results = metadata_eval_result( - ... model_pretty_name="RoBERTa fine-tuned on ReactionGIF", - ... task_pretty_name="Text Classification", - ... task_id="text-classification", - ... metrics_pretty_name="Accuracy", - ... metrics_id="accuracy", - ... metrics_value=0.2662102282047272, - ... dataset_pretty_name="ReactionJPEG", - ... dataset_id="julien-c/reactionjpeg", - ... dataset_config="default", - ... dataset_split="test", - ... ) - >>> results == { - ... 'model-index': [ - ... { - ... 'name': 'RoBERTa fine-tuned on ReactionGIF', - ... 'results': [ - ... { - ... 'task': { - ... 'type': 'text-classification', - ... 'name': 'Text Classification' - ... }, - ... 'dataset': { - ... 'name': 'ReactionJPEG', - ... 'type': 'julien-c/reactionjpeg', - ... 'config': 'default', - ... 'split': 'test' - ... }, - ... 'metrics': [ - ... { - ... 'type': 'accuracy', - ... 'value': 0.2662102282047272, - ... 'name': 'Accuracy', - ... 'verified': False - ... } - ... ] - ... } - ... ] - ... } - ... ] - ... } - True - - ``` - """ - - return { - "model-index": eval_results_to_model_index( - model_name=model_pretty_name, - eval_results=[ - EvalResult( - task_name=task_pretty_name, - task_type=task_id, - metric_name=metrics_pretty_name, - metric_type=metrics_id, - metric_value=metrics_value, - dataset_name=dataset_pretty_name, - dataset_type=dataset_id, - metric_config=metrics_config, - verified=metrics_verified, - verify_token=metrics_verification_token, - dataset_config=dataset_config, - dataset_split=dataset_split, - dataset_revision=dataset_revision, - ) - ], - ) - } - - -@validate_hf_hub_args -def metadata_update( - repo_id: str, - metadata: Dict, - *, - repo_type: Optional[str] = None, - overwrite: bool = False, - token: Optional[str] = None, - commit_message: Optional[str] = None, - commit_description: Optional[str] = None, - revision: Optional[str] = None, - create_pr: bool = False, - parent_commit: Optional[str] = None, -) -> str: - """ - Updates the metadata in the README.md of a repository on the Hugging Face Hub. - If the README.md file doesn't exist yet, a new one is created with metadata and an - the default ModelCard or DatasetCard template. For `space` repo, an error is thrown - as a Space cannot exist without a `README.md` file. - - Args: - repo_id (`str`): - The name of the repository. - metadata (`dict`): - A dictionary containing the metadata to be updated. - repo_type (`str`, *optional*): - Set to `"dataset"` or `"space"` if updating to a dataset or space, - `None` or `"model"` if updating to a model. Default is `None`. - overwrite (`bool`, *optional*, defaults to `False`): - If set to `True` an existing field can be overwritten, otherwise - attempting to overwrite an existing field will cause an error. - token (`str`, *optional*): - The Hugging Face authentication token. - commit_message (`str`, *optional*): - The summary / title / first line of the generated commit. Defaults to - `f"Update metadata with huggingface_hub"` - commit_description (`str` *optional*) - The description of the generated commit - revision (`str`, *optional*): - The git revision to commit from. Defaults to the head of the - `"main"` branch. - create_pr (`boolean`, *optional*): - Whether or not to create a Pull Request from `revision` with that commit. - Defaults to `False`. - parent_commit (`str`, *optional*): - The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. - If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. - If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. - Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be - especially useful if the repo is updated / committed to concurrently. - Returns: - `str`: URL of the commit which updated the card metadata. - - Example: - ```python - >>> from huggingface_hub import metadata_update - >>> metadata = {'model-index': [{'name': 'RoBERTa fine-tuned on ReactionGIF', - ... 'results': [{'dataset': {'name': 'ReactionGIF', - ... 'type': 'julien-c/reactiongif'}, - ... 'metrics': [{'name': 'Recall', - ... 'type': 'recall', - ... 'value': 0.7762102282047272}], - ... 'task': {'name': 'Text Classification', - ... 'type': 'text-classification'}}]}]} - >>> url = metadata_update("hf-internal-testing/reactiongif-roberta-card", metadata) - - ``` - """ - commit_message = commit_message if commit_message is not None else "Update metadata with huggingface_hub" - - # Card class given repo_type - card_class: Type[RepoCard] - if repo_type is None or repo_type == "model": - card_class = ModelCard - elif repo_type == "dataset": - card_class = DatasetCard - elif repo_type == "space": - card_class = RepoCard - else: - raise ValueError(f"Unknown repo_type: {repo_type}") - - # Either load repo_card from the Hub or create an empty one. - # NOTE: Will not create the repo if it doesn't exist. - try: - card = card_class.load(repo_id, token=token, repo_type=repo_type) - except EntryNotFoundError: - if repo_type == "space": - raise ValueError("Cannot update metadata on a Space that doesn't contain a `README.md` file.") - - # Initialize a ModelCard or DatasetCard from default template and no data. - # Cast to the concrete expected card type to satisfy type checkers. - card = card_class.from_template(CardData()) # type: ignore[return-value] - - for key, value in metadata.items(): - if key == "model-index": - # if the new metadata doesn't include a name, either use existing one or repo name - if "name" not in value[0]: - value[0]["name"] = getattr(card, "model_name", repo_id) - model_name, new_results = model_index_to_eval_results(value) - if card.data.eval_results is None: - card.data.eval_results = new_results - card.data.model_name = model_name - else: - existing_results = card.data.eval_results - - # Iterate over new results - # Iterate over existing results - # If both results describe the same metric but value is different: - # If overwrite=True: overwrite the metric value - # Else: raise ValueError - # Else: append new result to existing ones. - for new_result in new_results: - result_found = False - for existing_result in existing_results: - if new_result.is_equal_except_value(existing_result): - if new_result != existing_result and not overwrite: - raise ValueError( - "You passed a new value for the existing metric" - f" 'name: {new_result.metric_name}, type: " - f"{new_result.metric_type}'. Set `overwrite=True`" - " to overwrite existing metrics." - ) - result_found = True - existing_result.metric_value = new_result.metric_value - if existing_result.verified is True: - existing_result.verify_token = new_result.verify_token - if not result_found: - card.data.eval_results.append(new_result) - else: - # Any metadata that is not a result metric - if card.data.get(key) is not None and not overwrite and card.data.get(key) != value: - raise ValueError( - f"You passed a new value for the existing meta data field '{key}'." - " Set `overwrite=True` to overwrite existing metadata." - ) - else: - card.data[key] = value - - return card.push_to_hub( - repo_id, - token=token, - repo_type=repo_type, - commit_message=commit_message, - commit_description=commit_description, - create_pr=create_pr, - revision=revision, - parent_commit=parent_commit, - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/repocard_data.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/repocard_data.py deleted file mode 100644 index 62215f2274e482d4ed69a1d6deeafdf34fc5a6a4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/repocard_data.py +++ /dev/null @@ -1,770 +0,0 @@ -import copy -from collections import defaultdict -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union - -from huggingface_hub.utils import logging, yaml_dump - - -logger = logging.get_logger(__name__) - - -@dataclass -class EvalResult: - """ - Flattened representation of individual evaluation results found in model-index of Model Cards. - - For more information on the model-index spec, see https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1. - - Args: - task_type (`str`): - The task identifier. Example: "image-classification". - dataset_type (`str`): - The dataset identifier. Example: "common_voice". Use dataset id from https://hf.co/datasets. - dataset_name (`str`): - A pretty name for the dataset. Example: "Common Voice (French)". - metric_type (`str`): - The metric identifier. Example: "wer". Use metric id from https://hf.co/metrics. - metric_value (`Any`): - The metric value. Example: 0.9 or "20.0 ± 1.2". - task_name (`str`, *optional*): - A pretty name for the task. Example: "Speech Recognition". - dataset_config (`str`, *optional*): - The name of the dataset configuration used in `load_dataset()`. - Example: fr in `load_dataset("common_voice", "fr")`. See the `datasets` docs for more info: - https://hf.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name - dataset_split (`str`, *optional*): - The split used in `load_dataset()`. Example: "test". - dataset_revision (`str`, *optional*): - The revision (AKA Git Sha) of the dataset used in `load_dataset()`. - Example: 5503434ddd753f426f4b38109466949a1217c2bb - dataset_args (`Dict[str, Any]`, *optional*): - The arguments passed during `Metric.compute()`. Example for `bleu`: `{"max_order": 4}` - metric_name (`str`, *optional*): - A pretty name for the metric. Example: "Test WER". - metric_config (`str`, *optional*): - The name of the metric configuration used in `load_metric()`. - Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`. - See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations - metric_args (`Dict[str, Any]`, *optional*): - The arguments passed during `Metric.compute()`. Example for `bleu`: max_order: 4 - verified (`bool`, *optional*): - Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set. - verify_token (`str`, *optional*): - A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. - source_name (`str`, *optional*): - The name of the source of the evaluation result. Example: "Open LLM Leaderboard". - source_url (`str`, *optional*): - The URL of the source of the evaluation result. Example: "https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard". - """ - - # Required - - # The task identifier - # Example: automatic-speech-recognition - task_type: str - - # The dataset identifier - # Example: common_voice. Use dataset id from https://hf.co/datasets - dataset_type: str - - # A pretty name for the dataset. - # Example: Common Voice (French) - dataset_name: str - - # The metric identifier - # Example: wer. Use metric id from https://hf.co/metrics - metric_type: str - - # Value of the metric. - # Example: 20.0 or "20.0 ± 1.2" - metric_value: Any - - # Optional - - # A pretty name for the task. - # Example: Speech Recognition - task_name: Optional[str] = None - - # The name of the dataset configuration used in `load_dataset()`. - # Example: fr in `load_dataset("common_voice", "fr")`. - # See the `datasets` docs for more info: - # https://huggingface.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name - dataset_config: Optional[str] = None - - # The split used in `load_dataset()`. - # Example: test - dataset_split: Optional[str] = None - - # The revision (AKA Git Sha) of the dataset used in `load_dataset()`. - # Example: 5503434ddd753f426f4b38109466949a1217c2bb - dataset_revision: Optional[str] = None - - # The arguments passed during `Metric.compute()`. - # Example for `bleu`: max_order: 4 - dataset_args: Optional[Dict[str, Any]] = None - - # A pretty name for the metric. - # Example: Test WER - metric_name: Optional[str] = None - - # The name of the metric configuration used in `load_metric()`. - # Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`. - # See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations - metric_config: Optional[str] = None - - # The arguments passed during `Metric.compute()`. - # Example for `bleu`: max_order: 4 - metric_args: Optional[Dict[str, Any]] = None - - # Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set. - verified: Optional[bool] = None - - # A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. - verify_token: Optional[str] = None - - # The name of the source of the evaluation result. - # Example: Open LLM Leaderboard - source_name: Optional[str] = None - - # The URL of the source of the evaluation result. - # Example: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard - source_url: Optional[str] = None - - @property - def unique_identifier(self) -> tuple: - """Returns a tuple that uniquely identifies this evaluation.""" - return ( - self.task_type, - self.dataset_type, - self.dataset_config, - self.dataset_split, - self.dataset_revision, - ) - - def is_equal_except_value(self, other: "EvalResult") -> bool: - """ - Return True if `self` and `other` describe exactly the same metric but with a - different value. - """ - for key, _ in self.__dict__.items(): - if key == "metric_value": - continue - # For metrics computed by Hugging Face's evaluation service, `verify_token` is derived from `metric_value`, - # so we exclude it here in the comparison. - if key != "verify_token" and getattr(self, key) != getattr(other, key): - return False - return True - - def __post_init__(self) -> None: - if self.source_name is not None and self.source_url is None: - raise ValueError("If `source_name` is provided, `source_url` must also be provided.") - - -@dataclass -class CardData: - """Structure containing metadata from a RepoCard. - - [`CardData`] is the parent class of [`ModelCardData`] and [`DatasetCardData`]. - - Metadata can be exported as a dictionary or YAML. Export can be customized to alter the representation of the data - (example: flatten evaluation results). `CardData` behaves as a dictionary (can get, pop, set values) but do not - inherit from `dict` to allow this export step. - """ - - def __init__(self, ignore_metadata_errors: bool = False, **kwargs): - self.__dict__.update(kwargs) - - def to_dict(self): - """Converts CardData to a dict. - - Returns: - `dict`: CardData represented as a dictionary ready to be dumped to a YAML - block for inclusion in a README.md file. - """ - - data_dict = copy.deepcopy(self.__dict__) - self._to_dict(data_dict) - return {key: value for key, value in data_dict.items() if value is not None} - - def _to_dict(self, data_dict): - """Use this method in child classes to alter the dict representation of the data. Alter the dict in-place. - - Args: - data_dict (`dict`): The raw dict representation of the card data. - """ - pass - - def to_yaml(self, line_break=None, original_order: Optional[List[str]] = None) -> str: - """Dumps CardData to a YAML block for inclusion in a README.md file. - - Args: - line_break (str, *optional*): - The line break to use when dumping to yaml. - - Returns: - `str`: CardData represented as a YAML block. - """ - if original_order: - self.__dict__ = { - k: self.__dict__[k] - for k in original_order + list(set(self.__dict__.keys()) - set(original_order)) - if k in self.__dict__ - } - return yaml_dump(self.to_dict(), sort_keys=False, line_break=line_break).strip() - - def __repr__(self): - return repr(self.__dict__) - - def __str__(self): - return self.to_yaml() - - def get(self, key: str, default: Any = None) -> Any: - """Get value for a given metadata key.""" - value = self.__dict__.get(key) - return default if value is None else value - - def pop(self, key: str, default: Any = None) -> Any: - """Pop value for a given metadata key.""" - return self.__dict__.pop(key, default) - - def __getitem__(self, key: str) -> Any: - """Get value for a given metadata key.""" - return self.__dict__[key] - - def __setitem__(self, key: str, value: Any) -> None: - """Set value for a given metadata key.""" - self.__dict__[key] = value - - def __contains__(self, key: str) -> bool: - """Check if a given metadata key is set.""" - return key in self.__dict__ - - def __len__(self) -> int: - """Return the number of metadata keys set.""" - return len(self.__dict__) - - -def _validate_eval_results( - eval_results: Optional[Union[EvalResult, List[EvalResult]]], - model_name: Optional[str], -) -> List[EvalResult]: - if eval_results is None: - return [] - if isinstance(eval_results, EvalResult): - eval_results = [eval_results] - if not isinstance(eval_results, list) or not all(isinstance(r, EvalResult) for r in eval_results): - raise ValueError( - f"`eval_results` should be of type `EvalResult` or a list of `EvalResult`, got {type(eval_results)}." - ) - if model_name is None: - raise ValueError("Passing `eval_results` requires `model_name` to be set.") - return eval_results - - -class ModelCardData(CardData): - """Model Card Metadata that is used by Hugging Face Hub when included at the top of your README.md - - Args: - base_model (`str` or `List[str]`, *optional*): - The identifier of the base model from which the model derives. This is applicable for example if your model is a - fine-tune or adapter of an existing model. The value must be the ID of a model on the Hub (or a list of IDs - if your model derives from multiple models). Defaults to None. - datasets (`Union[str, List[str]]`, *optional*): - Dataset or list of datasets that were used to train this model. Should be a dataset ID - found on https://hf.co/datasets. Defaults to None. - eval_results (`Union[List[EvalResult], EvalResult]`, *optional*): - List of `huggingface_hub.EvalResult` that define evaluation results of the model. If provided, - `model_name` is used to as a name on PapersWithCode's leaderboards. Defaults to `None`. - language (`Union[str, List[str]]`, *optional*): - Language of model's training data or metadata. It must be an ISO 639-1, 639-2 or - 639-3 code (two/three letters), or a special value like "code", "multilingual". Defaults to `None`. - library_name (`str`, *optional*): - Name of library used by this model. Example: keras or any library from - https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/model-libraries.ts. - Defaults to None. - license (`str`, *optional*): - License of this model. Example: apache-2.0 or any license from - https://huggingface.co/docs/hub/repositories-licenses. Defaults to None. - license_name (`str`, *optional*): - Name of the license of this model. Defaults to None. To be used in conjunction with `license_link`. - Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a name. In that case, use `license` instead. - license_link (`str`, *optional*): - Link to the license of this model. Defaults to None. To be used in conjunction with `license_name`. - Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a link. In that case, use `license` instead. - metrics (`List[str]`, *optional*): - List of metrics used to evaluate this model. Should be a metric name that can be found - at https://hf.co/metrics. Example: 'accuracy'. Defaults to None. - model_name (`str`, *optional*): - A name for this model. It is used along with - `eval_results` to construct the `model-index` within the card's metadata. The name - you supply here is what will be used on PapersWithCode's leaderboards. If None is provided - then the repo name is used as a default. Defaults to None. - pipeline_tag (`str`, *optional*): - The pipeline tag associated with the model. Example: "text-classification". - tags (`List[str]`, *optional*): - List of tags to add to your model that can be used when filtering on the Hugging - Face Hub. Defaults to None. - ignore_metadata_errors (`str`): - If True, errors while parsing the metadata section will be ignored. Some information might be lost during - the process. Use it at your own risk. - kwargs (`dict`, *optional*): - Additional metadata that will be added to the model card. Defaults to None. - - Example: - ```python - >>> from huggingface_hub import ModelCardData - >>> card_data = ModelCardData( - ... language="en", - ... license="mit", - ... library_name="timm", - ... tags=['image-classification', 'resnet'], - ... ) - >>> card_data.to_dict() - {'language': 'en', 'license': 'mit', 'library_name': 'timm', 'tags': ['image-classification', 'resnet']} - - ``` - """ - - def __init__( - self, - *, - base_model: Optional[Union[str, List[str]]] = None, - datasets: Optional[Union[str, List[str]]] = None, - eval_results: Optional[List[EvalResult]] = None, - language: Optional[Union[str, List[str]]] = None, - library_name: Optional[str] = None, - license: Optional[str] = None, - license_name: Optional[str] = None, - license_link: Optional[str] = None, - metrics: Optional[List[str]] = None, - model_name: Optional[str] = None, - pipeline_tag: Optional[str] = None, - tags: Optional[List[str]] = None, - ignore_metadata_errors: bool = False, - **kwargs, - ): - self.base_model = base_model - self.datasets = datasets - self.eval_results = eval_results - self.language = language - self.library_name = library_name - self.license = license - self.license_name = license_name - self.license_link = license_link - self.metrics = metrics - self.model_name = model_name - self.pipeline_tag = pipeline_tag - self.tags = _to_unique_list(tags) - - model_index = kwargs.pop("model-index", None) - if model_index: - try: - model_name, eval_results = model_index_to_eval_results(model_index) - self.model_name = model_name - self.eval_results = eval_results - except (KeyError, TypeError) as error: - if ignore_metadata_errors: - logger.warning("Invalid model-index. Not loading eval results into CardData.") - else: - raise ValueError( - f"Invalid `model_index` in metadata cannot be parsed: {error.__class__} {error}. Pass" - " `ignore_metadata_errors=True` to ignore this error while loading a Model Card. Warning:" - " some information will be lost. Use it at your own risk." - ) - - super().__init__(**kwargs) - - if self.eval_results: - try: - self.eval_results = _validate_eval_results(self.eval_results, self.model_name) - except Exception as e: - if ignore_metadata_errors: - logger.warning(f"Failed to validate eval_results: {e}. Not loading eval results into CardData.") - else: - raise ValueError(f"Failed to validate eval_results: {e}") from e - - def _to_dict(self, data_dict): - """Format the internal data dict. In this case, we convert eval results to a valid model index""" - if self.eval_results is not None: - data_dict["model-index"] = eval_results_to_model_index(self.model_name, self.eval_results) - del data_dict["eval_results"], data_dict["model_name"] - - -class DatasetCardData(CardData): - """Dataset Card Metadata that is used by Hugging Face Hub when included at the top of your README.md - - Args: - language (`List[str]`, *optional*): - Language of dataset's data or metadata. It must be an ISO 639-1, 639-2 or - 639-3 code (two/three letters), or a special value like "code", "multilingual". - license (`Union[str, List[str]]`, *optional*): - License(s) of this dataset. Example: apache-2.0 or any license from - https://huggingface.co/docs/hub/repositories-licenses. - annotations_creators (`Union[str, List[str]]`, *optional*): - How the annotations for the dataset were created. - Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'no-annotation', 'other'. - language_creators (`Union[str, List[str]]`, *optional*): - How the text-based data in the dataset was created. - Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'other' - multilinguality (`Union[str, List[str]]`, *optional*): - Whether the dataset is multilingual. - Options are: 'monolingual', 'multilingual', 'translation', 'other'. - size_categories (`Union[str, List[str]]`, *optional*): - The number of examples in the dataset. Options are: 'n<1K', '1K1T', and 'other'. - source_datasets (`List[str]]`, *optional*): - Indicates whether the dataset is an original dataset or extended from another existing dataset. - Options are: 'original' and 'extended'. - task_categories (`Union[str, List[str]]`, *optional*): - What categories of task does the dataset support? - task_ids (`Union[str, List[str]]`, *optional*): - What specific tasks does the dataset support? - paperswithcode_id (`str`, *optional*): - ID of the dataset on PapersWithCode. - pretty_name (`str`, *optional*): - A more human-readable name for the dataset. (ex. "Cats vs. Dogs") - train_eval_index (`Dict`, *optional*): - A dictionary that describes the necessary spec for doing evaluation on the Hub. - If not provided, it will be gathered from the 'train-eval-index' key of the kwargs. - config_names (`Union[str, List[str]]`, *optional*): - A list of the available dataset configs for the dataset. - """ - - def __init__( - self, - *, - language: Optional[Union[str, List[str]]] = None, - license: Optional[Union[str, List[str]]] = None, - annotations_creators: Optional[Union[str, List[str]]] = None, - language_creators: Optional[Union[str, List[str]]] = None, - multilinguality: Optional[Union[str, List[str]]] = None, - size_categories: Optional[Union[str, List[str]]] = None, - source_datasets: Optional[List[str]] = None, - task_categories: Optional[Union[str, List[str]]] = None, - task_ids: Optional[Union[str, List[str]]] = None, - paperswithcode_id: Optional[str] = None, - pretty_name: Optional[str] = None, - train_eval_index: Optional[Dict] = None, - config_names: Optional[Union[str, List[str]]] = None, - ignore_metadata_errors: bool = False, - **kwargs, - ): - self.annotations_creators = annotations_creators - self.language_creators = language_creators - self.language = language - self.license = license - self.multilinguality = multilinguality - self.size_categories = size_categories - self.source_datasets = source_datasets - self.task_categories = task_categories - self.task_ids = task_ids - self.paperswithcode_id = paperswithcode_id - self.pretty_name = pretty_name - self.config_names = config_names - - # TODO - maybe handle this similarly to EvalResult? - self.train_eval_index = train_eval_index or kwargs.pop("train-eval-index", None) - super().__init__(**kwargs) - - def _to_dict(self, data_dict): - data_dict["train-eval-index"] = data_dict.pop("train_eval_index") - - -class SpaceCardData(CardData): - """Space Card Metadata that is used by Hugging Face Hub when included at the top of your README.md - - To get an exhaustive reference of Spaces configuration, please visit https://huggingface.co/docs/hub/spaces-config-reference#spaces-configuration-reference. - - Args: - title (`str`, *optional*) - Title of the Space. - sdk (`str`, *optional*) - SDK of the Space (one of `gradio`, `streamlit`, `docker`, or `static`). - sdk_version (`str`, *optional*) - Version of the used SDK (if Gradio/Streamlit sdk). - python_version (`str`, *optional*) - Python version used in the Space (if Gradio/Streamlit sdk). - app_file (`str`, *optional*) - Path to your main application file (which contains either gradio or streamlit Python code, or static html code). - Path is relative to the root of the repository. - app_port (`str`, *optional*) - Port on which your application is running. Used only if sdk is `docker`. - license (`str`, *optional*) - License of this model. Example: apache-2.0 or any license from - https://huggingface.co/docs/hub/repositories-licenses. - duplicated_from (`str`, *optional*) - ID of the original Space if this is a duplicated Space. - models (List[`str`], *optional*) - List of models related to this Space. Should be a dataset ID found on https://hf.co/models. - datasets (`List[str]`, *optional*) - List of datasets related to this Space. Should be a dataset ID found on https://hf.co/datasets. - tags (`List[str]`, *optional*) - List of tags to add to your Space that can be used when filtering on the Hub. - ignore_metadata_errors (`str`): - If True, errors while parsing the metadata section will be ignored. Some information might be lost during - the process. Use it at your own risk. - kwargs (`dict`, *optional*): - Additional metadata that will be added to the space card. - - Example: - ```python - >>> from huggingface_hub import SpaceCardData - >>> card_data = SpaceCardData( - ... title="Dreambooth Training", - ... license="mit", - ... sdk="gradio", - ... duplicated_from="multimodalart/dreambooth-training" - ... ) - >>> card_data.to_dict() - {'title': 'Dreambooth Training', 'sdk': 'gradio', 'license': 'mit', 'duplicated_from': 'multimodalart/dreambooth-training'} - ``` - """ - - def __init__( - self, - *, - title: Optional[str] = None, - sdk: Optional[str] = None, - sdk_version: Optional[str] = None, - python_version: Optional[str] = None, - app_file: Optional[str] = None, - app_port: Optional[int] = None, - license: Optional[str] = None, - duplicated_from: Optional[str] = None, - models: Optional[List[str]] = None, - datasets: Optional[List[str]] = None, - tags: Optional[List[str]] = None, - ignore_metadata_errors: bool = False, - **kwargs, - ): - self.title = title - self.sdk = sdk - self.sdk_version = sdk_version - self.python_version = python_version - self.app_file = app_file - self.app_port = app_port - self.license = license - self.duplicated_from = duplicated_from - self.models = models - self.datasets = datasets - self.tags = _to_unique_list(tags) - super().__init__(**kwargs) - - -def model_index_to_eval_results(model_index: List[Dict[str, Any]]) -> Tuple[str, List[EvalResult]]: - """Takes in a model index and returns the model name and a list of `huggingface_hub.EvalResult` objects. - - A detailed spec of the model index can be found here: - https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 - - Args: - model_index (`List[Dict[str, Any]]`): - A model index data structure, likely coming from a README.md file on the - Hugging Face Hub. - - Returns: - model_name (`str`): - The name of the model as found in the model index. This is used as the - identifier for the model on leaderboards like PapersWithCode. - eval_results (`List[EvalResult]`): - A list of `huggingface_hub.EvalResult` objects containing the metrics - reported in the provided model_index. - - Example: - ```python - >>> from huggingface_hub.repocard_data import model_index_to_eval_results - >>> # Define a minimal model index - >>> model_index = [ - ... { - ... "name": "my-cool-model", - ... "results": [ - ... { - ... "task": { - ... "type": "image-classification" - ... }, - ... "dataset": { - ... "type": "beans", - ... "name": "Beans" - ... }, - ... "metrics": [ - ... { - ... "type": "accuracy", - ... "value": 0.9 - ... } - ... ] - ... } - ... ] - ... } - ... ] - >>> model_name, eval_results = model_index_to_eval_results(model_index) - >>> model_name - 'my-cool-model' - >>> eval_results[0].task_type - 'image-classification' - >>> eval_results[0].metric_type - 'accuracy' - - ``` - """ - - eval_results = [] - for elem in model_index: - name = elem["name"] - results = elem["results"] - for result in results: - task_type = result["task"]["type"] - task_name = result["task"].get("name") - dataset_type = result["dataset"]["type"] - dataset_name = result["dataset"]["name"] - dataset_config = result["dataset"].get("config") - dataset_split = result["dataset"].get("split") - dataset_revision = result["dataset"].get("revision") - dataset_args = result["dataset"].get("args") - source_name = result.get("source", {}).get("name") - source_url = result.get("source", {}).get("url") - - for metric in result["metrics"]: - metric_type = metric["type"] - metric_value = metric["value"] - metric_name = metric.get("name") - metric_args = metric.get("args") - metric_config = metric.get("config") - verified = metric.get("verified") - verify_token = metric.get("verifyToken") - - eval_result = EvalResult( - task_type=task_type, # Required - dataset_type=dataset_type, # Required - dataset_name=dataset_name, # Required - metric_type=metric_type, # Required - metric_value=metric_value, # Required - task_name=task_name, - dataset_config=dataset_config, - dataset_split=dataset_split, - dataset_revision=dataset_revision, - dataset_args=dataset_args, - metric_name=metric_name, - metric_args=metric_args, - metric_config=metric_config, - verified=verified, - verify_token=verify_token, - source_name=source_name, - source_url=source_url, - ) - eval_results.append(eval_result) - return name, eval_results - - -def _remove_none(obj): - """ - Recursively remove `None` values from a dict. Borrowed from: https://stackoverflow.com/a/20558778 - """ - if isinstance(obj, (list, tuple, set)): - return type(obj)(_remove_none(x) for x in obj if x is not None) - elif isinstance(obj, dict): - return type(obj)((_remove_none(k), _remove_none(v)) for k, v in obj.items() if k is not None and v is not None) - else: - return obj - - -def eval_results_to_model_index(model_name: str, eval_results: List[EvalResult]) -> List[Dict[str, Any]]: - """Takes in given model name and list of `huggingface_hub.EvalResult` and returns a - valid model-index that will be compatible with the format expected by the - Hugging Face Hub. - - Args: - model_name (`str`): - Name of the model (ex. "my-cool-model"). This is used as the identifier - for the model on leaderboards like PapersWithCode. - eval_results (`List[EvalResult]`): - List of `huggingface_hub.EvalResult` objects containing the metrics to be - reported in the model-index. - - Returns: - model_index (`List[Dict[str, Any]]`): The eval_results converted to a model-index. - - Example: - ```python - >>> from huggingface_hub.repocard_data import eval_results_to_model_index, EvalResult - >>> # Define minimal eval_results - >>> eval_results = [ - ... EvalResult( - ... task_type="image-classification", # Required - ... dataset_type="beans", # Required - ... dataset_name="Beans", # Required - ... metric_type="accuracy", # Required - ... metric_value=0.9, # Required - ... ) - ... ] - >>> eval_results_to_model_index("my-cool-model", eval_results) - [{'name': 'my-cool-model', 'results': [{'task': {'type': 'image-classification'}, 'dataset': {'name': 'Beans', 'type': 'beans'}, 'metrics': [{'type': 'accuracy', 'value': 0.9}]}]}] - - ``` - """ - - # Metrics are reported on a unique task-and-dataset basis. - # Here, we make a map of those pairs and the associated EvalResults. - task_and_ds_types_map: Dict[Any, List[EvalResult]] = defaultdict(list) - for eval_result in eval_results: - task_and_ds_types_map[eval_result.unique_identifier].append(eval_result) - - # Use the map from above to generate the model index data. - model_index_data = [] - for results in task_and_ds_types_map.values(): - # All items from `results` share same metadata - sample_result = results[0] - data = { - "task": { - "type": sample_result.task_type, - "name": sample_result.task_name, - }, - "dataset": { - "name": sample_result.dataset_name, - "type": sample_result.dataset_type, - "config": sample_result.dataset_config, - "split": sample_result.dataset_split, - "revision": sample_result.dataset_revision, - "args": sample_result.dataset_args, - }, - "metrics": [ - { - "type": result.metric_type, - "value": result.metric_value, - "name": result.metric_name, - "config": result.metric_config, - "args": result.metric_args, - "verified": result.verified, - "verifyToken": result.verify_token, - } - for result in results - ], - } - if sample_result.source_url is not None: - source = { - "url": sample_result.source_url, - } - if sample_result.source_name is not None: - source["name"] = sample_result.source_name - data["source"] = source - model_index_data.append(data) - - # TODO - Check if there cases where this list is longer than one? - # Finally, the model index itself is list of dicts. - model_index = [ - { - "name": model_name, - "results": model_index_data, - } - ] - return _remove_none(model_index) - - -def _to_unique_list(tags: Optional[List[str]]) -> Optional[List[str]]: - if tags is None: - return tags - unique_tags = [] # make tags unique + keep order explicitly - for tag in tags: - if tag not in unique_tags: - unique_tags.append(tag) - return unique_tags diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/repository.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/repository.py deleted file mode 100644 index 56e2bce619dff404476f95343fb039a0dae9fc56..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/repository.py +++ /dev/null @@ -1,1471 +0,0 @@ -import atexit -import os -import re -import subprocess -import threading -import time -from contextlib import contextmanager -from pathlib import Path -from typing import Callable, Dict, Iterator, List, Optional, Tuple, TypedDict, Union -from urllib.parse import urlparse - -from huggingface_hub import constants -from huggingface_hub.repocard import metadata_load, metadata_save - -from .hf_api import HfApi, repo_type_and_id_from_hf_id -from .lfs import LFS_MULTIPART_UPLOAD_COMMAND -from .utils import ( - SoftTemporaryDirectory, - get_token, - logging, - run_subprocess, - tqdm, - validate_hf_hub_args, -) -from .utils._deprecation import _deprecate_method - - -logger = logging.get_logger(__name__) - - -class CommandInProgress: - """ - Utility to follow commands launched asynchronously. - """ - - def __init__( - self, - title: str, - is_done_method: Callable, - status_method: Callable, - process: subprocess.Popen, - post_method: Optional[Callable] = None, - ): - self.title = title - self._is_done = is_done_method - self._status = status_method - self._process = process - self._stderr = "" - self._stdout = "" - self._post_method = post_method - - @property - def is_done(self) -> bool: - """ - Whether the process is done. - """ - result = self._is_done() - - if result and self._post_method is not None: - self._post_method() - self._post_method = None - - return result - - @property - def status(self) -> int: - """ - The exit code/status of the current action. Will return `0` if the - command has completed successfully, and a number between 1 and 255 if - the process errored-out. - - Will return -1 if the command is still ongoing. - """ - return self._status() - - @property - def failed(self) -> bool: - """ - Whether the process errored-out. - """ - return self.status > 0 - - @property - def stderr(self) -> str: - """ - The current output message on the standard error. - """ - if self._process.stderr is not None: - self._stderr += self._process.stderr.read() - return self._stderr - - @property - def stdout(self) -> str: - """ - The current output message on the standard output. - """ - if self._process.stdout is not None: - self._stdout += self._process.stdout.read() - return self._stdout - - def __repr__(self): - status = self.status - - if status == -1: - status = "running" - - return ( - f"[{self.title} command, status code: {status}," - f" {'in progress.' if not self.is_done else 'finished.'} PID:" - f" {self._process.pid}]" - ) - - -def is_git_repo(folder: Union[str, Path]) -> bool: - """ - Check if the folder is the root or part of a git repository - - Args: - folder (`str`): - The folder in which to run the command. - - Returns: - `bool`: `True` if the repository is part of a repository, `False` - otherwise. - """ - folder_exists = os.path.exists(os.path.join(folder, ".git")) - git_branch = subprocess.run("git branch".split(), cwd=folder, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - return folder_exists and git_branch.returncode == 0 - - -def is_local_clone(folder: Union[str, Path], remote_url: str) -> bool: - """ - Check if the folder is a local clone of the remote_url - - Args: - folder (`str` or `Path`): - The folder in which to run the command. - remote_url (`str`): - The url of a git repository. - - Returns: - `bool`: `True` if the repository is a local clone of the remote - repository specified, `False` otherwise. - """ - if not is_git_repo(folder): - return False - - remotes = run_subprocess("git remote -v", folder).stdout - - # Remove token for the test with remotes. - remote_url = re.sub(r"https://.*@", "https://", remote_url) - remotes = [re.sub(r"https://.*@", "https://", remote) for remote in remotes.split()] - return remote_url in remotes - - -def is_tracked_with_lfs(filename: Union[str, Path]) -> bool: - """ - Check if the file passed is tracked with git-lfs. - - Args: - filename (`str` or `Path`): - The filename to check. - - Returns: - `bool`: `True` if the file passed is tracked with git-lfs, `False` - otherwise. - """ - folder = Path(filename).parent - filename = Path(filename).name - - try: - p = run_subprocess("git check-attr -a".split() + [filename], folder) - attributes = p.stdout.strip() - except subprocess.CalledProcessError as exc: - if not is_git_repo(folder): - return False - else: - raise OSError(exc.stderr) - - if len(attributes) == 0: - return False - - found_lfs_tag = {"diff": False, "merge": False, "filter": False} - - for attribute in attributes.split("\n"): - for tag in found_lfs_tag.keys(): - if tag in attribute and "lfs" in attribute: - found_lfs_tag[tag] = True - - return all(found_lfs_tag.values()) - - -def is_git_ignored(filename: Union[str, Path]) -> bool: - """ - Check if file is git-ignored. Supports nested .gitignore files. - - Args: - filename (`str` or `Path`): - The filename to check. - - Returns: - `bool`: `True` if the file passed is ignored by `git`, `False` - otherwise. - """ - folder = Path(filename).parent - filename = Path(filename).name - - try: - p = run_subprocess("git check-ignore".split() + [filename], folder, check=False) - # Will return exit code 1 if not gitignored - is_ignored = not bool(p.returncode) - except subprocess.CalledProcessError as exc: - raise OSError(exc.stderr) - - return is_ignored - - -def is_binary_file(filename: Union[str, Path]) -> bool: - """ - Check if file is a binary file. - - Args: - filename (`str` or `Path`): - The filename to check. - - Returns: - `bool`: `True` if the file passed is a binary file, `False` otherwise. - """ - try: - with open(filename, "rb") as f: - content = f.read(10 * (1024**2)) # Read a maximum of 10MB - - # Code sample taken from the following stack overflow thread - # https://stackoverflow.com/questions/898669/how-can-i-detect-if-a-file-is-binary-non-text-in-python/7392391#7392391 - text_chars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7F}) - return bool(content.translate(None, text_chars)) - except UnicodeDecodeError: - return True - - -def files_to_be_staged(pattern: str = ".", folder: Union[str, Path, None] = None) -> List[str]: - """ - Returns a list of filenames that are to be staged. - - Args: - pattern (`str` or `Path`): - The pattern of filenames to check. Put `.` to get all files. - folder (`str` or `Path`): - The folder in which to run the command. - - Returns: - `List[str]`: List of files that are to be staged. - """ - try: - p = run_subprocess("git ls-files --exclude-standard -mo".split() + [pattern], folder) - if len(p.stdout.strip()): - files = p.stdout.strip().split("\n") - else: - files = [] - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - return files - - -def is_tracked_upstream(folder: Union[str, Path]) -> bool: - """ - Check if the current checked-out branch is tracked upstream. - - Args: - folder (`str` or `Path`): - The folder in which to run the command. - - Returns: - `bool`: `True` if the current checked-out branch is tracked upstream, - `False` otherwise. - """ - try: - run_subprocess("git rev-parse --symbolic-full-name --abbrev-ref @{u}", folder) - return True - except subprocess.CalledProcessError as exc: - if "HEAD" in exc.stderr: - raise OSError("No branch checked out") - - return False - - -def commits_to_push(folder: Union[str, Path], upstream: Optional[str] = None) -> int: - """ - Check the number of commits that would be pushed upstream - - Args: - folder (`str` or `Path`): - The folder in which to run the command. - upstream (`str`, *optional*): - The name of the upstream repository with which the comparison should be - made. - - Returns: - `int`: Number of commits that would be pushed upstream were a `git - push` to proceed. - """ - try: - result = run_subprocess(f"git cherry -v {upstream or ''}", folder) - return len(result.stdout.split("\n")) - 1 - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - -class PbarT(TypedDict): - # Used to store an opened progress bar in `_lfs_log_progress` - bar: tqdm - past_bytes: int - - -@contextmanager -def _lfs_log_progress(): - """ - This is a context manager that will log the Git LFS progress of cleaning, - smudging, pulling and pushing. - """ - - if logger.getEffectiveLevel() >= logging.ERROR: - try: - yield - except Exception: - pass - return - - def output_progress(stopping_event: threading.Event): - """ - To be launched as a separate thread with an event meaning it should stop - the tail. - """ - # Key is tuple(state, filename), value is a dict(tqdm bar and a previous value) - pbars: Dict[Tuple[str, str], PbarT] = {} - - def close_pbars(): - for pbar in pbars.values(): - pbar["bar"].update(pbar["bar"].total - pbar["past_bytes"]) - pbar["bar"].refresh() - pbar["bar"].close() - - def tail_file(filename) -> Iterator[str]: - """ - Creates a generator to be iterated through, which will return each - line one by one. Will stop tailing the file if the stopping_event is - set. - """ - with open(filename, "r") as file: - current_line = "" - while True: - if stopping_event.is_set(): - close_pbars() - break - - line_bit = file.readline() - if line_bit is not None and not len(line_bit.strip()) == 0: - current_line += line_bit - if current_line.endswith("\n"): - yield current_line - current_line = "" - else: - time.sleep(1) - - # If the file isn't created yet, wait for a few seconds before trying again. - # Can be interrupted with the stopping_event. - while not os.path.exists(os.environ["GIT_LFS_PROGRESS"]): - if stopping_event.is_set(): - close_pbars() - return - - time.sleep(2) - - for line in tail_file(os.environ["GIT_LFS_PROGRESS"]): - try: - state, file_progress, byte_progress, filename = line.split() - except ValueError as error: - # Try/except to ease debugging. See https://github.com/huggingface/huggingface_hub/issues/1373. - raise ValueError(f"Cannot unpack LFS progress line:\n{line}") from error - description = f"{state.capitalize()} file {filename}" - - current_bytes, total_bytes = byte_progress.split("/") - current_bytes_int = int(current_bytes) - total_bytes_int = int(total_bytes) - - pbar = pbars.get((state, filename)) - if pbar is None: - # Initialize progress bar - pbars[(state, filename)] = { - "bar": tqdm( - desc=description, - initial=current_bytes_int, - total=total_bytes_int, - unit="B", - unit_scale=True, - unit_divisor=1024, - name="huggingface_hub.lfs_upload", - ), - "past_bytes": int(current_bytes), - } - else: - # Update progress bar - pbar["bar"].update(current_bytes_int - pbar["past_bytes"]) - pbar["past_bytes"] = current_bytes_int - - current_lfs_progress_value = os.environ.get("GIT_LFS_PROGRESS", "") - - with SoftTemporaryDirectory() as tmpdir: - os.environ["GIT_LFS_PROGRESS"] = os.path.join(tmpdir, "lfs_progress") - logger.debug(f"Following progress in {os.environ['GIT_LFS_PROGRESS']}") - - exit_event = threading.Event() - x = threading.Thread(target=output_progress, args=(exit_event,), daemon=True) - x.start() - - try: - yield - finally: - exit_event.set() - x.join() - - os.environ["GIT_LFS_PROGRESS"] = current_lfs_progress_value - - -class Repository: - """ - Helper class to wrap the git and git-lfs commands. - - The aim is to facilitate interacting with huggingface.co hosted model or - dataset repos, though not a lot here (if any) is actually specific to - huggingface.co. - - > [!WARNING] - > [`Repository`] is deprecated in favor of the http-based alternatives implemented in - > [`HfApi`]. Given its large adoption in legacy code, the complete removal of - > [`Repository`] will only happen in release `v1.0`. For more details, please read - > https://huggingface.co/docs/huggingface_hub/concepts/git_vs_http. - """ - - command_queue: List[CommandInProgress] - - @validate_hf_hub_args - @_deprecate_method( - version="1.0", - message=( - "Please prefer the http-based alternatives instead. Given its large adoption in legacy code, the complete" - " removal is only planned on next major release.\nFor more details, please read" - " https://huggingface.co/docs/huggingface_hub/concepts/git_vs_http." - ), - ) - def __init__( - self, - local_dir: Union[str, Path], - clone_from: Optional[str] = None, - repo_type: Optional[str] = None, - token: Union[bool, str] = True, - git_user: Optional[str] = None, - git_email: Optional[str] = None, - revision: Optional[str] = None, - skip_lfs_files: bool = False, - client: Optional[HfApi] = None, - ): - """ - Instantiate a local clone of a git repo. - - If `clone_from` is set, the repo will be cloned from an existing remote repository. - If the remote repo does not exist, a `EnvironmentError` exception will be thrown. - Please create the remote repo first using [`create_repo`]. - - `Repository` uses the local git credentials by default. If explicitly set, the `token` - or the `git_user`/`git_email` pair will be used instead. - - Args: - local_dir (`str` or `Path`): - path (e.g. `'my_trained_model/'`) to the local directory, where - the `Repository` will be initialized. - clone_from (`str`, *optional*): - Either a repository url or `repo_id`. - Example: - - `"https://huggingface.co/philschmid/playground-tests"` - - `"philschmid/playground-tests"` - repo_type (`str`, *optional*): - To set when cloning a repo from a repo_id. Default is model. - token (`bool` or `str`, *optional*): - A valid authentication token (see https://huggingface.co/settings/token). - If `None` or `True` and machine is logged in (through `hf auth login` - or [`~huggingface_hub.login`]), token will be retrieved from the cache. - If `False`, token is not sent in the request header. - git_user (`str`, *optional*): - will override the `git config user.name` for committing and - pushing files to the hub. - git_email (`str`, *optional*): - will override the `git config user.email` for committing and - pushing files to the hub. - revision (`str`, *optional*): - Revision to checkout after initializing the repository. If the - revision doesn't exist, a branch will be created with that - revision name from the default branch's current HEAD. - skip_lfs_files (`bool`, *optional*, defaults to `False`): - whether to skip git-LFS files or not. - client (`HfApi`, *optional*): - Instance of [`HfApi`] to use when calling the HF Hub API. A new - instance will be created if this is left to `None`. - - Raises: - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) - If the remote repository set in `clone_from` does not exist. - """ - if isinstance(local_dir, Path): - local_dir = str(local_dir) - os.makedirs(local_dir, exist_ok=True) - self.local_dir = os.path.join(os.getcwd(), local_dir) - self._repo_type = repo_type - self.command_queue = [] - self.skip_lfs_files = skip_lfs_files - self.client = client if client is not None else HfApi() - - self.check_git_versions() - - if isinstance(token, str): - self.huggingface_token: Optional[str] = token - elif token is False: - self.huggingface_token = None - else: - # if `True` -> explicit use of the cached token - # if `None` -> implicit use of the cached token - self.huggingface_token = get_token() - - if clone_from is not None: - self.clone_from(repo_url=clone_from) - else: - if is_git_repo(self.local_dir): - logger.debug("[Repository] is a valid git repo") - else: - raise ValueError("If not specifying `clone_from`, you need to pass Repository a valid git clone.") - - if self.huggingface_token is not None and (git_email is None or git_user is None): - user = self.client.whoami(self.huggingface_token) - - if git_email is None: - git_email = user.get("email") - - if git_user is None: - git_user = user.get("fullname") - - if git_user is not None or git_email is not None: - self.git_config_username_and_email(git_user, git_email) - - self.lfs_enable_largefiles() - self.git_credential_helper_store() - - if revision is not None: - self.git_checkout(revision, create_branch_ok=True) - - # This ensures that all commands exit before exiting the Python runtime. - # This will ensure all pushes register on the hub, even if other errors happen in subsequent operations. - atexit.register(self.wait_for_commands) - - @property - def current_branch(self) -> str: - """ - Returns the current checked out branch. - - Returns: - `str`: Current checked out branch. - """ - try: - result = run_subprocess("git rev-parse --abbrev-ref HEAD", self.local_dir).stdout.strip() - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - return result - - def check_git_versions(self): - """ - Checks that `git` and `git-lfs` can be run. - - Raises: - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) - If `git` or `git-lfs` are not installed. - """ - try: - git_version = run_subprocess("git --version", self.local_dir).stdout.strip() - except FileNotFoundError: - raise EnvironmentError("Looks like you do not have git installed, please install.") - - try: - lfs_version = run_subprocess("git-lfs --version", self.local_dir).stdout.strip() - except FileNotFoundError: - raise EnvironmentError( - "Looks like you do not have git-lfs installed, please install." - " You can install from https://git-lfs.github.com/." - " Then run `git lfs install` (you only have to do this once)." - ) - logger.info(git_version + "\n" + lfs_version) - - @validate_hf_hub_args - def clone_from(self, repo_url: str, token: Union[bool, str, None] = None): - """ - Clone from a remote. If the folder already exists, will try to clone the - repository within it. - - If this folder is a git repository with linked history, will try to - update the repository. - - Args: - repo_url (`str`): - The URL from which to clone the repository - token (`Union[str, bool]`, *optional*): - Whether to use the authentication token. It can be: - - a string which is the token itself - - `False`, which would not use the authentication token - - `True`, which would fetch the authentication token from the - local folder and use it (you should be logged in for this to - work). - - `None`, which would retrieve the value of - `self.huggingface_token`. - - > [!TIP] - > Raises the following error: - > - > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > if an organization token (starts with "api_org") is passed. Use must use - > your own personal access token (see https://hf.co/settings/tokens). - > - > - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) - > if you are trying to clone the repository in a non-empty folder, or if the - > `git` operations raise errors. - """ - token = ( - token # str -> use it - if isinstance(token, str) - else ( - None # `False` -> explicit no token - if token is False - else self.huggingface_token # `None` or `True` -> use default - ) - ) - if token is not None and token.startswith("api_org"): - raise ValueError( - "You must use your personal access token, not an Organization token" - " (see https://hf.co/settings/tokens)." - ) - - hub_url = self.client.endpoint - if hub_url in repo_url or ("http" not in repo_url and len(repo_url.split("/")) <= 2): - repo_type, namespace, repo_name = repo_type_and_id_from_hf_id(repo_url, hub_url=hub_url) - repo_id = f"{namespace}/{repo_name}" if namespace is not None else repo_name - - if repo_type is not None: - self._repo_type = repo_type - - repo_url = hub_url + "/" - - if self._repo_type in constants.REPO_TYPES_URL_PREFIXES: - repo_url += constants.REPO_TYPES_URL_PREFIXES[self._repo_type] - - if token is not None: - # Add token in git url when provided - scheme = urlparse(repo_url).scheme - repo_url = repo_url.replace(f"{scheme}://", f"{scheme}://user:{token}@") - - repo_url += repo_id - - # For error messages, it's cleaner to show the repo url without the token. - clean_repo_url = re.sub(r"(https?)://.*@", r"\1://", repo_url) - try: - run_subprocess("git lfs install", self.local_dir) - - # checks if repository is initialized in a empty repository or in one with files - if len(os.listdir(self.local_dir)) == 0: - logger.warning(f"Cloning {clean_repo_url} into local empty directory.") - - with _lfs_log_progress(): - env = os.environ.copy() - - if self.skip_lfs_files: - env.update({"GIT_LFS_SKIP_SMUDGE": "1"}) - - run_subprocess( - # 'git lfs clone' is deprecated (will display a warning in the terminal) - # but we still use it as it provides a nicer UX when downloading large - # files (shows progress). - f"{'git clone' if self.skip_lfs_files else 'git lfs clone'} {repo_url} .", - self.local_dir, - env=env, - ) - else: - # Check if the folder is the root of a git repository - if not is_git_repo(self.local_dir): - raise EnvironmentError( - "Tried to clone a repository in a non-empty folder that isn't" - f" a git repository ('{self.local_dir}'). If you really want to" - f" do this, do it manually:\n cd {self.local_dir} && git init" - " && git remote add origin && git pull origin main\n or clone" - " repo to a new folder and move your existing files there" - " afterwards." - ) - - if is_local_clone(self.local_dir, repo_url): - logger.warning( - f"{self.local_dir} is already a clone of {clean_repo_url}." - " Make sure you pull the latest changes with" - " `repo.git_pull()`." - ) - else: - output = run_subprocess("git remote get-url origin", self.local_dir, check=False) - - error_msg = ( - f"Tried to clone {clean_repo_url} in an unrelated git" - " repository.\nIf you believe this is an error, please add" - f" a remote with the following URL: {clean_repo_url}." - ) - if output.returncode == 0: - clean_local_remote_url = re.sub(r"https://.*@", "https://", output.stdout) - error_msg += f"\nLocal path has its origin defined as: {clean_local_remote_url}" - raise EnvironmentError(error_msg) - - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def git_config_username_and_email(self, git_user: Optional[str] = None, git_email: Optional[str] = None): - """ - Sets git username and email (only in the current repo). - - Args: - git_user (`str`, *optional*): - The username to register through `git`. - git_email (`str`, *optional*): - The email to register through `git`. - """ - try: - if git_user is not None: - run_subprocess("git config user.name".split() + [git_user], self.local_dir) - - if git_email is not None: - run_subprocess(f"git config user.email {git_email}".split(), self.local_dir) - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def git_credential_helper_store(self): - """ - Sets the git credential helper to `store` - """ - try: - run_subprocess("git config credential.helper store", self.local_dir) - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def git_head_hash(self) -> str: - """ - Get commit sha on top of HEAD. - - Returns: - `str`: The current checked out commit SHA. - """ - try: - p = run_subprocess("git rev-parse HEAD", self.local_dir) - return p.stdout.strip() - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def git_remote_url(self) -> str: - """ - Get URL to origin remote. - - Returns: - `str`: The URL of the `origin` remote. - """ - try: - p = run_subprocess("git config --get remote.origin.url", self.local_dir) - url = p.stdout.strip() - # Strip basic auth info. - return re.sub(r"https://.*@", "https://", url) - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def git_head_commit_url(self) -> str: - """ - Get URL to last commit on HEAD. We assume it's been pushed, and the url - scheme is the same one as for GitHub or HuggingFace. - - Returns: - `str`: The URL to the current checked-out commit. - """ - sha = self.git_head_hash() - url = self.git_remote_url() - if url.endswith("/"): - url = url[:-1] - return f"{url}/commit/{sha}" - - def list_deleted_files(self) -> List[str]: - """ - Returns a list of the files that are deleted in the working directory or - index. - - Returns: - `List[str]`: A list of files that have been deleted in the working - directory or index. - """ - try: - git_status = run_subprocess("git status -s", self.local_dir).stdout.strip() - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - if len(git_status) == 0: - return [] - - # Receives a status like the following - # D .gitignore - # D new_file.json - # AD new_file1.json - # ?? new_file2.json - # ?? new_file4.json - - # Strip each line of whitespaces - modified_files_statuses = [status.strip() for status in git_status.split("\n")] - - # Only keep files that are deleted using the D prefix - deleted_files_statuses = [status for status in modified_files_statuses if "D" in status.split()[0]] - - # Remove the D prefix and strip to keep only the relevant filename - deleted_files = [status.split()[-1].strip() for status in deleted_files_statuses] - - return deleted_files - - def lfs_track(self, patterns: Union[str, List[str]], filename: bool = False): - """ - Tell git-lfs to track files according to a pattern. - - Setting the `filename` argument to `True` will treat the arguments as - literal filenames, not as patterns. Any special glob characters in the - filename will be escaped when writing to the `.gitattributes` file. - - Args: - patterns (`Union[str, List[str]]`): - The pattern, or list of patterns, to track with git-lfs. - filename (`bool`, *optional*, defaults to `False`): - Whether to use the patterns as literal filenames. - """ - if isinstance(patterns, str): - patterns = [patterns] - try: - for pattern in patterns: - run_subprocess( - f"git lfs track {'--filename' if filename else ''} {pattern}", - self.local_dir, - ) - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def lfs_untrack(self, patterns: Union[str, List[str]]): - """ - Tell git-lfs to untrack those files. - - Args: - patterns (`Union[str, List[str]]`): - The pattern, or list of patterns, to untrack with git-lfs. - """ - if isinstance(patterns, str): - patterns = [patterns] - try: - for pattern in patterns: - run_subprocess("git lfs untrack".split() + [pattern], self.local_dir) - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def lfs_enable_largefiles(self): - """ - HF-specific. This enables upload support of files >5GB. - """ - try: - lfs_config = "git config lfs.customtransfer.multipart" - run_subprocess(f"{lfs_config}.path hf", self.local_dir) - run_subprocess( - f"{lfs_config}.args {LFS_MULTIPART_UPLOAD_COMMAND}", - self.local_dir, - ) - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def auto_track_binary_files(self, pattern: str = ".") -> List[str]: - """ - Automatically track binary files with git-lfs. - - Args: - pattern (`str`, *optional*, defaults to "."): - The pattern with which to track files that are binary. - - Returns: - `List[str]`: List of filenames that are now tracked due to being - binary files - """ - files_to_be_tracked_with_lfs = [] - - deleted_files = self.list_deleted_files() - - for filename in files_to_be_staged(pattern, folder=self.local_dir): - if filename in deleted_files: - continue - - path_to_file = os.path.join(os.getcwd(), self.local_dir, filename) - - if not (is_tracked_with_lfs(path_to_file) or is_git_ignored(path_to_file)): - size_in_mb = os.path.getsize(path_to_file) / (1024 * 1024) - - if size_in_mb >= 10: - logger.warning( - "Parsing a large file to check if binary or not. Tracking large" - " files using `repository.auto_track_large_files` is" - " recommended so as to not load the full file in memory." - ) - - is_binary = is_binary_file(path_to_file) - - if is_binary: - self.lfs_track(filename) - files_to_be_tracked_with_lfs.append(filename) - - # Cleanup the .gitattributes if files were deleted - self.lfs_untrack(deleted_files) - - return files_to_be_tracked_with_lfs - - def auto_track_large_files(self, pattern: str = ".") -> List[str]: - """ - Automatically track large files (files that weigh more than 10MBs) with - git-lfs. - - Args: - pattern (`str`, *optional*, defaults to "."): - The pattern with which to track files that are above 10MBs. - - Returns: - `List[str]`: List of filenames that are now tracked due to their - size. - """ - files_to_be_tracked_with_lfs = [] - - deleted_files = self.list_deleted_files() - - for filename in files_to_be_staged(pattern, folder=self.local_dir): - if filename in deleted_files: - continue - - path_to_file = os.path.join(os.getcwd(), self.local_dir, filename) - size_in_mb = os.path.getsize(path_to_file) / (1024 * 1024) - - if size_in_mb >= 10 and not is_tracked_with_lfs(path_to_file) and not is_git_ignored(path_to_file): - self.lfs_track(filename) - files_to_be_tracked_with_lfs.append(filename) - - # Cleanup the .gitattributes if files were deleted - self.lfs_untrack(deleted_files) - - return files_to_be_tracked_with_lfs - - def lfs_prune(self, recent=False): - """ - git lfs prune - - Args: - recent (`bool`, *optional*, defaults to `False`): - Whether to prune files even if they were referenced by recent - commits. See the following - [link](https://github.com/git-lfs/git-lfs/blob/f3d43f0428a84fc4f1e5405b76b5a73ec2437e65/docs/man/git-lfs-prune.1.ronn#recent-files) - for more information. - """ - try: - with _lfs_log_progress(): - result = run_subprocess(f"git lfs prune {'--recent' if recent else ''}", self.local_dir) - logger.info(result.stdout) - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def git_pull(self, rebase: bool = False, lfs: bool = False): - """ - git pull - - Args: - rebase (`bool`, *optional*, defaults to `False`): - Whether to rebase the current branch on top of the upstream - branch after fetching. - lfs (`bool`, *optional*, defaults to `False`): - Whether to fetch the LFS files too. This option only changes the - behavior when a repository was cloned without fetching the LFS - files; calling `repo.git_pull(lfs=True)` will then fetch the LFS - file from the remote repository. - """ - command = "git pull" if not lfs else "git lfs pull" - if rebase: - command += " --rebase" - try: - with _lfs_log_progress(): - result = run_subprocess(command, self.local_dir) - logger.info(result.stdout) - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def git_add(self, pattern: str = ".", auto_lfs_track: bool = False): - """ - git add - - Setting the `auto_lfs_track` parameter to `True` will automatically - track files that are larger than 10MB with `git-lfs`. - - Args: - pattern (`str`, *optional*, defaults to "."): - The pattern with which to add files to staging. - auto_lfs_track (`bool`, *optional*, defaults to `False`): - Whether to automatically track large and binary files with - git-lfs. Any file over 10MB in size, or in binary format, will - be automatically tracked. - """ - if auto_lfs_track: - # Track files according to their size (>=10MB) - tracked_files = self.auto_track_large_files(pattern) - - # Read the remaining files and track them if they're binary - tracked_files.extend(self.auto_track_binary_files(pattern)) - - if tracked_files: - logger.warning( - f"Adding files tracked by Git LFS: {tracked_files}. This may take a" - " bit of time if the files are large." - ) - - try: - result = run_subprocess("git add -v".split() + [pattern], self.local_dir) - logger.info(f"Adding to index:\n{result.stdout}\n") - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def git_commit(self, commit_message: str = "commit files to HF hub"): - """ - git commit - - Args: - commit_message (`str`, *optional*, defaults to "commit files to HF hub"): - The message attributed to the commit. - """ - try: - result = run_subprocess("git commit -v -m".split() + [commit_message], self.local_dir) - logger.info(f"Committed:\n{result.stdout}\n") - except subprocess.CalledProcessError as exc: - if len(exc.stderr) > 0: - raise EnvironmentError(exc.stderr) - else: - raise EnvironmentError(exc.stdout) - - def git_push( - self, - upstream: Optional[str] = None, - blocking: bool = True, - auto_lfs_prune: bool = False, - ) -> Union[str, Tuple[str, CommandInProgress]]: - """ - git push - - If used without setting `blocking`, will return url to commit on remote - repo. If used with `blocking=True`, will return a tuple containing the - url to commit and the command object to follow for information about the - process. - - Args: - upstream (`str`, *optional*): - Upstream to which this should push. If not specified, will push - to the lastly defined upstream or to the default one (`origin - main`). - blocking (`bool`, *optional*, defaults to `True`): - Whether the function should return only when the push has - finished. Setting this to `False` will return an - `CommandInProgress` object which has an `is_done` property. This - property will be set to `True` when the push is finished. - auto_lfs_prune (`bool`, *optional*, defaults to `False`): - Whether to automatically prune files once they have been pushed - to the remote. - """ - command = "git push" - - if upstream: - command += f" --set-upstream {upstream}" - - number_of_commits = commits_to_push(self.local_dir, upstream) - - if number_of_commits > 1: - logger.warning(f"Several commits ({number_of_commits}) will be pushed upstream.") - if blocking: - logger.warning("The progress bars may be unreliable.") - - try: - with _lfs_log_progress(): - process = subprocess.Popen( - command.split(), - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - encoding="utf-8", - cwd=self.local_dir, - ) - - if blocking: - stdout, stderr = process.communicate() - return_code = process.poll() - process.kill() - - if len(stderr): - logger.warning(stderr) - - if return_code: - raise subprocess.CalledProcessError(return_code, process.args, output=stdout, stderr=stderr) - - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - if not blocking: - - def status_method(): - status = process.poll() - if status is None: - return -1 - else: - return status - - command_in_progress = CommandInProgress( - "push", - is_done_method=lambda: process.poll() is not None, - status_method=status_method, - process=process, - post_method=self.lfs_prune if auto_lfs_prune else None, - ) - - self.command_queue.append(command_in_progress) - - return self.git_head_commit_url(), command_in_progress - - if auto_lfs_prune: - self.lfs_prune() - - return self.git_head_commit_url() - - def git_checkout(self, revision: str, create_branch_ok: bool = False): - """ - git checkout a given revision - - Specifying `create_branch_ok` to `True` will create the branch to the - given revision if that revision doesn't exist. - - Args: - revision (`str`): - The revision to checkout. - create_branch_ok (`str`, *optional*, defaults to `False`): - Whether creating a branch named with the `revision` passed at - the current checked-out reference if `revision` isn't an - existing revision is allowed. - """ - try: - result = run_subprocess(f"git checkout {revision}", self.local_dir) - logger.warning(f"Checked out {revision} from {self.current_branch}.") - logger.warning(result.stdout) - except subprocess.CalledProcessError as exc: - if not create_branch_ok: - raise EnvironmentError(exc.stderr) - else: - try: - result = run_subprocess(f"git checkout -b {revision}", self.local_dir) - logger.warning( - f"Revision `{revision}` does not exist. Created and checked out branch `{revision}`." - ) - logger.warning(result.stdout) - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def tag_exists(self, tag_name: str, remote: Optional[str] = None) -> bool: - """ - Check if a tag exists or not. - - Args: - tag_name (`str`): - The name of the tag to check. - remote (`str`, *optional*): - Whether to check if the tag exists on a remote. This parameter - should be the identifier of the remote. - - Returns: - `bool`: Whether the tag exists. - """ - if remote: - try: - result = run_subprocess(f"git ls-remote origin refs/tags/{tag_name}", self.local_dir).stdout.strip() - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - return len(result) != 0 - else: - try: - git_tags = run_subprocess("git tag", self.local_dir).stdout.strip() - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - git_tags = git_tags.split("\n") - return tag_name in git_tags - - def delete_tag(self, tag_name: str, remote: Optional[str] = None) -> bool: - """ - Delete a tag, both local and remote, if it exists - - Args: - tag_name (`str`): - The tag name to delete. - remote (`str`, *optional*): - The remote on which to delete the tag. - - Returns: - `bool`: `True` if deleted, `False` if the tag didn't exist. - If remote is not passed, will just be updated locally - """ - delete_locally = True - delete_remotely = True - - if not self.tag_exists(tag_name): - delete_locally = False - - if not self.tag_exists(tag_name, remote=remote): - delete_remotely = False - - if delete_locally: - try: - run_subprocess(["git", "tag", "-d", tag_name], self.local_dir).stdout.strip() - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - if remote and delete_remotely: - try: - run_subprocess(f"git push {remote} --delete {tag_name}", self.local_dir).stdout.strip() - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - return True - - def add_tag(self, tag_name: str, message: Optional[str] = None, remote: Optional[str] = None): - """ - Add a tag at the current head and push it - - If remote is None, will just be updated locally - - If no message is provided, the tag will be lightweight. if a message is - provided, the tag will be annotated. - - Args: - tag_name (`str`): - The name of the tag to be added. - message (`str`, *optional*): - The message that accompanies the tag. The tag will turn into an - annotated tag if a message is passed. - remote (`str`, *optional*): - The remote on which to add the tag. - """ - if message: - tag_args = ["git", "tag", "-a", tag_name, "-m", message] - else: - tag_args = ["git", "tag", tag_name] - - try: - run_subprocess(tag_args, self.local_dir).stdout.strip() - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - if remote: - try: - run_subprocess(f"git push {remote} {tag_name}", self.local_dir).stdout.strip() - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - def is_repo_clean(self) -> bool: - """ - Return whether or not the git status is clean or not - - Returns: - `bool`: `True` if the git status is clean, `False` otherwise. - """ - try: - git_status = run_subprocess("git status --porcelain", self.local_dir).stdout.strip() - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - return len(git_status) == 0 - - def push_to_hub( - self, - commit_message: str = "commit files to HF hub", - blocking: bool = True, - clean_ok: bool = True, - auto_lfs_prune: bool = False, - ) -> Union[None, str, Tuple[str, CommandInProgress]]: - """ - Helper to add, commit, and push files to remote repository on the - HuggingFace Hub. Will automatically track large files (>10MB). - - Args: - commit_message (`str`): - Message to use for the commit. - blocking (`bool`, *optional*, defaults to `True`): - Whether the function should return only when the `git push` has - finished. - clean_ok (`bool`, *optional*, defaults to `True`): - If True, this function will return None if the repo is - untouched. Default behavior is to fail because the git command - fails. - auto_lfs_prune (`bool`, *optional*, defaults to `False`): - Whether to automatically prune files once they have been pushed - to the remote. - """ - if clean_ok and self.is_repo_clean(): - logger.info("Repo currently clean. Ignoring push_to_hub") - return None - self.git_add(auto_lfs_track=True) - self.git_commit(commit_message) - return self.git_push( - upstream=f"origin {self.current_branch}", - blocking=blocking, - auto_lfs_prune=auto_lfs_prune, - ) - - @contextmanager - def commit( - self, - commit_message: str, - branch: Optional[str] = None, - track_large_files: bool = True, - blocking: bool = True, - auto_lfs_prune: bool = False, - ): - """ - Context manager utility to handle committing to a repository. This - automatically tracks large files (>10Mb) with git-lfs. Set the - `track_large_files` argument to `False` if you wish to ignore that - behavior. - - Args: - commit_message (`str`): - Message to use for the commit. - branch (`str`, *optional*): - The branch on which the commit will appear. This branch will be - checked-out before any operation. - track_large_files (`bool`, *optional*, defaults to `True`): - Whether to automatically track large files or not. Will do so by - default. - blocking (`bool`, *optional*, defaults to `True`): - Whether the function should return only when the `git push` has - finished. - auto_lfs_prune (`bool`, defaults to `True`): - Whether to automatically prune files once they have been pushed - to the remote. - - Examples: - - ```python - >>> with Repository( - ... "text-files", - ... clone_from="/text-files", - ... token=True, - >>> ).commit("My first file :)"): - ... with open("file.txt", "w+") as f: - ... f.write(json.dumps({"hey": 8})) - - >>> import torch - - >>> model = torch.nn.Transformer() - >>> with Repository( - ... "torch-model", - ... clone_from="/torch-model", - ... token=True, - >>> ).commit("My cool model :)"): - ... torch.save(model.state_dict(), "model.pt") - ``` - - """ - - files_to_stage = files_to_be_staged(".", folder=self.local_dir) - - if len(files_to_stage): - files_in_msg = str(files_to_stage[:5])[:-1] + ", ...]" if len(files_to_stage) > 5 else str(files_to_stage) - logger.error( - "There exists some updated files in the local repository that are not" - f" committed: {files_in_msg}. This may lead to errors if checking out" - " a branch. These files and their modifications will be added to the" - " current commit." - ) - - if branch is not None: - self.git_checkout(branch, create_branch_ok=True) - - if is_tracked_upstream(self.local_dir): - logger.warning("Pulling changes ...") - self.git_pull(rebase=True) - else: - logger.warning(f"The current branch has no upstream branch. Will push to 'origin {self.current_branch}'") - - current_working_directory = os.getcwd() - os.chdir(os.path.join(current_working_directory, self.local_dir)) - - try: - yield self - finally: - self.git_add(auto_lfs_track=track_large_files) - - try: - self.git_commit(commit_message) - except OSError as e: - # If no changes are detected, there is nothing to commit. - if "nothing to commit" not in str(e): - raise e - - try: - self.git_push( - upstream=f"origin {self.current_branch}", - blocking=blocking, - auto_lfs_prune=auto_lfs_prune, - ) - except OSError as e: - # If no changes are detected, there is nothing to commit. - if "could not read Username" in str(e): - raise OSError("Couldn't authenticate user for push. Did you set `token` to `True`?") from e - else: - raise e - - os.chdir(current_working_directory) - - def repocard_metadata_load(self) -> Optional[Dict]: - filepath = os.path.join(self.local_dir, constants.REPOCARD_NAME) - if os.path.isfile(filepath): - return metadata_load(filepath) - return None - - def repocard_metadata_save(self, data: Dict) -> None: - return metadata_save(os.path.join(self.local_dir, constants.REPOCARD_NAME), data) - - @property - def commands_failed(self): - """ - Returns the asynchronous commands that failed. - """ - return [c for c in self.command_queue if c.status > 0] - - @property - def commands_in_progress(self): - """ - Returns the asynchronous commands that are currently in progress. - """ - return [c for c in self.command_queue if not c.is_done] - - def wait_for_commands(self): - """ - Blocking method: blocks all subsequent execution until all commands have - been processed. - """ - index = 0 - for command_failed in self.commands_failed: - logger.error(f"The {command_failed.title} command with PID {command_failed._process.pid} failed.") - logger.error(command_failed.stderr) - - while self.commands_in_progress: - if index % 10 == 0: - logger.warning( - f"Waiting for the following commands to finish before shutting down: {self.commands_in_progress}." - ) - - index += 1 - - time.sleep(1) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/__init__.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/__init__.py deleted file mode 100644 index 8949a22a5f65ab29b7df65aa6a9df9bce0544b7e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ruff: noqa: F401 -"""Contains helpers to serialize tensors.""" - -from ._base import StateDictSplit, split_state_dict_into_shards_factory -from ._tensorflow import get_tf_storage_size, split_tf_state_dict_into_shards -from ._torch import ( - get_torch_storage_id, - get_torch_storage_size, - load_state_dict_from_file, - load_torch_model, - save_torch_model, - save_torch_state_dict, - split_torch_state_dict_into_shards, -) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_base.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_base.py deleted file mode 100644 index b79c82f5dba58d252b5c3a7345f0df09794b55ce..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_base.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright 2024 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains helpers to split tensors into shards.""" - -from dataclasses import dataclass, field -from typing import Any, Callable, Dict, List, Optional, TypeVar, Union - -from .. import logging - - -TensorT = TypeVar("TensorT") -TensorSizeFn_T = Callable[[TensorT], int] -StorageIDFn_T = Callable[[TensorT], Optional[Any]] - -MAX_SHARD_SIZE = "5GB" -SIZE_UNITS = { - "TB": 10**12, - "GB": 10**9, - "MB": 10**6, - "KB": 10**3, -} - - -logger = logging.get_logger(__file__) - - -@dataclass -class StateDictSplit: - is_sharded: bool = field(init=False) - metadata: Dict[str, Any] - filename_to_tensors: Dict[str, List[str]] - tensor_to_filename: Dict[str, str] - - def __post_init__(self): - self.is_sharded = len(self.filename_to_tensors) > 1 - - -def split_state_dict_into_shards_factory( - state_dict: Dict[str, TensorT], - *, - get_storage_size: TensorSizeFn_T, - filename_pattern: str, - get_storage_id: StorageIDFn_T = lambda tensor: None, - max_shard_size: Union[int, str] = MAX_SHARD_SIZE, -) -> StateDictSplit: - """ - Split a model state dictionary in shards so that each shard is smaller than a given size. - - The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization - made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we - have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not - [6+2+2GB], [6+2GB], [6GB]. - - > [!WARNING] - > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a - > size greater than `max_shard_size`. - - Args: - state_dict (`Dict[str, Tensor]`): - The state dictionary to save. - get_storage_size (`Callable[[Tensor], int]`): - A function that returns the size of a tensor when saved on disk in bytes. - get_storage_id (`Callable[[Tensor], Optional[Any]]`, *optional*): - A function that returns a unique identifier to a tensor storage. Multiple different tensors can share the - same underlying storage. This identifier is guaranteed to be unique and constant for this tensor's storage - during its lifetime. Two tensor storages with non-overlapping lifetimes may have the same id. - filename_pattern (`str`, *optional*): - The pattern to generate the files names in which the model will be saved. Pattern must be a string that - can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` - max_shard_size (`int` or `str`, *optional*): - The maximum size of each shard, in bytes. Defaults to 5GB. - - Returns: - [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them. - """ - storage_id_to_tensors: Dict[Any, List[str]] = {} - - shard_list: List[Dict[str, TensorT]] = [] - current_shard: Dict[str, TensorT] = {} - current_shard_size = 0 - total_size = 0 - - if isinstance(max_shard_size, str): - max_shard_size = parse_size_to_int(max_shard_size) - - for key, tensor in state_dict.items(): - # when bnb serialization is used the weights in the state dict can be strings - # check: https://github.com/huggingface/transformers/pull/24416 for more details - if isinstance(tensor, str): - logger.info("Skipping tensor %s as it is a string (bnb serialization)", key) - continue - - # If a `tensor` shares the same underlying storage as another tensor, we put `tensor` in the same `block` - storage_id = get_storage_id(tensor) - if storage_id is not None: - if storage_id in storage_id_to_tensors: - # We skip this tensor for now and will reassign to correct shard later - storage_id_to_tensors[storage_id].append(key) - continue - else: - # This is the first tensor with this storage_id, we create a new entry - # in the storage_id_to_tensors dict => we will assign the shard id later - storage_id_to_tensors[storage_id] = [key] - - # Compute tensor size - tensor_size = get_storage_size(tensor) - - # If this tensor is bigger than the maximal size, we put it in its own shard - if tensor_size > max_shard_size: - total_size += tensor_size - shard_list.append({key: tensor}) - continue - - # If this tensor is going to tip up over the maximal size, we split. - # Current shard already has some tensors, we add it to the list of shards and create a new one. - if current_shard_size + tensor_size > max_shard_size: - shard_list.append(current_shard) - current_shard = {} - current_shard_size = 0 - - # Add the tensor to the current shard - current_shard[key] = tensor - current_shard_size += tensor_size - total_size += tensor_size - - # Add the last shard - if len(current_shard) > 0: - shard_list.append(current_shard) - nb_shards = len(shard_list) - - # Loop over the tensors that share the same storage and assign them together - for storage_id, keys in storage_id_to_tensors.items(): - # Let's try to find the shard where the first tensor of this storage is and put all tensors in the same shard - for shard in shard_list: - if keys[0] in shard: - for key in keys: - shard[key] = state_dict[key] - break - - # If we only have one shard, we return it => no need to build the index - if nb_shards == 1: - filename = filename_pattern.format(suffix="") - return StateDictSplit( - metadata={"total_size": total_size}, - filename_to_tensors={filename: list(state_dict.keys())}, - tensor_to_filename={key: filename for key in state_dict.keys()}, - ) - - # Now that each tensor is assigned to a shard, let's assign a filename to each shard - tensor_name_to_filename = {} - filename_to_tensors = {} - for idx, shard in enumerate(shard_list): - filename = filename_pattern.format(suffix=f"-{idx + 1:05d}-of-{nb_shards:05d}") - for key in shard: - tensor_name_to_filename[key] = filename - filename_to_tensors[filename] = list(shard.keys()) - - # Build the index and return - return StateDictSplit( - metadata={"total_size": total_size}, - filename_to_tensors=filename_to_tensors, - tensor_to_filename=tensor_name_to_filename, - ) - - -def parse_size_to_int(size_as_str: str) -> int: - """ - Parse a size expressed as a string with digits and unit (like `"5MB"`) to an integer (in bytes). - - Supported units are "TB", "GB", "MB", "KB". - - Args: - size_as_str (`str`): The size to convert. Will be directly returned if an `int`. - - Example: - - ```py - >>> parse_size_to_int("5MB") - 5000000 - ``` - """ - size_as_str = size_as_str.strip() - - # Parse unit - unit = size_as_str[-2:].upper() - if unit not in SIZE_UNITS: - raise ValueError(f"Unit '{unit}' not supported. Supported units are TB, GB, MB, KB. Got '{size_as_str}'.") - multiplier = SIZE_UNITS[unit] - - # Parse value - try: - value = float(size_as_str[:-2].strip()) - except ValueError as e: - raise ValueError(f"Could not parse the size value from '{size_as_str}': {e}") from e - - return int(value * multiplier) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_dduf.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_dduf.py deleted file mode 100644 index a1debadb3ac8a45716f0359b932dc065f09edb84..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_dduf.py +++ /dev/null @@ -1,387 +0,0 @@ -import json -import logging -import mmap -import os -import shutil -import zipfile -from contextlib import contextmanager -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Dict, Generator, Iterable, Tuple, Union - -from ..errors import DDUFCorruptedFileError, DDUFExportError, DDUFInvalidEntryNameError - - -logger = logging.getLogger(__name__) - -DDUF_ALLOWED_ENTRIES = { - # Allowed file extensions in a DDUF file - ".json", - ".model", - ".safetensors", - ".txt", -} - -DDUF_FOLDER_REQUIRED_ENTRIES = { - # Each folder must contain at least one of these entries - "config.json", - "tokenizer_config.json", - "preprocessor_config.json", - "scheduler_config.json", -} - - -@dataclass -class DDUFEntry: - """Object representing a file entry in a DDUF file. - - See [`read_dduf_file`] for how to read a DDUF file. - - Attributes: - filename (str): - The name of the file in the DDUF archive. - offset (int): - The offset of the file in the DDUF archive. - length (int): - The length of the file in the DDUF archive. - dduf_path (str): - The path to the DDUF archive (for internal use). - """ - - filename: str - length: int - offset: int - - dduf_path: Path = field(repr=False) - - @contextmanager - def as_mmap(self) -> Generator[bytes, None, None]: - """Open the file as a memory-mapped file. - - Useful to load safetensors directly from the file. - - Example: - ```py - >>> import safetensors.torch - >>> with entry.as_mmap() as mm: - ... tensors = safetensors.torch.load(mm) - ``` - """ - with self.dduf_path.open("rb") as f: - with mmap.mmap(f.fileno(), length=0, access=mmap.ACCESS_READ) as mm: - yield mm[self.offset : self.offset + self.length] - - def read_text(self, encoding: str = "utf-8") -> str: - """Read the file as text. - - Useful for '.txt' and '.json' entries. - - Example: - ```py - >>> import json - >>> index = json.loads(entry.read_text()) - ``` - """ - with self.dduf_path.open("rb") as f: - f.seek(self.offset) - return f.read(self.length).decode(encoding=encoding) - - -def read_dduf_file(dduf_path: Union[os.PathLike, str]) -> Dict[str, DDUFEntry]: - """ - Read a DDUF file and return a dictionary of entries. - - Only the metadata is read, the data is not loaded in memory. - - Args: - dduf_path (`str` or `os.PathLike`): - The path to the DDUF file to read. - - Returns: - `Dict[str, DDUFEntry]`: - A dictionary of [`DDUFEntry`] indexed by filename. - - Raises: - - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format). - - Example: - ```python - >>> import json - >>> import safetensors.torch - >>> from huggingface_hub import read_dduf_file - - # Read DDUF metadata - >>> dduf_entries = read_dduf_file("FLUX.1-dev.dduf") - - # Returns a mapping filename <> DDUFEntry - >>> dduf_entries["model_index.json"] - DDUFEntry(filename='model_index.json', offset=66, length=587) - - # Load model index as JSON - >>> json.loads(dduf_entries["model_index.json"].read_text()) - {'_class_name': 'FluxPipeline', '_diffusers_version': '0.32.0.dev0', '_name_or_path': 'black-forest-labs/FLUX.1-dev', ... - - # Load VAE weights using safetensors - >>> with dduf_entries["vae/diffusion_pytorch_model.safetensors"].as_mmap() as mm: - ... state_dict = safetensors.torch.load(mm) - ``` - """ - entries = {} - dduf_path = Path(dduf_path) - logger.info(f"Reading DDUF file {dduf_path}") - with zipfile.ZipFile(str(dduf_path), "r") as zf: - for info in zf.infolist(): - logger.debug(f"Reading entry {info.filename}") - if info.compress_type != zipfile.ZIP_STORED: - raise DDUFCorruptedFileError("Data must not be compressed in DDUF file.") - - try: - _validate_dduf_entry_name(info.filename) - except DDUFInvalidEntryNameError as e: - raise DDUFCorruptedFileError(f"Invalid entry name in DDUF file: {info.filename}") from e - - offset = _get_data_offset(zf, info) - - entries[info.filename] = DDUFEntry( - filename=info.filename, offset=offset, length=info.file_size, dduf_path=dduf_path - ) - - # Consistency checks on the DDUF file - if "model_index.json" not in entries: - raise DDUFCorruptedFileError("Missing required 'model_index.json' entry in DDUF file.") - index = json.loads(entries["model_index.json"].read_text()) - _validate_dduf_structure(index, entries.keys()) - - logger.info(f"Done reading DDUF file {dduf_path}. Found {len(entries)} entries") - return entries - - -def export_entries_as_dduf( - dduf_path: Union[str, os.PathLike], entries: Iterable[Tuple[str, Union[str, Path, bytes]]] -) -> None: - """Write a DDUF file from an iterable of entries. - - This is a lower-level helper than [`export_folder_as_dduf`] that allows more flexibility when serializing data. - In particular, you don't need to save the data on disk before exporting it in the DDUF file. - - Args: - dduf_path (`str` or `os.PathLike`): - The path to the DDUF file to write. - entries (`Iterable[Tuple[str, Union[str, Path, bytes]]]`): - An iterable of entries to write in the DDUF file. Each entry is a tuple with the filename and the content. - The filename should be the path to the file in the DDUF archive. - The content can be a string or a pathlib.Path representing a path to a file on the local disk or directly the content as bytes. - - Raises: - - [`DDUFExportError`]: If anything goes wrong during the export (e.g. invalid entry name, missing 'model_index.json', etc.). - - Example: - ```python - # Export specific files from the local disk. - >>> from huggingface_hub import export_entries_as_dduf - >>> export_entries_as_dduf( - ... dduf_path="stable-diffusion-v1-4-FP16.dduf", - ... entries=[ # List entries to add to the DDUF file (here, only FP16 weights) - ... ("model_index.json", "path/to/model_index.json"), - ... ("vae/config.json", "path/to/vae/config.json"), - ... ("vae/diffusion_pytorch_model.fp16.safetensors", "path/to/vae/diffusion_pytorch_model.fp16.safetensors"), - ... ("text_encoder/config.json", "path/to/text_encoder/config.json"), - ... ("text_encoder/model.fp16.safetensors", "path/to/text_encoder/model.fp16.safetensors"), - ... # ... add more entries here - ... ] - ... ) - ``` - - ```python - # Export state_dicts one by one from a loaded pipeline - >>> from diffusers import DiffusionPipeline - >>> from typing import Generator, Tuple - >>> import safetensors.torch - >>> from huggingface_hub import export_entries_as_dduf - >>> pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") - ... # ... do some work with the pipeline - - >>> def as_entries(pipe: DiffusionPipeline) -> Generator[Tuple[str, bytes], None, None]: - ... # Build an generator that yields the entries to add to the DDUF file. - ... # The first element of the tuple is the filename in the DDUF archive (must use UNIX separator!). The second element is the content of the file. - ... # Entries will be evaluated lazily when the DDUF file is created (only 1 entry is loaded in memory at a time) - ... yield "vae/config.json", pipe.vae.to_json_string().encode() - ... yield "vae/diffusion_pytorch_model.safetensors", safetensors.torch.save(pipe.vae.state_dict()) - ... yield "text_encoder/config.json", pipe.text_encoder.config.to_json_string().encode() - ... yield "text_encoder/model.safetensors", safetensors.torch.save(pipe.text_encoder.state_dict()) - ... # ... add more entries here - - >>> export_entries_as_dduf(dduf_path="stable-diffusion-v1-4.dduf", entries=as_entries(pipe)) - ``` - """ - logger.info(f"Exporting DDUF file '{dduf_path}'") - filenames = set() - index = None - with zipfile.ZipFile(str(dduf_path), "w", zipfile.ZIP_STORED) as archive: - for filename, content in entries: - if filename in filenames: - raise DDUFExportError(f"Can't add duplicate entry: {filename}") - filenames.add(filename) - - if filename == "model_index.json": - try: - index = json.loads(_load_content(content).decode()) - except json.JSONDecodeError as e: - raise DDUFExportError("Failed to parse 'model_index.json'.") from e - - try: - filename = _validate_dduf_entry_name(filename) - except DDUFInvalidEntryNameError as e: - raise DDUFExportError(f"Invalid entry name: {filename}") from e - logger.debug(f"Adding entry '{filename}' to DDUF file") - _dump_content_in_archive(archive, filename, content) - - # Consistency checks on the DDUF file - if index is None: - raise DDUFExportError("Missing required 'model_index.json' entry in DDUF file.") - try: - _validate_dduf_structure(index, filenames) - except DDUFCorruptedFileError as e: - raise DDUFExportError("Invalid DDUF file structure.") from e - - logger.info(f"Done writing DDUF file {dduf_path}") - - -def export_folder_as_dduf(dduf_path: Union[str, os.PathLike], folder_path: Union[str, os.PathLike]) -> None: - """ - Export a folder as a DDUF file. - - AUses [`export_entries_as_dduf`] under the hood. - - Args: - dduf_path (`str` or `os.PathLike`): - The path to the DDUF file to write. - folder_path (`str` or `os.PathLike`): - The path to the folder containing the diffusion model. - - Example: - ```python - >>> from huggingface_hub import export_folder_as_dduf - >>> export_folder_as_dduf(dduf_path="FLUX.1-dev.dduf", folder_path="path/to/FLUX.1-dev") - ``` - """ - folder_path = Path(folder_path) - - def _iterate_over_folder() -> Iterable[Tuple[str, Path]]: - for path in Path(folder_path).glob("**/*"): - if not path.is_file(): - continue - if path.suffix not in DDUF_ALLOWED_ENTRIES: - logger.debug(f"Skipping file '{path}' (file type not allowed)") - continue - path_in_archive = path.relative_to(folder_path) - if len(path_in_archive.parts) >= 3: - logger.debug(f"Skipping file '{path}' (nested directories not allowed)") - continue - yield path_in_archive.as_posix(), path - - export_entries_as_dduf(dduf_path, _iterate_over_folder()) - - -def _dump_content_in_archive(archive: zipfile.ZipFile, filename: str, content: Union[str, os.PathLike, bytes]) -> None: - with archive.open(filename, "w", force_zip64=True) as archive_fh: - if isinstance(content, (str, Path)): - content_path = Path(content) - with content_path.open("rb") as content_fh: - shutil.copyfileobj(content_fh, archive_fh, 1024 * 1024 * 8) # type: ignore[misc] - elif isinstance(content, bytes): - archive_fh.write(content) - else: - raise DDUFExportError(f"Invalid content type for {filename}. Must be str, Path or bytes.") - - -def _load_content(content: Union[str, Path, bytes]) -> bytes: - """Load the content of an entry as bytes. - - Used only for small checks (not to dump content into archive). - """ - if isinstance(content, (str, Path)): - return Path(content).read_bytes() - elif isinstance(content, bytes): - return content - else: - raise DDUFExportError(f"Invalid content type. Must be str, Path or bytes. Got {type(content)}.") - - -def _validate_dduf_entry_name(entry_name: str) -> str: - if "." + entry_name.split(".")[-1] not in DDUF_ALLOWED_ENTRIES: - raise DDUFInvalidEntryNameError(f"File type not allowed: {entry_name}") - if "\\" in entry_name: - raise DDUFInvalidEntryNameError(f"Entry names must use UNIX separators ('/'). Got {entry_name}.") - entry_name = entry_name.strip("/") - if entry_name.count("/") > 1: - raise DDUFInvalidEntryNameError(f"DDUF only supports 1 level of directory. Got {entry_name}.") - return entry_name - - -def _validate_dduf_structure(index: Any, entry_names: Iterable[str]) -> None: - """ - Consistency checks on the DDUF file structure. - - Rules: - - The 'model_index.json' entry is required and must contain a dictionary. - - Each folder name must correspond to an entry in 'model_index.json'. - - Each folder must contain at least a config file ('config.json', 'tokenizer_config.json', 'preprocessor_config.json', 'scheduler_config.json'). - - Args: - index (Any): - The content of the 'model_index.json' entry. - entry_names (Iterable[str]): - The list of entry names in the DDUF file. - - Raises: - - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format). - """ - if not isinstance(index, dict): - raise DDUFCorruptedFileError(f"Invalid 'model_index.json' content. Must be a dictionary. Got {type(index)}.") - - dduf_folders = {entry.split("/")[0] for entry in entry_names if "/" in entry} - for folder in dduf_folders: - if folder not in index: - raise DDUFCorruptedFileError(f"Missing required entry '{folder}' in 'model_index.json'.") - if not any(f"{folder}/{required_entry}" in entry_names for required_entry in DDUF_FOLDER_REQUIRED_ENTRIES): - raise DDUFCorruptedFileError( - f"Missing required file in folder '{folder}'. Must contains at least one of {DDUF_FOLDER_REQUIRED_ENTRIES}." - ) - - -def _get_data_offset(zf: zipfile.ZipFile, info: zipfile.ZipInfo) -> int: - """ - Calculate the data offset for a file in a ZIP archive. - - Args: - zf (`zipfile.ZipFile`): - The opened ZIP file. Must be opened in read mode. - info (`zipfile.ZipInfo`): - The file info. - - Returns: - int: The offset of the file data in the ZIP archive. - """ - if zf.fp is None: - raise DDUFCorruptedFileError("ZipFile object must be opened in read mode.") - - # Step 1: Get the local file header offset - header_offset = info.header_offset - - # Step 2: Read the local file header - zf.fp.seek(header_offset) - local_file_header = zf.fp.read(30) # Fixed-size part of the local header - - if len(local_file_header) < 30: - raise DDUFCorruptedFileError("Incomplete local file header.") - - # Step 3: Parse the header fields to calculate the start of file data - # Local file header: https://en.wikipedia.org/wiki/ZIP_(file_format)#File_headers - filename_len = int.from_bytes(local_file_header[26:28], "little") - extra_field_len = int.from_bytes(local_file_header[28:30], "little") - - # Data offset is after the fixed header, filename, and extra fields - data_offset = header_offset + 30 + filename_len + extra_field_len - - return data_offset diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_tensorflow.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_tensorflow.py deleted file mode 100644 index 1173e34a28b2d7f9d879e01ffdae8ce09e9d5b5c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_tensorflow.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2024 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains tensorflow-specific helpers.""" - -import math -import re -from typing import TYPE_CHECKING, Dict, Union - -from .. import constants -from ._base import MAX_SHARD_SIZE, StateDictSplit, split_state_dict_into_shards_factory - - -if TYPE_CHECKING: - import tensorflow as tf - - -def split_tf_state_dict_into_shards( - state_dict: Dict[str, "tf.Tensor"], - *, - filename_pattern: str = constants.TF2_WEIGHTS_FILE_PATTERN, - max_shard_size: Union[int, str] = MAX_SHARD_SIZE, -) -> StateDictSplit: - """ - Split a model state dictionary in shards so that each shard is smaller than a given size. - - The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization - made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we - have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not - [6+2+2GB], [6+2GB], [6GB]. - - > [!WARNING] - > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a - > size greater than `max_shard_size`. - - Args: - state_dict (`Dict[str, Tensor]`): - The state dictionary to save. - filename_pattern (`str`, *optional*): - The pattern to generate the files names in which the model will be saved. Pattern must be a string that - can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` - Defaults to `"tf_model{suffix}.h5"`. - max_shard_size (`int` or `str`, *optional*): - The maximum size of each shard, in bytes. Defaults to 5GB. - - Returns: - [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them. - """ - return split_state_dict_into_shards_factory( - state_dict, - max_shard_size=max_shard_size, - filename_pattern=filename_pattern, - get_storage_size=get_tf_storage_size, - ) - - -def get_tf_storage_size(tensor: "tf.Tensor") -> int: - # Return `math.ceil` since dtype byte size can be a float (e.g., 0.125 for tf.bool). - # Better to overestimate than underestimate. - return math.ceil(tensor.numpy().size * _dtype_byte_size_tf(tensor.dtype)) - - -def _dtype_byte_size_tf(dtype) -> float: - """ - Returns the size (in bytes) occupied by one parameter of type `dtype`. - Taken from https://github.com/huggingface/transformers/blob/74d9d0cebb0263a3f8ab9c280569170cc74651d0/src/transformers/modeling_tf_utils.py#L608. - NOTE: why not `tensor.numpy().nbytes`? - Example: - ```py - >>> _dtype_byte_size(tf.float32) - 4 - ``` - """ - import tensorflow as tf - - if dtype == tf.bool: - return 1 / 8 - bit_search = re.search(r"[^\d](\d+)$", dtype.name) - if bit_search is None: - raise ValueError(f"`dtype` is not a valid dtype: {dtype}.") - bit_size = int(bit_search.groups()[0]) - return bit_size // 8 diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_torch.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_torch.py deleted file mode 100644 index e24d46ab4e14415104922681cd64944a33a3d9ab..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/serialization/_torch.py +++ /dev/null @@ -1,1015 +0,0 @@ -# Copyright 2024 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains pytorch-specific helpers.""" - -import importlib -import json -import os -import re -from collections import defaultdict, namedtuple -from functools import lru_cache -from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple, Union - -from packaging import version - -from .. import constants, logging -from ._base import MAX_SHARD_SIZE, StateDictSplit, split_state_dict_into_shards_factory - - -logger = logging.get_logger(__file__) - -if TYPE_CHECKING: - import torch - -# SAVING - - -def save_torch_model( - model: "torch.nn.Module", - save_directory: Union[str, Path], - *, - filename_pattern: Optional[str] = None, - force_contiguous: bool = True, - max_shard_size: Union[int, str] = MAX_SHARD_SIZE, - metadata: Optional[Dict[str, str]] = None, - safe_serialization: bool = True, - is_main_process: bool = True, - shared_tensors_to_discard: Optional[List[str]] = None, -): - """ - Saves a given torch model to disk, handling sharding and shared tensors issues. - - See also [`save_torch_state_dict`] to save a state dict with more flexibility. - - For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors). - - The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are - saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard, - an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses - [`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as - safetensors (the default). Otherwise, the shards are saved as pickle. - - Before saving the model, the `save_directory` is cleaned from any previous shard files. - - > [!WARNING] - > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a - > size greater than `max_shard_size`. - - > [!WARNING] - > If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving. - - Args: - model (`torch.nn.Module`): - The model to save on disk. - save_directory (`str` or `Path`): - The directory in which the model will be saved. - filename_pattern (`str`, *optional*): - The pattern to generate the files names in which the model will be saved. Pattern must be a string that - can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` - Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization` - parameter. - force_contiguous (`boolean`, *optional*): - Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the - model, but it could potentially change performance if the layout of the tensor was chosen specifically for - that reason. Defaults to `True`. - max_shard_size (`int` or `str`, *optional*): - The maximum size of each shard, in bytes. Defaults to 5GB. - metadata (`Dict[str, str]`, *optional*): - Extra information to save along with the model. Some metadata will be added for each dropped tensors. - This information will not be enough to recover the entire shared structure but might help understanding - things. - safe_serialization (`bool`, *optional*): - Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle. - Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed - in a future version. - is_main_process (`bool`, *optional*): - Whether the process calling this is the main process or not. Useful when in distributed training like - TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on - the main process to avoid race conditions. Defaults to True. - shared_tensors_to_discard (`List[str]`, *optional*): - List of tensor names to drop when saving shared tensors. If not provided and shared tensors are - detected, it will drop the first name alphabetically. - - Example: - - ```py - >>> from huggingface_hub import save_torch_model - >>> model = ... # A PyTorch model - - # Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors. - >>> save_torch_model(model, "path/to/folder") - - # Load model back - >>> from huggingface_hub import load_torch_model # TODO - >>> load_torch_model(model, "path/to/folder") - >>> - ``` - """ - save_torch_state_dict( - state_dict=model.state_dict(), - filename_pattern=filename_pattern, - force_contiguous=force_contiguous, - max_shard_size=max_shard_size, - metadata=metadata, - safe_serialization=safe_serialization, - save_directory=save_directory, - is_main_process=is_main_process, - shared_tensors_to_discard=shared_tensors_to_discard, - ) - - -def save_torch_state_dict( - state_dict: Dict[str, "torch.Tensor"], - save_directory: Union[str, Path], - *, - filename_pattern: Optional[str] = None, - force_contiguous: bool = True, - max_shard_size: Union[int, str] = MAX_SHARD_SIZE, - metadata: Optional[Dict[str, str]] = None, - safe_serialization: bool = True, - is_main_process: bool = True, - shared_tensors_to_discard: Optional[List[str]] = None, -) -> None: - """ - Save a model state dictionary to the disk, handling sharding and shared tensors issues. - - See also [`save_torch_model`] to directly save a PyTorch model. - - For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors). - - The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are - saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard, - an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses - [`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as - safetensors (the default). Otherwise, the shards are saved as pickle. - - Before saving the model, the `save_directory` is cleaned from any previous shard files. - - > [!WARNING] - > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a - > size greater than `max_shard_size`. - - > [!WARNING] - > If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving. - - Args: - state_dict (`Dict[str, torch.Tensor]`): - The state dictionary to save. - save_directory (`str` or `Path`): - The directory in which the model will be saved. - filename_pattern (`str`, *optional*): - The pattern to generate the files names in which the model will be saved. Pattern must be a string that - can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` - Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization` - parameter. - force_contiguous (`boolean`, *optional*): - Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the - model, but it could potentially change performance if the layout of the tensor was chosen specifically for - that reason. Defaults to `True`. - max_shard_size (`int` or `str`, *optional*): - The maximum size of each shard, in bytes. Defaults to 5GB. - metadata (`Dict[str, str]`, *optional*): - Extra information to save along with the model. Some metadata will be added for each dropped tensors. - This information will not be enough to recover the entire shared structure but might help understanding - things. - safe_serialization (`bool`, *optional*): - Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle. - Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed - in a future version. - is_main_process (`bool`, *optional*): - Whether the process calling this is the main process or not. Useful when in distributed training like - TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on - the main process to avoid race conditions. Defaults to True. - shared_tensors_to_discard (`List[str]`, *optional*): - List of tensor names to drop when saving shared tensors. If not provided and shared tensors are - detected, it will drop the first name alphabetically. - - Example: - - ```py - >>> from huggingface_hub import save_torch_state_dict - >>> model = ... # A PyTorch model - - # Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors. - >>> state_dict = model_to_save.state_dict() - >>> save_torch_state_dict(state_dict, "path/to/folder") - ``` - """ - save_directory = str(save_directory) - - if filename_pattern is None: - filename_pattern = ( - constants.SAFETENSORS_WEIGHTS_FILE_PATTERN - if safe_serialization - else constants.PYTORCH_WEIGHTS_FILE_PATTERN - ) - - if metadata is None: - metadata = {} - if safe_serialization: - try: - from safetensors.torch import save_file as save_file_fn - except ImportError as e: - raise ImportError( - "Please install `safetensors` to use safe serialization. " - "You can install it with `pip install safetensors`." - ) from e - # Clean state dict for safetensors - state_dict = _clean_state_dict_for_safetensors( - state_dict, - metadata, - force_contiguous=force_contiguous, - shared_tensors_to_discard=shared_tensors_to_discard, - ) - else: - from torch import save as save_file_fn # type: ignore[assignment, no-redef] - - logger.warning( - "You are using unsafe serialization. Due to security reasons, it is recommended not to load " - "pickled models from untrusted sources. If you intend to share your model, we strongly recommend " - "using safe serialization by installing `safetensors` with `pip install safetensors`." - ) - # Split dict - state_dict_split = split_torch_state_dict_into_shards( - state_dict, filename_pattern=filename_pattern, max_shard_size=max_shard_size - ) - - # Only main process should clean up existing files to avoid race conditions in distributed environment - if is_main_process: - existing_files_regex = re.compile(filename_pattern.format(suffix=r"(-\d{5}-of-\d{5})?") + r"(\.index\.json)?") - for filename in os.listdir(save_directory): - if existing_files_regex.match(filename): - try: - logger.debug(f"Removing existing file '{filename}' from folder.") - os.remove(os.path.join(save_directory, filename)) - except Exception as e: - logger.warning( - f"Error when trying to remove existing '{filename}' from folder: {e}. Continuing..." - ) - - # Save each shard - per_file_metadata = {"format": "pt"} - if not state_dict_split.is_sharded: - per_file_metadata.update(metadata) - safe_file_kwargs = {"metadata": per_file_metadata} if safe_serialization else {} - for filename, tensors in state_dict_split.filename_to_tensors.items(): - shard = {tensor: state_dict[tensor] for tensor in tensors} - save_file_fn(shard, os.path.join(save_directory, filename), **safe_file_kwargs) # ty: ignore[invalid-argument-type] - logger.debug(f"Shard saved to {filename}") - - # Save the index (if any) - if state_dict_split.is_sharded: - index_path = filename_pattern.format(suffix="") + ".index.json" - index = { - "metadata": {**state_dict_split.metadata, **metadata}, - "weight_map": state_dict_split.tensor_to_filename, - } - with open(os.path.join(save_directory, index_path), "w") as f: - json.dump(index, f, indent=2) - logger.info( - f"The model is bigger than the maximum size per checkpoint ({max_shard_size}). " - f"Model weighs have been saved in {len(state_dict_split.filename_to_tensors)} checkpoint shards. " - f"You can find where each parameters has been saved in the index located at {index_path}." - ) - - logger.info(f"Model weights successfully saved to {save_directory}!") - - -def split_torch_state_dict_into_shards( - state_dict: Dict[str, "torch.Tensor"], - *, - filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN, - max_shard_size: Union[int, str] = MAX_SHARD_SIZE, -) -> StateDictSplit: - """ - Split a model state dictionary in shards so that each shard is smaller than a given size. - - The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization - made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we - have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not - [6+2+2GB], [6+2GB], [6GB]. - - - > [!TIP] - > To save a model state dictionary to the disk, see [`save_torch_state_dict`]. This helper uses - > `split_torch_state_dict_into_shards` under the hood. - - > [!WARNING] - > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a - > size greater than `max_shard_size`. - - Args: - state_dict (`Dict[str, torch.Tensor]`): - The state dictionary to save. - filename_pattern (`str`, *optional*): - The pattern to generate the files names in which the model will be saved. Pattern must be a string that - can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` - Defaults to `"model{suffix}.safetensors"`. - max_shard_size (`int` or `str`, *optional*): - The maximum size of each shard, in bytes. Defaults to 5GB. - - Returns: - [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them. - - Example: - ```py - >>> import json - >>> import os - >>> from safetensors.torch import save_file as safe_save_file - >>> from huggingface_hub import split_torch_state_dict_into_shards - - >>> def save_state_dict(state_dict: Dict[str, torch.Tensor], save_directory: str): - ... state_dict_split = split_torch_state_dict_into_shards(state_dict) - ... for filename, tensors in state_dict_split.filename_to_tensors.items(): - ... shard = {tensor: state_dict[tensor] for tensor in tensors} - ... safe_save_file( - ... shard, - ... os.path.join(save_directory, filename), - ... metadata={"format": "pt"}, - ... ) - ... if state_dict_split.is_sharded: - ... index = { - ... "metadata": state_dict_split.metadata, - ... "weight_map": state_dict_split.tensor_to_filename, - ... } - ... with open(os.path.join(save_directory, "model.safetensors.index.json"), "w") as f: - ... f.write(json.dumps(index, indent=2)) - ``` - """ - return split_state_dict_into_shards_factory( - state_dict, - max_shard_size=max_shard_size, - filename_pattern=filename_pattern, - get_storage_size=get_torch_storage_size, - get_storage_id=get_torch_storage_id, - ) - - -# LOADING - - -def load_torch_model( - model: "torch.nn.Module", - checkpoint_path: Union[str, os.PathLike], - *, - strict: bool = False, - safe: bool = True, - weights_only: bool = False, - map_location: Optional[Union[str, "torch.device"]] = None, - mmap: bool = False, - filename_pattern: Optional[str] = None, -) -> NamedTuple: - """ - Load a checkpoint into a model, handling both sharded and non-sharded checkpoints. - - Args: - model (`torch.nn.Module`): - The model in which to load the checkpoint. - checkpoint_path (`str` or `os.PathLike`): - Path to either the checkpoint file or directory containing the checkpoint(s). - strict (`bool`, *optional*, defaults to `False`): - Whether to strictly enforce that the keys in the model state dict match the keys in the checkpoint. - safe (`bool`, *optional*, defaults to `True`): - If `safe` is True, the safetensors files will be loaded. If `safe` is False, the function - will first attempt to load safetensors files if they are available, otherwise it will fall back to loading - pickle files. `filename_pattern` parameter takes precedence over `safe` parameter. - weights_only (`bool`, *optional*, defaults to `False`): - If True, only loads the model weights without optimizer states and other metadata. - Only supported in PyTorch >= 1.13. - map_location (`str` or `torch.device`, *optional*): - A `torch.device` object, string or a dict specifying how to remap storage locations. It - indicates the location where all tensors should be loaded. - mmap (`bool`, *optional*, defaults to `False`): - Whether to use memory-mapped file loading. Memory mapping can improve loading performance - for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints. - filename_pattern (`str`, *optional*): - The pattern to look for the index file. Pattern must be a string that - can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` - Defaults to `"model{suffix}.safetensors"`. - Returns: - `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields. - - `missing_keys` is a list of str containing the missing keys, i.e. keys that are in the model but not in the checkpoint. - - `unexpected_keys` is a list of str containing the unexpected keys, i.e. keys that are in the checkpoint but not in the model. - - Raises: - [`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError) - If the checkpoint file or directory does not exist. - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) - If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If the checkpoint path is invalid or if the checkpoint format cannot be determined. - - Example: - ```python - >>> from huggingface_hub import load_torch_model - >>> model = ... # A PyTorch model - >>> load_torch_model(model, "path/to/checkpoint") - ``` - """ - checkpoint_path = Path(checkpoint_path) - - if not checkpoint_path.exists(): - raise ValueError(f"Checkpoint path {checkpoint_path} does not exist") - # 1. Check if checkpoint is a single file - if checkpoint_path.is_file(): - state_dict = load_state_dict_from_file( - checkpoint_file=checkpoint_path, - map_location=map_location, - weights_only=weights_only, - ) - return model.load_state_dict(state_dict, strict=strict) - - # 2. If not, checkpoint_path is a directory - if filename_pattern is None: - filename_pattern = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN - index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json") - # Only fallback to pickle format if safetensors index is not found and safe is False. - if not index_path.is_file() and not safe: - filename_pattern = constants.PYTORCH_WEIGHTS_FILE_PATTERN - - index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json") - - if index_path.is_file(): - return _load_sharded_checkpoint( - model=model, - save_directory=checkpoint_path, - strict=strict, - weights_only=weights_only, - filename_pattern=filename_pattern, - ) - - # Look for single model file - model_files = list(checkpoint_path.glob("*.safetensors" if safe else "*.bin")) - if len(model_files) == 1: - state_dict = load_state_dict_from_file( - checkpoint_file=model_files[0], - map_location=map_location, - weights_only=weights_only, - mmap=mmap, - ) - return model.load_state_dict(state_dict, strict=strict) - - raise ValueError( - f"Directory '{checkpoint_path}' does not contain a valid checkpoint. " - "Expected either a sharded checkpoint with an index file, or a single model file." - ) - - -def _load_sharded_checkpoint( - model: "torch.nn.Module", - save_directory: os.PathLike, - *, - strict: bool = False, - weights_only: bool = False, - filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN, -) -> NamedTuple: - """ - Loads a sharded checkpoint into a model. This is the same as - [`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict) - but for a sharded checkpoint. Each shard is loaded one by one and removed from memory after being loaded into the model. - - Args: - model (`torch.nn.Module`): - The model in which to load the checkpoint. - save_directory (`str` or `os.PathLike`): - A path to a folder containing the sharded checkpoint. - strict (`bool`, *optional*, defaults to `False`): - Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint. - weights_only (`bool`, *optional*, defaults to `False`): - If True, only loads the model weights without optimizer states and other metadata. - Only supported in PyTorch >= 1.13. - filename_pattern (`str`, *optional*, defaults to `"model{suffix}.safetensors"`): - The pattern to look for the index file. Pattern must be a string that - can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` - Defaults to `"model{suffix}.safetensors"`. - - Returns: - `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields, - - `missing_keys` is a list of str containing the missing keys - - `unexpected_keys` is a list of str containing the unexpected keys - """ - - # 1. Load and validate index file - # The index file contains mapping of parameter names to shard files - index_path = filename_pattern.format(suffix="") + ".index.json" - index_file = os.path.join(save_directory, index_path) - with open(index_file, "r", encoding="utf-8") as f: - index = json.load(f) - - # 2. Validate keys if in strict mode - # This is done before loading any shards to fail fast - if strict: - _validate_keys_for_strict_loading(model, index["weight_map"].keys()) - - # 3. Load each shard using `load_state_dict` - # Get unique shard files (multiple parameters can be in same shard) - shard_files = list(set(index["weight_map"].values())) - for shard_file in shard_files: - # Load shard into memory - shard_path = os.path.join(save_directory, shard_file) - state_dict = load_state_dict_from_file( - shard_path, - map_location="cpu", - weights_only=weights_only, - ) - # Update model with parameters from this shard - model.load_state_dict(state_dict, strict=strict) - # Explicitly remove the state dict from memory - del state_dict - - # 4. Return compatibility info - loaded_keys = set(index["weight_map"].keys()) - model_keys = set(model.state_dict().keys()) - return _IncompatibleKeys( - missing_keys=list(model_keys - loaded_keys), unexpected_keys=list(loaded_keys - model_keys) - ) - - -def load_state_dict_from_file( - checkpoint_file: Union[str, os.PathLike], - map_location: Optional[Union[str, "torch.device"]] = None, - weights_only: bool = False, - mmap: bool = False, -) -> Union[Dict[str, "torch.Tensor"], Any]: - """ - Loads a checkpoint file, handling both safetensors and pickle checkpoint formats. - - Args: - checkpoint_file (`str` or `os.PathLike`): - Path to the checkpoint file to load. Can be either a safetensors or pickle (`.bin`) checkpoint. - map_location (`str` or `torch.device`, *optional*): - A `torch.device` object, string or a dict specifying how to remap storage locations. It - indicates the location where all tensors should be loaded. - weights_only (`bool`, *optional*, defaults to `False`): - If True, only loads the model weights without optimizer states and other metadata. - Only supported for pickle (`.bin`) checkpoints with PyTorch >= 1.13. Has no effect when - loading safetensors files. - mmap (`bool`, *optional*, defaults to `False`): - Whether to use memory-mapped file loading. Memory mapping can improve loading performance - for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints. Has no effect when - loading safetensors files, as the `safetensors` library uses memory mapping by default. - - Returns: - `Union[Dict[str, "torch.Tensor"], Any]`: The loaded checkpoint. - - For safetensors files: always returns a dictionary mapping parameter names to tensors. - - For pickle files: returns any Python object that was pickled (commonly a state dict, but could be - an entire model, optimizer state, or any other Python object). - - Raises: - [`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError) - If the checkpoint file does not exist. - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) - If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively. - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) - If the checkpoint file format is invalid or if git-lfs files are not properly downloaded. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If the checkpoint file path is empty or invalid. - - Example: - ```python - >>> from huggingface_hub import load_state_dict_from_file - - # Load a PyTorch checkpoint - >>> state_dict = load_state_dict_from_file("path/to/model.bin", map_location="cpu") - >>> model.load_state_dict(state_dict) - - # Load a safetensors checkpoint - >>> state_dict = load_state_dict_from_file("path/to/model.safetensors") - >>> model.load_state_dict(state_dict) - ``` - """ - checkpoint_path = Path(checkpoint_file) - - # Check if file exists and is a regular file (not a directory) - if not checkpoint_path.is_file(): - raise FileNotFoundError( - f"No checkpoint file found at '{checkpoint_path}'. Please verify the path is correct and " - "the file has been properly downloaded." - ) - - # Load safetensors checkpoint - if checkpoint_path.suffix == ".safetensors": - try: - from safetensors import safe_open - from safetensors.torch import load_file - except ImportError as e: - raise ImportError( - "Please install `safetensors` to load safetensors checkpoint. " - "You can install it with `pip install safetensors`." - ) from e - - # Check format of the archive - with safe_open(checkpoint_file, framework="pt") as f: # type: ignore[attr-defined] - metadata = f.metadata() - # see comment: https://github.com/huggingface/transformers/blob/3d213b57fe74302e5902d68ed9478c3ad1aaa713/src/transformers/modeling_utils.py#L3966 - if metadata is not None and metadata.get("format") not in ["pt", "mlx"]: - raise OSError( - f"The safetensors archive passed at {checkpoint_file} does not contain the valid metadata. Make sure " - "you save your model with the `save_torch_model` method." - ) - device = str(map_location.type) if map_location is not None and hasattr(map_location, "type") else map_location - # meta device is not supported with safetensors, falling back to CPU - if device == "meta": - logger.warning("Meta device is not supported with safetensors. Falling back to CPU device.") - device = "cpu" - return load_file(checkpoint_file, device=device) # type: ignore[arg-type] - # Otherwise, load from pickle - try: - import torch - from torch import load - except ImportError as e: - raise ImportError( - "Please install `torch` to load torch tensors. You can install it with `pip install torch`." - ) from e - # Add additional kwargs, mmap is only supported in torch >= 2.1.0 - additional_kwargs = {} - if version.parse(torch.__version__) >= version.parse("2.1.0"): - additional_kwargs["mmap"] = mmap - - # weights_only is only supported in torch >= 1.13.0 - if version.parse(torch.__version__) >= version.parse("1.13.0"): - additional_kwargs["weights_only"] = weights_only - - return load( - checkpoint_file, - map_location=map_location, - **additional_kwargs, - ) - - -# HELPERS - - -def _validate_keys_for_strict_loading( - model: "torch.nn.Module", - loaded_keys: Iterable[str], -) -> None: - """ - Validate that model keys match loaded keys when strict loading is enabled. - - Args: - model: The PyTorch model being loaded - loaded_keys: The keys present in the checkpoint - - Raises: - RuntimeError: If there are missing or unexpected keys in strict mode - """ - loaded_keys_set = set(loaded_keys) - model_keys = set(model.state_dict().keys()) - missing_keys = model_keys - loaded_keys_set # Keys in model but not in checkpoint - unexpected_keys = loaded_keys_set - model_keys # Keys in checkpoint but not in model - - if missing_keys or unexpected_keys: - error_message = f"Error(s) in loading state_dict for {model.__class__.__name__}" - if missing_keys: - str_missing_keys = ",".join([f'"{k}"' for k in sorted(missing_keys)]) - error_message += f"\nMissing key(s): {str_missing_keys}." - if unexpected_keys: - str_unexpected_keys = ",".join([f'"{k}"' for k in sorted(unexpected_keys)]) - error_message += f"\nUnexpected key(s): {str_unexpected_keys}." - raise RuntimeError(error_message) - - -def _get_unique_id(tensor: "torch.Tensor") -> Union[int, Tuple[Any, ...]]: - """Returns a unique id for plain tensor - or a (potentially nested) Tuple of unique id for the flattened Tensor - if the input is a wrapper tensor subclass Tensor - """ - - try: - from torch.distributed.tensor import DTensor - - if isinstance(tensor, DTensor): - local_tensor = tensor.to_local() - return local_tensor.storage().data_ptr() - except ImportError: - pass - - try: - # for torch 2.1 and above we can also handle tensor subclasses - from torch.utils._python_dispatch import is_traceable_wrapper_subclass - - if is_traceable_wrapper_subclass(tensor): - attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] - return tuple(_get_unique_id(getattr(tensor, attr)) for attr in attrs) - - except ImportError: - # for torch version less than 2.1, we can fallback to original implementation - pass - - if tensor.device.type == "xla" and is_torch_tpu_available(): - # NOTE: xla tensors dont have storage - # use some other unique id to distinguish. - # this is a XLA tensor, it must be created using torch_xla's - # device. So the following import is safe: - import torch_xla # type: ignore[import] - - unique_id = torch_xla._XLAC._xla_get_tensor_id(tensor) - else: - unique_id = storage_ptr(tensor) - - return unique_id - - -def get_torch_storage_id(tensor: "torch.Tensor") -> Optional[Tuple["torch.device", Union[int, Tuple[Any, ...]], int]]: - """ - Return unique identifier to a tensor storage. - - Multiple different tensors can share the same underlying storage. This identifier is - guaranteed to be unique and constant for this tensor's storage during its lifetime. Two tensor storages with - non-overlapping lifetimes may have the same id. - In the case of meta tensors, we return None since we can't tell if they share the same storage. - - Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/pytorch_utils.py#L278. - """ - if tensor.device.type == "meta": - return None - else: - return tensor.device, _get_unique_id(tensor), get_torch_storage_size(tensor) - - -def get_torch_storage_size(tensor: "torch.Tensor") -> int: - """ - Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L31C1-L41C59 - """ - try: - from torch.distributed.tensor import DTensor - - if isinstance(tensor, DTensor): - # this returns the size of the FULL tensor in bytes - return tensor.nbytes - except ImportError: - pass - - try: - # for torch 2.1 and above we can also handle tensor subclasses - from torch.utils._python_dispatch import is_traceable_wrapper_subclass - - if is_traceable_wrapper_subclass(tensor): - attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] - return sum(get_torch_storage_size(getattr(tensor, attr)) for attr in attrs) - except ImportError: - # for torch version less than 2.1, we can fallback to original implementation - pass - - try: - return tensor.untyped_storage().nbytes() - except AttributeError: - # Fallback for torch==1.10 - try: - return tensor.storage().size() * _get_dtype_size(tensor.dtype) - except NotImplementedError: - # Fallback for meta storage - # On torch >=2.0 this is the tensor size - return tensor.nelement() * _get_dtype_size(tensor.dtype) - - -@lru_cache() -def is_torch_tpu_available(check_device=True): - """ - Checks if `torch_xla` is installed and potentially if a TPU is in the environment - - Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/utils/import_utils.py#L463. - """ - if importlib.util.find_spec("torch_xla") is not None: - if check_device: - # We need to check if `xla_device` can be found, will raise a RuntimeError if not - try: - import torch_xla.core.xla_model as xm # type: ignore[import] - - _ = xm.xla_device() - return True - except RuntimeError: - return False - return True - return False - - -def storage_ptr(tensor: "torch.Tensor") -> Union[int, Tuple[Any, ...]]: - """ - Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L11. - """ - try: - # for torch 2.1 and above we can also handle tensor subclasses - from torch.utils._python_dispatch import is_traceable_wrapper_subclass - - if is_traceable_wrapper_subclass(tensor): - return _get_unique_id(tensor) # type: ignore - except ImportError: - # for torch version less than 2.1, we can fallback to original implementation - pass - - try: - return tensor.untyped_storage().data_ptr() - except Exception: - # Fallback for torch==1.10 - try: - return tensor.storage().data_ptr() - except NotImplementedError: - # Fallback for meta storage - return 0 - - -def _clean_state_dict_for_safetensors( - state_dict: Dict[str, "torch.Tensor"], - metadata: Dict[str, str], - force_contiguous: bool = True, - shared_tensors_to_discard: Optional[List[str]] = None, -): - """Remove shared tensors from state_dict and update metadata accordingly (for reloading). - - Warning: `state_dict` and `metadata` are mutated in-place! - - Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L155. - """ - to_removes = _remove_duplicate_names(state_dict, discard_names=shared_tensors_to_discard) - for kept_name, to_remove_group in to_removes.items(): - for to_remove in to_remove_group: - if metadata is None: - metadata = {} - - if to_remove not in metadata: - # Do not override user data - metadata[to_remove] = kept_name - del state_dict[to_remove] - if force_contiguous: - state_dict = {k: v.contiguous() for k, v in state_dict.items()} - return state_dict - - -def _end_ptr(tensor: "torch.Tensor") -> int: - """ - Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L23. - """ - if tensor.nelement(): - stop = tensor.view(-1)[-1].data_ptr() + _get_dtype_size(tensor.dtype) - else: - stop = tensor.data_ptr() - return stop - - -def _filter_shared_not_shared(tensors: List[Set[str]], state_dict: Dict[str, "torch.Tensor"]) -> List[Set[str]]: - """ - Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L44 - """ - filtered_tensors = [] - for shared in tensors: - if len(shared) < 2: - filtered_tensors.append(shared) - continue - - areas = [] - for name in shared: - tensor = state_dict[name] - areas.append((tensor.data_ptr(), _end_ptr(tensor), name)) - areas.sort() - - _, last_stop, last_name = areas[0] - filtered_tensors.append({last_name}) - for start, stop, name in areas[1:]: - if start >= last_stop: - filtered_tensors.append({name}) - else: - filtered_tensors[-1].add(name) - last_stop = stop - - return filtered_tensors - - -def _find_shared_tensors(state_dict: Dict[str, "torch.Tensor"]) -> List[Set[str]]: - """ - Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L69. - """ - import torch - - tensors_dict = defaultdict(set) - for k, v in state_dict.items(): - if v.device != torch.device("meta") and storage_ptr(v) != 0 and get_torch_storage_size(v) != 0: - # Need to add device as key because of multiple GPU. - tensors_dict[(v.device, storage_ptr(v), get_torch_storage_size(v))].add(k) - tensors = list(sorted(tensors_dict.values())) - tensors = _filter_shared_not_shared(tensors, state_dict) - return tensors - - -def _is_complete(tensor: "torch.Tensor") -> bool: - """ - Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80 - """ - try: - # for torch 2.1 and above we can also handle tensor subclasses - from torch.utils._python_dispatch import is_traceable_wrapper_subclass - - if is_traceable_wrapper_subclass(tensor): - attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] - return all(_is_complete(getattr(tensor, attr)) for attr in attrs) - except ImportError: - # for torch version less than 2.1, we can fallback to original implementation - pass - - return tensor.data_ptr() == storage_ptr(tensor) and tensor.nelement() * _get_dtype_size( - tensor.dtype - ) == get_torch_storage_size(tensor) - - -def _remove_duplicate_names( - state_dict: Dict[str, "torch.Tensor"], - *, - preferred_names: Optional[List[str]] = None, - discard_names: Optional[List[str]] = None, -) -> Dict[str, List[str]]: - """ - Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80 - """ - if preferred_names is None: - preferred_names = [] - unique_preferred_names = set(preferred_names) - if discard_names is None: - discard_names = [] - unique_discard_names = set(discard_names) - - shareds = _find_shared_tensors(state_dict) - to_remove = defaultdict(list) - for shared in shareds: - complete_names = set([name for name in shared if _is_complete(state_dict[name])]) - if not complete_names: - raise RuntimeError( - "Error while trying to find names to remove to save state dict, but found no suitable name to keep" - f" for saving amongst: {shared}. None is covering the entire storage. Refusing to save/load the model" - " since you could be storing much more memory than needed. Please refer to" - " https://huggingface.co/docs/safetensors/torch_shared_tensors for more information. Or open an" - " issue." - ) - - keep_name = sorted(list(complete_names))[0] - - # Mechanism to preferentially select keys to keep - # coming from the on-disk file to allow - # loading models saved with a different choice - # of keep_name - preferred = complete_names.difference(unique_discard_names) - if preferred: - keep_name = sorted(list(preferred))[0] - - if unique_preferred_names: - preferred = unique_preferred_names.intersection(complete_names) - if preferred: - keep_name = sorted(list(preferred))[0] - for name in sorted(shared): - if name != keep_name: - to_remove[keep_name].append(name) - return to_remove - - -@lru_cache() -def _get_dtype_size(dtype: "torch.dtype") -> int: - """ - Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L344 - """ - import torch - - # torch.float8 formats require 2.1; we do not support these dtypes on earlier versions - _float8_e4m3fn = getattr(torch, "float8_e4m3fn", None) - _float8_e5m2 = getattr(torch, "float8_e5m2", None) - _SIZE = { - torch.int64: 8, - torch.float32: 4, - torch.int32: 4, - torch.bfloat16: 2, - torch.float16: 2, - torch.int16: 2, - torch.uint8: 1, - torch.int8: 1, - torch.bool: 1, - torch.float64: 8, - _float8_e4m3fn: 1, - _float8_e5m2: 1, - } - return _SIZE[dtype] - - -class _IncompatibleKeys(namedtuple("IncompatibleKeys", ["missing_keys", "unexpected_keys"])): - """ - This is used to report missing and unexpected keys in the state dict. - Taken from https://github.com/pytorch/pytorch/blob/main/torch/nn/modules/module.py#L52. - - """ - - def __repr__(self) -> str: - if not self.missing_keys and not self.unexpected_keys: - return "" - return super().__repr__() - - __str__ = __repr__ diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/templates/datasetcard_template.md b/bundle/python-cpu/Lib/site-packages/huggingface_hub/templates/datasetcard_template.md deleted file mode 100644 index 9af29ebbed93653ec74a8952e314e7554323ef15..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/templates/datasetcard_template.md +++ /dev/null @@ -1,143 +0,0 @@ ---- -# For reference on dataset card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/datasetcard.md?plain=1 -# Doc / guide: https://huggingface.co/docs/hub/datasets-cards -{{ card_data }} ---- - -# Dataset Card for {{ pretty_name | default("Dataset Name", true) }} - - - -{{ dataset_summary | default("", true) }} - -## Dataset Details - -### Dataset Description - - - -{{ dataset_description | default("", true) }} - -- **Curated by:** {{ curators | default("[More Information Needed]", true)}} -- **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}} -- **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}} -- **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}} -- **License:** {{ license | default("[More Information Needed]", true)}} - -### Dataset Sources [optional] - - - -- **Repository:** {{ repo | default("[More Information Needed]", true)}} -- **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}} -- **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}} - -## Uses - - - -### Direct Use - - - -{{ direct_use | default("[More Information Needed]", true)}} - -### Out-of-Scope Use - - - -{{ out_of_scope_use | default("[More Information Needed]", true)}} - -## Dataset Structure - - - -{{ dataset_structure | default("[More Information Needed]", true)}} - -## Dataset Creation - -### Curation Rationale - - - -{{ curation_rationale_section | default("[More Information Needed]", true)}} - -### Source Data - - - -#### Data Collection and Processing - - - -{{ data_collection_and_processing_section | default("[More Information Needed]", true)}} - -#### Who are the source data producers? - - - -{{ source_data_producers_section | default("[More Information Needed]", true)}} - -### Annotations [optional] - - - -#### Annotation process - - - -{{ annotation_process_section | default("[More Information Needed]", true)}} - -#### Who are the annotators? - - - -{{ who_are_annotators_section | default("[More Information Needed]", true)}} - -#### Personal and Sensitive Information - - - -{{ personal_and_sensitive_information | default("[More Information Needed]", true)}} - -## Bias, Risks, and Limitations - - - -{{ bias_risks_limitations | default("[More Information Needed]", true)}} - -### Recommendations - - - -{{ bias_recommendations | default("Users should be made aware of the risks, biases and limitations of the dataset. More information needed for further recommendations.", true)}} - -## Citation [optional] - - - -**BibTeX:** - -{{ citation_bibtex | default("[More Information Needed]", true)}} - -**APA:** - -{{ citation_apa | default("[More Information Needed]", true)}} - -## Glossary [optional] - - - -{{ glossary | default("[More Information Needed]", true)}} - -## More Information [optional] - -{{ more_information | default("[More Information Needed]", true)}} - -## Dataset Card Authors [optional] - -{{ dataset_card_authors | default("[More Information Needed]", true)}} - -## Dataset Card Contact - -{{ dataset_card_contact | default("[More Information Needed]", true)}} diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/templates/modelcard_template.md b/bundle/python-cpu/Lib/site-packages/huggingface_hub/templates/modelcard_template.md deleted file mode 100644 index 79ca15e4547debac763b390ef8e4b715e6f6403f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/templates/modelcard_template.md +++ /dev/null @@ -1,200 +0,0 @@ ---- -# For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 -# Doc / guide: https://huggingface.co/docs/hub/model-cards -{{ card_data }} ---- - -# Model Card for {{ model_id | default("Model ID", true) }} - - - -{{ model_summary | default("", true) }} - -## Model Details - -### Model Description - - - -{{ model_description | default("", true) }} - -- **Developed by:** {{ developers | default("[More Information Needed]", true)}} -- **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}} -- **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}} -- **Model type:** {{ model_type | default("[More Information Needed]", true)}} -- **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}} -- **License:** {{ license | default("[More Information Needed]", true)}} -- **Finetuned from model [optional]:** {{ base_model | default("[More Information Needed]", true)}} - -### Model Sources [optional] - - - -- **Repository:** {{ repo | default("[More Information Needed]", true)}} -- **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}} -- **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}} - -## Uses - - - -### Direct Use - - - -{{ direct_use | default("[More Information Needed]", true)}} - -### Downstream Use [optional] - - - -{{ downstream_use | default("[More Information Needed]", true)}} - -### Out-of-Scope Use - - - -{{ out_of_scope_use | default("[More Information Needed]", true)}} - -## Bias, Risks, and Limitations - - - -{{ bias_risks_limitations | default("[More Information Needed]", true)}} - -### Recommendations - - - -{{ bias_recommendations | default("Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.", true)}} - -## How to Get Started with the Model - -Use the code below to get started with the model. - -{{ get_started_code | default("[More Information Needed]", true)}} - -## Training Details - -### Training Data - - - -{{ training_data | default("[More Information Needed]", true)}} - -### Training Procedure - - - -#### Preprocessing [optional] - -{{ preprocessing | default("[More Information Needed]", true)}} - - -#### Training Hyperparameters - -- **Training regime:** {{ training_regime | default("[More Information Needed]", true)}} - -#### Speeds, Sizes, Times [optional] - - - -{{ speeds_sizes_times | default("[More Information Needed]", true)}} - -## Evaluation - - - -### Testing Data, Factors & Metrics - -#### Testing Data - - - -{{ testing_data | default("[More Information Needed]", true)}} - -#### Factors - - - -{{ testing_factors | default("[More Information Needed]", true)}} - -#### Metrics - - - -{{ testing_metrics | default("[More Information Needed]", true)}} - -### Results - -{{ results | default("[More Information Needed]", true)}} - -#### Summary - -{{ results_summary | default("", true) }} - -## Model Examination [optional] - - - -{{ model_examination | default("[More Information Needed]", true)}} - -## Environmental Impact - - - -Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). - -- **Hardware Type:** {{ hardware_type | default("[More Information Needed]", true)}} -- **Hours used:** {{ hours_used | default("[More Information Needed]", true)}} -- **Cloud Provider:** {{ cloud_provider | default("[More Information Needed]", true)}} -- **Compute Region:** {{ cloud_region | default("[More Information Needed]", true)}} -- **Carbon Emitted:** {{ co2_emitted | default("[More Information Needed]", true)}} - -## Technical Specifications [optional] - -### Model Architecture and Objective - -{{ model_specs | default("[More Information Needed]", true)}} - -### Compute Infrastructure - -{{ compute_infrastructure | default("[More Information Needed]", true)}} - -#### Hardware - -{{ hardware_requirements | default("[More Information Needed]", true)}} - -#### Software - -{{ software | default("[More Information Needed]", true)}} - -## Citation [optional] - - - -**BibTeX:** - -{{ citation_bibtex | default("[More Information Needed]", true)}} - -**APA:** - -{{ citation_apa | default("[More Information Needed]", true)}} - -## Glossary [optional] - - - -{{ glossary | default("[More Information Needed]", true)}} - -## More Information [optional] - -{{ more_information | default("[More Information Needed]", true)}} - -## Model Card Authors [optional] - -{{ model_card_authors | default("[More Information Needed]", true)}} - -## Model Card Contact - -{{ model_card_contact | default("[More Information Needed]", true)}} diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/__init__.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/__init__.py deleted file mode 100644 index 992eac104bd80de97444003172e926d5ad4522a0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/__init__.py +++ /dev/null @@ -1,117 +0,0 @@ -# coding=utf-8 -# Copyright 2021 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - -# ruff: noqa: F401 -from huggingface_hub.errors import ( - BadRequestError, - CacheNotFound, - CorruptedCacheException, - DisabledRepoError, - EntryNotFoundError, - FileMetadataError, - GatedRepoError, - HfHubHTTPError, - HFValidationError, - LocalEntryNotFoundError, - LocalTokenNotFoundError, - NotASafetensorsRepoError, - OfflineModeIsEnabled, - RepositoryNotFoundError, - RevisionNotFoundError, - SafetensorsParsingError, -) - -from . import tqdm as _tqdm # _tqdm is the module -from ._auth import get_stored_tokens, get_token -from ._cache_assets import cached_assets_path -from ._cache_manager import ( - CachedFileInfo, - CachedRepoInfo, - CachedRevisionInfo, - DeleteCacheStrategy, - HFCacheInfo, - scan_cache_dir, -) -from ._chunk_utils import chunk_iterable -from ._datetime import parse_datetime -from ._experimental import experimental -from ._fixes import SoftTemporaryDirectory, WeakFileLock, yaml_dump -from ._git_credential import list_credential_helpers, set_git_credential, unset_git_credential -from ._headers import build_hf_headers, get_token_to_send -from ._hf_folder import HfFolder -from ._http import ( - configure_http_backend, - fix_hf_endpoint_in_url, - get_session, - hf_raise_for_status, - http_backoff, - reset_sessions, -) -from ._pagination import paginate -from ._paths import DEFAULT_IGNORE_PATTERNS, FORBIDDEN_FOLDERS, filter_repo_objects -from ._runtime import ( - dump_environment_info, - get_aiohttp_version, - get_fastai_version, - get_fastapi_version, - get_fastcore_version, - get_gradio_version, - get_graphviz_version, - get_hf_hub_version, - get_hf_transfer_version, - get_jinja_version, - get_numpy_version, - get_pillow_version, - get_pydantic_version, - get_pydot_version, - get_python_version, - get_tensorboard_version, - get_tf_version, - get_torch_version, - is_aiohttp_available, - is_colab_enterprise, - is_fastai_available, - is_fastapi_available, - is_fastcore_available, - is_google_colab, - is_gradio_available, - is_graphviz_available, - is_hf_transfer_available, - is_jinja_available, - is_notebook, - is_numpy_available, - is_package_available, - is_pillow_available, - is_pydantic_available, - is_pydot_available, - is_safetensors_available, - is_tensorboard_available, - is_tf_available, - is_torch_available, -) -from ._safetensors import SafetensorsFileMetadata, SafetensorsRepoMetadata, TensorInfo -from ._subprocess import capture_output, run_interactive_subprocess, run_subprocess -from ._telemetry import send_telemetry -from ._typing import is_jsonable, is_simple_optional_type, unwrap_simple_optional_type -from ._validators import smoothly_deprecate_use_auth_token, validate_hf_hub_args, validate_repo_id -from ._xet import ( - XetConnectionInfo, - XetFileData, - XetTokenType, - fetch_xet_connection_info_from_repo_info, - parse_xet_file_data_from_response, - refresh_xet_connection_info, -) -from .tqdm import are_progress_bars_disabled, disable_progress_bars, enable_progress_bars, tqdm, tqdm_stream_file diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_auth.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_auth.py deleted file mode 100644 index 72be4dedbd94421ee2b4b2ba1073569d71b50569..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_auth.py +++ /dev/null @@ -1,214 +0,0 @@ -# Copyright 2023 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains an helper to get the token from machine (env variable, secret or config file).""" - -import configparser -import logging -import os -import warnings -from pathlib import Path -from threading import Lock -from typing import Dict, Optional - -from .. import constants -from ._runtime import is_colab_enterprise, is_google_colab - - -_IS_GOOGLE_COLAB_CHECKED = False -_GOOGLE_COLAB_SECRET_LOCK = Lock() -_GOOGLE_COLAB_SECRET: Optional[str] = None - -logger = logging.getLogger(__name__) - - -def get_token() -> Optional[str]: - """ - Get token if user is logged in. - - Note: in most cases, you should use [`huggingface_hub.utils.build_hf_headers`] instead. This method is only useful - if you want to retrieve the token for other purposes than sending an HTTP request. - - Token is retrieved in priority from the `HF_TOKEN` environment variable. Otherwise, we read the token file located - in the Hugging Face home folder. Returns None if user is not logged in. To log in, use [`login`] or - `hf auth login`. - - Returns: - `str` or `None`: The token, `None` if it doesn't exist. - """ - return _get_token_from_google_colab() or _get_token_from_environment() or _get_token_from_file() - - -def _get_token_from_google_colab() -> Optional[str]: - """Get token from Google Colab secrets vault using `google.colab.userdata.get(...)`. - - Token is read from the vault only once per session and then stored in a global variable to avoid re-requesting - access to the vault. - """ - # If it's not a Google Colab or it's Colab Enterprise, fallback to environment variable or token file authentication - if not is_google_colab() or is_colab_enterprise(): - return None - - # `google.colab.userdata` is not thread-safe - # This can lead to a deadlock if multiple threads try to access it at the same time - # (typically when using `snapshot_download`) - # => use a lock - # See https://github.com/huggingface/huggingface_hub/issues/1952 for more details. - with _GOOGLE_COLAB_SECRET_LOCK: - global _GOOGLE_COLAB_SECRET - global _IS_GOOGLE_COLAB_CHECKED - - if _IS_GOOGLE_COLAB_CHECKED: # request access only once - return _GOOGLE_COLAB_SECRET - - try: - from google.colab import userdata # type: ignore - from google.colab.errors import Error as ColabError # type: ignore - except ImportError: - return None - - try: - token = userdata.get("HF_TOKEN") - _GOOGLE_COLAB_SECRET = _clean_token(token) - except userdata.NotebookAccessError: - # Means the user has a secret call `HF_TOKEN` and got a popup "please grand access to HF_TOKEN" and refused it - # => warn user but ignore error => do not re-request access to user - warnings.warn( - "\nAccess to the secret `HF_TOKEN` has not been granted on this notebook." - "\nYou will not be requested again." - "\nPlease restart the session if you want to be prompted again." - ) - _GOOGLE_COLAB_SECRET = None - except userdata.SecretNotFoundError: - # Means the user did not define a `HF_TOKEN` secret => warn - warnings.warn( - "\nThe secret `HF_TOKEN` does not exist in your Colab secrets." - "\nTo authenticate with the Hugging Face Hub, create a token in your settings tab " - "(https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session." - "\nYou will be able to reuse this secret in all of your notebooks." - "\nPlease note that authentication is recommended but still optional to access public models or datasets." - ) - _GOOGLE_COLAB_SECRET = None - except ColabError as e: - # Something happen but we don't know what => recommend to open a GitHub issue - warnings.warn( - f"\nError while fetching `HF_TOKEN` secret value from your vault: '{str(e)}'." - "\nYou are not authenticated with the Hugging Face Hub in this notebook." - "\nIf the error persists, please let us know by opening an issue on GitHub " - "(https://github.com/huggingface/huggingface_hub/issues/new)." - ) - _GOOGLE_COLAB_SECRET = None - - _IS_GOOGLE_COLAB_CHECKED = True - return _GOOGLE_COLAB_SECRET - - -def _get_token_from_environment() -> Optional[str]: - # `HF_TOKEN` has priority (keep `HUGGING_FACE_HUB_TOKEN` for backward compatibility) - return _clean_token(os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")) - - -def _get_token_from_file() -> Optional[str]: - try: - return _clean_token(Path(constants.HF_TOKEN_PATH).read_text()) - except FileNotFoundError: - return None - - -def get_stored_tokens() -> Dict[str, str]: - """ - Returns the parsed INI file containing the access tokens. - The file is located at `HF_STORED_TOKENS_PATH`, defaulting to `~/.cache/huggingface/stored_tokens`. - If the file does not exist, an empty dictionary is returned. - - Returns: `Dict[str, str]` - Key is the token name and value is the token. - """ - tokens_path = Path(constants.HF_STORED_TOKENS_PATH) - if not tokens_path.exists(): - stored_tokens = {} - config = configparser.ConfigParser() - try: - config.read(tokens_path) - stored_tokens = {token_name: config.get(token_name, "hf_token") for token_name in config.sections()} - except configparser.Error as e: - logger.error(f"Error parsing stored tokens file: {e}") - stored_tokens = {} - return stored_tokens - - -def _save_stored_tokens(stored_tokens: Dict[str, str]) -> None: - """ - Saves the given configuration to the stored tokens file. - - Args: - stored_tokens (`Dict[str, str]`): - The stored tokens to save. Key is the token name and value is the token. - """ - stored_tokens_path = Path(constants.HF_STORED_TOKENS_PATH) - - # Write the stored tokens into an INI file - config = configparser.ConfigParser() - for token_name in sorted(stored_tokens.keys()): - config.add_section(token_name) - config.set(token_name, "hf_token", stored_tokens[token_name]) - - stored_tokens_path.parent.mkdir(parents=True, exist_ok=True) - with stored_tokens_path.open("w") as config_file: - config.write(config_file) - - -def _get_token_by_name(token_name: str) -> Optional[str]: - """ - Get the token by name. - - Args: - token_name (`str`): - The name of the token to get. - - Returns: - `str` or `None`: The token, `None` if it doesn't exist. - - """ - stored_tokens = get_stored_tokens() - if token_name not in stored_tokens: - return None - return _clean_token(stored_tokens[token_name]) - - -def _save_token(token: str, token_name: str) -> None: - """ - Save the given token. - - If the stored tokens file does not exist, it will be created. - Args: - token (`str`): - The token to save. - token_name (`str`): - The name of the token. - """ - tokens_path = Path(constants.HF_STORED_TOKENS_PATH) - stored_tokens = get_stored_tokens() - stored_tokens[token_name] = token - _save_stored_tokens(stored_tokens) - logger.info(f"The token `{token_name}` has been saved to {tokens_path}") - - -def _clean_token(token: Optional[str]) -> Optional[str]: - """Clean token by removing trailing and leading spaces and newlines. - - If token is an empty string, return None. - """ - if token is None: - return None - return token.replace("\r", "").replace("\n", "").strip() or None diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_cache_assets.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_cache_assets.py deleted file mode 100644 index e5d435df9b0bb0c67c0bcb5ef65711e9aef367f6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_cache_assets.py +++ /dev/null @@ -1,135 +0,0 @@ -# coding=utf-8 -# Copyright 2019-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from pathlib import Path -from typing import Union - -from ..constants import HF_ASSETS_CACHE - - -def cached_assets_path( - library_name: str, - namespace: str = "default", - subfolder: str = "default", - *, - assets_dir: Union[str, Path, None] = None, -): - """Return a folder path to cache arbitrary files. - - `huggingface_hub` provides a canonical folder path to store assets. This is the - recommended way to integrate cache in a downstream library as it will benefit from - the builtins tools to scan and delete the cache properly. - - The distinction is made between files cached from the Hub and assets. Files from the - Hub are cached in a git-aware manner and entirely managed by `huggingface_hub`. See - [related documentation](https://huggingface.co/docs/huggingface_hub/how-to-cache). - All other files that a downstream library caches are considered to be "assets" - (files downloaded from external sources, extracted from a .tar archive, preprocessed - for training,...). - - Once the folder path is generated, it is guaranteed to exist and to be a directory. - The path is based on 3 levels of depth: the library name, a namespace and a - subfolder. Those 3 levels grants flexibility while allowing `huggingface_hub` to - expect folders when scanning/deleting parts of the assets cache. Within a library, - it is expected that all namespaces share the same subset of subfolder names but this - is not a mandatory rule. The downstream library has then full control on which file - structure to adopt within its cache. Namespace and subfolder are optional (would - default to a `"default/"` subfolder) but library name is mandatory as we want every - downstream library to manage its own cache. - - Expected tree: - ```text - assets/ - └── datasets/ - │ ├── SQuAD/ - │ │ ├── downloaded/ - │ │ ├── extracted/ - │ │ └── processed/ - │ ├── Helsinki-NLP--tatoeba_mt/ - │ ├── downloaded/ - │ ├── extracted/ - │ └── processed/ - └── transformers/ - ├── default/ - │ ├── something/ - ├── bert-base-cased/ - │ ├── default/ - │ └── training/ - hub/ - └── models--julien-c--EsperBERTo-small/ - ├── blobs/ - │ ├── (...) - │ ├── (...) - ├── refs/ - │ └── (...) - └── [ 128] snapshots/ - ├── 2439f60ef33a0d46d85da5001d52aeda5b00ce9f/ - │ ├── (...) - └── bbc77c8132af1cc5cf678da3f1ddf2de43606d48/ - └── (...) - ``` - - - Args: - library_name (`str`): - Name of the library that will manage the cache folder. Example: `"dataset"`. - namespace (`str`, *optional*, defaults to "default"): - Namespace to which the data belongs. Example: `"SQuAD"`. - subfolder (`str`, *optional*, defaults to "default"): - Subfolder in which the data will be stored. Example: `extracted`. - assets_dir (`str`, `Path`, *optional*): - Path to the folder where assets are cached. This must not be the same folder - where Hub files are cached. Defaults to `HF_HOME / "assets"` if not provided. - Can also be set with `HF_ASSETS_CACHE` environment variable. - - Returns: - Path to the cache folder (`Path`). - - Example: - ```py - >>> from huggingface_hub import cached_assets_path - - >>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="download") - PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/download') - - >>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="extracted") - PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/extracted') - - >>> cached_assets_path(library_name="datasets", namespace="Helsinki-NLP/tatoeba_mt") - PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/Helsinki-NLP--tatoeba_mt/default') - - >>> cached_assets_path(library_name="datasets", assets_dir="/tmp/tmp123456") - PosixPath('/tmp/tmp123456/datasets/default/default') - ``` - """ - # Resolve assets_dir - if assets_dir is None: - assets_dir = HF_ASSETS_CACHE - assets_dir = Path(assets_dir).expanduser().resolve() - - # Avoid names that could create path issues - for part in (" ", "/", "\\"): - library_name = library_name.replace(part, "--") - namespace = namespace.replace(part, "--") - subfolder = subfolder.replace(part, "--") - - # Path to subfolder is created - path = assets_dir / library_name / namespace / subfolder - try: - path.mkdir(exist_ok=True, parents=True) - except (FileExistsError, NotADirectoryError): - raise ValueError(f"Corrupted assets folder: cannot create directory because of an existing file ({path}).") - - # Return - return path diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_cache_manager.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_cache_manager.py deleted file mode 100644 index 90d0e01f74812c5c3e65ba9313a155ee8e517927..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_cache_manager.py +++ /dev/null @@ -1,866 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains utilities to manage the HF cache directory.""" - -import os -import shutil -import time -from collections import defaultdict -from dataclasses import dataclass -from pathlib import Path -from typing import Dict, FrozenSet, List, Literal, Optional, Set, Union - -from huggingface_hub.errors import CacheNotFound, CorruptedCacheException - -from ..commands._cli_utils import tabulate -from ..constants import HF_HUB_CACHE -from . import logging - - -logger = logging.get_logger(__name__) - -REPO_TYPE_T = Literal["model", "dataset", "space"] - -# List of OS-created helper files that need to be ignored -FILES_TO_IGNORE = [".DS_Store"] - - -@dataclass(frozen=True) -class CachedFileInfo: - """Frozen data structure holding information about a single cached file. - - Args: - file_name (`str`): - Name of the file. Example: `config.json`. - file_path (`Path`): - Path of the file in the `snapshots` directory. The file path is a symlink - referring to a blob in the `blobs` folder. - blob_path (`Path`): - Path of the blob file. This is equivalent to `file_path.resolve()`. - size_on_disk (`int`): - Size of the blob file in bytes. - blob_last_accessed (`float`): - Timestamp of the last time the blob file has been accessed (from any - revision). - blob_last_modified (`float`): - Timestamp of the last time the blob file has been modified/created. - - > [!WARNING] - > `blob_last_accessed` and `blob_last_modified` reliability can depend on the OS you - > are using. See [python documentation](https://docs.python.org/3/library/os.html#os.stat_result) - > for more details. - """ - - file_name: str - file_path: Path - blob_path: Path - size_on_disk: int - - blob_last_accessed: float - blob_last_modified: float - - @property - def blob_last_accessed_str(self) -> str: - """ - (property) Timestamp of the last time the blob file has been accessed (from any - revision), returned as a human-readable string. - - Example: "2 weeks ago". - """ - return _format_timesince(self.blob_last_accessed) - - @property - def blob_last_modified_str(self) -> str: - """ - (property) Timestamp of the last time the blob file has been modified, returned - as a human-readable string. - - Example: "2 weeks ago". - """ - return _format_timesince(self.blob_last_modified) - - @property - def size_on_disk_str(self) -> str: - """ - (property) Size of the blob file as a human-readable string. - - Example: "42.2K". - """ - return _format_size(self.size_on_disk) - - -@dataclass(frozen=True) -class CachedRevisionInfo: - """Frozen data structure holding information about a revision. - - A revision correspond to a folder in the `snapshots` folder and is populated with - the exact tree structure as the repo on the Hub but contains only symlinks. A - revision can be either referenced by 1 or more `refs` or be "detached" (no refs). - - Args: - commit_hash (`str`): - Hash of the revision (unique). - Example: `"9338f7b671827df886678df2bdd7cc7b4f36dffd"`. - snapshot_path (`Path`): - Path to the revision directory in the `snapshots` folder. It contains the - exact tree structure as the repo on the Hub. - files: (`FrozenSet[CachedFileInfo]`): - Set of [`~CachedFileInfo`] describing all files contained in the snapshot. - refs (`FrozenSet[str]`): - Set of `refs` pointing to this revision. If the revision has no `refs`, it - is considered detached. - Example: `{"main", "2.4.0"}` or `{"refs/pr/1"}`. - size_on_disk (`int`): - Sum of the blob file sizes that are symlink-ed by the revision. - last_modified (`float`): - Timestamp of the last time the revision has been created/modified. - - > [!WARNING] - > `last_accessed` cannot be determined correctly on a single revision as blob files - > are shared across revisions. - - > [!WARNING] - > `size_on_disk` is not necessarily the sum of all file sizes because of possible - > duplicated files. Besides, only blobs are taken into account, not the (negligible) - > size of folders and symlinks. - """ - - commit_hash: str - snapshot_path: Path - size_on_disk: int - files: FrozenSet[CachedFileInfo] - refs: FrozenSet[str] - - last_modified: float - - @property - def last_modified_str(self) -> str: - """ - (property) Timestamp of the last time the revision has been modified, returned - as a human-readable string. - - Example: "2 weeks ago". - """ - return _format_timesince(self.last_modified) - - @property - def size_on_disk_str(self) -> str: - """ - (property) Sum of the blob file sizes as a human-readable string. - - Example: "42.2K". - """ - return _format_size(self.size_on_disk) - - @property - def nb_files(self) -> int: - """ - (property) Total number of files in the revision. - """ - return len(self.files) - - -@dataclass(frozen=True) -class CachedRepoInfo: - """Frozen data structure holding information about a cached repository. - - Args: - repo_id (`str`): - Repo id of the repo on the Hub. Example: `"google/fleurs"`. - repo_type (`Literal["dataset", "model", "space"]`): - Type of the cached repo. - repo_path (`Path`): - Local path to the cached repo. - size_on_disk (`int`): - Sum of the blob file sizes in the cached repo. - nb_files (`int`): - Total number of blob files in the cached repo. - revisions (`FrozenSet[CachedRevisionInfo]`): - Set of [`~CachedRevisionInfo`] describing all revisions cached in the repo. - last_accessed (`float`): - Timestamp of the last time a blob file of the repo has been accessed. - last_modified (`float`): - Timestamp of the last time a blob file of the repo has been modified/created. - - > [!WARNING] - > `size_on_disk` is not necessarily the sum of all revisions sizes because of - > duplicated files. Besides, only blobs are taken into account, not the (negligible) - > size of folders and symlinks. - - > [!WARNING] - > `last_accessed` and `last_modified` reliability can depend on the OS you are using. - > See [python documentation](https://docs.python.org/3/library/os.html#os.stat_result) - > for more details. - """ - - repo_id: str - repo_type: REPO_TYPE_T - repo_path: Path - size_on_disk: int - nb_files: int - revisions: FrozenSet[CachedRevisionInfo] - - last_accessed: float - last_modified: float - - @property - def last_accessed_str(self) -> str: - """ - (property) Last time a blob file of the repo has been accessed, returned as a - human-readable string. - - Example: "2 weeks ago". - """ - return _format_timesince(self.last_accessed) - - @property - def last_modified_str(self) -> str: - """ - (property) Last time a blob file of the repo has been modified, returned as a - human-readable string. - - Example: "2 weeks ago". - """ - return _format_timesince(self.last_modified) - - @property - def size_on_disk_str(self) -> str: - """ - (property) Sum of the blob file sizes as a human-readable string. - - Example: "42.2K". - """ - return _format_size(self.size_on_disk) - - @property - def refs(self) -> Dict[str, CachedRevisionInfo]: - """ - (property) Mapping between `refs` and revision data structures. - """ - return {ref: revision for revision in self.revisions for ref in revision.refs} - - -@dataclass(frozen=True) -class DeleteCacheStrategy: - """Frozen data structure holding the strategy to delete cached revisions. - - This object is not meant to be instantiated programmatically but to be returned by - [`~utils.HFCacheInfo.delete_revisions`]. See documentation for usage example. - - Args: - expected_freed_size (`float`): - Expected freed size once strategy is executed. - blobs (`FrozenSet[Path]`): - Set of blob file paths to be deleted. - refs (`FrozenSet[Path]`): - Set of reference file paths to be deleted. - repos (`FrozenSet[Path]`): - Set of entire repo paths to be deleted. - snapshots (`FrozenSet[Path]`): - Set of snapshots to be deleted (directory of symlinks). - """ - - expected_freed_size: int - blobs: FrozenSet[Path] - refs: FrozenSet[Path] - repos: FrozenSet[Path] - snapshots: FrozenSet[Path] - - @property - def expected_freed_size_str(self) -> str: - """ - (property) Expected size that will be freed as a human-readable string. - - Example: "42.2K". - """ - return _format_size(self.expected_freed_size) - - def execute(self) -> None: - """Execute the defined strategy. - - > [!WARNING] - > If this method is interrupted, the cache might get corrupted. Deletion order is - > implemented so that references and symlinks are deleted before the actual blob - > files. - - > [!WARNING] - > This method is irreversible. If executed, cached files are erased and must be - > downloaded again. - """ - # Deletion order matters. Blobs are deleted in last so that the user can't end - # up in a state where a `ref`` refers to a missing snapshot or a snapshot - # symlink refers to a deleted blob. - - # Delete entire repos - for path in self.repos: - _try_delete_path(path, path_type="repo") - - # Delete snapshot directories - for path in self.snapshots: - _try_delete_path(path, path_type="snapshot") - - # Delete refs files - for path in self.refs: - _try_delete_path(path, path_type="ref") - - # Delete blob files - for path in self.blobs: - _try_delete_path(path, path_type="blob") - - logger.info(f"Cache deletion done. Saved {self.expected_freed_size_str}.") - - -@dataclass(frozen=True) -class HFCacheInfo: - """Frozen data structure holding information about the entire cache-system. - - This data structure is returned by [`scan_cache_dir`] and is immutable. - - Args: - size_on_disk (`int`): - Sum of all valid repo sizes in the cache-system. - repos (`FrozenSet[CachedRepoInfo]`): - Set of [`~CachedRepoInfo`] describing all valid cached repos found on the - cache-system while scanning. - warnings (`List[CorruptedCacheException]`): - List of [`~CorruptedCacheException`] that occurred while scanning the cache. - Those exceptions are captured so that the scan can continue. Corrupted repos - are skipped from the scan. - - > [!WARNING] - > Here `size_on_disk` is equal to the sum of all repo sizes (only blobs). However if - > some cached repos are corrupted, their sizes are not taken into account. - """ - - size_on_disk: int - repos: FrozenSet[CachedRepoInfo] - warnings: List[CorruptedCacheException] - - @property - def size_on_disk_str(self) -> str: - """ - (property) Sum of all valid repo sizes in the cache-system as a human-readable - string. - - Example: "42.2K". - """ - return _format_size(self.size_on_disk) - - def delete_revisions(self, *revisions: str) -> DeleteCacheStrategy: - """Prepare the strategy to delete one or more revisions cached locally. - - Input revisions can be any revision hash. If a revision hash is not found in the - local cache, a warning is thrown but no error is raised. Revisions can be from - different cached repos since hashes are unique across repos, - - Examples: - ```py - >>> from huggingface_hub import scan_cache_dir - >>> cache_info = scan_cache_dir() - >>> delete_strategy = cache_info.delete_revisions( - ... "81fd1d6e7847c99f5862c9fb81387956d99ec7aa" - ... ) - >>> print(f"Will free {delete_strategy.expected_freed_size_str}.") - Will free 7.9K. - >>> delete_strategy.execute() - Cache deletion done. Saved 7.9K. - ``` - - ```py - >>> from huggingface_hub import scan_cache_dir - >>> scan_cache_dir().delete_revisions( - ... "81fd1d6e7847c99f5862c9fb81387956d99ec7aa", - ... "e2983b237dccf3ab4937c97fa717319a9ca1a96d", - ... "6c0e6080953db56375760c0471a8c5f2929baf11", - ... ).execute() - Cache deletion done. Saved 8.6G. - ``` - - > [!WARNING] - > `delete_revisions` returns a [`~utils.DeleteCacheStrategy`] object that needs to - > be executed. The [`~utils.DeleteCacheStrategy`] is not meant to be modified but - > allows having a dry run before actually executing the deletion. - """ - hashes_to_delete: Set[str] = set(revisions) - - repos_with_revisions: Dict[CachedRepoInfo, Set[CachedRevisionInfo]] = defaultdict(set) - - for repo in self.repos: - for revision in repo.revisions: - if revision.commit_hash in hashes_to_delete: - repos_with_revisions[repo].add(revision) - hashes_to_delete.remove(revision.commit_hash) - - if len(hashes_to_delete) > 0: - logger.warning(f"Revision(s) not found - cannot delete them: {', '.join(hashes_to_delete)}") - - delete_strategy_blobs: Set[Path] = set() - delete_strategy_refs: Set[Path] = set() - delete_strategy_repos: Set[Path] = set() - delete_strategy_snapshots: Set[Path] = set() - delete_strategy_expected_freed_size = 0 - - for affected_repo, revisions_to_delete in repos_with_revisions.items(): - other_revisions = affected_repo.revisions - revisions_to_delete - - # If no other revisions, it means all revisions are deleted - # -> delete the entire cached repo - if len(other_revisions) == 0: - delete_strategy_repos.add(affected_repo.repo_path) - delete_strategy_expected_freed_size += affected_repo.size_on_disk - continue - - # Some revisions of the repo will be deleted but not all. We need to filter - # which blob files will not be linked anymore. - for revision_to_delete in revisions_to_delete: - # Snapshot dir - delete_strategy_snapshots.add(revision_to_delete.snapshot_path) - - # Refs dir - for ref in revision_to_delete.refs: - delete_strategy_refs.add(affected_repo.repo_path / "refs" / ref) - - # Blobs dir - for file in revision_to_delete.files: - if file.blob_path not in delete_strategy_blobs: - is_file_alone = True - for revision in other_revisions: - for rev_file in revision.files: - if file.blob_path == rev_file.blob_path: - is_file_alone = False - break - if not is_file_alone: - break - - # Blob file not referenced by remaining revisions -> delete - if is_file_alone: - delete_strategy_blobs.add(file.blob_path) - delete_strategy_expected_freed_size += file.size_on_disk - - # Return the strategy instead of executing it. - return DeleteCacheStrategy( - blobs=frozenset(delete_strategy_blobs), - refs=frozenset(delete_strategy_refs), - repos=frozenset(delete_strategy_repos), - snapshots=frozenset(delete_strategy_snapshots), - expected_freed_size=delete_strategy_expected_freed_size, - ) - - def export_as_table(self, *, verbosity: int = 0) -> str: - """Generate a table from the [`HFCacheInfo`] object. - - Pass `verbosity=0` to get a table with a single row per repo, with columns - "repo_id", "repo_type", "size_on_disk", "nb_files", "last_accessed", "last_modified", "refs", "local_path". - - Pass `verbosity=1` to get a table with a row per repo and revision (thus multiple rows can appear for a single repo), with columns - "repo_id", "repo_type", "revision", "size_on_disk", "nb_files", "last_modified", "refs", "local_path". - - Example: - ```py - >>> from huggingface_hub.utils import scan_cache_dir - - >>> hf_cache_info = scan_cache_dir() - HFCacheInfo(...) - - >>> print(hf_cache_info.export_as_table()) - REPO ID REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED LAST_MODIFIED REFS LOCAL PATH - --------------------------------------------------- --------- ------------ -------- ------------- ------------- ---- -------------------------------------------------------------------------------------------------- - roberta-base model 2.7M 5 1 day ago 1 week ago main ~/.cache/huggingface/hub/models--roberta-base - suno/bark model 8.8K 1 1 week ago 1 week ago main ~/.cache/huggingface/hub/models--suno--bark - t5-base model 893.8M 4 4 days ago 7 months ago main ~/.cache/huggingface/hub/models--t5-base - t5-large model 3.0G 4 5 weeks ago 5 months ago main ~/.cache/huggingface/hub/models--t5-large - - >>> print(hf_cache_info.export_as_table(verbosity=1)) - REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED REFS LOCAL PATH - --------------------------------------------------- --------- ---------------------------------------- ------------ -------- ------------- ---- ----------------------------------------------------------------------------------------------------------------------------------------------------- - roberta-base model e2da8e2f811d1448a5b465c236feacd80ffbac7b 2.7M 5 1 week ago main ~/.cache/huggingface/hub/models--roberta-base/snapshots/e2da8e2f811d1448a5b465c236feacd80ffbac7b - suno/bark model 70a8a7d34168586dc5d028fa9666aceade177992 8.8K 1 1 week ago main ~/.cache/huggingface/hub/models--suno--bark/snapshots/70a8a7d34168586dc5d028fa9666aceade177992 - t5-base model a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 893.8M 4 7 months ago main ~/.cache/huggingface/hub/models--t5-base/snapshots/a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 - t5-large model 150ebc2c4b72291e770f58e6057481c8d2ed331a 3.0G 4 5 months ago main ~/.cache/huggingface/hub/models--t5-large/snapshots/150ebc2c4b72291e770f58e6057481c8d2ed331a - ``` - - Args: - verbosity (`int`, *optional*): - The verbosity level. Defaults to 0. - - Returns: - `str`: The table as a string. - """ - if verbosity == 0: - return tabulate( - rows=[ - [ - repo.repo_id, - repo.repo_type, - "{:>12}".format(repo.size_on_disk_str), - repo.nb_files, - repo.last_accessed_str, - repo.last_modified_str, - ", ".join(sorted(repo.refs)), - str(repo.repo_path), - ] - for repo in sorted(self.repos, key=lambda repo: repo.repo_path) - ], - headers=[ - "REPO ID", - "REPO TYPE", - "SIZE ON DISK", - "NB FILES", - "LAST_ACCESSED", - "LAST_MODIFIED", - "REFS", - "LOCAL PATH", - ], - ) - else: - return tabulate( - rows=[ - [ - repo.repo_id, - repo.repo_type, - revision.commit_hash, - "{:>12}".format(revision.size_on_disk_str), - revision.nb_files, - revision.last_modified_str, - ", ".join(sorted(revision.refs)), - str(revision.snapshot_path), - ] - for repo in sorted(self.repos, key=lambda repo: repo.repo_path) - for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash) - ], - headers=[ - "REPO ID", - "REPO TYPE", - "REVISION", - "SIZE ON DISK", - "NB FILES", - "LAST_MODIFIED", - "REFS", - "LOCAL PATH", - ], - ) - - -def scan_cache_dir(cache_dir: Optional[Union[str, Path]] = None) -> HFCacheInfo: - """Scan the entire HF cache-system and return a [`~HFCacheInfo`] structure. - - Use `scan_cache_dir` in order to programmatically scan your cache-system. The cache - will be scanned repo by repo. If a repo is corrupted, a [`~CorruptedCacheException`] - will be thrown internally but captured and returned in the [`~HFCacheInfo`] - structure. Only valid repos get a proper report. - - ```py - >>> from huggingface_hub import scan_cache_dir - - >>> hf_cache_info = scan_cache_dir() - HFCacheInfo( - size_on_disk=3398085269, - repos=frozenset({ - CachedRepoInfo( - repo_id='t5-small', - repo_type='model', - repo_path=PosixPath(...), - size_on_disk=970726914, - nb_files=11, - revisions=frozenset({ - CachedRevisionInfo( - commit_hash='d78aea13fa7ecd06c29e3e46195d6341255065d5', - size_on_disk=970726339, - snapshot_path=PosixPath(...), - files=frozenset({ - CachedFileInfo( - file_name='config.json', - size_on_disk=1197 - file_path=PosixPath(...), - blob_path=PosixPath(...), - ), - CachedFileInfo(...), - ... - }), - ), - CachedRevisionInfo(...), - ... - }), - ), - CachedRepoInfo(...), - ... - }), - warnings=[ - CorruptedCacheException("Snapshots dir doesn't exist in cached repo: ..."), - CorruptedCacheException(...), - ... - ], - ) - ``` - - You can also print a detailed report directly from the `hf` command line using: - ```text - > hf cache scan - REPO ID REPO TYPE SIZE ON DISK NB FILES REFS LOCAL PATH - --------------------------- --------- ------------ -------- ------------------- ------------------------------------------------------------------------- - glue dataset 116.3K 15 1.17.0, main, 2.4.0 /Users/lucain/.cache/huggingface/hub/datasets--glue - google/fleurs dataset 64.9M 6 main, refs/pr/1 /Users/lucain/.cache/huggingface/hub/datasets--google--fleurs - Jean-Baptiste/camembert-ner model 441.0M 7 main /Users/lucain/.cache/huggingface/hub/models--Jean-Baptiste--camembert-ner - bert-base-cased model 1.9G 13 main /Users/lucain/.cache/huggingface/hub/models--bert-base-cased - t5-base model 10.1K 3 main /Users/lucain/.cache/huggingface/hub/models--t5-base - t5-small model 970.7M 11 refs/pr/1, main /Users/lucain/.cache/huggingface/hub/models--t5-small - - Done in 0.0s. Scanned 6 repo(s) for a total of 3.4G. - Got 1 warning(s) while scanning. Use -vvv to print details. - ``` - - Args: - cache_dir (`str` or `Path`, `optional`): - Cache directory to cache. Defaults to the default HF cache directory. - - > [!WARNING] - > Raises: - > - > `CacheNotFound` - > If the cache directory does not exist. - > - > [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - > If the cache directory is a file, instead of a directory. - - Returns: a [`~HFCacheInfo`] object. - """ - if cache_dir is None: - cache_dir = HF_HUB_CACHE - - cache_dir = Path(cache_dir).expanduser().resolve() - if not cache_dir.exists(): - raise CacheNotFound( - f"Cache directory not found: {cache_dir}. Please use `cache_dir` argument or set `HF_HUB_CACHE` environment variable.", - cache_dir=cache_dir, - ) - - if cache_dir.is_file(): - raise ValueError( - f"Scan cache expects a directory but found a file: {cache_dir}. Please use `cache_dir` argument or set `HF_HUB_CACHE` environment variable." - ) - - repos: Set[CachedRepoInfo] = set() - warnings: List[CorruptedCacheException] = [] - for repo_path in cache_dir.iterdir(): - if repo_path.name == ".locks": # skip './.locks/' folder - continue - try: - repos.add(_scan_cached_repo(repo_path)) - except CorruptedCacheException as e: - warnings.append(e) - - return HFCacheInfo( - repos=frozenset(repos), - size_on_disk=sum(repo.size_on_disk for repo in repos), - warnings=warnings, - ) - - -def _scan_cached_repo(repo_path: Path) -> CachedRepoInfo: - """Scan a single cache repo and return information about it. - - Any unexpected behavior will raise a [`~CorruptedCacheException`]. - """ - if not repo_path.is_dir(): - raise CorruptedCacheException(f"Repo path is not a directory: {repo_path}") - - if "--" not in repo_path.name: - raise CorruptedCacheException(f"Repo path is not a valid HuggingFace cache directory: {repo_path}") - - repo_type, repo_id = repo_path.name.split("--", maxsplit=1) - repo_type = repo_type[:-1] # "models" -> "model" - repo_id = repo_id.replace("--", "/") # google/fleurs -> "google/fleurs" - - if repo_type not in {"dataset", "model", "space"}: - raise CorruptedCacheException( - f"Repo type must be `dataset`, `model` or `space`, found `{repo_type}` ({repo_path})." - ) - - blob_stats: Dict[Path, os.stat_result] = {} # Key is blob_path, value is blob stats - - snapshots_path = repo_path / "snapshots" - refs_path = repo_path / "refs" - - if not snapshots_path.exists() or not snapshots_path.is_dir(): - raise CorruptedCacheException(f"Snapshots dir doesn't exist in cached repo: {snapshots_path}") - - # Scan over `refs` directory - - # key is revision hash, value is set of refs - refs_by_hash: Dict[str, Set[str]] = defaultdict(set) - if refs_path.exists(): - # Example of `refs` directory - # ── refs - # ├── main - # └── refs - # └── pr - # └── 1 - if refs_path.is_file(): - raise CorruptedCacheException(f"Refs directory cannot be a file: {refs_path}") - - for ref_path in refs_path.glob("**/*"): - # glob("**/*") iterates over all files and directories -> skip directories - if ref_path.is_dir() or ref_path.name in FILES_TO_IGNORE: - continue - - ref_name = str(ref_path.relative_to(refs_path)) - with ref_path.open() as f: - commit_hash = f.read() - - refs_by_hash[commit_hash].add(ref_name) - - # Scan snapshots directory - cached_revisions: Set[CachedRevisionInfo] = set() - for revision_path in snapshots_path.iterdir(): - # Ignore OS-created helper files - if revision_path.name in FILES_TO_IGNORE: - continue - if revision_path.is_file(): - raise CorruptedCacheException(f"Snapshots folder corrupted. Found a file: {revision_path}") - - cached_files = set() - for file_path in revision_path.glob("**/*"): - # glob("**/*") iterates over all files and directories -> skip directories - if file_path.is_dir(): - continue - - blob_path = Path(file_path).resolve() - if not blob_path.exists(): - raise CorruptedCacheException(f"Blob missing (broken symlink): {blob_path}") - - if blob_path not in blob_stats: - blob_stats[blob_path] = blob_path.stat() - - cached_files.add( - CachedFileInfo( - file_name=file_path.name, - file_path=file_path, - size_on_disk=blob_stats[blob_path].st_size, - blob_path=blob_path, - blob_last_accessed=blob_stats[blob_path].st_atime, - blob_last_modified=blob_stats[blob_path].st_mtime, - ) - ) - - # Last modified is either the last modified blob file or the revision folder - # itself if it is empty - if len(cached_files) > 0: - revision_last_modified = max(blob_stats[file.blob_path].st_mtime for file in cached_files) - else: - revision_last_modified = revision_path.stat().st_mtime - - cached_revisions.add( - CachedRevisionInfo( - commit_hash=revision_path.name, - files=frozenset(cached_files), - refs=frozenset(refs_by_hash.pop(revision_path.name, set())), - size_on_disk=sum( - blob_stats[blob_path].st_size for blob_path in set(file.blob_path for file in cached_files) - ), - snapshot_path=revision_path, - last_modified=revision_last_modified, - ) - ) - - # Check that all refs referred to an existing revision - if len(refs_by_hash) > 0: - raise CorruptedCacheException( - f"Reference(s) refer to missing commit hashes: {dict(refs_by_hash)} ({repo_path})." - ) - - # Last modified is either the last modified blob file or the repo folder itself if - # no blob files has been found. Same for last accessed. - if len(blob_stats) > 0: - repo_last_accessed = max(stat.st_atime for stat in blob_stats.values()) - repo_last_modified = max(stat.st_mtime for stat in blob_stats.values()) - else: - repo_stats = repo_path.stat() - repo_last_accessed = repo_stats.st_atime - repo_last_modified = repo_stats.st_mtime - - # Build and return frozen structure - return CachedRepoInfo( - nb_files=len(blob_stats), - repo_id=repo_id, - repo_path=repo_path, - repo_type=repo_type, # type: ignore - revisions=frozenset(cached_revisions), - size_on_disk=sum(stat.st_size for stat in blob_stats.values()), - last_accessed=repo_last_accessed, - last_modified=repo_last_modified, - ) - - -def _format_size(num: int) -> str: - """Format size in bytes into a human-readable string. - - Taken from https://stackoverflow.com/a/1094933 - """ - num_f = float(num) - for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: - if abs(num_f) < 1000.0: - return f"{num_f:3.1f}{unit}" - num_f /= 1000.0 - return f"{num_f:.1f}Y" - - -_TIMESINCE_CHUNKS = ( - # Label, divider, max value - ("second", 1, 60), - ("minute", 60, 60), - ("hour", 60 * 60, 24), - ("day", 60 * 60 * 24, 6), - ("week", 60 * 60 * 24 * 7, 6), - ("month", 60 * 60 * 24 * 30, 11), - ("year", 60 * 60 * 24 * 365, None), -) - - -def _format_timesince(ts: float) -> str: - """Format timestamp in seconds into a human-readable string, relative to now. - - Vaguely inspired by Django's `timesince` formatter. - """ - delta = time.time() - ts - if delta < 20: - return "a few seconds ago" - for label, divider, max_value in _TIMESINCE_CHUNKS: # noqa: B007 - value = round(delta / divider) - if max_value is not None and value <= max_value: - break - return f"{value} {label}{'s' if value > 1 else ''} ago" - - -def _try_delete_path(path: Path, path_type: str) -> None: - """Try to delete a local file or folder. - - If the path does not exists, error is logged as a warning and then ignored. - - Args: - path (`Path`) - Path to delete. Can be a file or a folder. - path_type (`str`) - What path are we deleting ? Only for logging purposes. Example: "snapshot". - """ - logger.info(f"Delete {path_type}: {path}") - try: - if path.is_file(): - os.remove(path) - else: - shutil.rmtree(path) - except FileNotFoundError: - logger.warning(f"Couldn't delete {path_type}: file not found ({path})", exc_info=True) - except PermissionError: - logger.warning(f"Couldn't delete {path_type}: permission denied ({path})", exc_info=True) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_chunk_utils.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_chunk_utils.py deleted file mode 100644 index fe8ecc9c94f9c09503761e734a005124d3291a52..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_chunk_utils.py +++ /dev/null @@ -1,64 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains a utility to iterate by chunks over an iterator.""" - -import itertools -from typing import Iterable, TypeVar - - -T = TypeVar("T") - - -def chunk_iterable(iterable: Iterable[T], chunk_size: int) -> Iterable[Iterable[T]]: - """Iterates over an iterator chunk by chunk. - - Taken from https://stackoverflow.com/a/8998040. - See also https://github.com/huggingface/huggingface_hub/pull/920#discussion_r938793088. - - Args: - iterable (`Iterable`): - The iterable on which we want to iterate. - chunk_size (`int`): - Size of the chunks. Must be a strictly positive integer (e.g. >0). - - Example: - - ```python - >>> from huggingface_hub.utils import chunk_iterable - - >>> for items in chunk_iterable(range(17), chunk_size=8): - ... print(items) - # [0, 1, 2, 3, 4, 5, 6, 7] - # [8, 9, 10, 11, 12, 13, 14, 15] - # [16] # smaller last chunk - ``` - - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If `chunk_size` <= 0. - - > [!WARNING] - > The last chunk can be smaller than `chunk_size`. - """ - if not isinstance(chunk_size, int) or chunk_size <= 0: - raise ValueError("`chunk_size` must be a strictly positive integer (>0).") - - iterator = iter(iterable) - while True: - try: - next_item = next(iterator) - except StopIteration: - return - yield itertools.chain((next_item,), itertools.islice(iterator, chunk_size - 1)) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_datetime.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_datetime.py deleted file mode 100644 index 1a7f44285d1c826006c97176ca66c3e9c33f61c0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_datetime.py +++ /dev/null @@ -1,67 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains utilities to handle datetimes in Huggingface Hub.""" - -from datetime import datetime, timezone - - -def parse_datetime(date_string: str) -> datetime: - """ - Parses a date_string returned from the server to a datetime object. - - This parser is a weak-parser is the sense that it handles only a single format of - date_string. It is expected that the server format will never change. The - implementation depends only on the standard lib to avoid an external dependency - (python-dateutil). See full discussion about this decision on PR: - https://github.com/huggingface/huggingface_hub/pull/999. - - Example: - ```py - > parse_datetime('2022-08-19T07:19:38.123Z') - datetime.datetime(2022, 8, 19, 7, 19, 38, 123000, tzinfo=timezone.utc) - ``` - - Args: - date_string (`str`): - A string representing a datetime returned by the Hub server. - String is expected to follow '%Y-%m-%dT%H:%M:%S.%fZ' pattern. - - Returns: - A python datetime object. - - Raises: - :class:`ValueError`: - If `date_string` cannot be parsed. - """ - try: - # Normalize the string to always have 6 digits of fractional seconds - if date_string.endswith("Z"): - # Case 1: No decimal point (e.g., "2024-11-16T00:27:02Z") - if "." not in date_string: - # No fractional seconds - insert .000000 - date_string = date_string[:-1] + ".000000Z" - # Case 2: Has decimal point (e.g., "2022-08-19T07:19:38.123456789Z") - else: - # Get the fractional and base parts - base, fraction = date_string[:-1].split(".") - # fraction[:6] takes first 6 digits and :0<6 pads with zeros if less than 6 digits - date_string = f"{base}.{fraction[:6]:0<6}Z" - - return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc) - except ValueError as e: - raise ValueError( - f"Cannot parse '{date_string}' as a datetime. Date string is expected to" - " follow '%Y-%m-%dT%H:%M:%S.%fZ' pattern." - ) from e diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_deprecation.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_deprecation.py deleted file mode 100644 index 4cb8d6e418c76accd1ecd61158b4bdd265e12f71..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_deprecation.py +++ /dev/null @@ -1,136 +0,0 @@ -import warnings -from functools import wraps -from inspect import Parameter, signature -from typing import Iterable, Optional - - -def _deprecate_positional_args(*, version: str): - """Decorator for methods that issues warnings for positional arguments. - Using the keyword-only argument syntax in pep 3102, arguments after the - * will issue a warning when passed as a positional argument. - - Args: - version (`str`): - The version when positional arguments will result in error. - """ - - def _inner_deprecate_positional_args(f): - sig = signature(f) - kwonly_args = [] - all_args = [] - for name, param in sig.parameters.items(): - if param.kind == Parameter.POSITIONAL_OR_KEYWORD: - all_args.append(name) - elif param.kind == Parameter.KEYWORD_ONLY: - kwonly_args.append(name) - - @wraps(f) - def inner_f(*args, **kwargs): - extra_args = len(args) - len(all_args) - if extra_args <= 0: - return f(*args, **kwargs) - # extra_args > 0 - args_msg = [ - f"{name}='{arg}'" if isinstance(arg, str) else f"{name}={arg}" - for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:]) - ] - args_msg = ", ".join(args_msg) - warnings.warn( - f"Deprecated positional argument(s) used in '{f.__name__}': pass" - f" {args_msg} as keyword args. From version {version} passing these" - " as positional arguments will result in an error,", - FutureWarning, - ) - kwargs.update(zip(sig.parameters, args)) - return f(**kwargs) - - return inner_f - - return _inner_deprecate_positional_args - - -def _deprecate_arguments( - *, - version: str, - deprecated_args: Iterable[str], - custom_message: Optional[str] = None, -): - """Decorator to issue warnings when using deprecated arguments. - - TODO: could be useful to be able to set a custom error message. - - Args: - version (`str`): - The version when deprecated arguments will result in error. - deprecated_args (`List[str]`): - List of the arguments to be deprecated. - custom_message (`str`, *optional*): - Warning message that is raised. If not passed, a default warning message - will be created. - """ - - def _inner_deprecate_positional_args(f): - sig = signature(f) - - @wraps(f) - def inner_f(*args, **kwargs): - # Check for used deprecated arguments - used_deprecated_args = [] - for _, parameter in zip(args, sig.parameters.values()): - if parameter.name in deprecated_args: - used_deprecated_args.append(parameter.name) - for kwarg_name, kwarg_value in kwargs.items(): - if ( - # If argument is deprecated but still used - kwarg_name in deprecated_args - # And then the value is not the default value - and kwarg_value != sig.parameters[kwarg_name].default - ): - used_deprecated_args.append(kwarg_name) - - # Warn and proceed - if len(used_deprecated_args) > 0: - message = ( - f"Deprecated argument(s) used in '{f.__name__}':" - f" {', '.join(used_deprecated_args)}. Will not be supported from" - f" version '{version}'." - ) - if custom_message is not None: - message += "\n\n" + custom_message - warnings.warn(message, FutureWarning) - return f(*args, **kwargs) - - return inner_f - - return _inner_deprecate_positional_args - - -def _deprecate_method(*, version: str, message: Optional[str] = None): - """Decorator to issue warnings when using a deprecated method. - - Args: - version (`str`): - The version when deprecated arguments will result in error. - message (`str`, *optional*): - Warning message that is raised. If not passed, a default warning message - will be created. - """ - - def _inner_deprecate_method(f): - name = f.__name__ - if name == "__init__": - name = f.__qualname__.split(".")[0] # class name instead of method name - - @wraps(f) - def inner_f(*args, **kwargs): - warning_message = ( - f"'{name}' (from '{f.__module__}') is deprecated and will be removed from version '{version}'." - ) - if message is not None: - warning_message += " " + message - warnings.warn(warning_message, FutureWarning) - return f(*args, **kwargs) - - return inner_f - - return _inner_deprecate_method diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_dotenv.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_dotenv.py deleted file mode 100644 index 23b8a1b70a4827fc8ae4149c2b1b1e4b00ed7ca2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_dotenv.py +++ /dev/null @@ -1,55 +0,0 @@ -# AI-generated module (ChatGPT) -import re -from typing import Dict, Optional - - -def load_dotenv(dotenv_str: str, environ: Optional[Dict[str, str]] = None) -> Dict[str, str]: - """ - Parse a DOTENV-format string and return a dictionary of key-value pairs. - Handles quoted values, comments, export keyword, and blank lines. - """ - env: Dict[str, str] = {} - line_pattern = re.compile( - r""" - ^\s* - (?:export[^\S\n]+)? # optional export - ([A-Za-z_][A-Za-z0-9_]*) # key - [^\S\n]*(=)?[^\S\n]* - ( # value group - (?: - '(?:\\'|[^'])*' # single-quoted value - | \"(?:\\\"|[^\"])*\" # double-quoted value - | [^#\n\r]+? # unquoted value - ) - )? - [^\S\n]*(?:\#.*)?$ # optional inline comment - """, - re.VERBOSE, - ) - - for line in dotenv_str.splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue # Skip comments and empty lines - - match = line_pattern.match(line) - if match: - key = match.group(1) - val = None - if match.group(2): # if there is '=' - raw_val = match.group(3) or "" - val = raw_val.strip() - # Remove surrounding quotes if quoted - if (val.startswith('"') and val.endswith('"')) or (val.startswith("'") and val.endswith("'")): - val = val[1:-1] - val = val.replace(r"\n", "\n").replace(r"\t", "\t").replace(r"\"", '"').replace(r"\\", "\\") - if raw_val.startswith('"'): - val = val.replace(r"\$", "$") # only in double quotes - elif environ is not None: - # Get it from the current environment - val = environ.get(key) - - if val is not None: - env[key] = val - - return env diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_experimental.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_experimental.py deleted file mode 100644 index 40b0ed90ff8af6797758d59b93019498cd72f9ad..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_experimental.py +++ /dev/null @@ -1,68 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains utilities to flag a feature as "experimental" in Huggingface Hub.""" - -import warnings -from functools import wraps -from typing import Callable - -from .. import constants - - -def experimental(fn: Callable) -> Callable: - """Decorator to flag a feature as experimental. - - An experimental feature triggers a warning when used as it might be subject to breaking changes without prior notice - in the future. - - Warnings can be disabled by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment variable. - - Args: - fn (`Callable`): - The function to flag as experimental. - - Returns: - `Callable`: The decorated function. - - Example: - - ```python - >>> from huggingface_hub.utils import experimental - - >>> @experimental - ... def my_function(): - ... print("Hello world!") - - >>> my_function() - UserWarning: 'my_function' is experimental and might be subject to breaking changes in the future without prior - notice. You can disable this warning by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment variable. - Hello world! - ``` - """ - # For classes, put the "experimental" around the "__new__" method => __new__ will be removed in warning message - name = fn.__qualname__[: -len(".__new__")] if fn.__qualname__.endswith(".__new__") else fn.__qualname__ - - @wraps(fn) - def _inner_fn(*args, **kwargs): - if not constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING: - warnings.warn( - f"'{name}' is experimental and might be subject to breaking changes in the future without prior notice." - " You can disable this warning by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment" - " variable.", - UserWarning, - ) - return fn(*args, **kwargs) - - return _inner_fn diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_fixes.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_fixes.py deleted file mode 100644 index 560003b6222058b03791491b1ce70ea9d7a94404..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_fixes.py +++ /dev/null @@ -1,133 +0,0 @@ -# JSONDecodeError was introduced in requests=2.27 released in 2022. -# This allows us to support older requests for users -# More information: https://github.com/psf/requests/pull/5856 -try: - from requests import JSONDecodeError # type: ignore # noqa: F401 -except ImportError: - try: - from simplejson import JSONDecodeError # type: ignore # noqa: F401 - except ImportError: - from json import JSONDecodeError # type: ignore # noqa: F401 -import contextlib -import os -import shutil -import stat -import tempfile -import time -from functools import partial -from pathlib import Path -from typing import Callable, Generator, Optional, Union - -import yaml -from filelock import BaseFileLock, FileLock, SoftFileLock, Timeout - -from .. import constants -from . import logging - - -logger = logging.get_logger(__name__) - -# Wrap `yaml.dump` to set `allow_unicode=True` by default. -# -# Example: -# ```py -# >>> yaml.dump({"emoji": "👀", "some unicode": "日本か"}) -# 'emoji: "\\U0001F440"\nsome unicode: "\\u65E5\\u672C\\u304B"\n' -# -# >>> yaml_dump({"emoji": "👀", "some unicode": "日本か"}) -# 'emoji: "👀"\nsome unicode: "日本か"\n' -# ``` -yaml_dump: Callable[..., str] = partial(yaml.dump, stream=None, allow_unicode=True) # type: ignore - - -@contextlib.contextmanager -def SoftTemporaryDirectory( - suffix: Optional[str] = None, - prefix: Optional[str] = None, - dir: Optional[Union[Path, str]] = None, - **kwargs, -) -> Generator[Path, None, None]: - """ - Context manager to create a temporary directory and safely delete it. - - If tmp directory cannot be deleted normally, we set the WRITE permission and retry. - If cleanup still fails, we give up but don't raise an exception. This is equivalent - to `tempfile.TemporaryDirectory(..., ignore_cleanup_errors=True)` introduced in - Python 3.10. - - See https://www.scivision.dev/python-tempfile-permission-error-windows/. - """ - tmpdir = tempfile.TemporaryDirectory(prefix=prefix, suffix=suffix, dir=dir, **kwargs) - yield Path(tmpdir.name).resolve() - - try: - # First once with normal cleanup - shutil.rmtree(tmpdir.name) - except Exception: - # If failed, try to set write permission and retry - try: - shutil.rmtree(tmpdir.name, onerror=_set_write_permission_and_retry) - except Exception: - pass - - # And finally, cleanup the tmpdir. - # If it fails again, give up but do not throw error - try: - tmpdir.cleanup() - except Exception: - pass - - -def _set_write_permission_and_retry(func, path, excinfo): - os.chmod(path, stat.S_IWRITE) - func(path) - - -@contextlib.contextmanager -def WeakFileLock( - lock_file: Union[str, Path], *, timeout: Optional[float] = None -) -> Generator[BaseFileLock, None, None]: - """A filelock with some custom logic. - - This filelock is weaker than the default filelock in that: - 1. It won't raise an exception if release fails. - 2. It will default to a SoftFileLock if the filesystem does not support flock. - - An INFO log message is emitted every 10 seconds if the lock is not acquired immediately. - If a timeout is provided, a `filelock.Timeout` exception is raised if the lock is not acquired within the timeout. - """ - log_interval = constants.FILELOCK_LOG_EVERY_SECONDS - lock = FileLock(lock_file, timeout=log_interval) - start_time = time.time() - - while True: - elapsed_time = time.time() - start_time - if timeout is not None and elapsed_time >= timeout: - raise Timeout(str(lock_file)) - - try: - lock.acquire(timeout=min(log_interval, timeout - elapsed_time) if timeout else log_interval) - except Timeout: - logger.info( - f"Still waiting to acquire lock on {lock_file} (elapsed: {time.time() - start_time:.1f} seconds)" - ) - except NotImplementedError as e: - if "use SoftFileLock instead" in str(e): - logger.warning( - "FileSystem does not appear to support flock. Falling back to SoftFileLock for %s", lock_file - ) - lock = SoftFileLock(lock_file, timeout=log_interval) - continue - else: - break - - try: - yield lock - finally: - try: - lock.release() - except OSError: - try: - Path(lock_file).unlink() - except OSError: - pass diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_git_credential.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_git_credential.py deleted file mode 100644 index 5ad84648a0093de6e6defc178e4ffffe985f50e4..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_git_credential.py +++ /dev/null @@ -1,121 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains utilities to manage Git credentials.""" - -import re -import subprocess -from typing import List, Optional - -from ..constants import ENDPOINT -from ._subprocess import run_interactive_subprocess, run_subprocess - - -GIT_CREDENTIAL_REGEX = re.compile( - r""" - ^\s* # start of line - credential\.helper # credential.helper value - \s*=\s* # separator - ([\w\-\/]+) # the helper name or absolute path (group 1) - (\s|$) # whitespace or end of line - """, - flags=re.MULTILINE | re.IGNORECASE | re.VERBOSE, -) - - -def list_credential_helpers(folder: Optional[str] = None) -> List[str]: - """Return the list of git credential helpers configured. - - See https://git-scm.com/docs/gitcredentials. - - Credentials are saved in all configured helpers (store, cache, macOS keychain,...). - Calls "`git credential approve`" internally. See https://git-scm.com/docs/git-credential. - - Args: - folder (`str`, *optional*): - The folder in which to check the configured helpers. - """ - try: - output = run_subprocess("git config --list", folder=folder).stdout - parsed = _parse_credential_output(output) - return parsed - except subprocess.CalledProcessError as exc: - raise EnvironmentError(exc.stderr) - - -def set_git_credential(token: str, username: str = "hf_user", folder: Optional[str] = None) -> None: - """Save a username/token pair in git credential for HF Hub registry. - - Credentials are saved in all configured helpers (store, cache, macOS keychain,...). - Calls "`git credential approve`" internally. See https://git-scm.com/docs/git-credential. - - Args: - username (`str`, defaults to `"hf_user"`): - A git username. Defaults to `"hf_user"`, the default user used in the Hub. - token (`str`, defaults to `"hf_user"`): - A git password. In practice, the User Access Token for the Hub. - See https://huggingface.co/settings/tokens. - folder (`str`, *optional*): - The folder in which to check the configured helpers. - """ - with run_interactive_subprocess("git credential approve", folder=folder) as ( - stdin, - _, - ): - stdin.write(f"url={ENDPOINT}\nusername={username.lower()}\npassword={token}\n\n") - stdin.flush() - - -def unset_git_credential(username: str = "hf_user", folder: Optional[str] = None) -> None: - """Erase credentials from git credential for HF Hub registry. - - Credentials are erased from the configured helpers (store, cache, macOS - keychain,...), if any. If `username` is not provided, any credential configured for - HF Hub endpoint is erased. - Calls "`git credential erase`" internally. See https://git-scm.com/docs/git-credential. - - Args: - username (`str`, defaults to `"hf_user"`): - A git username. Defaults to `"hf_user"`, the default user used in the Hub. - folder (`str`, *optional*): - The folder in which to check the configured helpers. - """ - with run_interactive_subprocess("git credential reject", folder=folder) as ( - stdin, - _, - ): - standard_input = f"url={ENDPOINT}\n" - if username is not None: - standard_input += f"username={username.lower()}\n" - standard_input += "\n" - - stdin.write(standard_input) - stdin.flush() - - -def _parse_credential_output(output: str) -> List[str]: - """Parse the output of `git credential fill` to extract the password. - - Args: - output (`str`): - The output of `git credential fill`. - """ - # NOTE: If user has set an helper for a custom URL, it will not we caught here. - # Example: `credential.https://huggingface.co.helper=store` - # See: https://github.com/huggingface/huggingface_hub/pull/1138#discussion_r1013324508 - return sorted( # Sort for nice printing - set( # Might have some duplicates - match[0] for match in GIT_CREDENTIAL_REGEX.findall(output) - ) - ) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_headers.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_headers.py deleted file mode 100644 index 053a92a398f8734ee14cd67e4b514dfc350fcecd..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_headers.py +++ /dev/null @@ -1,228 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains utilities to handle headers to send in calls to Huggingface Hub.""" - -from typing import Dict, Optional, Union - -from huggingface_hub.errors import LocalTokenNotFoundError - -from .. import constants -from ._auth import get_token -from ._deprecation import _deprecate_arguments -from ._runtime import ( - get_fastai_version, - get_fastcore_version, - get_hf_hub_version, - get_python_version, - get_tf_version, - get_torch_version, - is_fastai_available, - is_fastcore_available, - is_tf_available, - is_torch_available, -) -from ._validators import validate_hf_hub_args - - -@_deprecate_arguments( - version="1.0", - deprecated_args="is_write_action", - custom_message="This argument is ignored and we let the server handle the permission error instead (if any).", -) -@validate_hf_hub_args -def build_hf_headers( - *, - token: Optional[Union[bool, str]] = None, - library_name: Optional[str] = None, - library_version: Optional[str] = None, - user_agent: Union[Dict, str, None] = None, - headers: Optional[Dict[str, str]] = None, - is_write_action: bool = False, -) -> Dict[str, str]: - """ - Build headers dictionary to send in a HF Hub call. - - By default, authorization token is always provided either from argument (explicit - use) or retrieved from the cache (implicit use). To explicitly avoid sending the - token to the Hub, set `token=False` or set the `HF_HUB_DISABLE_IMPLICIT_TOKEN` - environment variable. - - In case of an API call that requires write access, an error is thrown if token is - `None` or token is an organization token (starting with `"api_org***"`). - - In addition to the auth header, a user-agent is added to provide information about - the installed packages (versions of python, huggingface_hub, torch, tensorflow, - fastai and fastcore). - - Args: - token (`str`, `bool`, *optional*): - The token to be sent in authorization header for the Hub call: - - if a string, it is used as the Hugging Face token - - if `True`, the token is read from the machine (cache or env variable) - - if `False`, authorization header is not set - - if `None`, the token is read from the machine only except if - `HF_HUB_DISABLE_IMPLICIT_TOKEN` env variable is set. - library_name (`str`, *optional*): - The name of the library that is making the HTTP request. Will be added to - the user-agent header. - library_version (`str`, *optional*): - The version of the library that is making the HTTP request. Will be added - to the user-agent header. - user_agent (`str`, `dict`, *optional*): - The user agent info in the form of a dictionary or a single string. It will - be completed with information about the installed packages. - headers (`dict`, *optional*): - Additional headers to include in the request. Those headers take precedence - over the ones generated by this function. - is_write_action (`bool`): - Ignored and deprecated argument. - - Returns: - A `Dict` of headers to pass in your API call. - - Example: - ```py - >>> build_hf_headers(token="hf_***") # explicit token - {"authorization": "Bearer hf_***", "user-agent": ""} - - >>> build_hf_headers(token=True) # explicitly use cached token - {"authorization": "Bearer hf_***",...} - - >>> build_hf_headers(token=False) # explicitly don't use cached token - {"user-agent": ...} - - >>> build_hf_headers() # implicit use of the cached token - {"authorization": "Bearer hf_***",...} - - # HF_HUB_DISABLE_IMPLICIT_TOKEN=True # to set as env variable - >>> build_hf_headers() # token is not sent - {"user-agent": ...} - - >>> build_hf_headers(library_name="transformers", library_version="1.2.3") - {"authorization": ..., "user-agent": "transformers/1.2.3; hf_hub/0.10.2; python/3.10.4; tensorflow/1.55"} - ``` - - Raises: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If organization token is passed and "write" access is required. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If "write" access is required but token is not passed and not saved locally. - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) - If `token=True` but token is not saved locally. - """ - # Get auth token to send - token_to_send = get_token_to_send(token) - - # Combine headers - hf_headers = { - "user-agent": _http_user_agent( - library_name=library_name, - library_version=library_version, - user_agent=user_agent, - ) - } - if token_to_send is not None: - hf_headers["authorization"] = f"Bearer {token_to_send}" - if headers is not None: - hf_headers.update(headers) - return hf_headers - - -def get_token_to_send(token: Optional[Union[bool, str]]) -> Optional[str]: - """Select the token to send from either `token` or the cache.""" - # Case token is explicitly provided - if isinstance(token, str): - return token - - # Case token is explicitly forbidden - if token is False: - return None - - # Token is not provided: we get it from local cache - cached_token = get_token() - - # Case token is explicitly required - if token is True: - if cached_token is None: - raise LocalTokenNotFoundError( - "Token is required (`token=True`), but no token found. You" - " need to provide a token or be logged in to Hugging Face with" - " `hf auth login` or `huggingface_hub.login`. See" - " https://huggingface.co/settings/tokens." - ) - return cached_token - - # Case implicit use of the token is forbidden by env variable - if constants.HF_HUB_DISABLE_IMPLICIT_TOKEN: - return None - - # Otherwise: we use the cached token as the user has not explicitly forbidden it - return cached_token - - -def _http_user_agent( - *, - library_name: Optional[str] = None, - library_version: Optional[str] = None, - user_agent: Union[Dict, str, None] = None, -) -> str: - """Format a user-agent string containing information about the installed packages. - - Args: - library_name (`str`, *optional*): - The name of the library that is making the HTTP request. - library_version (`str`, *optional*): - The version of the library that is making the HTTP request. - user_agent (`str`, `dict`, *optional*): - The user agent info in the form of a dictionary or a single string. - - Returns: - The formatted user-agent string. - """ - if library_name is not None: - ua = f"{library_name}/{library_version}" - else: - ua = "unknown/None" - ua += f"; hf_hub/{get_hf_hub_version()}" - ua += f"; python/{get_python_version()}" - - if not constants.HF_HUB_DISABLE_TELEMETRY: - if is_torch_available(): - ua += f"; torch/{get_torch_version()}" - if is_tf_available(): - ua += f"; tensorflow/{get_tf_version()}" - if is_fastai_available(): - ua += f"; fastai/{get_fastai_version()}" - if is_fastcore_available(): - ua += f"; fastcore/{get_fastcore_version()}" - - if isinstance(user_agent, dict): - ua += "; " + "; ".join(f"{k}/{v}" for k, v in user_agent.items()) - elif isinstance(user_agent, str): - ua += "; " + user_agent - - # Retrieve user-agent origin headers from environment variable - origin = constants.HF_HUB_USER_AGENT_ORIGIN - if origin is not None: - ua += "; origin/" + origin - - return _deduplicate_user_agent(ua) - - -def _deduplicate_user_agent(user_agent: str) -> str: - """Deduplicate redundant information in the generated user-agent.""" - # Split around ";" > Strip whitespaces > Store as dict keys (ensure unicity) > format back as string - # Order is implicitly preserved by dictionary structure (see https://stackoverflow.com/a/53657523). - return "; ".join({key.strip(): None for key in user_agent.split(";")}.keys()) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_hf_folder.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_hf_folder.py deleted file mode 100644 index 6418bf2fd2c59b4bcf301c1dd82bc468f2f42ddf..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_hf_folder.py +++ /dev/null @@ -1,68 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contain helper class to retrieve/store token from/to local cache.""" - -from pathlib import Path -from typing import Optional - -from .. import constants -from ._auth import get_token - - -class HfFolder: - # TODO: deprecate when adapted in transformers/datasets/gradio - # @_deprecate_method(version="1.0", message="Use `huggingface_hub.login` instead.") - @classmethod - def save_token(cls, token: str) -> None: - """ - Save token, creating folder as needed. - - Token is saved in the huggingface home folder. You can configure it by setting - the `HF_HOME` environment variable. - - Args: - token (`str`): - The token to save to the [`HfFolder`] - """ - path_token = Path(constants.HF_TOKEN_PATH) - path_token.parent.mkdir(parents=True, exist_ok=True) - path_token.write_text(token) - - # TODO: deprecate when adapted in transformers/datasets/gradio - # @_deprecate_method(version="1.0", message="Use `huggingface_hub.get_token` instead.") - @classmethod - def get_token(cls) -> Optional[str]: - """ - Get token or None if not existent. - - This method is deprecated in favor of [`huggingface_hub.get_token`] but is kept for backward compatibility. - Its behavior is the same as [`huggingface_hub.get_token`]. - - Returns: - `str` or `None`: The token, `None` if it doesn't exist. - """ - return get_token() - - # TODO: deprecate when adapted in transformers/datasets/gradio - # @_deprecate_method(version="1.0", message="Use `huggingface_hub.logout` instead.") - @classmethod - def delete_token(cls) -> None: - """ - Deletes the token from storage. Does not fail if token does not exist. - """ - try: - Path(constants.HF_TOKEN_PATH).unlink() - except FileNotFoundError: - pass diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_http.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_http.py deleted file mode 100644 index 3471031b34b15efe9d5fc76077eac467bbb03500..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_http.py +++ /dev/null @@ -1,638 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains utilities to handle HTTP requests in Huggingface Hub.""" - -import io -import os -import re -import threading -import time -import uuid -from functools import lru_cache -from shlex import quote -from typing import Any, Callable, List, Optional, Tuple, Type, Union - -import requests -from requests import HTTPError, Response -from requests.adapters import HTTPAdapter -from requests.models import PreparedRequest - -from huggingface_hub.errors import OfflineModeIsEnabled - -from .. import constants -from ..errors import ( - BadRequestError, - DisabledRepoError, - EntryNotFoundError, - GatedRepoError, - HfHubHTTPError, - RepositoryNotFoundError, - RevisionNotFoundError, -) -from . import logging -from ._fixes import JSONDecodeError -from ._lfs import SliceFileObj -from ._typing import HTTP_METHOD_T - - -logger = logging.get_logger(__name__) - -# Both headers are used by the Hub to debug failed requests. -# `X_AMZN_TRACE_ID` is better as it also works to debug on Cloudfront and ALB. -# If `X_AMZN_TRACE_ID` is set, the Hub will use it as well. -X_AMZN_TRACE_ID = "X-Amzn-Trace-Id" -X_REQUEST_ID = "x-request-id" -X_AMZ_CF_ID = "x-amz-cf-id" - -REPO_API_REGEX = re.compile( - r""" - # staging or production endpoint - ^https://[^/]+ - ( - # on /api/repo_type/repo_id - /api/(models|datasets|spaces)/(.+) - | - # or /repo_id/resolve/revision/... - /(.+)/resolve/(.+) - ) - """, - flags=re.VERBOSE, -) - - -class UniqueRequestIdAdapter(HTTPAdapter): - X_AMZN_TRACE_ID = "X-Amzn-Trace-Id" - - def add_headers(self, request, **kwargs): - super().add_headers(request, **kwargs) - - # Add random request ID => easier for server-side debug - if X_AMZN_TRACE_ID not in request.headers: - request.headers[X_AMZN_TRACE_ID] = request.headers.get(X_REQUEST_ID) or str(uuid.uuid4()) - - # Add debug log - has_token = len(str(request.headers.get("authorization", ""))) > 0 - logger.debug( - f"Request {request.headers[X_AMZN_TRACE_ID]}: {request.method} {request.url} (authenticated: {has_token})" - ) - - def send(self, request: PreparedRequest, *args, **kwargs) -> Response: - """Catch any RequestException to append request id to the error message for debugging.""" - if constants.HF_DEBUG: - logger.debug(f"Send: {_curlify(request)}") - try: - return super().send(request, *args, **kwargs) - except requests.RequestException as e: - request_id = request.headers.get(X_AMZN_TRACE_ID) - if request_id is not None: - # Taken from https://stackoverflow.com/a/58270258 - e.args = (*e.args, f"(Request ID: {request_id})") - raise - - -class OfflineAdapter(HTTPAdapter): - def send(self, request: PreparedRequest, *args, **kwargs) -> Response: - raise OfflineModeIsEnabled( - f"Cannot reach {request.url}: offline mode is enabled. To disable it, please unset the `HF_HUB_OFFLINE` environment variable." - ) - - -def _default_backend_factory() -> requests.Session: - session = requests.Session() - if constants.HF_HUB_OFFLINE: - session.mount("http://", OfflineAdapter()) - session.mount("https://", OfflineAdapter()) - else: - session.mount("http://", UniqueRequestIdAdapter()) - session.mount("https://", UniqueRequestIdAdapter()) - return session - - -BACKEND_FACTORY_T = Callable[[], requests.Session] -_GLOBAL_BACKEND_FACTORY: BACKEND_FACTORY_T = _default_backend_factory - - -def configure_http_backend(backend_factory: BACKEND_FACTORY_T = _default_backend_factory) -> None: - """ - Configure the HTTP backend by providing a `backend_factory`. Any HTTP calls made by `huggingface_hub` will use a - Session object instantiated by this factory. This can be useful if you are running your scripts in a specific - environment requiring custom configuration (e.g. custom proxy or certifications). - - Use [`get_session`] to get a configured Session. Since `requests.Session` is not guaranteed to be thread-safe, - `huggingface_hub` creates 1 Session instance per thread. They are all instantiated using the same `backend_factory` - set in [`configure_http_backend`]. A LRU cache is used to cache the created sessions (and connections) between - calls. Max size is 128 to avoid memory leaks if thousands of threads are spawned. - - See [this issue](https://github.com/psf/requests/issues/2766) to know more about thread-safety in `requests`. - - Example: - ```py - import requests - from huggingface_hub import configure_http_backend, get_session - - # Create a factory function that returns a Session with configured proxies - def backend_factory() -> requests.Session: - session = requests.Session() - session.proxies = {"http": "http://10.10.1.10:3128", "https": "https://10.10.1.11:1080"} - return session - - # Set it as the default session factory - configure_http_backend(backend_factory=backend_factory) - - # In practice, this is mostly done internally in `huggingface_hub` - session = get_session() - ``` - """ - global _GLOBAL_BACKEND_FACTORY - _GLOBAL_BACKEND_FACTORY = backend_factory - reset_sessions() - - -def get_session() -> requests.Session: - """ - Get a `requests.Session` object, using the session factory from the user. - - Use [`get_session`] to get a configured Session. Since `requests.Session` is not guaranteed to be thread-safe, - `huggingface_hub` creates 1 Session instance per thread. They are all instantiated using the same `backend_factory` - set in [`configure_http_backend`]. A LRU cache is used to cache the created sessions (and connections) between - calls. Max size is 128 to avoid memory leaks if thousands of threads are spawned. - - See [this issue](https://github.com/psf/requests/issues/2766) to know more about thread-safety in `requests`. - - Example: - ```py - import requests - from huggingface_hub import configure_http_backend, get_session - - # Create a factory function that returns a Session with configured proxies - def backend_factory() -> requests.Session: - session = requests.Session() - session.proxies = {"http": "http://10.10.1.10:3128", "https": "https://10.10.1.11:1080"} - return session - - # Set it as the default session factory - configure_http_backend(backend_factory=backend_factory) - - # In practice, this is mostly done internally in `huggingface_hub` - session = get_session() - ``` - """ - return _get_session_from_cache(process_id=os.getpid(), thread_id=threading.get_ident()) - - -def reset_sessions() -> None: - """Reset the cache of sessions. - - Mostly used internally when sessions are reconfigured or an SSLError is raised. - See [`configure_http_backend`] for more details. - """ - _get_session_from_cache.cache_clear() - - -@lru_cache -def _get_session_from_cache(process_id: int, thread_id: int) -> requests.Session: - """ - Create a new session per thread using global factory. Using LRU cache (maxsize 128) to avoid memory leaks when - using thousands of threads. Cache is cleared when `configure_http_backend` is called. - """ - return _GLOBAL_BACKEND_FACTORY() - - -def http_backoff( - method: HTTP_METHOD_T, - url: str, - *, - max_retries: int = 5, - base_wait_time: float = 1, - max_wait_time: float = 8, - retry_on_exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]] = ( - requests.Timeout, - requests.ConnectionError, - requests.exceptions.ChunkedEncodingError, - ), - retry_on_status_codes: Union[int, Tuple[int, ...]] = (500, 502, 503, 504), - **kwargs, -) -> Response: - """Wrapper around requests to retry calls on an endpoint, with exponential backoff. - - Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...) - and/or on specific status codes (ex: service unavailable). If the call failed more - than `max_retries`, the exception is thrown or `raise_for_status` is called on the - response object. - - Re-implement mechanisms from the `backoff` library to avoid adding an external - dependencies to `hugging_face_hub`. See https://github.com/litl/backoff. - - Args: - method (`Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]`): - HTTP method to perform. - url (`str`): - The URL of the resource to fetch. - max_retries (`int`, *optional*, defaults to `5`): - Maximum number of retries, defaults to 5 (no retries). - base_wait_time (`float`, *optional*, defaults to `1`): - Duration (in seconds) to wait before retrying the first time. - Wait time between retries then grows exponentially, capped by - `max_wait_time`. - max_wait_time (`float`, *optional*, defaults to `8`): - Maximum duration (in seconds) to wait before retrying. - retry_on_exceptions (`Type[Exception]` or `Tuple[Type[Exception]]`, *optional*): - Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types. - By default, retry on `requests.Timeout`, `requests.ConnectionError` and `requests.exceptions.ChunkedEncodingError`. - retry_on_status_codes (`int` or `Tuple[int]`, *optional*, defaults to `(500, 502, 503, 504)`): - Define on which status codes the request must be retried. By default, 5xx errors are retried. - **kwargs (`dict`, *optional*): - kwargs to pass to `requests.request`. - - Example: - ``` - >>> from huggingface_hub.utils import http_backoff - - # Same usage as "requests.request". - >>> response = http_backoff("GET", "https://www.google.com") - >>> response.raise_for_status() - - # If you expect a Gateway Timeout from time to time - >>> http_backoff("PUT", upload_url, data=data, retry_on_status_codes=504) - >>> response.raise_for_status() - ``` - - > [!WARNING] - > When using `requests` it is possible to stream data by passing an iterator to the - > `data` argument. On http backoff this is a problem as the iterator is not reset - > after a failed call. This issue is mitigated for file objects or any IO streams - > by saving the initial position of the cursor (with `data.tell()`) and resetting the - > cursor between each call (with `data.seek()`). For arbitrary iterators, http backoff - > will fail. If this is a hard constraint for you, please let us know by opening an - > issue on [Github](https://github.com/huggingface/huggingface_hub). - """ - if isinstance(retry_on_exceptions, type): # Tuple from single exception type - retry_on_exceptions = (retry_on_exceptions,) - - if isinstance(retry_on_status_codes, int): # Tuple from single status code - retry_on_status_codes = (retry_on_status_codes,) - - nb_tries = 0 - sleep_time = base_wait_time - - # If `data` is used and is a file object (or any IO), it will be consumed on the - # first HTTP request. We need to save the initial position so that the full content - # of the file is re-sent on http backoff. See warning tip in docstring. - io_obj_initial_pos = None - if "data" in kwargs and isinstance(kwargs["data"], (io.IOBase, SliceFileObj)): - io_obj_initial_pos = kwargs["data"].tell() - - session = get_session() - while True: - nb_tries += 1 - try: - # If `data` is used and is a file object (or any IO), set back cursor to - # initial position. - if io_obj_initial_pos is not None: - kwargs["data"].seek(io_obj_initial_pos) - - # Perform request and return if status_code is not in the retry list. - response = session.request(method=method, url=url, **kwargs) - if response.status_code not in retry_on_status_codes: - return response - - # Wrong status code returned (HTTP 503 for instance) - logger.warning(f"HTTP Error {response.status_code} thrown while requesting {method} {url}") - if nb_tries > max_retries: - response.raise_for_status() # Will raise uncaught exception - # We return response to avoid infinite loop in the corner case where the - # user ask for retry on a status code that doesn't raise_for_status. - return response - - except retry_on_exceptions as err: - logger.warning(f"'{err}' thrown while requesting {method} {url}") - - if isinstance(err, requests.ConnectionError): - reset_sessions() # In case of SSLError it's best to reset the shared requests.Session objects - - if nb_tries > max_retries: - raise err - - # Sleep for X seconds - logger.warning(f"Retrying in {sleep_time}s [Retry {nb_tries}/{max_retries}].") - time.sleep(sleep_time) - - # Update sleep time for next retry - sleep_time = min(max_wait_time, sleep_time * 2) # Exponential backoff - - -def fix_hf_endpoint_in_url(url: str, endpoint: Optional[str]) -> str: - """Replace the default endpoint in a URL by a custom one. - - This is useful when using a proxy and the Hugging Face Hub returns a URL with the default endpoint. - """ - endpoint = endpoint.rstrip("/") if endpoint else constants.ENDPOINT - # check if a proxy has been set => if yes, update the returned URL to use the proxy - if endpoint not in (constants._HF_DEFAULT_ENDPOINT, constants._HF_DEFAULT_STAGING_ENDPOINT): - url = url.replace(constants._HF_DEFAULT_ENDPOINT, endpoint) - url = url.replace(constants._HF_DEFAULT_STAGING_ENDPOINT, endpoint) - return url - - -def hf_raise_for_status(response: Response, endpoint_name: Optional[str] = None) -> None: - """ - Internal version of `response.raise_for_status()` that will refine a - potential HTTPError. Raised exception will be an instance of `HfHubHTTPError`. - - This helper is meant to be the unique method to raise_for_status when making a call - to the Hugging Face Hub. - - - Example: - ```py - import requests - from huggingface_hub.utils import get_session, hf_raise_for_status, HfHubHTTPError - - response = get_session().post(...) - try: - hf_raise_for_status(response) - except HfHubHTTPError as e: - print(str(e)) # formatted message - e.request_id, e.server_message # details returned by server - - # Complete the error message with additional information once it's raised - e.append_to_message("\n`create_commit` expects the repository to exist.") - raise - ``` - - Args: - response (`Response`): - Response from the server. - endpoint_name (`str`, *optional*): - Name of the endpoint that has been called. If provided, the error message - will be more complete. - - > [!WARNING] - > Raises when the request has failed: - > - > - [`~utils.RepositoryNotFoundError`] - > If the repository to download from cannot be found. This may be because it - > doesn't exist, because `repo_type` is not set correctly, or because the repo - > is `private` and you do not have access. - > - [`~utils.GatedRepoError`] - > If the repository exists but is gated and the user is not on the authorized - > list. - > - [`~utils.RevisionNotFoundError`] - > If the repository exists but the revision couldn't be find. - > - [`~utils.EntryNotFoundError`] - > If the repository exists but the entry (e.g. the requested file) couldn't be - > find. - > - [`~utils.BadRequestError`] - > If request failed with a HTTP 400 BadRequest error. - > - [`~utils.HfHubHTTPError`] - > If request failed for a reason not listed above. - """ - try: - response.raise_for_status() - except HTTPError as e: - error_code = response.headers.get("X-Error-Code") - error_message = response.headers.get("X-Error-Message") - - if error_code == "RevisionNotFound": - message = f"{response.status_code} Client Error." + "\n\n" + f"Revision Not Found for url: {response.url}." - raise _format(RevisionNotFoundError, message, response) from e - - elif error_code == "EntryNotFound": - message = f"{response.status_code} Client Error." + "\n\n" + f"Entry Not Found for url: {response.url}." - raise _format(EntryNotFoundError, message, response) from e - - elif error_code == "GatedRepo": - message = ( - f"{response.status_code} Client Error." + "\n\n" + f"Cannot access gated repo for url {response.url}." - ) - raise _format(GatedRepoError, message, response) from e - - elif error_message == "Access to this resource is disabled.": - message = ( - f"{response.status_code} Client Error." - + "\n\n" - + f"Cannot access repository for url {response.url}." - + "\n" - + "Access to this resource is disabled." - ) - raise _format(DisabledRepoError, message, response) from e - - elif error_code == "RepoNotFound" or ( - response.status_code == 401 - and error_message != "Invalid credentials in Authorization header" - and response.request is not None - and response.request.url is not None - and REPO_API_REGEX.search(response.request.url) is not None - ): - # 401 is misleading as it is returned for: - # - private and gated repos if user is not authenticated - # - missing repos - # => for now, we process them as `RepoNotFound` anyway. - # See https://gist.github.com/Wauplin/46c27ad266b15998ce56a6603796f0b9 - message = ( - f"{response.status_code} Client Error." - + "\n\n" - + f"Repository Not Found for url: {response.url}." - + "\nPlease make sure you specified the correct `repo_id` and" - " `repo_type`.\nIf you are trying to access a private or gated repo," - " make sure you are authenticated. For more details, see" - " https://huggingface.co/docs/huggingface_hub/authentication" - ) - raise _format(RepositoryNotFoundError, message, response) from e - - elif response.status_code == 400: - message = ( - f"\n\nBad request for {endpoint_name} endpoint:" if endpoint_name is not None else "\n\nBad request:" - ) - raise _format(BadRequestError, message, response) from e - - elif response.status_code == 403: - message = ( - f"\n\n{response.status_code} Forbidden: {error_message}." - + f"\nCannot access content at: {response.url}." - + "\nMake sure your token has the correct permissions." - ) - raise _format(HfHubHTTPError, message, response) from e - - elif response.status_code == 416: - range_header = response.request.headers.get("Range") - message = f"{e}. Requested range: {range_header}. Content-Range: {response.headers.get('Content-Range')}." - raise _format(HfHubHTTPError, message, response) from e - - # Convert `HTTPError` into a `HfHubHTTPError` to display request information - # as well (request id and/or server error message) - raise _format(HfHubHTTPError, str(e), response) from e - - -def _format(error_type: Type[HfHubHTTPError], custom_message: str, response: Response) -> HfHubHTTPError: - server_errors = [] - - # Retrieve server error from header - from_headers = response.headers.get("X-Error-Message") - if from_headers is not None: - server_errors.append(from_headers) - - # Retrieve server error from body - try: - # Case errors are returned in a JSON format - data = response.json() - - error = data.get("error") - if error is not None: - if isinstance(error, list): - # Case {'error': ['my error 1', 'my error 2']} - server_errors.extend(error) - else: - # Case {'error': 'my error'} - server_errors.append(error) - - errors = data.get("errors") - if errors is not None: - # Case {'errors': [{'message': 'my error 1'}, {'message': 'my error 2'}]} - for error in errors: - if "message" in error: - server_errors.append(error["message"]) - - except JSONDecodeError: - # If content is not JSON and not HTML, append the text - content_type = response.headers.get("Content-Type", "") - if response.text and "html" not in content_type.lower(): - server_errors.append(response.text) - - # Strip all server messages - server_errors = [str(line).strip() for line in server_errors if str(line).strip()] - - # Deduplicate server messages (keep order) - # taken from https://stackoverflow.com/a/17016257 - server_errors = list(dict.fromkeys(server_errors)) - - # Format server error - server_message = "\n".join(server_errors) - - # Add server error to custom message - final_error_message = custom_message - if server_message and server_message.lower() not in custom_message.lower(): - if "\n\n" in custom_message: - final_error_message += "\n" + server_message - else: - final_error_message += "\n\n" + server_message - - # Prepare Request ID message - request_id = "" - request_id_message = "" - for header, label in ( - (X_REQUEST_ID, "Request ID"), - (X_AMZN_TRACE_ID, "Amzn Trace ID"), - (X_AMZ_CF_ID, "Amz CF ID"), - ): - value = response.headers.get(header) - if value: - request_id = str(value) - request_id_message = f" ({label}: {value})" - break - - # Add Request ID - if request_id and request_id.lower() not in final_error_message.lower(): - if "\n" in final_error_message: - newline_index = final_error_message.index("\n") - final_error_message = ( - final_error_message[:newline_index] + request_id_message + final_error_message[newline_index:] - ) - else: - final_error_message += request_id_message - - # Return - return error_type(final_error_message.strip(), response=response, server_message=server_message or None) - - -def _curlify(request: requests.PreparedRequest) -> str: - """Convert a `requests.PreparedRequest` into a curl command (str). - - Used for debug purposes only. - - Implementation vendored from https://github.com/ofw/curlify/blob/master/curlify.py. - MIT License Copyright (c) 2016 Egor. - """ - parts: List[Tuple[Any, Any]] = [ - ("curl", None), - ("-X", request.method), - ] - - for k, v in sorted(request.headers.items()): - if k.lower() == "authorization": - v = "" # Hide authorization header, no matter its value (can be Bearer, Key, etc.) - parts += [("-H", "{0}: {1}".format(k, v))] - - if request.body: - body = request.body - if isinstance(body, bytes): - body = body.decode("utf-8", errors="ignore") - elif hasattr(body, "read"): - body = "" # Don't try to read it to avoid consuming the stream - if len(body) > 1000: - body = body[:1000] + " ... [truncated]" - parts += [("-d", body.replace("\n", ""))] - - parts += [(None, request.url)] - - flat_parts = [] - for k, v in parts: - if k: - flat_parts.append(quote(k)) - if v: - flat_parts.append(quote(v)) - - return " ".join(flat_parts) - - -# Regex to parse HTTP Range header -RANGE_REGEX = re.compile(r"^\s*bytes\s*=\s*(\d*)\s*-\s*(\d*)\s*$", re.IGNORECASE) - - -def _adjust_range_header(original_range: Optional[str], resume_size: int) -> Optional[str]: - """ - Adjust HTTP Range header to account for resume position. - """ - if not original_range: - return f"bytes={resume_size}-" - - if "," in original_range: - raise ValueError(f"Multiple ranges detected - {original_range!r}, not supported yet.") - - match = RANGE_REGEX.match(original_range) - if not match: - raise RuntimeError(f"Invalid range format - {original_range!r}.") - start, end = match.groups() - - if not start: - if not end: - raise RuntimeError(f"Invalid range format - {original_range!r}.") - - new_suffix = int(end) - resume_size - new_range = f"bytes=-{new_suffix}" - if new_suffix <= 0: - raise RuntimeError(f"Empty new range - {new_range!r}.") - return new_range - - start = int(start) - new_start = start + resume_size - if end: - end = int(end) - new_range = f"bytes={new_start}-{end}" - if new_start > end: - raise RuntimeError(f"Empty new range - {new_range!r}.") - return new_range - - return f"bytes={new_start}-" diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_lfs.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_lfs.py deleted file mode 100644 index 307f371ffa79a8ae726ee03458c52e230a792898..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_lfs.py +++ /dev/null @@ -1,110 +0,0 @@ -# coding=utf-8 -# Copyright 2019-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Git LFS related utilities""" - -import io -import os -from contextlib import AbstractContextManager -from typing import BinaryIO - - -class SliceFileObj(AbstractContextManager): - """ - Utility context manager to read a *slice* of a seekable file-like object as a seekable, file-like object. - - This is NOT thread safe - - Inspired by stackoverflow.com/a/29838711/593036 - - Credits to @julien-c - - Args: - fileobj (`BinaryIO`): - A file-like object to slice. MUST implement `tell()` and `seek()` (and `read()` of course). - `fileobj` will be reset to its original position when exiting the context manager. - seek_from (`int`): - The start of the slice (offset from position 0 in bytes). - read_limit (`int`): - The maximum number of bytes to read from the slice. - - Attributes: - previous_position (`int`): - The previous position - - Examples: - - Reading 200 bytes with an offset of 128 bytes from a file (ie bytes 128 to 327): - ```python - >>> with open("path/to/file", "rb") as file: - ... with SliceFileObj(file, seek_from=128, read_limit=200) as fslice: - ... fslice.read(...) - ``` - - Reading a file in chunks of 512 bytes - ```python - >>> import os - >>> chunk_size = 512 - >>> file_size = os.getsize("path/to/file") - >>> with open("path/to/file", "rb") as file: - ... for chunk_idx in range(ceil(file_size / chunk_size)): - ... with SliceFileObj(file, seek_from=chunk_idx * chunk_size, read_limit=chunk_size) as fslice: - ... chunk = fslice.read(...) - - ``` - """ - - def __init__(self, fileobj: BinaryIO, seek_from: int, read_limit: int): - self.fileobj = fileobj - self.seek_from = seek_from - self.read_limit = read_limit - - def __enter__(self): - self._previous_position = self.fileobj.tell() - end_of_stream = self.fileobj.seek(0, os.SEEK_END) - self._len = min(self.read_limit, end_of_stream - self.seek_from) - # ^^ The actual number of bytes that can be read from the slice - self.fileobj.seek(self.seek_from, io.SEEK_SET) - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.fileobj.seek(self._previous_position, io.SEEK_SET) - - def read(self, n: int = -1): - pos = self.tell() - if pos >= self._len: - return b"" - remaining_amount = self._len - pos - data = self.fileobj.read(remaining_amount if n < 0 else min(n, remaining_amount)) - return data - - def tell(self) -> int: - return self.fileobj.tell() - self.seek_from - - def seek(self, offset: int, whence: int = os.SEEK_SET) -> int: - start = self.seek_from - end = start + self._len - if whence in (os.SEEK_SET, os.SEEK_END): - offset = start + offset if whence == os.SEEK_SET else end + offset - offset = max(start, min(offset, end)) - whence = os.SEEK_SET - elif whence == os.SEEK_CUR: - cur_pos = self.fileobj.tell() - offset = max(start - cur_pos, min(offset, end - cur_pos)) - else: - raise ValueError(f"whence value {whence} is not supported") - return self.fileobj.seek(offset, whence) - self.seek_from - - def __iter__(self): - yield self.read(n=4 * 1024 * 1024) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_pagination.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_pagination.py deleted file mode 100644 index 3ef2b6668ba09d4c6a715509131d157139a1fac0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_pagination.py +++ /dev/null @@ -1,52 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains utilities to handle pagination on Huggingface Hub.""" - -from typing import Dict, Iterable, Optional - -import requests - -from . import get_session, hf_raise_for_status, http_backoff, logging - - -logger = logging.get_logger(__name__) - - -def paginate(path: str, params: Dict, headers: Dict) -> Iterable: - """Fetch a list of models/datasets/spaces and paginate through results. - - This is using the same "Link" header format as GitHub. - See: - - https://requests.readthedocs.io/en/latest/api/#requests.Response.links - - https://docs.github.com/en/rest/guides/traversing-with-pagination#link-header - """ - session = get_session() - r = session.get(path, params=params, headers=headers) - hf_raise_for_status(r) - yield from r.json() - - # Follow pages - # Next link already contains query params - next_page = _get_next_page(r) - while next_page is not None: - logger.debug(f"Pagination detected. Requesting next page: {next_page}") - r = http_backoff("GET", next_page, max_retries=20, retry_on_status_codes=429, headers=headers) - hf_raise_for_status(r) - yield from r.json() - next_page = _get_next_page(r) - - -def _get_next_page(response: requests.Response) -> Optional[str]: - return response.links.get("next", {}).get("url") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_paths.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_paths.py deleted file mode 100644 index 4f2c0ebce070bbde4900e919a3aca7cfc331e747..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_paths.py +++ /dev/null @@ -1,141 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains utilities to handle paths in Huggingface Hub.""" - -from fnmatch import fnmatch -from pathlib import Path -from typing import Callable, Generator, Iterable, List, Optional, TypeVar, Union - - -T = TypeVar("T") - -# Always ignore `.git` and `.cache/huggingface` folders in commits -DEFAULT_IGNORE_PATTERNS = [ - ".git", - ".git/*", - "*/.git", - "**/.git/**", - ".cache/huggingface", - ".cache/huggingface/*", - "*/.cache/huggingface", - "**/.cache/huggingface/**", -] -# Forbidden to commit these folders -FORBIDDEN_FOLDERS = [".git", ".cache"] - - -def filter_repo_objects( - items: Iterable[T], - *, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, - key: Optional[Callable[[T], str]] = None, -) -> Generator[T, None, None]: - """Filter repo objects based on an allowlist and a denylist. - - Input must be a list of paths (`str` or `Path`) or a list of arbitrary objects. - In the later case, `key` must be provided and specifies a function of one argument - that is used to extract a path from each element in iterable. - - Patterns are Unix shell-style wildcards which are NOT regular expressions. See - https://docs.python.org/3/library/fnmatch.html for more details. - - Args: - items (`Iterable`): - List of items to filter. - allow_patterns (`str` or `List[str]`, *optional*): - Patterns constituting the allowlist. If provided, item paths must match at - least one pattern from the allowlist. - ignore_patterns (`str` or `List[str]`, *optional*): - Patterns constituting the denylist. If provided, item paths must not match - any patterns from the denylist. - key (`Callable[[T], str]`, *optional*): - Single-argument function to extract a path from each item. If not provided, - the `items` must already be `str` or `Path`. - - Returns: - Filtered list of objects, as a generator. - - Raises: - :class:`ValueError`: - If `key` is not provided and items are not `str` or `Path`. - - Example usage with paths: - ```python - >>> # Filter only PDFs that are not hidden. - >>> list(filter_repo_objects( - ... ["aaa.PDF", "bbb.jpg", ".ccc.pdf", ".ddd.png"], - ... allow_patterns=["*.pdf"], - ... ignore_patterns=[".*"], - ... )) - ["aaa.pdf"] - ``` - - Example usage with objects: - ```python - >>> list(filter_repo_objects( - ... [ - ... CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf") - ... CommitOperationAdd(path_or_fileobj="/tmp/bbb.jpg", path_in_repo="bbb.jpg") - ... CommitOperationAdd(path_or_fileobj="/tmp/.ccc.pdf", path_in_repo=".ccc.pdf") - ... CommitOperationAdd(path_or_fileobj="/tmp/.ddd.png", path_in_repo=".ddd.png") - ... ], - ... allow_patterns=["*.pdf"], - ... ignore_patterns=[".*"], - ... key=lambda x: x.repo_in_path - ... )) - [CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf")] - ``` - """ - if isinstance(allow_patterns, str): - allow_patterns = [allow_patterns] - - if isinstance(ignore_patterns, str): - ignore_patterns = [ignore_patterns] - - if allow_patterns is not None: - allow_patterns = [_add_wildcard_to_directories(p) for p in allow_patterns] - if ignore_patterns is not None: - ignore_patterns = [_add_wildcard_to_directories(p) for p in ignore_patterns] - - if key is None: - - def _identity(item: T) -> str: - if isinstance(item, str): - return item - if isinstance(item, Path): - return str(item) - raise ValueError(f"Please provide `key` argument in `filter_repo_objects`: `{item}` is not a string.") - - key = _identity # Items must be `str` or `Path`, otherwise raise ValueError - - for item in items: - path = key(item) - - # Skip if there's an allowlist and path doesn't match any - if allow_patterns is not None and not any(fnmatch(path, r) for r in allow_patterns): - continue - - # Skip if there's a denylist and path matches any - if ignore_patterns is not None and any(fnmatch(path, r) for r in ignore_patterns): - continue - - yield item - - -def _add_wildcard_to_directories(pattern: str) -> str: - if pattern[-1] == "/": - return pattern + "*" - return pattern diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_runtime.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_runtime.py deleted file mode 100644 index 9e38e6da7493074703032150b8b7d6766ed0fed6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_runtime.py +++ /dev/null @@ -1,395 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Check presence of installed packages at runtime.""" - -import importlib.metadata -import os -import platform -import sys -import warnings -from typing import Any, Dict - -from .. import __version__, constants - - -_PY_VERSION: str = sys.version.split()[0].rstrip("+") - -_package_versions = {} - -_CANDIDATES = { - "aiohttp": {"aiohttp"}, - "fastai": {"fastai"}, - "fastapi": {"fastapi"}, - "fastcore": {"fastcore"}, - "gradio": {"gradio"}, - "graphviz": {"graphviz"}, - "hf_transfer": {"hf_transfer"}, - "hf_xet": {"hf_xet"}, - "jinja": {"Jinja2"}, - "keras": {"keras"}, - "numpy": {"numpy"}, - "pillow": {"Pillow"}, - "pydantic": {"pydantic"}, - "pydot": {"pydot"}, - "safetensors": {"safetensors"}, - "tensorboard": {"tensorboardX"}, - "tensorflow": ( - "tensorflow", - "tensorflow-cpu", - "tensorflow-gpu", - "tf-nightly", - "tf-nightly-cpu", - "tf-nightly-gpu", - "intel-tensorflow", - "intel-tensorflow-avx512", - "tensorflow-rocm", - "tensorflow-macos", - ), - "torch": {"torch"}, -} - -# Check once at runtime -for candidate_name, package_names in _CANDIDATES.items(): - _package_versions[candidate_name] = "N/A" - for name in package_names: - try: - _package_versions[candidate_name] = importlib.metadata.version(name) - break - except importlib.metadata.PackageNotFoundError: - pass - - -def _get_version(package_name: str) -> str: - return _package_versions.get(package_name, "N/A") - - -def is_package_available(package_name: str) -> bool: - return _get_version(package_name) != "N/A" - - -# Python -def get_python_version() -> str: - return _PY_VERSION - - -# Huggingface Hub -def get_hf_hub_version() -> str: - return __version__ - - -# aiohttp -def is_aiohttp_available() -> bool: - return is_package_available("aiohttp") - - -def get_aiohttp_version() -> str: - return _get_version("aiohttp") - - -# FastAI -def is_fastai_available() -> bool: - return is_package_available("fastai") - - -def get_fastai_version() -> str: - return _get_version("fastai") - - -# FastAPI -def is_fastapi_available() -> bool: - return is_package_available("fastapi") - - -def get_fastapi_version() -> str: - return _get_version("fastapi") - - -# Fastcore -def is_fastcore_available() -> bool: - return is_package_available("fastcore") - - -def get_fastcore_version() -> str: - return _get_version("fastcore") - - -# FastAI -def is_gradio_available() -> bool: - return is_package_available("gradio") - - -def get_gradio_version() -> str: - return _get_version("gradio") - - -# Graphviz -def is_graphviz_available() -> bool: - return is_package_available("graphviz") - - -def get_graphviz_version() -> str: - return _get_version("graphviz") - - -# hf_transfer -def is_hf_transfer_available() -> bool: - return is_package_available("hf_transfer") - - -def get_hf_transfer_version() -> str: - return _get_version("hf_transfer") - - -# xet -def is_xet_available() -> bool: - # since hf_xet is automatically used if available, allow explicit disabling via environment variable - if constants.HF_HUB_DISABLE_XET: - return False - - return is_package_available("hf_xet") - - -def get_xet_version() -> str: - return _get_version("hf_xet") - - -# keras -def is_keras_available() -> bool: - return is_package_available("keras") - - -def get_keras_version() -> str: - return _get_version("keras") - - -# Numpy -def is_numpy_available() -> bool: - return is_package_available("numpy") - - -def get_numpy_version() -> str: - return _get_version("numpy") - - -# Jinja -def is_jinja_available() -> bool: - return is_package_available("jinja") - - -def get_jinja_version() -> str: - return _get_version("jinja") - - -# Pillow -def is_pillow_available() -> bool: - return is_package_available("pillow") - - -def get_pillow_version() -> str: - return _get_version("pillow") - - -# Pydantic -def is_pydantic_available() -> bool: - if not is_package_available("pydantic"): - return False - # For Pydantic, we add an extra check to test whether it is correctly installed or not. If both pydantic 2.x and - # typing_extensions<=4.5.0 are installed, then pydantic will fail at import time. This should not happen when - # it is installed with `pip install huggingface_hub[inference]` but it can happen when it is installed manually - # by the user in an environment that we don't control. - # - # Usually we won't need to do this kind of check on optional dependencies. However, pydantic is a special case - # as it is automatically imported when doing `from huggingface_hub import ...` even if the user doesn't use it. - # - # See https://github.com/huggingface/huggingface_hub/pull/1829 for more details. - try: - from pydantic import validator # noqa: F401 - except ImportError: - # Example: "ImportError: cannot import name 'TypeAliasType' from 'typing_extensions'" - warnings.warn( - "Pydantic is installed but cannot be imported. Please check your installation. `huggingface_hub` will " - "default to not using Pydantic. Error message: '{e}'" - ) - return False - return True - - -def get_pydantic_version() -> str: - return _get_version("pydantic") - - -# Pydot -def is_pydot_available() -> bool: - return is_package_available("pydot") - - -def get_pydot_version() -> str: - return _get_version("pydot") - - -# Tensorboard -def is_tensorboard_available() -> bool: - return is_package_available("tensorboard") - - -def get_tensorboard_version() -> str: - return _get_version("tensorboard") - - -# Tensorflow -def is_tf_available() -> bool: - return is_package_available("tensorflow") - - -def get_tf_version() -> str: - return _get_version("tensorflow") - - -# Torch -def is_torch_available() -> bool: - return is_package_available("torch") - - -def get_torch_version() -> str: - return _get_version("torch") - - -# Safetensors -def is_safetensors_available() -> bool: - return is_package_available("safetensors") - - -# Shell-related helpers -try: - # Set to `True` if script is running in a Google Colab notebook. - # If running in Google Colab, git credential store is set globally which makes the - # warning disappear. See https://github.com/huggingface/huggingface_hub/issues/1043 - # - # Taken from https://stackoverflow.com/a/63519730. - _is_google_colab = "google.colab" in str(get_ipython()) # type: ignore # noqa: F821 -except NameError: - _is_google_colab = False - - -def is_notebook() -> bool: - """Return `True` if code is executed in a notebook (Jupyter, Colab, QTconsole). - - Taken from https://stackoverflow.com/a/39662359. - Adapted to make it work with Google colab as well. - """ - try: - shell_class = get_ipython().__class__ # type: ignore # noqa: F821 - for parent_class in shell_class.__mro__: # e.g. "is subclass of" - if parent_class.__name__ == "ZMQInteractiveShell": - return True # Jupyter notebook, Google colab or qtconsole - return False - except NameError: - return False # Probably standard Python interpreter - - -def is_google_colab() -> bool: - """Return `True` if code is executed in a Google colab. - - Taken from https://stackoverflow.com/a/63519730. - """ - return _is_google_colab - - -def is_colab_enterprise() -> bool: - """Return `True` if code is executed in a Google Colab Enterprise environment.""" - return os.environ.get("VERTEX_PRODUCT") == "COLAB_ENTERPRISE" - - -def dump_environment_info() -> Dict[str, Any]: - """Dump information about the machine to help debugging issues. - - Similar helper exist in: - - `datasets` (https://github.com/huggingface/datasets/blob/main/src/datasets/commands/env.py) - - `diffusers` (https://github.com/huggingface/diffusers/blob/main/src/diffusers/commands/env.py) - - `transformers` (https://github.com/huggingface/transformers/blob/main/src/transformers/commands/env.py) - """ - from huggingface_hub import get_token, whoami - from huggingface_hub.utils import list_credential_helpers - - token = get_token() - - # Generic machine info - info: Dict[str, Any] = { - "huggingface_hub version": get_hf_hub_version(), - "Platform": platform.platform(), - "Python version": get_python_version(), - } - - # Interpreter info - try: - shell_class = get_ipython().__class__ # type: ignore # noqa: F821 - info["Running in iPython ?"] = "Yes" - info["iPython shell"] = shell_class.__name__ - except NameError: - info["Running in iPython ?"] = "No" - info["Running in notebook ?"] = "Yes" if is_notebook() else "No" - info["Running in Google Colab ?"] = "Yes" if is_google_colab() else "No" - info["Running in Google Colab Enterprise ?"] = "Yes" if is_colab_enterprise() else "No" - # Login info - info["Token path ?"] = constants.HF_TOKEN_PATH - info["Has saved token ?"] = token is not None - if token is not None: - try: - info["Who am I ?"] = whoami()["name"] - except Exception: - pass - - try: - info["Configured git credential helpers"] = ", ".join(list_credential_helpers()) - except Exception: - pass - - # Installed dependencies - info["FastAI"] = get_fastai_version() - info["Tensorflow"] = get_tf_version() - info["Torch"] = get_torch_version() - info["Jinja2"] = get_jinja_version() - info["Graphviz"] = get_graphviz_version() - info["keras"] = get_keras_version() - info["Pydot"] = get_pydot_version() - info["Pillow"] = get_pillow_version() - info["hf_transfer"] = get_hf_transfer_version() - info["gradio"] = get_gradio_version() - info["tensorboard"] = get_tensorboard_version() - info["numpy"] = get_numpy_version() - info["pydantic"] = get_pydantic_version() - info["aiohttp"] = get_aiohttp_version() - info["hf_xet"] = get_xet_version() - - # Environment variables - info["ENDPOINT"] = constants.ENDPOINT - info["HF_HUB_CACHE"] = constants.HF_HUB_CACHE - info["HF_ASSETS_CACHE"] = constants.HF_ASSETS_CACHE - info["HF_TOKEN_PATH"] = constants.HF_TOKEN_PATH - info["HF_STORED_TOKENS_PATH"] = constants.HF_STORED_TOKENS_PATH - info["HF_HUB_OFFLINE"] = constants.HF_HUB_OFFLINE - info["HF_HUB_DISABLE_TELEMETRY"] = constants.HF_HUB_DISABLE_TELEMETRY - info["HF_HUB_DISABLE_PROGRESS_BARS"] = constants.HF_HUB_DISABLE_PROGRESS_BARS - info["HF_HUB_DISABLE_SYMLINKS_WARNING"] = constants.HF_HUB_DISABLE_SYMLINKS_WARNING - info["HF_HUB_DISABLE_EXPERIMENTAL_WARNING"] = constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING - info["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = constants.HF_HUB_DISABLE_IMPLICIT_TOKEN - info["HF_HUB_DISABLE_XET"] = constants.HF_HUB_DISABLE_XET - info["HF_HUB_ENABLE_HF_TRANSFER"] = constants.HF_HUB_ENABLE_HF_TRANSFER - info["HF_HUB_ETAG_TIMEOUT"] = constants.HF_HUB_ETAG_TIMEOUT - info["HF_HUB_DOWNLOAD_TIMEOUT"] = constants.HF_HUB_DOWNLOAD_TIMEOUT - - print("\nCopy-and-paste the text below in your GitHub issue.\n") - print("\n".join([f"- {prop}: {val}" for prop, val in info.items()]) + "\n") - return info diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_safetensors.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_safetensors.py deleted file mode 100644 index 38546c6d34db786c62861e1706f747a21b7012bf..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_safetensors.py +++ /dev/null @@ -1,111 +0,0 @@ -import functools -import operator -from collections import defaultdict -from dataclasses import dataclass, field -from typing import Dict, List, Literal, Optional, Tuple - - -FILENAME_T = str -TENSOR_NAME_T = str -DTYPE_T = Literal["F64", "F32", "F16", "BF16", "I64", "I32", "I16", "I8", "U8", "BOOL"] - - -@dataclass -class TensorInfo: - """Information about a tensor. - - For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. - - Attributes: - dtype (`str`): - The data type of the tensor ("F64", "F32", "F16", "BF16", "I64", "I32", "I16", "I8", "U8", "BOOL"). - shape (`List[int]`): - The shape of the tensor. - data_offsets (`Tuple[int, int]`): - The offsets of the data in the file as a tuple `[BEGIN, END]`. - parameter_count (`int`): - The number of parameters in the tensor. - """ - - dtype: DTYPE_T - shape: List[int] - data_offsets: Tuple[int, int] - parameter_count: int = field(init=False) - - def __post_init__(self) -> None: - # Taken from https://stackoverflow.com/a/13840436 - try: - self.parameter_count = functools.reduce(operator.mul, self.shape) - except TypeError: - self.parameter_count = 1 # scalar value has no shape - - -@dataclass -class SafetensorsFileMetadata: - """Metadata for a Safetensors file hosted on the Hub. - - This class is returned by [`parse_safetensors_file_metadata`]. - - For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. - - Attributes: - metadata (`Dict`): - The metadata contained in the file. - tensors (`Dict[str, TensorInfo]`): - A map of all tensors. Keys are tensor names and values are information about the corresponding tensor, as a - [`TensorInfo`] object. - parameter_count (`Dict[str, int]`): - A map of the number of parameters per data type. Keys are data types and values are the number of parameters - of that data type. - """ - - metadata: Dict[str, str] - tensors: Dict[TENSOR_NAME_T, TensorInfo] - parameter_count: Dict[DTYPE_T, int] = field(init=False) - - def __post_init__(self) -> None: - parameter_count: Dict[DTYPE_T, int] = defaultdict(int) - for tensor in self.tensors.values(): - parameter_count[tensor.dtype] += tensor.parameter_count - self.parameter_count = dict(parameter_count) - - -@dataclass -class SafetensorsRepoMetadata: - """Metadata for a Safetensors repo. - - A repo is considered to be a Safetensors repo if it contains either a 'model.safetensors' weight file (non-shared - model) or a 'model.safetensors.index.json' index file (sharded model) at its root. - - This class is returned by [`get_safetensors_metadata`]. - - For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. - - Attributes: - metadata (`Dict`, *optional*): - The metadata contained in the 'model.safetensors.index.json' file, if it exists. Only populated for sharded - models. - sharded (`bool`): - Whether the repo contains a sharded model or not. - weight_map (`Dict[str, str]`): - A map of all weights. Keys are tensor names and values are filenames of the files containing the tensors. - files_metadata (`Dict[str, SafetensorsFileMetadata]`): - A map of all files metadata. Keys are filenames and values are the metadata of the corresponding file, as - a [`SafetensorsFileMetadata`] object. - parameter_count (`Dict[str, int]`): - A map of the number of parameters per data type. Keys are data types and values are the number of parameters - of that data type. - """ - - metadata: Optional[Dict] - sharded: bool - weight_map: Dict[TENSOR_NAME_T, FILENAME_T] # tensor name -> filename - files_metadata: Dict[FILENAME_T, SafetensorsFileMetadata] # filename -> metadata - parameter_count: Dict[DTYPE_T, int] = field(init=False) - - def __post_init__(self) -> None: - parameter_count: Dict[DTYPE_T, int] = defaultdict(int) - for file_metadata in self.files_metadata.values(): - for dtype, nb_parameters_ in file_metadata.parameter_count.items(): - parameter_count[dtype] += nb_parameters_ - self.parameter_count = dict(parameter_count) diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_subprocess.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_subprocess.py deleted file mode 100644 index fdabf1c4df3b61dc610ae08eb7842df6af3552f3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_subprocess.py +++ /dev/null @@ -1,144 +0,0 @@ -# coding=utf-8 -# Copyright 2021 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License -"""Contains utilities to easily handle subprocesses in `huggingface_hub`.""" - -import os -import subprocess -import sys -from contextlib import contextmanager -from io import StringIO -from pathlib import Path -from typing import IO, Generator, List, Optional, Tuple, Union - -from .logging import get_logger - - -logger = get_logger(__name__) - - -@contextmanager -def capture_output() -> Generator[StringIO, None, None]: - """Capture output that is printed to terminal. - - Taken from https://stackoverflow.com/a/34738440 - - Example: - ```py - >>> with capture_output() as output: - ... print("hello world") - >>> assert output.getvalue() == "hello world\n" - ``` - """ - output = StringIO() - previous_output = sys.stdout - sys.stdout = output - try: - yield output - finally: - sys.stdout = previous_output - - -def run_subprocess( - command: Union[str, List[str]], - folder: Optional[Union[str, Path]] = None, - check=True, - **kwargs, -) -> subprocess.CompletedProcess: - """ - Method to run subprocesses. Calling this will capture the `stderr` and `stdout`, - please call `subprocess.run` manually in case you would like for them not to - be captured. - - Args: - command (`str` or `List[str]`): - The command to execute as a string or list of strings. - folder (`str`, *optional*): - The folder in which to run the command. Defaults to current working - directory (from `os.getcwd()`). - check (`bool`, *optional*, defaults to `True`): - Setting `check` to `True` will raise a `subprocess.CalledProcessError` - when the subprocess has a non-zero exit code. - kwargs (`Dict[str]`): - Keyword arguments to be passed to the `subprocess.run` underlying command. - - Returns: - `subprocess.CompletedProcess`: The completed process. - """ - if isinstance(command, str): - command = command.split() - - if isinstance(folder, Path): - folder = str(folder) - - return subprocess.run( - command, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - check=check, - encoding="utf-8", - errors="replace", # if not utf-8, replace char by � - cwd=folder or os.getcwd(), - **kwargs, - ) - - -@contextmanager -def run_interactive_subprocess( - command: Union[str, List[str]], - folder: Optional[Union[str, Path]] = None, - **kwargs, -) -> Generator[Tuple[IO[str], IO[str]], None, None]: - """Run a subprocess in an interactive mode in a context manager. - - Args: - command (`str` or `List[str]`): - The command to execute as a string or list of strings. - folder (`str`, *optional*): - The folder in which to run the command. Defaults to current working - directory (from `os.getcwd()`). - kwargs (`Dict[str]`): - Keyword arguments to be passed to the `subprocess.run` underlying command. - - Returns: - `Tuple[IO[str], IO[str]]`: A tuple with `stdin` and `stdout` to interact - with the process (input and output are utf-8 encoded). - - Example: - ```python - with _interactive_subprocess("git credential-store get") as (stdin, stdout): - # Write to stdin - stdin.write("url=hf.co\nusername=obama\n".encode("utf-8")) - stdin.flush() - - # Read from stdout - output = stdout.read().decode("utf-8") - ``` - """ - if isinstance(command, str): - command = command.split() - - with subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - encoding="utf-8", - errors="replace", # if not utf-8, replace char by � - cwd=folder or os.getcwd(), - **kwargs, - ) as process: - assert process.stdin is not None, "subprocess is opened as subprocess.PIPE" - assert process.stdout is not None, "subprocess is opened as subprocess.PIPE" - yield process.stdin, process.stdout diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_telemetry.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_telemetry.py deleted file mode 100644 index 2ba4a6349a8de1c565263ec73d235d36f88b68cf..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_telemetry.py +++ /dev/null @@ -1,126 +0,0 @@ -from queue import Queue -from threading import Lock, Thread -from typing import Dict, Optional, Union -from urllib.parse import quote - -from .. import constants, logging -from . import build_hf_headers, get_session, hf_raise_for_status - - -logger = logging.get_logger(__name__) - -# Telemetry is sent by a separate thread to avoid blocking the main thread. -# A daemon thread is started once and consume tasks from the _TELEMETRY_QUEUE. -# If the thread stops for some reason -shouldn't happen-, we restart a new one. -_TELEMETRY_THREAD: Optional[Thread] = None -_TELEMETRY_THREAD_LOCK = Lock() # Lock to avoid starting multiple threads in parallel -_TELEMETRY_QUEUE: Queue = Queue() - - -def send_telemetry( - topic: str, - *, - library_name: Optional[str] = None, - library_version: Optional[str] = None, - user_agent: Union[Dict, str, None] = None, -) -> None: - """ - Sends telemetry that helps tracking usage of different HF libraries. - - This usage data helps us debug issues and prioritize new features. However, we understand that not everyone wants - to share additional information, and we respect your privacy. You can disable telemetry collection by setting the - `HF_HUB_DISABLE_TELEMETRY=1` as environment variable. Telemetry is also disabled in offline mode (i.e. when setting - `HF_HUB_OFFLINE=1`). - - Telemetry collection is run in a separate thread to minimize impact for the user. - - Args: - topic (`str`): - Name of the topic that is monitored. The topic is directly used to build the URL. If you want to monitor - subtopics, just use "/" separation. Examples: "gradio", "transformers/examples",... - library_name (`str`, *optional*): - The name of the library that is making the HTTP request. Will be added to the user-agent header. - library_version (`str`, *optional*): - The version of the library that is making the HTTP request. Will be added to the user-agent header. - user_agent (`str`, `dict`, *optional*): - The user agent info in the form of a dictionary or a single string. It will be completed with information about the installed packages. - - Example: - ```py - >>> from huggingface_hub.utils import send_telemetry - - # Send telemetry without library information - >>> send_telemetry("ping") - - # Send telemetry to subtopic with library information - >>> send_telemetry("gradio/local_link", library_name="gradio", library_version="3.22.1") - - # Send telemetry with additional data - >>> send_telemetry( - ... topic="examples", - ... library_name="transformers", - ... library_version="4.26.0", - ... user_agent={"pipeline": "text_classification", "framework": "flax"}, - ... ) - ``` - """ - if constants.HF_HUB_OFFLINE or constants.HF_HUB_DISABLE_TELEMETRY: - return - - _start_telemetry_thread() # starts thread only if doesn't exist yet - _TELEMETRY_QUEUE.put( - {"topic": topic, "library_name": library_name, "library_version": library_version, "user_agent": user_agent} - ) - - -def _start_telemetry_thread(): - """Start a daemon thread to consume tasks from the telemetry queue. - - If the thread is interrupted, start a new one. - """ - with _TELEMETRY_THREAD_LOCK: # avoid to start multiple threads if called concurrently - global _TELEMETRY_THREAD - if _TELEMETRY_THREAD is None or not _TELEMETRY_THREAD.is_alive(): - _TELEMETRY_THREAD = Thread(target=_telemetry_worker, daemon=True) - _TELEMETRY_THREAD.start() - - -def _telemetry_worker(): - """Wait for a task and consume it.""" - while True: - kwargs = _TELEMETRY_QUEUE.get() - _send_telemetry_in_thread(**kwargs) - _TELEMETRY_QUEUE.task_done() - - -def _send_telemetry_in_thread( - topic: str, - *, - library_name: Optional[str] = None, - library_version: Optional[str] = None, - user_agent: Union[Dict, str, None] = None, -) -> None: - """Contains the actual data sending data to the Hub. - - This function is called directly in gradio's analytics because - it is not possible to send telemetry from a daemon thread. - - See here: https://github.com/gradio-app/gradio/pull/8180 - - Please do not rename or remove this function. - """ - path = "/".join(quote(part) for part in topic.split("/") if len(part) > 0) - try: - r = get_session().head( - f"{constants.ENDPOINT}/api/telemetry/{path}", - headers=build_hf_headers( - token=False, # no need to send a token for telemetry - library_name=library_name, - library_version=library_version, - user_agent=user_agent, - ), - ) - hf_raise_for_status(r) - except Exception as e: - # We don't want to error in case of connection errors of any kind. - logger.debug(f"Error while sending telemetry: {e}") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_typing.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_typing.py deleted file mode 100644 index 8c5d6381a2a73afa08698bb99193f1774fb02f64..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_typing.py +++ /dev/null @@ -1,95 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Handle typing imports based on system compatibility.""" - -import sys -from typing import Any, Callable, List, Literal, Optional, Set, Type, TypeVar, Union, get_args, get_origin - - -UNION_TYPES: List[Any] = [Union] -if sys.version_info >= (3, 10): - from types import UnionType - - UNION_TYPES += [UnionType] - - -HTTP_METHOD_T = Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"] - -# type hint meaning "function signature not changed by decorator" -CallableT = TypeVar("CallableT", bound=Callable) - -_JSON_SERIALIZABLE_TYPES = (int, float, str, bool, type(None)) - - -def is_jsonable(obj: Any, _visited: Optional[Set[int]] = None) -> bool: - """Check if an object is JSON serializable. - - This is a weak check, as it does not check for the actual JSON serialization, but only for the types of the object. - It works correctly for basic use cases but do not guarantee an exhaustive check. - - Object is considered to be recursively json serializable if: - - it is an instance of int, float, str, bool, or NoneType - - it is a list or tuple and all its items are json serializable - - it is a dict and all its keys are strings and all its values are json serializable - - Uses a visited set to avoid infinite recursion on circular references. If object has already been visited, it is - considered not json serializable. - """ - # Initialize visited set to track object ids and detect circular references - if _visited is None: - _visited = set() - - # Detect circular reference - obj_id = id(obj) - if obj_id in _visited: - return False - - # Add current object to visited before recursive checks - _visited.add(obj_id) - try: - if isinstance(obj, _JSON_SERIALIZABLE_TYPES): - return True - if isinstance(obj, (list, tuple)): - return all(is_jsonable(item, _visited) for item in obj) - if isinstance(obj, dict): - return all( - isinstance(key, _JSON_SERIALIZABLE_TYPES) and is_jsonable(value, _visited) - for key, value in obj.items() - ) - if hasattr(obj, "__json__"): - return True - return False - except RecursionError: - return False - finally: - # Remove the object id from visited to avoid side‑effects for other branches - _visited.discard(obj_id) - - -def is_simple_optional_type(type_: Type) -> bool: - """Check if a type is optional, i.e. Optional[Type] or Union[Type, None] or Type | None, where Type is a non-composite type.""" - if get_origin(type_) in UNION_TYPES: - union_args = get_args(type_) - if len(union_args) == 2 and type(None) in union_args: - return True - return False - - -def unwrap_simple_optional_type(optional_type: Type) -> Type: - """Unwraps a simple optional type, i.e. returns Type from Optional[Type].""" - for arg in get_args(optional_type): - if arg is not type(None): - return arg - raise ValueError(f"'{optional_type}' is not an optional type") diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_validators.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_validators.py deleted file mode 100644 index 4bc219611b2132d699643975d00cf99853e03e47..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_validators.py +++ /dev/null @@ -1,226 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains utilities to validate argument values in `huggingface_hub`.""" - -import inspect -import re -import warnings -from functools import wraps -from itertools import chain -from typing import Any, Dict - -from huggingface_hub.errors import HFValidationError - -from ._typing import CallableT - - -REPO_ID_REGEX = re.compile( - r""" - ^ - (\b[\w\-.]+\b/)? # optional namespace (username or organization) - \b # starts with a word boundary - [\w\-.]{1,96} # repo_name: alphanumeric + . _ - - \b # ends with a word boundary - $ - """, - flags=re.VERBOSE, -) - - -def validate_hf_hub_args(fn: CallableT) -> CallableT: - """Validate values received as argument for any public method of `huggingface_hub`. - - The goal of this decorator is to harmonize validation of arguments reused - everywhere. By default, all defined validators are tested. - - Validators: - - [`~utils.validate_repo_id`]: `repo_id` must be `"repo_name"` - or `"namespace/repo_name"`. Namespace is a username or an organization. - - [`~utils.smoothly_deprecate_use_auth_token`]: Use `token` instead of - `use_auth_token` (only if `use_auth_token` is not expected by the decorated - function - in practice, always the case in `huggingface_hub`). - - Example: - ```py - >>> from huggingface_hub.utils import validate_hf_hub_args - - >>> @validate_hf_hub_args - ... def my_cool_method(repo_id: str): - ... print(repo_id) - - >>> my_cool_method(repo_id="valid_repo_id") - valid_repo_id - - >>> my_cool_method("other..repo..id") - huggingface_hub.utils._validators.HFValidationError: Cannot have -- or .. in repo_id: 'other..repo..id'. - - >>> my_cool_method(repo_id="other..repo..id") - huggingface_hub.utils._validators.HFValidationError: Cannot have -- or .. in repo_id: 'other..repo..id'. - - >>> @validate_hf_hub_args - ... def my_cool_auth_method(token: str): - ... print(token) - - >>> my_cool_auth_method(token="a token") - "a token" - - >>> my_cool_auth_method(use_auth_token="a use_auth_token") - "a use_auth_token" - - >>> my_cool_auth_method(token="a token", use_auth_token="a use_auth_token") - UserWarning: Both `token` and `use_auth_token` are passed (...) - "a token" - ``` - - Raises: - [`~utils.HFValidationError`]: - If an input is not valid. - """ - # TODO: add an argument to opt-out validation for specific argument? - signature = inspect.signature(fn) - - # Should the validator switch `use_auth_token` values to `token`? In practice, always - # True in `huggingface_hub`. Might not be the case in a downstream library. - check_use_auth_token = "use_auth_token" not in signature.parameters and "token" in signature.parameters - - @wraps(fn) - def _inner_fn(*args, **kwargs): - has_token = False - for arg_name, arg_value in chain( - zip(signature.parameters, args), # Args values - kwargs.items(), # Kwargs values - ): - if arg_name in ["repo_id", "from_id", "to_id"]: - validate_repo_id(arg_value) - - elif arg_name == "token" and arg_value is not None: - has_token = True - - if check_use_auth_token: - kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.__name__, has_token=has_token, kwargs=kwargs) - - return fn(*args, **kwargs) - - return _inner_fn # type: ignore - - -def validate_repo_id(repo_id: str) -> None: - """Validate `repo_id` is valid. - - This is not meant to replace the proper validation made on the Hub but rather to - avoid local inconsistencies whenever possible (example: passing `repo_type` in the - `repo_id` is forbidden). - - Rules: - - Between 1 and 96 characters. - - Either "repo_name" or "namespace/repo_name" - - [a-zA-Z0-9] or "-", "_", "." - - "--" and ".." are forbidden - - Valid: `"foo"`, `"foo/bar"`, `"123"`, `"Foo-BAR_foo.bar123"` - - Not valid: `"datasets/foo/bar"`, `".repo_id"`, `"foo--bar"`, `"foo.git"` - - Example: - ```py - >>> from huggingface_hub.utils import validate_repo_id - >>> validate_repo_id(repo_id="valid_repo_id") - >>> validate_repo_id(repo_id="other..repo..id") - huggingface_hub.utils._validators.HFValidationError: Cannot have -- or .. in repo_id: 'other..repo..id'. - ``` - - Discussed in https://github.com/huggingface/huggingface_hub/issues/1008. - In moon-landing (internal repository): - - https://github.com/huggingface/moon-landing/blob/main/server/lib/Names.ts#L27 - - https://github.com/huggingface/moon-landing/blob/main/server/views/components/NewRepoForm/NewRepoForm.svelte#L138 - """ - if not isinstance(repo_id, str): - # Typically, a Path is not a repo_id - raise HFValidationError(f"Repo id must be a string, not {type(repo_id)}: '{repo_id}'.") - - if repo_id.count("/") > 1: - raise HFValidationError( - "Repo id must be in the form 'repo_name' or 'namespace/repo_name':" - f" '{repo_id}'. Use `repo_type` argument if needed." - ) - - if not REPO_ID_REGEX.match(repo_id): - raise HFValidationError( - "Repo id must use alphanumeric chars, '-', '_' or '.'." - " The name cannot start or end with '-' or '.' and the maximum length is 96:" - f" '{repo_id}'." - ) - - if "--" in repo_id or ".." in repo_id: - raise HFValidationError(f"Cannot have -- or .. in repo_id: '{repo_id}'.") - - if repo_id.endswith(".git"): - raise HFValidationError(f"Repo_id cannot end by '.git': '{repo_id}'.") - - -def smoothly_deprecate_use_auth_token(fn_name: str, has_token: bool, kwargs: Dict[str, Any]) -> Dict[str, Any]: - """Smoothly deprecate `use_auth_token` in the `huggingface_hub` codebase. - - The long-term goal is to remove any mention of `use_auth_token` in the codebase in - favor of a unique and less verbose `token` argument. This will be done a few steps: - - 0. Step 0: methods that require a read-access to the Hub use the `use_auth_token` - argument (`str`, `bool` or `None`). Methods requiring write-access have a `token` - argument (`str`, `None`). This implicit rule exists to be able to not send the - token when not necessary (`use_auth_token=False`) even if logged in. - - 1. Step 1: we want to harmonize everything and use `token` everywhere (supporting - `token=False` for read-only methods). In order not to break existing code, if - `use_auth_token` is passed to a function, the `use_auth_token` value is passed - as `token` instead, without any warning. - a. Corner case: if both `use_auth_token` and `token` values are passed, a warning - is thrown and the `use_auth_token` value is ignored. - - 2. Step 2: Once it is release, we should push downstream libraries to switch from - `use_auth_token` to `token` as much as possible, but without throwing a warning - (e.g. manually create issues on the corresponding repos). - - 3. Step 3: After a transitional period (6 months e.g. until April 2023?), we update - `huggingface_hub` to throw a warning on `use_auth_token`. Hopefully, very few - users will be impacted as it would have already been fixed. - In addition, unit tests in `huggingface_hub` must be adapted to expect warnings - to be thrown (but still use `use_auth_token` as before). - - 4. Step 4: After a normal deprecation cycle (3 releases ?), remove this validator. - `use_auth_token` will definitely not be supported. - In addition, we update unit tests in `huggingface_hub` to use `token` everywhere. - - This has been discussed in: - - https://github.com/huggingface/huggingface_hub/issues/1094. - - https://github.com/huggingface/huggingface_hub/pull/928 - - (related) https://github.com/huggingface/huggingface_hub/pull/1064 - """ - new_kwargs = kwargs.copy() # do not mutate input ! - - use_auth_token = new_kwargs.pop("use_auth_token", None) # remove from kwargs - if use_auth_token is not None: - if has_token: - warnings.warn( - "Both `token` and `use_auth_token` are passed to" - f" `{fn_name}` with non-None values. `token` is now the" - " preferred argument to pass a User Access Token." - " `use_auth_token` value will be ignored." - ) - else: - # `token` argument is not passed and a non-None value is passed in - # `use_auth_token` => use `use_auth_token` value as `token` kwarg. - new_kwargs["token"] = use_auth_token - - return new_kwargs diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_xet.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_xet.py deleted file mode 100644 index 3dcf99068f87eebdf3c684edf6026a576bd34eaf..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_xet.py +++ /dev/null @@ -1,192 +0,0 @@ -from dataclasses import dataclass -from enum import Enum -from typing import Dict, Optional - -import requests - -from .. import constants -from . import get_session, hf_raise_for_status, validate_hf_hub_args - - -class XetTokenType(str, Enum): - READ = "read" - WRITE = "write" - - -@dataclass(frozen=True) -class XetFileData: - file_hash: str - refresh_route: str - - -@dataclass(frozen=True) -class XetConnectionInfo: - access_token: str - expiration_unix_epoch: int - endpoint: str - - -def parse_xet_file_data_from_response( - response: requests.Response, endpoint: Optional[str] = None -) -> Optional[XetFileData]: - """ - Parse XET file metadata from an HTTP response. - - This function extracts XET file metadata from the HTTP headers or HTTP links - of a given response object. If the required metadata is not found, it returns `None`. - - Args: - response (`requests.Response`): - The HTTP response object containing headers dict and links dict to extract the XET metadata from. - Returns: - `Optional[XetFileData]`: - An instance of `XetFileData` containing the file hash and refresh route if the metadata - is found. Returns `None` if the required metadata is missing. - """ - if response is None: - return None - try: - file_hash = response.headers[constants.HUGGINGFACE_HEADER_X_XET_HASH] - - if constants.HUGGINGFACE_HEADER_LINK_XET_AUTH_KEY in response.links: - refresh_route = response.links[constants.HUGGINGFACE_HEADER_LINK_XET_AUTH_KEY]["url"] - else: - refresh_route = response.headers[constants.HUGGINGFACE_HEADER_X_XET_REFRESH_ROUTE] - except KeyError: - return None - endpoint = endpoint if endpoint is not None else constants.ENDPOINT - if refresh_route.startswith(constants.HUGGINGFACE_CO_URL_HOME): - refresh_route = refresh_route.replace(constants.HUGGINGFACE_CO_URL_HOME.rstrip("/"), endpoint.rstrip("/")) - return XetFileData( - file_hash=file_hash, - refresh_route=refresh_route, - ) - - -def parse_xet_connection_info_from_headers(headers: Dict[str, str]) -> Optional[XetConnectionInfo]: - """ - Parse XET connection info from the HTTP headers or return None if not found. - Args: - headers (`Dict`): - HTTP headers to extract the XET metadata from. - Returns: - `XetConnectionInfo` or `None`: - The information needed to connect to the XET storage service. - Returns `None` if the headers do not contain the XET connection info. - """ - try: - endpoint = headers[constants.HUGGINGFACE_HEADER_X_XET_ENDPOINT] - access_token = headers[constants.HUGGINGFACE_HEADER_X_XET_ACCESS_TOKEN] - expiration_unix_epoch = int(headers[constants.HUGGINGFACE_HEADER_X_XET_EXPIRATION]) - except (KeyError, ValueError, TypeError): - return None - - return XetConnectionInfo( - endpoint=endpoint, - access_token=access_token, - expiration_unix_epoch=expiration_unix_epoch, - ) - - -@validate_hf_hub_args -def refresh_xet_connection_info( - *, - file_data: XetFileData, - headers: Dict[str, str], -) -> XetConnectionInfo: - """ - Utilizes the information in the parsed metadata to request the Hub xet connection information. - This includes the access token, expiration, and XET service URL. - Args: - file_data: (`XetFileData`): - The file data needed to refresh the xet connection information. - headers (`Dict[str, str]`): - Headers to use for the request, including authorization headers and user agent. - Returns: - `XetConnectionInfo`: - The connection information needed to make the request to the xet storage service. - Raises: - [`~utils.HfHubHTTPError`] - If the Hub API returned an error. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If the Hub API response is improperly formatted. - """ - if file_data.refresh_route is None: - raise ValueError("The provided xet metadata does not contain a refresh endpoint.") - return _fetch_xet_connection_info_with_url(file_data.refresh_route, headers) - - -@validate_hf_hub_args -def fetch_xet_connection_info_from_repo_info( - *, - token_type: XetTokenType, - repo_id: str, - repo_type: str, - revision: Optional[str] = None, - headers: Dict[str, str], - endpoint: Optional[str] = None, - params: Optional[Dict[str, str]] = None, -) -> XetConnectionInfo: - """ - Uses the repo info to request a xet access token from Hub. - Args: - token_type (`XetTokenType`): - Type of the token to request: `"read"` or `"write"`. - repo_id (`str`): - A namespace (user or an organization) and a repo name separated by a `/`. - repo_type (`str`): - Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. - revision (`str`, `optional`): - The revision of the repo to get the token for. - headers (`Dict[str, str]`): - Headers to use for the request, including authorization headers and user agent. - endpoint (`str`, `optional`): - The endpoint to use for the request. Defaults to the Hub endpoint. - params (`Dict[str, str]`, `optional`): - Additional parameters to pass with the request. - Returns: - `XetConnectionInfo`: - The connection information needed to make the request to the xet storage service. - Raises: - [`~utils.HfHubHTTPError`] - If the Hub API returned an error. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If the Hub API response is improperly formatted. - """ - endpoint = endpoint if endpoint is not None else constants.ENDPOINT - url = f"{endpoint}/api/{repo_type}s/{repo_id}/xet-{token_type.value}-token/{revision}" - return _fetch_xet_connection_info_with_url(url, headers, params) - - -@validate_hf_hub_args -def _fetch_xet_connection_info_with_url( - url: str, - headers: Dict[str, str], - params: Optional[Dict[str, str]] = None, -) -> XetConnectionInfo: - """ - Requests the xet connection info from the supplied URL. This includes the - access token, expiration time, and endpoint to use for the xet storage service. - Args: - url: (`str`): - The access token endpoint URL. - headers (`Dict[str, str]`): - Headers to use for the request, including authorization headers and user agent. - params (`Dict[str, str]`, `optional`): - Additional parameters to pass with the request. - Returns: - `XetConnectionInfo`: - The connection information needed to make the request to the xet storage service. - Raises: - [`~utils.HfHubHTTPError`] - If the Hub API returned an error. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) - If the Hub API response is improperly formatted. - """ - resp = get_session().get(headers=headers, url=url, params=params) - hf_raise_for_status(resp) - - metadata = parse_xet_connection_info_from_headers(resp.headers) # type: ignore - if metadata is None: - raise ValueError("Xet headers have not been correctly set by the server.") - return metadata diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_xet_progress_reporting.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_xet_progress_reporting.py deleted file mode 100644 index e47740d5c5ea27253debc9e29c1eae9d10a6034f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/_xet_progress_reporting.py +++ /dev/null @@ -1,162 +0,0 @@ -from collections import OrderedDict -from typing import List - -from hf_xet import PyItemProgressUpdate, PyTotalProgressUpdate - -from . import is_google_colab, is_notebook -from .tqdm import tqdm - - -class XetProgressReporter: - """ - Reports on progress for Xet uploads. - - Shows summary progress bars when running in notebooks or GUIs, and detailed per-file progress in console environments. - """ - - def __init__(self, n_lines: int = 10, description_width: int = 30): - self.n_lines = n_lines - self.description_width = description_width - - self.per_file_progress = is_google_colab() or not is_notebook() - - self.tqdm_settings = { - "unit": "B", - "unit_scale": True, - "leave": True, - "unit_divisor": 1000, - "nrows": n_lines + 3 if self.per_file_progress else 3, - "miniters": 1, - "bar_format": "{l_bar}{bar}| {n_fmt:>5}B / {total_fmt:>5}B{postfix:>12}", - } - - # Overall progress bars - self.data_processing_bar = tqdm( - total=0, desc=self.format_desc("Processing Files (0 / 0)", False), position=0, **self.tqdm_settings - ) - - self.upload_bar = tqdm( - total=0, desc=self.format_desc("New Data Upload", False), position=1, **self.tqdm_settings - ) - - self.known_items: set[str] = set() - self.completed_items: set[str] = set() - - # Item bars (scrolling view) - self.item_state: OrderedDict[str, PyItemProgressUpdate] = OrderedDict() - self.current_bars: List = [None] * self.n_lines - - def format_desc(self, name: str, indent: bool) -> str: - """ - if name is longer than width characters, prints ... at the start and then the last width-3 characters of the name, otherwise - the whole name right justified into description_width characters. Also adds some padding. - """ - - if not self.per_file_progress: - # Here we just use the defaults. - return name - - padding = " " if indent else "" - width = self.description_width - len(padding) - - if len(name) > width: - name = f"...{name[-(width - 3) :]}" - - return f"{padding}{name.ljust(width)}" - - def update_progress(self, total_update: PyTotalProgressUpdate, item_updates: List[PyItemProgressUpdate]): - # Update all the per-item values. - for item in item_updates: - item_name = item.item_name - - self.known_items.add(item_name) - - # Only care about items where the processing has already started. - if item.bytes_completed == 0: - continue - - # Overwrite the existing value in there. - self.item_state[item_name] = item - - bar_idx = 0 - new_completed = [] - - # Now, go through and update all the bars - for name, item in self.item_state.items(): - # Is this ready to be removed on the next update? - if item.bytes_completed == item.total_bytes: - self.completed_items.add(name) - new_completed.append(name) - - # If we're only showing summary information, then don't update the individual bars - if not self.per_file_progress: - continue - - # If we've run out of bars to use, then collapse the last ones together. - if bar_idx >= len(self.current_bars): - bar = self.current_bars[-1] - in_final_bar_mode = True - final_bar_aggregation_count = bar_idx + 1 - len(self.current_bars) - else: - bar = self.current_bars[bar_idx] - in_final_bar_mode = False - - if bar is None: - self.current_bars[bar_idx] = tqdm( - desc=self.format_desc(name, True), - position=2 + bar_idx, # Set to the position past the initial bars. - total=item.total_bytes, - initial=item.bytes_completed, - **self.tqdm_settings, - ) - - elif in_final_bar_mode: - bar.n += item.bytes_completed - bar.total += item.total_bytes - bar.set_description(self.format_desc(f"[+ {final_bar_aggregation_count} files]", True), refresh=False) - else: - bar.set_description(self.format_desc(name, True), refresh=False) - bar.n = item.bytes_completed - bar.total = item.total_bytes - - bar_idx += 1 - - # Remove all the completed ones from the ordered dictionary - for name in new_completed: - # Only remove ones from consideration to make room for more items coming in. - if len(self.item_state) <= self.n_lines: - break - - del self.item_state[name] - - if self.per_file_progress: - # Now manually refresh each of the bars - for bar in self.current_bars: - if bar: - bar.refresh() - - # Update overall bars - def postfix(speed): - s = tqdm.format_sizeof(speed) if speed is not None else "???" - return f"{s}B/s ".rjust(10, " ") - - self.data_processing_bar.total = total_update.total_bytes - self.data_processing_bar.set_description( - self.format_desc(f"Processing Files ({len(self.completed_items)} / {len(self.known_items)})", False), - refresh=False, - ) - self.data_processing_bar.set_postfix_str(postfix(total_update.total_bytes_completion_rate), refresh=False) - self.data_processing_bar.update(total_update.total_bytes_completion_increment) - - self.upload_bar.total = total_update.total_transfer_bytes - self.upload_bar.set_postfix_str(postfix(total_update.total_transfer_bytes_completion_rate), refresh=False) - self.upload_bar.update(total_update.total_transfer_bytes_completion_increment) - - def close(self, _success): - self.data_processing_bar.close() - self.upload_bar.close() - - if self.per_file_progress: - for bar in self.current_bars: - if bar: - bar.close() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/endpoint_helpers.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/endpoint_helpers.py deleted file mode 100644 index 85cd86011b78bcdc57034aeebc3c01e9e721ab50..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/endpoint_helpers.py +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Helpful utility functions and classes in relation to exploring API endpoints -with the aim for a user-friendly interface. -""" - -import math -import re -from typing import TYPE_CHECKING - -from ..repocard_data import ModelCardData - - -if TYPE_CHECKING: - from ..hf_api import ModelInfo - - -def _is_emission_within_threshold(model_info: "ModelInfo", minimum_threshold: float, maximum_threshold: float) -> bool: - """Checks if a model's emission is within a given threshold. - - Args: - model_info (`ModelInfo`): - A model info object containing the model's emission information. - minimum_threshold (`float`): - A minimum carbon threshold to filter by, such as 1. - maximum_threshold (`float`): - A maximum carbon threshold to filter by, such as 10. - - Returns: - `bool`: Whether the model's emission is within the given threshold. - """ - if minimum_threshold is None and maximum_threshold is None: - raise ValueError("Both `minimum_threshold` and `maximum_threshold` cannot both be `None`") - if minimum_threshold is None: - minimum_threshold = -1 - if maximum_threshold is None: - maximum_threshold = math.inf - - card_data = getattr(model_info, "card_data", None) - if card_data is None or not isinstance(card_data, (dict, ModelCardData)): - return False - - # Get CO2 emission metadata - emission = card_data.get("co2_eq_emissions", None) - if isinstance(emission, dict): - emission = emission["emissions"] - if not emission: - return False - - # Filter out if value is missing or out of range - matched = re.search(r"\d+\.\d+|\d+", str(emission)) - if matched is None: - return False - - emission_value = float(matched.group(0)) - return minimum_threshold <= emission_value <= maximum_threshold diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/insecure_hashlib.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/insecure_hashlib.py deleted file mode 100644 index 6901b6d647cc706b85333a66f3bcb7d8c5e2ee9e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/insecure_hashlib.py +++ /dev/null @@ -1,38 +0,0 @@ -# Taken from https://github.com/mlflow/mlflow/pull/10119 -# -# DO NOT use this function for security purposes (e.g., password hashing). -# -# In Python >= 3.9, insecure hashing algorithms such as MD5 fail in FIPS-compliant -# environments unless `usedforsecurity=False` is explicitly passed. -# -# References: -# - https://github.com/mlflow/mlflow/issues/9905 -# - https://github.com/mlflow/mlflow/pull/10119 -# - https://docs.python.org/3/library/hashlib.html -# - https://github.com/huggingface/transformers/pull/27038 -# -# Usage: -# ```python -# # Use -# from huggingface_hub.utils.insecure_hashlib import sha256 -# # instead of -# from hashlib import sha256 -# -# # Use -# from huggingface_hub.utils import insecure_hashlib -# # instead of -# import hashlib -# ``` -import functools -import hashlib -import sys - - -if sys.version_info >= (3, 9): - md5 = functools.partial(hashlib.md5, usedforsecurity=False) - sha1 = functools.partial(hashlib.sha1, usedforsecurity=False) - sha256 = functools.partial(hashlib.sha256, usedforsecurity=False) -else: - md5 = hashlib.md5 - sha1 = hashlib.sha1 - sha256 = hashlib.sha256 diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/logging.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/logging.py deleted file mode 100644 index 1e2f8ded83074b251a72c83edcf33205808250b9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/logging.py +++ /dev/null @@ -1,185 +0,0 @@ -# coding=utf-8 -# Copyright 2020 Optuna, Hugging Face -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Logging utilities.""" - -import logging -import os -from logging import ( - CRITICAL, # NOQA - DEBUG, # NOQA - ERROR, # NOQA - FATAL, # NOQA - INFO, # NOQA - NOTSET, # NOQA - WARN, # NOQA - WARNING, # NOQA -) -from typing import Optional - -from .. import constants - - -log_levels = { - "debug": logging.DEBUG, - "info": logging.INFO, - "warning": logging.WARNING, - "error": logging.ERROR, - "critical": logging.CRITICAL, -} - -_default_log_level = logging.WARNING - - -def _get_library_name() -> str: - return __name__.split(".")[0] - - -def _get_library_root_logger() -> logging.Logger: - return logging.getLogger(_get_library_name()) - - -def _get_default_logging_level(): - """ - If `HF_HUB_VERBOSITY` env var is set to one of the valid choices return that as the new default level. If it is not - - fall back to `_default_log_level` - """ - env_level_str = os.getenv("HF_HUB_VERBOSITY", None) - if env_level_str: - if env_level_str in log_levels: - return log_levels[env_level_str] - else: - logging.getLogger().warning( - f"Unknown option HF_HUB_VERBOSITY={env_level_str}, has to be one of: {', '.join(log_levels.keys())}" - ) - return _default_log_level - - -def _configure_library_root_logger() -> None: - library_root_logger = _get_library_root_logger() - library_root_logger.addHandler(logging.StreamHandler()) - library_root_logger.setLevel(_get_default_logging_level()) - - -def _reset_library_root_logger() -> None: - library_root_logger = _get_library_root_logger() - library_root_logger.setLevel(logging.NOTSET) - - -def get_logger(name: Optional[str] = None) -> logging.Logger: - """ - Returns a logger with the specified name. This function is not supposed - to be directly accessed by library users. - - Args: - name (`str`, *optional*): - The name of the logger to get, usually the filename - - Example: - - ```python - >>> from huggingface_hub import get_logger - - >>> logger = get_logger(__file__) - >>> logger.set_verbosity_info() - ``` - """ - - if name is None: - name = _get_library_name() - - return logging.getLogger(name) - - -def get_verbosity() -> int: - """Return the current level for the HuggingFace Hub's root logger. - - Returns: - Logging level, e.g., `huggingface_hub.logging.DEBUG` and - `huggingface_hub.logging.INFO`. - - > [!TIP] - > HuggingFace Hub has following logging levels: - > - > - `huggingface_hub.logging.CRITICAL`, `huggingface_hub.logging.FATAL` - > - `huggingface_hub.logging.ERROR` - > - `huggingface_hub.logging.WARNING`, `huggingface_hub.logging.WARN` - > - `huggingface_hub.logging.INFO` - > - `huggingface_hub.logging.DEBUG` - """ - return _get_library_root_logger().getEffectiveLevel() - - -def set_verbosity(verbosity: int) -> None: - """ - Sets the level for the HuggingFace Hub's root logger. - - Args: - verbosity (`int`): - Logging level, e.g., `huggingface_hub.logging.DEBUG` and - `huggingface_hub.logging.INFO`. - """ - _get_library_root_logger().setLevel(verbosity) - - -def set_verbosity_info(): - """ - Sets the verbosity to `logging.INFO`. - """ - return set_verbosity(INFO) - - -def set_verbosity_warning(): - """ - Sets the verbosity to `logging.WARNING`. - """ - return set_verbosity(WARNING) - - -def set_verbosity_debug(): - """ - Sets the verbosity to `logging.DEBUG`. - """ - return set_verbosity(DEBUG) - - -def set_verbosity_error(): - """ - Sets the verbosity to `logging.ERROR`. - """ - return set_verbosity(ERROR) - - -def disable_propagation() -> None: - """ - Disable propagation of the library log outputs. Note that log propagation is - disabled by default. - """ - _get_library_root_logger().propagate = False - - -def enable_propagation() -> None: - """ - Enable propagation of the library log outputs. Please disable the - HuggingFace Hub's default handler to prevent double logging if the root - logger has been configured. - """ - _get_library_root_logger().propagate = True - - -_configure_library_root_logger() - -if constants.HF_DEBUG: - # If `HF_DEBUG` environment variable is set, set the verbosity of `huggingface_hub` logger to `DEBUG`. - set_verbosity_debug() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/sha.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/sha.py deleted file mode 100644 index 001c3fe8b2f37a64e890888ca3d521c10ec8f03b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/sha.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Utilities to efficiently compute the SHA 256 hash of a bunch of bytes.""" - -from typing import BinaryIO, Optional - -from .insecure_hashlib import sha1, sha256 - - -def sha_fileobj(fileobj: BinaryIO, chunk_size: Optional[int] = None) -> bytes: - """ - Computes the sha256 hash of the given file object, by chunks of size `chunk_size`. - - Args: - fileobj (file-like object): - The File object to compute sha256 for, typically obtained with `open(path, "rb")` - chunk_size (`int`, *optional*): - The number of bytes to read from `fileobj` at once, defaults to 1MB. - - Returns: - `bytes`: `fileobj`'s sha256 hash as bytes - """ - chunk_size = chunk_size if chunk_size is not None else 1024 * 1024 - - sha = sha256() - while True: - chunk = fileobj.read(chunk_size) - sha.update(chunk) - if not chunk: - break - return sha.digest() - - -def git_hash(data: bytes) -> str: - """ - Computes the git-sha1 hash of the given bytes, using the same algorithm as git. - - This is equivalent to running `git hash-object`. See https://git-scm.com/docs/git-hash-object - for more details. - - Note: this method is valid for regular files. For LFS files, the proper git hash is supposed to be computed on the - pointer file content, not the actual file content. However, for simplicity, we directly compare the sha256 of - the LFS file content when we want to compare LFS files. - - Args: - data (`bytes`): - The data to compute the git-hash for. - - Returns: - `str`: the git-hash of `data` as an hexadecimal string. - - Example: - ```python - >>> from huggingface_hub.utils.sha import git_hash - >>> git_hash(b"Hello, World!") - 'b45ef6fec89518d314f546fd6c3025367b721684' - ``` - """ - # Taken from https://gist.github.com/msabramo/763200 - # Note: no need to optimize by reading the file in chunks as we're not supposed to hash huge files (5MB maximum). - sha = sha1() - sha.update(b"blob ") - sha.update(str(len(data)).encode()) - sha.update(b"\0") - sha.update(data) - return sha.hexdigest() diff --git a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/tqdm.py b/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/tqdm.py deleted file mode 100644 index 4c1fcef4beb73bae13c57b3f66c5828e775b7cd9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/huggingface_hub/utils/tqdm.py +++ /dev/null @@ -1,307 +0,0 @@ -# coding=utf-8 -# Copyright 2021 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License -"""Utility helpers to handle progress bars in `huggingface_hub`. - -Example: - 1. Use `huggingface_hub.utils.tqdm` as you would use `tqdm.tqdm` or `tqdm.auto.tqdm`. - 2. To disable progress bars, either use `disable_progress_bars()` helper or set the - environment variable `HF_HUB_DISABLE_PROGRESS_BARS` to 1. - 3. To re-enable progress bars, use `enable_progress_bars()`. - 4. To check whether progress bars are disabled, use `are_progress_bars_disabled()`. - -NOTE: Environment variable `HF_HUB_DISABLE_PROGRESS_BARS` has the priority. - -Example: - ```py - >>> from huggingface_hub.utils import are_progress_bars_disabled, disable_progress_bars, enable_progress_bars, tqdm - - # Disable progress bars globally - >>> disable_progress_bars() - - # Use as normal `tqdm` - >>> for _ in tqdm(range(5)): - ... pass - - # Still not showing progress bars, as `disable=False` is overwritten to `True`. - >>> for _ in tqdm(range(5), disable=False): - ... pass - - >>> are_progress_bars_disabled() - True - - # Re-enable progress bars globally - >>> enable_progress_bars() - - # Progress bar will be shown ! - >>> for _ in tqdm(range(5)): - ... pass - 100%|███████████████████████████████████████| 5/5 [00:00<00:00, 117817.53it/s] - ``` - -Group-based control: - ```python - # Disable progress bars for a specific group - >>> disable_progress_bars("peft.foo") - - # Check state of different groups - >>> assert not are_progress_bars_disabled("peft")) - >>> assert not are_progress_bars_disabled("peft.something") - >>> assert are_progress_bars_disabled("peft.foo")) - >>> assert are_progress_bars_disabled("peft.foo.bar")) - - # Enable progress bars for a subgroup - >>> enable_progress_bars("peft.foo.bar") - - # Check if enabling a subgroup affects the parent group - >>> assert are_progress_bars_disabled("peft.foo")) - >>> assert not are_progress_bars_disabled("peft.foo.bar")) - - # No progress bar for `name="peft.foo"` - >>> for _ in tqdm(range(5), name="peft.foo"): - ... pass - - # Progress bar will be shown for `name="peft.foo.bar"` - >>> for _ in tqdm(range(5), name="peft.foo.bar"): - ... pass - 100%|███████████████████████████████████████| 5/5 [00:00<00:00, 117817.53it/s] - - ``` -""" - -import io -import logging -import os -import warnings -from contextlib import contextmanager, nullcontext -from pathlib import Path -from typing import ContextManager, Dict, Iterator, Optional, Union - -from tqdm.auto import tqdm as old_tqdm - -from ..constants import HF_HUB_DISABLE_PROGRESS_BARS - - -# The `HF_HUB_DISABLE_PROGRESS_BARS` environment variable can be True, False, or not set (None), -# allowing for control over progress bar visibility. When set, this variable takes precedence -# over programmatic settings, dictating whether progress bars should be shown or hidden globally. -# Essentially, the environment variable's setting overrides any code-based configurations. -# -# If `HF_HUB_DISABLE_PROGRESS_BARS` is not defined (None), it implies that users can manage -# progress bar visibility through code. By default, progress bars are turned on. - - -progress_bar_states: Dict[str, bool] = {} - - -def disable_progress_bars(name: Optional[str] = None) -> None: - """ - Disable progress bars either globally or for a specified group. - - This function updates the state of progress bars based on a group name. - If no group name is provided, all progress bars are disabled. The operation - respects the `HF_HUB_DISABLE_PROGRESS_BARS` environment variable's setting. - - Args: - name (`str`, *optional*): - The name of the group for which to disable the progress bars. If None, - progress bars are disabled globally. - - Raises: - Warning: If the environment variable precludes changes. - """ - if HF_HUB_DISABLE_PROGRESS_BARS is False: - warnings.warn( - "Cannot disable progress bars: environment variable `HF_HUB_DISABLE_PROGRESS_BARS=0` is set and has priority." - ) - return - - if name is None: - progress_bar_states.clear() - progress_bar_states["_global"] = False - else: - keys_to_remove = [key for key in progress_bar_states if key.startswith(f"{name}.")] - for key in keys_to_remove: - del progress_bar_states[key] - progress_bar_states[name] = False - - -def enable_progress_bars(name: Optional[str] = None) -> None: - """ - Enable progress bars either globally or for a specified group. - - This function sets the progress bars to enabled for the specified group or globally - if no group is specified. The operation is subject to the `HF_HUB_DISABLE_PROGRESS_BARS` - environment setting. - - Args: - name (`str`, *optional*): - The name of the group for which to enable the progress bars. If None, - progress bars are enabled globally. - - Raises: - Warning: If the environment variable precludes changes. - """ - if HF_HUB_DISABLE_PROGRESS_BARS is True: - warnings.warn( - "Cannot enable progress bars: environment variable `HF_HUB_DISABLE_PROGRESS_BARS=1` is set and has priority." - ) - return - - if name is None: - progress_bar_states.clear() - progress_bar_states["_global"] = True - else: - keys_to_remove = [key for key in progress_bar_states if key.startswith(f"{name}.")] - for key in keys_to_remove: - del progress_bar_states[key] - progress_bar_states[name] = True - - -def are_progress_bars_disabled(name: Optional[str] = None) -> bool: - """ - Check if progress bars are disabled globally or for a specific group. - - This function returns whether progress bars are disabled for a given group or globally. - It checks the `HF_HUB_DISABLE_PROGRESS_BARS` environment variable first, then the programmatic - settings. - - Args: - name (`str`, *optional*): - The group name to check; if None, checks the global setting. - - Returns: - `bool`: True if progress bars are disabled, False otherwise. - """ - if HF_HUB_DISABLE_PROGRESS_BARS is True: - return True - - if name is None: - return not progress_bar_states.get("_global", True) - - while name: - if name in progress_bar_states: - return not progress_bar_states[name] - name = ".".join(name.split(".")[:-1]) - - return not progress_bar_states.get("_global", True) - - -def is_tqdm_disabled(log_level: int) -> Optional[bool]: - """ - Determine if tqdm progress bars should be disabled based on logging level and environment settings. - - see https://github.com/huggingface/huggingface_hub/pull/2000 and https://github.com/huggingface/huggingface_hub/pull/2698. - """ - if log_level == logging.NOTSET: - return True - if os.getenv("TQDM_POSITION") == "-1": - return False - return None - - -class tqdm(old_tqdm): - """ - Class to override `disable` argument in case progress bars are globally disabled. - - Taken from https://github.com/tqdm/tqdm/issues/619#issuecomment-619639324. - """ - - def __init__(self, *args, **kwargs): - name = kwargs.pop("name", None) # do not pass `name` to `tqdm` - if are_progress_bars_disabled(name): - kwargs["disable"] = True - super().__init__(*args, **kwargs) - - def __delattr__(self, attr: str) -> None: - """Fix for https://github.com/huggingface/huggingface_hub/issues/1603""" - try: - super().__delattr__(attr) - except AttributeError: - if attr != "_lock": - raise - - -@contextmanager -def tqdm_stream_file(path: Union[Path, str]) -> Iterator[io.BufferedReader]: - """ - Open a file as binary and wrap the `read` method to display a progress bar when it's streamed. - - First implemented in `transformers` in 2019 but removed when switched to git-lfs. Used in `huggingface_hub` to show - progress bar when uploading an LFS file to the Hub. See github.com/huggingface/transformers/pull/2078#discussion_r354739608 - for implementation details. - - Note: currently implementation handles only files stored on disk as it is the most common use case. Could be - extended to stream any `BinaryIO` object but we might have to debug some corner cases. - - Example: - ```py - >>> with tqdm_stream_file("config.json") as f: - >>> requests.put(url, data=f) - config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s] - ``` - """ - if isinstance(path, str): - path = Path(path) - - with path.open("rb") as f: - total_size = path.stat().st_size - pbar = tqdm( - unit="B", - unit_scale=True, - total=total_size, - initial=0, - desc=path.name, - ) - - f_read = f.read - - def _inner_read(size: Optional[int] = -1) -> bytes: - data = f_read(size) - pbar.update(len(data)) - return data - - f.read = _inner_read # type: ignore - - yield f - - pbar.close() - - -def _get_progress_bar_context( - *, - desc: str, - log_level: int, - total: Optional[int] = None, - initial: int = 0, - unit: str = "B", - unit_scale: bool = True, - name: Optional[str] = None, - _tqdm_bar: Optional[tqdm] = None, -) -> ContextManager[tqdm]: - if _tqdm_bar is not None: - return nullcontext(_tqdm_bar) - # ^ `contextlib.nullcontext` mimics a context manager that does nothing - # Makes it easier to use the same code path for both cases but in the later - # case, the progress bar is not closed when exiting the context manager. - - return tqdm( - unit=unit, - unit_scale=unit_scale, - total=total, - initial=initial, - desc=desc, - disable=is_tqdm_disabled(log_level=log_level), - name=name, - ) diff --git a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/METADATA deleted file mode 100644 index 6c4bf89b25c79af2e9ecdc22c5ddb802effa85bc..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/METADATA +++ /dev/null @@ -1,155 +0,0 @@ -Metadata-Version: 2.4 -Name: idna -Version: 3.18 -Summary: Internationalized Domain Names in Applications (IDNA) -Author-email: Kim Davies -Requires-Python: >=3.9 -Description-Content-Type: text/markdown -License-Expression: BSD-3-Clause -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: System Administrators -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Topic :: Internet :: Name Service (DNS) -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Topic :: Utilities -License-File: LICENSE.md -Requires-Dist: ruff >= 0.6.2 ; extra == "all" -Requires-Dist: mypy >= 1.11.2 ; extra == "all" -Requires-Dist: pytest >= 8.3.2 ; extra == "all" -Project-URL: Changelog, https://github.com/kjd/idna/blob/master/HISTORY.md -Project-URL: Issue tracker, https://github.com/kjd/idna/issues -Project-URL: Source, https://github.com/kjd/idna -Provides-Extra: all - -# Internationalized Domain Names in Applications (IDNA) - -Support for [Internationalized Domain Names in Applications -(IDNA)](https://tools.ietf.org/html/rfc5891) and [Unicode IDNA -Compatibility Processing](https://unicode.org/reports/tr46/). It -supersedes the standard library's `encodings.idna`, which only -implements the 2003 specification, offering broader script coverage and -limiting domains with known security vulnerabilities. - -## Usage - -Package may be installed from [PyPI](https://pypi.org/project/idna/) via -the typical methods (e.g. `python3 -m pip install idna`) - -For typical usage, the `encode` and `decode` functions will take a -domain name argument and perform a conversion to ASCII-compatible encoding -(known as A-labels), or to Unicode strings (known as U-labels) -respectively. - -```pycon ->>> import idna ->>> idna.encode('ドメイン.テスト') -b'xn--eckwd4c7c.xn--zckzah' ->>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) -ドメイン.テスト -``` - -Conversions can be applied at a per-label basis using the `ulabel` or -`alabel` functions for specialized use cases. - - -### Compatibility Mapping (UTS #46) - -This library provides support for [Unicode IDNA Compatibility -Processing](https://unicode.org/reports/tr46/) which normalizes input from -different potential ways a user may input a domain prior to performing the IDNA -conversion operations. This functionality, known as a -[mapping](https://tools.ietf.org/html/rfc5895), is considered by the -specification to be a local user-interface issue distinct from IDNA -conversion functionality. - -For example, "Königsgäßchen" is not a permissible label as capital letters -are not allowed. UTS 46 will convert this into lower case prior to applying -the IDNA conversion. - -```pycon ->>> import idna ->>> idna.encode('Königsgäßchen') -... -idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed ->>> idna.encode('Königsgäßchen', uts46=True) -b'xn--knigsgchen-b4a3dun' ->>> idna.decode('xn--knigsgchen-b4a3dun') -'königsgäßchen' -``` - -When performing a decode operation for display purposes, `decode()` -accepts a `display=True` argument that leaves any `xn--` label that -fails to decode unchanged. This is useful for user interface display -where a domain is in use, the A-label form can be presented when it -is not a valid IDN. - - -## Exceptions - -All errors raised during conversion derive from the `idna.IDNAError` -base class. The more specific exceptions are: - -* `idna.IDNABidiError` — raised when a label contains an illegal - combination of left-to-right and right-to-left characters. -* `idna.InvalidCodepoint` — raised when a label contains a codepoint - that is INVALID for IDNA. -* `idna.InvalidCodepointContext` — raised when a CONTEXTO or CONTEXTJ - codepoint appears in a position whose contextual requirements are - not satisfied. - - -## Command-line tool - -The package supports command-line usage to convert domain names -between their Unicode and ASCII-compatible forms. It can be run either -as a module (`python3 -m idna`) or, once installed (such as with `uv -tool` or `pipx`), via the `idna` script: - -```bash -$ uv tool install idna -$ idna xn--e1afmkfd.xn--p1ai -пример.рф -$ idna пример.рф -xn--e1afmkfd.xn--p1ai -``` - -Mode can be specified with `-e`/`--encode` or `-d`/`--decode`, otherwise -it will be chosen automatically based on the first input. Multiple -domains can be supplied either as arguments or through standard input. -UTS #46 mapping is applied by default, which lets the tool accept -inputs that aren't strictly valid IDNA 2008 by normalising them first, -pass `--strict` to disable UTS #46. - -Conversion failures are reported on stderr together with the -offending input; processing continues with the remaining domains and -the tool exits with a non-zero status if any conversion failed. - - -## Additional Notes - -* **Version support**. This library supports Python 3.9 and higher. - As this library serves as a low-level toolkit for a variety of - applications, we strive to support all versions of Python that are - not beyond end-of-life. - -* **Emoji**. It is an occasional request to support emoji domains in - this library. Encoding of symbols like emoji is expressly prohibited by - the IDNA technical standard, and emoji domains are broadly phased - out across the domain industry due to associated security risks. - -* **Regenerating lookup tables**. The IDNA and UTS 46 functionality - relies upon pre-calculated lookup tables, generated using the - `idna-data` script in [`tools/`](tools/README.md). - diff --git a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/RECORD deleted file mode 100644 index 6a70bcc27fbcc1617d5ec9147e34c924baa58fec..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/RECORD +++ /dev/null @@ -1,19 +0,0 @@ -../../Scripts/idna.exe,sha256=ewYeJgzCmtxA88xfBSTAsxFl0H1omUQj5cniTLKP5j8,46080 -idna-3.18.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -idna-3.18.dist-info/METADATA,sha256=Rt_m5axGLQ9oDs2avPZugptqIzSCS02eOXmzETXK8oE,6119 -idna-3.18.dist-info/RECORD,, -idna-3.18.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -idna-3.18.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 -idna-3.18.dist-info/entry_points.txt,sha256=7H3nGOHap3jnLE5e7q7Ywr9Vq8axB7WIj5-C_4N2vhw,38 -idna-3.18.dist-info/licenses/LICENSE.md,sha256=GppPDj1HmickDd1ZqRN6ZqtKD539yMphiMwL_YUYfwQ,1541 -idna/__init__.py,sha256=MPqNDLZbXqGaNdXxAFhiqFPKEQXju2jNQhCey6-5eJM,868 -idna/__main__.py,sha256=4JMK66Wj4uLZTKbF-sT3LAxOsr6buig77PmOkJCRRxw,83 -idna/cli.py,sha256=swqJLMNc8Uzs60KziNpbWnHuqlG3WRQwJSbo4n8xDAo,4139 -idna/codec.py,sha256=JRbo-f7pEkLdWeiH89Z72UR4VBYhvKDFrQBeNX6sRDE,5040 -idna/compat.py,sha256=AepA39ceRHxkfHP41-FvKW5Ki-f4PfUZ90RUMlCNdmo,1353 -idna/core.py,sha256=SfOr1xO3PoE0RDYx7bMciAnjiyjJPbPw_93AB5IUYOw,24685 -idna/idnadata.py,sha256=Af-mo8WBmkhAK6TyXKOQH88OX0mQNDKtdL7UWtQpppk,44862 -idna/intranges.py,sha256=g49scLSkqJtAhLmOODa7hVHriSjmb60tiTsEoocJdBI,1851 -idna/package_data.py,sha256=TeI94EqAFAFaXfBJwsOPUMLn2969uirPa-DaeaceAyU,21 -idna/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -idna/uts46data.py,sha256=jujNz5QqWMcJf-XYLv4X1jBvb5FlI0t6-e1mILsgbPk,234325 diff --git a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/WHEEL deleted file mode 100644 index d8b9936dad9ab2513fa6979f411560d3b6b57e37..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: flit 3.12.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/entry_points.txt b/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/entry_points.txt deleted file mode 100644 index 59ca7ac02ea0a40938ca867305e020981f52a077..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/entry_points.txt +++ /dev/null @@ -1,3 +0,0 @@ -[console_scripts] -idna=idna.cli:main - diff --git a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/licenses/LICENSE.md b/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/licenses/LICENSE.md deleted file mode 100644 index f706835ab3a5dd709a6e1f57aee0a94ae1415df6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna-3.18.dist-info/licenses/LICENSE.md +++ /dev/null @@ -1,31 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2013-2026, Kim Davies and contributors. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bundle/python-cpu/Lib/site-packages/idna/__init__.py b/bundle/python-cpu/Lib/site-packages/idna/__init__.py deleted file mode 100644 index cfdc030a751b089fc7e38fc88093b791605d501d..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -from .core import ( - IDNABidiError, - IDNAError, - InvalidCodepoint, - InvalidCodepointContext, - alabel, - check_bidi, - check_hyphen_ok, - check_initial_combiner, - check_label, - check_nfc, - decode, - encode, - ulabel, - uts46_remap, - valid_contextj, - valid_contexto, - valid_label_length, - valid_string_length, -) -from .intranges import intranges_contain -from .package_data import __version__ - -__all__ = [ - "__version__", - "IDNABidiError", - "IDNAError", - "InvalidCodepoint", - "InvalidCodepointContext", - "alabel", - "check_bidi", - "check_hyphen_ok", - "check_initial_combiner", - "check_label", - "check_nfc", - "decode", - "encode", - "intranges_contain", - "ulabel", - "uts46_remap", - "valid_contextj", - "valid_contexto", - "valid_label_length", - "valid_string_length", -] diff --git a/bundle/python-cpu/Lib/site-packages/idna/__main__.py b/bundle/python-cpu/Lib/site-packages/idna/__main__.py deleted file mode 100644 index dbdd066172160ef2d47b29f0caa00e27c62ebc20..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -import sys - -from .cli import main - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bundle/python-cpu/Lib/site-packages/idna/cli.py b/bundle/python-cpu/Lib/site-packages/idna/cli.py deleted file mode 100644 index 4acda2c0f03561ee7e2a80611bdcb35900bb34e0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna/cli.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Command-line interface for the :mod:`idna` package. - -Invoked via ``python -m idna``. See :func:`main` for the entry point. -""" - -import argparse -import sys -from collections.abc import Iterable -from itertools import chain -from typing import IO, Optional - -from . import IDNAError, decode, encode -from .core import _alabel_prefix, _unicode_dots_re -from .package_data import __version__ - - -def _looks_like_alabel(s: str) -> bool: - """Return True if any label in ``s`` carries the ``xn--`` ACE prefix.""" - prefix = _alabel_prefix.decode("ascii") - return any(label.lower().startswith(prefix) for label in _unicode_dots_re.split(s)) - - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="python -m idna", - description=( - "Convert a domain name between its Unicode (U-label) and " - "ASCII-compatible (A-label) forms. With no mode flag, the " - "direction is chosen from the first input — if it contains " - "an xn-- label the stream is decoded, otherwise it is " - "encoded — and the same mode is applied to every remaining " - "input. UTS #46 mapping is applied by default; pass " - "--strict to disable it. When no domains are given on the " - "command line and stdin is piped, one domain per line is " - "read from stdin." - ), - ) - mode = parser.add_mutually_exclusive_group() - mode.add_argument( - "-e", - "--encode", - dest="mode", - action="store_const", - const="encode", - help="Encode the input to its ASCII A-label form.", - ) - mode.add_argument( - "-d", - "--decode", - dest="mode", - action="store_const", - const="decode", - help="Decode the input from its ASCII A-label form.", - ) - parser.add_argument( - "--strict", - action="store_true", - help="Disable the default UTS #46 mapping and apply IDNA 2008 rules verbatim.", - ) - parser.add_argument( - "--version", - action="version", - version=f"idna {__version__}", - ) - parser.add_argument( - "domain", - nargs="*", - help="One or more domain names to convert. Omit to read from stdin.", - ) - return parser - - -def _iter_stdin(stream: IO[str]) -> Iterable[str]: - """Yield non-empty stripped lines from ``stream``, ignoring blanks.""" - for line in stream: - stripped = line.strip() - if stripped: - yield stripped - - -def _convert_one(domain: str, mode: str, uts46: bool) -> bool: - """Convert ``domain`` and write the result; return ``False`` on failure.""" - try: - if mode == "decode": - print(decode(domain, uts46=uts46)) - else: - print(encode(domain, uts46=uts46).decode("ascii")) - except IDNAError as err: - print(f"idna: {mode} failed for {domain!r}: {err}", file=sys.stderr) - return False - return True - - -def main(argv: Optional[list[str]] = None) -> int: - """Entry point for ``python -m idna``. - - When more than one domain is supplied (via positional arguments or - piped stdin) and no mode flag is given, the first input determines - the direction and that mode is applied uniformly to the rest. - - :param argv: Argument list excluding the program name. Defaults to - :data:`sys.argv` when ``None``. - :returns: ``0`` on success, ``1`` if any conversion fails. - """ - parser = _build_parser() - args = parser.parse_args(argv) - uts46 = not args.strict - - if args.domain: - domains: Iterable[str] = args.domain - elif not sys.stdin.isatty(): - domains = _iter_stdin(sys.stdin) - else: - parser.error("a domain argument is required when stdin is a terminal") - - iterator = iter(domains) - first = next(iterator, None) - if first is None: - return 0 - - mode = args.mode or ("decode" if _looks_like_alabel(first) else "encode") - - results = [_convert_one(domain, mode, uts46) for domain in chain([first], iterator)] - return 0 if all(results) else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bundle/python-cpu/Lib/site-packages/idna/codec.py b/bundle/python-cpu/Lib/site-packages/idna/codec.py deleted file mode 100644 index 83b42fe42b51adbe1cf9dbea79c4c8d8dbf8a0b9..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna/codec.py +++ /dev/null @@ -1,159 +0,0 @@ -import codecs -from typing import Any, Optional - -from .core import IDNAError, _unicode_dots_re, alabel, decode, encode, ulabel - - -class Codec(codecs.Codec): - """Stateless IDNA 2008 codec. - - Implements the :class:`codecs.Codec` protocol so that the whole-domain - encoder (:func:`idna.encode`) and decoder (:func:`idna.decode`) are - accessible through the standard codec machinery as ``"idna2008"``. - - Only the ``"strict"`` error handler is supported; any other handler - raises :exc:`~idna.IDNAError`. - """ - - def encode(self, data: str, errors: str = "strict") -> tuple[bytes, int]: # ty: ignore[invalid-method-override] - if errors != "strict": - raise IDNAError(f'Unsupported error handling "{errors}"') - - if not data: - return b"", 0 - - return encode(data), len(data) - - def decode(self, data: bytes, errors: str = "strict") -> tuple[str, int]: # ty: ignore[invalid-method-override] - if errors != "strict": - raise IDNAError(f'Unsupported error handling "{errors}"') - - if not data: - return "", 0 - - return decode(data), len(data) - - -class IncrementalEncoder(codecs.BufferedIncrementalEncoder): - """Incremental IDNA 2008 encoder. - - Buffers a partial trailing label across calls until either the next - label separator is seen or ``final=True``, so that streamed input is - encoded one whole label at a time. Any of the four Unicode label - separators (``U+002E``, ``U+3002``, ``U+FF0E``, ``U+FF61``) ends a - label; the result always uses ``U+002E`` as the separator. - - Only the ``"strict"`` error handler is supported. - """ - - def _buffer_encode(self, data: str, errors: str, final: bool) -> tuple[bytes, int]: # ty: ignore[invalid-method-override] - if errors != "strict": - raise IDNAError(f'Unsupported error handling "{errors}"') - - if not data: - return b"", 0 - - labels = _unicode_dots_re.split(data) - trailing_dot = b"" - if labels: - if not labels[-1]: - trailing_dot = b"." - del labels[-1] - elif not final: - # Keep potentially unfinished label until the next call - del labels[-1] - if labels: - trailing_dot = b"." - - result = [] - size = 0 - for label in labels: - result.append(alabel(label)) - if size: - size += 1 - size += len(label) - - # Join with U+002E - result_bytes = b".".join(result) + trailing_dot - size += len(trailing_dot) - return result_bytes, size - - -class IncrementalDecoder(codecs.BufferedIncrementalDecoder): - """Incremental IDNA 2008 decoder. - - Buffers a partial trailing label across calls until either the next - label separator is seen or ``final=True``, so that streamed input is - decoded one whole label at a time. - - Only the ``"strict"`` error handler is supported. - """ - - def _buffer_decode(self, data: Any, errors: str, final: bool) -> tuple[str, int]: # ty: ignore[invalid-method-override] - if errors != "strict": - raise IDNAError(f'Unsupported error handling "{errors}"') - - if not data: - return ("", 0) - - if not isinstance(data, str): - data = str(data, "ascii") - - labels = _unicode_dots_re.split(data) - trailing_dot = "" - if labels: - if not labels[-1]: - trailing_dot = "." - del labels[-1] - elif not final: - # Keep potentially unfinished label until the next call - del labels[-1] - if labels: - trailing_dot = "." - - result = [] - size = 0 - for label in labels: - result.append(ulabel(label)) - if size: - size += 1 - size += len(label) - - result_str = ".".join(result) + trailing_dot - size += len(trailing_dot) - return (result_str, size) - - -class StreamWriter(Codec, codecs.StreamWriter): - pass - - -class StreamReader(Codec, codecs.StreamReader): - pass - - -def search_function(name: str) -> Optional[codecs.CodecInfo]: - """Codec search function registered with :mod:`codecs`. - - Returns a :class:`codecs.CodecInfo` for the ``"idna2008"`` codec name - so that ``str.encode("idna2008")`` and ``bytes.decode("idna2008")`` - invoke the IDNA 2008 codec defined in this module. - - :param name: The codec name being looked up. - :returns: A :class:`codecs.CodecInfo` instance if ``name`` is - ``"idna2008"``, otherwise ``None``. - """ - if name != "idna2008": - return None - return codecs.CodecInfo( - name=name, - encode=Codec().encode, - decode=Codec().decode, # type: ignore - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamwriter=StreamWriter, - streamreader=StreamReader, - ) - - -codecs.register(search_function) diff --git a/bundle/python-cpu/Lib/site-packages/idna/compat.py b/bundle/python-cpu/Lib/site-packages/idna/compat.py deleted file mode 100644 index 1d01e3d9750860e6ec87fb7c0955cee6bcd9fcc1..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna/compat.py +++ /dev/null @@ -1,41 +0,0 @@ -from typing import Any, Union - -from .core import decode, encode - - -def ToASCII(label: str) -> bytes: - """Compatibility shim for :rfc:`3490` ``ToASCII``. - - Delegates to :func:`idna.encode` (IDNA 2008). Provided to ease porting - of code written against the legacy :mod:`encodings.idna` API; new code - should call :func:`idna.encode` directly. - - :param label: The label or domain to encode. - :returns: The encoded form as ASCII :class:`bytes`. - """ - return encode(label) - - -def ToUnicode(label: Union[bytes, bytearray]) -> str: - """Compatibility shim for :rfc:`3490` ``ToUnicode``. - - Delegates to :func:`idna.decode` (IDNA 2008). Provided to ease porting - of code written against the legacy :mod:`encodings.idna` API; new code - should call :func:`idna.decode` directly. - - :param label: The label or domain to decode. - :returns: The decoded Unicode form. - """ - return decode(label) - - -def nameprep(s: Any) -> None: - """Stub for :rfc:`3491` Nameprep, which is not used by IDNA 2008. - - IDNA 2008 (:rfc:`5891`) replaces Nameprep with the per-codepoint - validity classes from :rfc:`5892`; this function exists only to - return a clear error if legacy code attempts to call it. - - :raises NotImplementedError: Always. - """ - raise NotImplementedError("IDNA 2008 does not utilise nameprep protocol") diff --git a/bundle/python-cpu/Lib/site-packages/idna/core.py b/bundle/python-cpu/Lib/site-packages/idna/core.py deleted file mode 100644 index 1ccbd1f3356e4b9f2aeb36b00fd5591d85459464..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna/core.py +++ /dev/null @@ -1,648 +0,0 @@ -import bisect -import re -import unicodedata -import warnings -from typing import Optional, Union - -from . import idnadata -from .intranges import intranges_contain - -_virama_combining_class = 9 -_alabel_prefix = b"xn--" -_max_input_length = 1024 -_unicode_dots_re = re.compile("[\u002e\u3002\uff0e\uff61]") - - -# Bidi category sets from RFC 5893, hoisted out of the per-codepoint loop -_bidi_rtl_first = frozenset({"R", "AL"}) -_bidi_rtl_categories = frozenset({"R", "AL", "AN"}) -_bidi_rtl_allowed = frozenset({"R", "AL", "AN", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"}) -_bidi_rtl_valid_ending = frozenset({"R", "AL", "EN", "AN"}) -_bidi_rtl_numeric = frozenset({"AN", "EN"}) -_bidi_ltr_allowed = frozenset({"L", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"}) -_bidi_ltr_valid_ending = frozenset({"L", "EN"}) -_bidi_joiner_l_or_d = frozenset({"L", "D"}) -_bidi_joiner_r_or_d = frozenset({"R", "D"}) - - -def _joining_type(cp: int) -> Optional[str]: - for jt, ranges in idnadata.joining_types.items(): - if intranges_contain(cp, ranges): - return jt - return None - - -class IDNAError(UnicodeError): - """Base exception for all IDNA-encoding related problems""" - - -class IDNABidiError(IDNAError): - """Exception when bidirectional requirements are not satisfied""" - - -class InvalidCodepoint(IDNAError): - """Exception when a disallowed or unallocated codepoint is used""" - - -class InvalidCodepointContext(IDNAError): - """Exception when the codepoint is not valid in the context it is used""" - - -def _combining_class(cp: int) -> int: - v = unicodedata.combining(chr(cp)) - if v == 0 and not unicodedata.name(chr(cp)): - raise ValueError("Unknown character in unicodedata") - return v - - -def _is_script(cp: str, script: str) -> bool: - return intranges_contain(ord(cp), idnadata.scripts[script]) - - -def _punycode(s: str) -> bytes: - return s.encode("punycode") - - -def _unot(s: int) -> str: - return f"U+{s:04X}" - - -def valid_label_length(label: Union[bytes, str]) -> bool: - """Check that a label does not exceed the maximum permitted length. - - Per :rfc:`1035` (and :rfc:`5891` §4.2.4) a DNS label must not exceed - 63 octets. The argument may be either a :class:`str` (a U-label, where - length is measured in characters) or :class:`bytes` (an A-label, where - length is measured in octets). - - :param label: The label to check. - :returns: ``True`` if the label is within the length limit, otherwise - ``False``. - """ - return len(label) <= 63 - - -def valid_string_length(domain: Union[bytes, str], trailing_dot: bool) -> bool: - """Check that a full domain name does not exceed the maximum length. - - Per :rfc:`1035`, a domain name is limited to 253 octets when no trailing - dot is present, or 254 octets when one is included. - - :param domain: The full (possibly multi-label) domain name. - :param trailing_dot: ``True`` if ``domain`` includes a trailing ``.``. - :returns: ``True`` if the domain is within the length limit, otherwise - ``False``. - """ - return len(domain) <= (254 if trailing_dot else 253) - - -def check_bidi(label: str, check_ltr: bool = False) -> bool: - """Validate the Bidi Rule from :rfc:`5893` for a single label. - - The Bidi Rule constrains how bidirectional characters (Hebrew, Arabic, - etc.) may appear within a label. By default the check is only applied - when the label contains at least one right-to-left character (Unicode - bidirectional categories ``R``, ``AL``, or ``AN``); set ``check_ltr`` - to ``True`` to apply it to LTR-only labels as well. - - :param label: The label to validate, as a Unicode string. - :param check_ltr: If ``True``, apply the rules even when the label - contains no RTL characters. - :returns: ``True`` if the label satisfies the Bidi Rule. - :raises IDNABidiError: If any of Bidi Rule conditions 1-6 are violated, - or if the directional category of a codepoint cannot be determined. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - # Bidi rules should only be applied if string contains RTL characters - bidi_label = False - for idx, cp in enumerate(label, 1): - direction = unicodedata.bidirectional(cp) - if direction == "": - # String likely comes from a newer version of Unicode - raise IDNABidiError(f"Unknown directionality in label {label!r} at position {idx}") - if direction in _bidi_rtl_categories: - bidi_label = True - if not bidi_label and not check_ltr: - return True - - # Bidi rule 1 - direction = unicodedata.bidirectional(label[0]) - if direction in _bidi_rtl_first: - rtl = True - elif direction == "L": - rtl = False - else: - raise IDNABidiError(f"First codepoint in label {label!r} must be directionality L, R or AL") - - valid_ending = False - number_type: Optional[str] = None - for idx, cp in enumerate(label, 1): - direction = unicodedata.bidirectional(cp) - - if rtl: - # Bidi rule 2 - if direction not in _bidi_rtl_allowed: - raise IDNABidiError(f"Invalid direction for codepoint at position {idx} in a right-to-left label") - # Bidi rule 3 - if direction in _bidi_rtl_valid_ending: - valid_ending = True - elif direction != "NSM": - valid_ending = False - # Bidi rule 4 - if direction in _bidi_rtl_numeric: - if not number_type: - number_type = direction - elif number_type != direction: - raise IDNABidiError("Can not mix numeral types in a right-to-left label") - else: - # Bidi rule 5 - if direction not in _bidi_ltr_allowed: - raise IDNABidiError(f"Invalid direction for codepoint at position {idx} in a left-to-right label") - # Bidi rule 6 - if direction in _bidi_ltr_valid_ending: - valid_ending = True - elif direction != "NSM": - valid_ending = False - - if not valid_ending: - raise IDNABidiError("Label ends with illegal codepoint directionality") - - return True - - -def check_initial_combiner(label: str) -> bool: - """Reject labels that begin with a combining mark. - - Per :rfc:`5891` §4.2.3.2 a label must not start with a character of - Unicode general category ``M`` (Mark). - - :param label: The label to check. - :returns: ``True`` if the first character is not a combining mark. - :raises IDNAError: If the label begins with a combining character. - """ - if unicodedata.category(label[0])[0] == "M": - raise IDNAError("Label begins with an illegal combining character") - return True - - -def check_hyphen_ok(label: str) -> bool: - """Validate the hyphen restrictions for a label. - - Per :rfc:`5891` §4.2.3.1 a label must not start or end with a hyphen - (``U+002D``), and must not have hyphens in both the third and fourth - positions (the prefix reserved for A-labels). - - :param label: The label to check. - :returns: ``True`` if the hyphen restrictions are satisfied. - :raises IDNAError: If any of the hyphen restrictions are violated. - """ - if label[2:4] == "--": - raise IDNAError("Label has disallowed hyphens in 3rd and 4th position") - if label[0] == "-" or label[-1] == "-": - raise IDNAError("Label must not start or end with a hyphen") - return True - - -def check_nfc(label: str) -> None: - """Require that a label is in Unicode Normalization Form C. - - :param label: The label to check. - :raises IDNAError: If ``label`` differs from its NFC normalisation. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - if unicodedata.normalize("NFC", label) != label: - raise IDNAError("Label must be in Normalization Form C") - - -def valid_contextj(label: str, pos: int) -> bool: - """Validate the CONTEXTJ rules from :rfc:`5892` Appendix A. - - These rules govern the contextual use of the joiner codepoints - ``U+200C`` (ZERO WIDTH NON-JOINER, Appendix A.1) and ``U+200D`` - (ZERO WIDTH JOINER, Appendix A.2) within a label. - - :param label: The label containing the codepoint. - :param pos: Index of the joiner codepoint within ``label``. - :returns: ``True`` if the codepoint at ``pos`` satisfies its CONTEXTJ - rule, ``False`` otherwise (including when the codepoint at - ``pos`` is not a recognised joiner). - :raises ValueError: If an adjacent codepoint has no Unicode name when - determining its combining class. - :raises IDNAError: If ``label`` exceeds the defensive input length limit. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - cp_value = ord(label[pos]) - - if cp_value == 0x200C: - if pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class: - return True - - ok = False - for i in range(pos - 1, -1, -1): - joining_type = _joining_type(ord(label[i])) - if joining_type == "T": - continue - if joining_type in _bidi_joiner_l_or_d: - ok = True - break - break - - if not ok: - return False - - ok = False - for i in range(pos + 1, len(label)): - joining_type = _joining_type(ord(label[i])) - if joining_type == "T": - continue - if joining_type in _bidi_joiner_r_or_d: - ok = True - break - break - return ok - - if cp_value == 0x200D: - return pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class - - return False - - -def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: - """Validate the CONTEXTO rules from :rfc:`5892` Appendix A. - - Covers the contextual rules for codepoints such as MIDDLE DOT - (``U+00B7``), Greek lower numeral sign, Hebrew punctuation, Katakana - middle dot, and the Arabic-Indic / Extended Arabic-Indic digit ranges. - - :param label: The label containing the codepoint. - :param pos: Index of the codepoint within ``label``. - :param exception: Reserved for forward compatibility; currently unused. - :returns: ``True`` if the codepoint at ``pos`` satisfies its CONTEXTO - rule, ``False`` otherwise (including when the codepoint is not a - recognised CONTEXTO codepoint). - :raises IDNAError: If ``label`` exceeds the defensive input length limit. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - cp_value = ord(label[pos]) - - if cp_value == 0x00B7: - return 0 < pos < len(label) - 1 and ord(label[pos - 1]) == 0x006C and ord(label[pos + 1]) == 0x006C - - if cp_value == 0x0375: - if pos < len(label) - 1 and len(label) > 1: - return _is_script(label[pos + 1], "Greek") - return False - - if cp_value in {0x05F3, 0x05F4}: - if pos > 0: - return _is_script(label[pos - 1], "Hebrew") - return False - - if cp_value == 0x30FB: - for cp in label: - if cp == "\u30fb": - continue - if _is_script(cp, "Hiragana") or _is_script(cp, "Katakana") or _is_script(cp, "Han"): - return True - return False - - if 0x660 <= cp_value <= 0x669: - return not any(0x6F0 <= ord(cp) <= 0x06F9 for cp in label) - - if 0x6F0 <= cp_value <= 0x6F9: - return not any(0x660 <= ord(cp) <= 0x0669 for cp in label) - - return False - - -def check_label(label: Union[str, bytes, bytearray]) -> None: - """Run the full set of IDNA 2008 validity checks on a single label. - - Applies, in order: NFC normalisation (:func:`check_nfc`), hyphen - restrictions (:func:`check_hyphen_ok`), the no-leading-combiner rule - (:func:`check_initial_combiner`), per-codepoint validity (PVALID, - CONTEXTJ, CONTEXTO classes from :rfc:`5892`), and the Bidi Rule - (:func:`check_bidi`). - - :param label: The label to validate. ``bytes`` or ``bytearray`` input - is decoded as UTF-8 first. - :raises IDNAError: If the label is empty or fails a structural rule. - :raises InvalidCodepoint: If the label contains a DISALLOWED or - UNASSIGNED codepoint. - :raises InvalidCodepointContext: If a CONTEXTJ or CONTEXTO codepoint - is not valid in its context. - :raises IDNABidiError: If the Bidi Rule is violated. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - if isinstance(label, (bytes, bytearray)): - label = label.decode("utf-8") - if len(label) == 0: - raise IDNAError("Empty Label") - - # Reject on domain length rather than label length so support some UTS 46 - # use cases, still reducing processing of label contextual rules - if not valid_string_length(label, trailing_dot=True): - raise IDNAError("Label too long") - - check_nfc(label) - check_hyphen_ok(label) - check_initial_combiner(label) - - for pos, cp in enumerate(label): - cp_value = ord(cp) - if intranges_contain(cp_value, idnadata.codepoint_classes["PVALID"]): - continue - if intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]): - try: - if not valid_contextj(label, pos): - raise InvalidCodepointContext(f"Joiner {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") - except ValueError as err: - raise IDNAError( - f"Unknown codepoint adjacent to joiner {_unot(cp_value)} at position {pos + 1} in {label!r}" - ) from err - elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTO"]): - if not valid_contexto(label, pos): - raise InvalidCodepointContext(f"Codepoint {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") - else: - raise InvalidCodepoint(f"Codepoint {_unot(cp_value)} at position {pos + 1} of {label!r} not allowed") - - check_bidi(label) - - -def alabel(label: str) -> bytes: - """Convert a single U-label into its A-label form. - - The result is the ASCII-Compatible Encoding (ACE) form per :rfc:`5891` - §4: the label is validated, Punycode-encoded, and prefixed with - ``xn--``. Pure ASCII labels that are already valid IDNA labels are - returned unchanged (as :class:`bytes`). - - :param label: The label to convert, as a Unicode string. - :returns: The A-label as ASCII-encoded :class:`bytes`. - :raises IDNAError: If the label is invalid or the resulting A-label - exceeds 63 octets. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - try: - label_bytes = label.encode("ascii") - except UnicodeEncodeError: - pass - else: - ulabel(label_bytes) - if not valid_label_length(label_bytes): - raise IDNAError("Label too long") - return label_bytes - - check_label(label) - label_bytes = _alabel_prefix + _punycode(label) - - if not valid_label_length(label_bytes): - raise IDNAError("Label too long") - - return label_bytes - - -def ulabel(label: Union[str, bytes, bytearray]) -> str: - """Convert a single A-label into its U-label form. - - Performs the inverse of :func:`alabel`: an ``xn--``-prefixed label is - Punycode-decoded and validated. Labels that are already Unicode (or - plain ASCII without the ACE prefix) are validated and returned as a - Unicode string. - - :param label: The label to convert. ``bytes`` or ``bytearray`` input - is treated as ASCII. - :returns: The U-label as a Unicode string. - :raises IDNAError: If the label is malformed or fails validation. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - if not isinstance(label, (bytes, bytearray)): - try: - label_bytes = label.encode("ascii") - except UnicodeEncodeError: - check_label(label) - return label - else: - label_bytes = bytes(label) - - label_bytes = label_bytes.lower() - if label_bytes.startswith(_alabel_prefix): - label_bytes = label_bytes[len(_alabel_prefix) :] - if not label_bytes: - raise IDNAError("Malformed A-label, no Punycode eligible content found") - if label_bytes.endswith(b"-"): - raise IDNAError("A-label must not end with a hyphen") - else: - check_label(label_bytes) - return label_bytes.decode("ascii") - - try: - label = label_bytes.decode("punycode") - except UnicodeError as err: - raise IDNAError("Invalid A-label") from err - check_label(label) - return label - - -def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False) -> str: - """Apply the UTS #46 character mapping to a domain string. - - Implements the mapping table from `UTS #46 §4 - `_: each character is kept, - replaced, or rejected based on its status (``V``, ``M``, ``D``, ``3``, - ``I``). The result is returned in Normalisation Form C. - - :param domain: The full domain name to remap. - :param std3_rules: If ``True``, apply the stricter STD3 ASCII rules - (status ``3`` codepoints raise instead of being kept or mapped). - :param transitional: If ``True``, use transitional processing (status - ``D`` codepoints are mapped instead of kept). Transitional - processing has been removed from UTS #46 and this option is - retained only for backwards compatibility. - :returns: The remapped domain, in Normalisation Form C. - :raises InvalidCodepoint: If the domain contains a disallowed - codepoint under the chosen rules. - :raises IDNAError: If ``domain`` exceeds the defensive input length limit. - """ - if len(domain) > _max_input_length: - raise IDNAError("Domain too long") - from .uts46data import uts46_replacements, uts46_starts, uts46_statuses - - output = "" - - for pos, char in enumerate(domain): - code_point = ord(char) - i = code_point if code_point < 256 else bisect.bisect_right(uts46_starts, code_point) - 1 - status = chr(uts46_statuses[i]) - replacement: Optional[str] = uts46_replacements[i] - - # UTS #46 §4: V is always valid, D is deviation (kept unless transitional), - # 3 is disallowed-STD3 (kept unmapped if std3_rules is off and no mapping). - keep_as_is = ( - status == "V" or (status == "D" and not transitional) or (status == "3" and not std3_rules and replacement is None) - ) - # M is mapped, 3-with-replacement and transitional D fall through to the - # same replacement output path. - use_replacement = replacement is not None and ( - status == "M" or (status == "3" and not std3_rules) or (status == "D" and transitional) - ) - - if keep_as_is: - output += char - elif use_replacement: - assert replacement is not None # narrowed by use_replacement - output += replacement - elif status == "I": - continue - else: - raise InvalidCodepoint(f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {domain!r}") - - return unicodedata.normalize("NFC", output) - - -def encode( - s: Union[str, bytes, bytearray], - strict: bool = False, - uts46: bool = False, - std3_rules: bool = False, - transitional: bool = False, -) -> bytes: - """Encode a Unicode domain name into its ASCII (A-label) form. - - Splits the input on label separators (only ``U+002E`` if ``strict`` is - set; otherwise also IDEOGRAPHIC FULL STOP ``U+3002``, FULLWIDTH FULL - STOP ``U+FF0E``, and HALFWIDTH IDEOGRAPHIC FULL STOP ``U+FF61``), - encodes each label with :func:`alabel`, and rejoins them with ``.``. - Optionally pre-processes the input through :func:`uts46_remap`. - - :param s: The domain name to encode. - :param strict: If ``True``, only ``U+002E`` is recognised as a label - separator. - :param uts46: If ``True``, apply UTS #46 mapping before encoding. - :param std3_rules: Forwarded to :func:`uts46_remap` when ``uts46`` is - ``True``. - :param transitional: Forwarded to :func:`uts46_remap` when ``uts46`` - is ``True``. Deprecated: emits a :class:`DeprecationWarning` and - will be removed in a future version. - :returns: The encoded domain as ASCII :class:`bytes`. - :raises IDNAError: If the domain is empty, contains an invalid label, - or exceeds the maximum domain length. - """ - if transitional: - warnings.warn( - "Transitional processing has been removed from UTS #46. " - "The transitional argument will be removed in a future version.", - DeprecationWarning, - stacklevel=2, - ) - if not isinstance(s, str): - try: - s = str(s, "ascii") - except (UnicodeDecodeError, TypeError) as err: - raise IDNAError("should pass a unicode string to the function rather than a byte string.") from err - if len(s) > _max_input_length: - raise IDNAError("Domain too long") - if uts46: - s = uts46_remap(s, std3_rules, transitional) - - # Reject inputs that exceed the maximum DNS domain length up-front - # to avoid expensive computation on long inputs. - if not valid_string_length(s, trailing_dot=True): - raise IDNAError("Domain too long") - - trailing_dot = False - result = [] - labels = s.split(".") if strict else _unicode_dots_re.split(s) - if not labels or labels == [""]: - raise IDNAError("Empty domain") - if labels[-1] == "": - del labels[-1] - trailing_dot = True - for label in labels: - s = alabel(label) - if s: - result.append(s) - else: - raise IDNAError("Empty label") - if trailing_dot: - result.append(b"") - s = b".".join(result) - if not valid_string_length(s, trailing_dot): - raise IDNAError("Domain too long") - return s - - -def decode( - s: Union[str, bytes, bytearray], - strict: bool = False, - uts46: bool = False, - std3_rules: bool = False, - display: bool = False, -) -> str: - """Decode an A-label-encoded domain name back to Unicode. - - Splits the input on label separators (see :func:`encode` for the - rules), decodes each label with :func:`ulabel`, and rejoins them - with ``.``. Optionally pre-processes the input through - :func:`uts46_remap`. - - :param s: The domain name to decode. - :param strict: If ``True``, only ``U+002E`` is recognised as a label - separator. - :param uts46: If ``True``, apply UTS #46 mapping before decoding. - :param std3_rules: Forwarded to :func:`uts46_remap` when ``uts46`` is - ``True``. - :param display: If ``True``, any ``xn--`` label that fails IDNA - validation is passed through unchanged (lowercased) rather than - aborting the whole call. Intended for "decode for display" - consumers (e.g. URL libraries, HTTP clients) that want to show - the user the label as it appears on the wire when it cannot be - rendered as Unicode. Matches the per-label recovery prescribed - by UTS #46 §4 and the WHATWG URL "domain to Unicode" algorithm. - :returns: The decoded domain as a Unicode string. - :raises IDNAError: If the input is not valid ASCII, contains an - invalid label, or is empty. - """ - if not isinstance(s, str): - try: - s = str(s, "ascii") - except (UnicodeDecodeError, TypeError) as err: - raise IDNAError("Invalid ASCII in A-label") from err - if len(s) > _max_input_length: - raise IDNAError("Domain too long") - if uts46: - s = uts46_remap(s, std3_rules, False) - # Reject inputs that exceed the maximum DNS domain length up-front - # to avoid expensive computation on long inputs. - if not valid_string_length(s, trailing_dot=True): - raise IDNAError("Domain too long") - trailing_dot = False - result = [] - labels = s.split(".") if strict else _unicode_dots_re.split(s) - if not labels or labels == [""]: - raise IDNAError("Empty domain") - if not labels[-1]: - del labels[-1] - trailing_dot = True - for label in labels: - try: - u = ulabel(label) - except IDNAError: - if display and label[:4].lower() == "xn--": - u = label.lower() - else: - raise - if u: - result.append(u) - else: - raise IDNAError("Empty label") - if trailing_dot: - result.append("") - return ".".join(result) diff --git a/bundle/python-cpu/Lib/site-packages/idna/idnadata.py b/bundle/python-cpu/Lib/site-packages/idna/idnadata.py deleted file mode 100644 index f2ab38897bf1f8fc6f0db50f237edec68ca89295..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna/idnadata.py +++ /dev/null @@ -1,1897 +0,0 @@ -# This file is automatically generated by tools/idna-data - -__version__ = "17.0.0" - -scripts = { - "Greek": ( - 0x37000000374, - 0x37500000378, - 0x37A0000037E, - 0x37F00000380, - 0x38400000385, - 0x38600000387, - 0x3880000038B, - 0x38C0000038D, - 0x38E000003A2, - 0x3A3000003E2, - 0x3F000000400, - 0x1D2600001D2B, - 0x1D5D00001D62, - 0x1D6600001D6B, - 0x1DBF00001DC0, - 0x1F0000001F16, - 0x1F1800001F1E, - 0x1F2000001F46, - 0x1F4800001F4E, - 0x1F5000001F58, - 0x1F5900001F5A, - 0x1F5B00001F5C, - 0x1F5D00001F5E, - 0x1F5F00001F7E, - 0x1F8000001FB5, - 0x1FB600001FC5, - 0x1FC600001FD4, - 0x1FD600001FDC, - 0x1FDD00001FF0, - 0x1FF200001FF5, - 0x1FF600001FFF, - 0x212600002127, - 0xAB650000AB66, - 0x101400001018F, - 0x101A0000101A1, - 0x1D2000001D246, - ), - "Han": ( - 0x2E8000002E9A, - 0x2E9B00002EF4, - 0x2F0000002FD6, - 0x300500003006, - 0x300700003008, - 0x30210000302A, - 0x30380000303C, - 0x340000004DC0, - 0x4E000000A000, - 0xF9000000FA6E, - 0xFA700000FADA, - 0x16FE200016FE4, - 0x16FF000016FF7, - 0x200000002A6E0, - 0x2A7000002B81E, - 0x2B8200002CEAE, - 0x2CEB00002EBE1, - 0x2EBF00002EE5E, - 0x2F8000002FA1E, - 0x300000003134B, - 0x313500003347A, - ), - "Hebrew": ( - 0x591000005C8, - 0x5D0000005EB, - 0x5EF000005F5, - 0xFB1D0000FB37, - 0xFB380000FB3D, - 0xFB3E0000FB3F, - 0xFB400000FB42, - 0xFB430000FB45, - 0xFB460000FB50, - ), - "Hiragana": ( - 0x304100003097, - 0x309D000030A0, - 0x1B0010001B120, - 0x1B1320001B133, - 0x1B1500001B153, - 0x1F2000001F201, - ), - "Katakana": ( - 0x30A1000030FB, - 0x30FD00003100, - 0x31F000003200, - 0x32D0000032FF, - 0x330000003358, - 0xFF660000FF70, - 0xFF710000FF9E, - 0x1AFF00001AFF4, - 0x1AFF50001AFFC, - 0x1AFFD0001AFFF, - 0x1B0000001B001, - 0x1B1200001B123, - 0x1B1550001B156, - 0x1B1640001B168, - ), -} - - -joining_types = { - "C": ( - 0x64000000641, - 0x7FA000007FB, - 0x88300000886, - 0x180A0000180B, - 0x200D0000200E, - ), - "D": ( - 0x62000000621, - 0x62600000627, - 0x62800000629, - 0x62A0000062F, - 0x63300000640, - 0x64100000648, - 0x6490000064B, - 0x66E00000670, - 0x67800000688, - 0x69A000006C0, - 0x6C1000006C3, - 0x6CC000006CD, - 0x6CE000006CF, - 0x6D0000006D2, - 0x6FA000006FD, - 0x6FF00000700, - 0x71200000715, - 0x71A0000071E, - 0x71F00000728, - 0x7290000072A, - 0x72B0000072C, - 0x72D0000072F, - 0x74E00000759, - 0x75C0000076B, - 0x76D00000771, - 0x77200000773, - 0x77500000778, - 0x77A00000780, - 0x7CA000007EB, - 0x84100000846, - 0x84800000849, - 0x84A00000854, - 0x85500000856, - 0x86000000861, - 0x86200000866, - 0x86800000869, - 0x88600000887, - 0x8890000088E, - 0x88F00000890, - 0x8A0000008AA, - 0x8AF000008B1, - 0x8B3000008B9, - 0x8BA000008C9, - 0x180700001808, - 0x182000001879, - 0x1887000018A9, - 0x18AA000018AB, - 0xA8400000A872, - 0x10AC000010AC5, - 0x10AD300010AD7, - 0x10AD800010ADD, - 0x10ADE00010AE1, - 0x10AEB00010AEF, - 0x10B8000010B81, - 0x10B8200010B83, - 0x10B8600010B89, - 0x10B8A00010B8C, - 0x10B8D00010B8E, - 0x10B9000010B91, - 0x10BAD00010BAF, - 0x10D0100010D22, - 0x10D2300010D24, - 0x10EC300010EC5, - 0x10EC600010EC8, - 0x10F3000010F33, - 0x10F3400010F45, - 0x10F5100010F54, - 0x10F7000010F74, - 0x10F7600010F82, - 0x10FB000010FB1, - 0x10FB200010FB4, - 0x10FB800010FB9, - 0x10FBB00010FBD, - 0x10FBE00010FC0, - 0x10FC100010FC2, - 0x10FC400010FC5, - 0x10FCA00010FCB, - 0x1E9000001E944, - ), - "L": ( - 0xA8720000A873, - 0x10ACD00010ACE, - 0x10AD700010AD8, - 0x10D0000010D01, - 0x10FCB00010FCC, - ), - "R": ( - 0x62200000626, - 0x62700000628, - 0x6290000062A, - 0x62F00000633, - 0x64800000649, - 0x67100000674, - 0x67500000678, - 0x6880000069A, - 0x6C0000006C1, - 0x6C3000006CC, - 0x6CD000006CE, - 0x6CF000006D0, - 0x6D2000006D4, - 0x6D5000006D6, - 0x6EE000006F0, - 0x71000000711, - 0x7150000071A, - 0x71E0000071F, - 0x72800000729, - 0x72A0000072B, - 0x72C0000072D, - 0x72F00000730, - 0x74D0000074E, - 0x7590000075C, - 0x76B0000076D, - 0x77100000772, - 0x77300000775, - 0x7780000077A, - 0x84000000841, - 0x84600000848, - 0x8490000084A, - 0x85400000855, - 0x85600000859, - 0x86700000868, - 0x8690000086B, - 0x87000000883, - 0x88E0000088F, - 0x8AA000008AD, - 0x8AE000008AF, - 0x8B1000008B3, - 0x8B9000008BA, - 0x10AC500010AC6, - 0x10AC700010AC8, - 0x10AC900010ACB, - 0x10ACE00010AD3, - 0x10ADD00010ADE, - 0x10AE100010AE2, - 0x10AE400010AE5, - 0x10AEF00010AF0, - 0x10B8100010B82, - 0x10B8300010B86, - 0x10B8900010B8A, - 0x10B8C00010B8D, - 0x10B8E00010B90, - 0x10B9100010B92, - 0x10BA900010BAD, - 0x10D2200010D23, - 0x10EC200010EC3, - 0x10F3300010F34, - 0x10F5400010F55, - 0x10F7400010F76, - 0x10FB400010FB7, - 0x10FB900010FBB, - 0x10FBD00010FBE, - 0x10FC200010FC4, - 0x10FC900010FCA, - ), - "T": ( - 0xAD000000AE, - 0x30000000370, - 0x4830000048A, - 0x591000005BE, - 0x5BF000005C0, - 0x5C1000005C3, - 0x5C4000005C6, - 0x5C7000005C8, - 0x6100000061B, - 0x61C0000061D, - 0x64B00000660, - 0x67000000671, - 0x6D6000006DD, - 0x6DF000006E5, - 0x6E7000006E9, - 0x6EA000006EE, - 0x70F00000710, - 0x71100000712, - 0x7300000074B, - 0x7A6000007B1, - 0x7EB000007F4, - 0x7FD000007FE, - 0x8160000081A, - 0x81B00000824, - 0x82500000828, - 0x8290000082E, - 0x8590000085C, - 0x897000008A0, - 0x8CA000008E2, - 0x8E300000903, - 0x93A0000093B, - 0x93C0000093D, - 0x94100000949, - 0x94D0000094E, - 0x95100000958, - 0x96200000964, - 0x98100000982, - 0x9BC000009BD, - 0x9C1000009C5, - 0x9CD000009CE, - 0x9E2000009E4, - 0x9FE000009FF, - 0xA0100000A03, - 0xA3C00000A3D, - 0xA4100000A43, - 0xA4700000A49, - 0xA4B00000A4E, - 0xA5100000A52, - 0xA7000000A72, - 0xA7500000A76, - 0xA8100000A83, - 0xABC00000ABD, - 0xAC100000AC6, - 0xAC700000AC9, - 0xACD00000ACE, - 0xAE200000AE4, - 0xAFA00000B00, - 0xB0100000B02, - 0xB3C00000B3D, - 0xB3F00000B40, - 0xB4100000B45, - 0xB4D00000B4E, - 0xB5500000B57, - 0xB6200000B64, - 0xB8200000B83, - 0xBC000000BC1, - 0xBCD00000BCE, - 0xC0000000C01, - 0xC0400000C05, - 0xC3C00000C3D, - 0xC3E00000C41, - 0xC4600000C49, - 0xC4A00000C4E, - 0xC5500000C57, - 0xC6200000C64, - 0xC8100000C82, - 0xCBC00000CBD, - 0xCBF00000CC0, - 0xCC600000CC7, - 0xCCC00000CCE, - 0xCE200000CE4, - 0xD0000000D02, - 0xD3B00000D3D, - 0xD4100000D45, - 0xD4D00000D4E, - 0xD6200000D64, - 0xD8100000D82, - 0xDCA00000DCB, - 0xDD200000DD5, - 0xDD600000DD7, - 0xE3100000E32, - 0xE3400000E3B, - 0xE4700000E4F, - 0xEB100000EB2, - 0xEB400000EBD, - 0xEC800000ECF, - 0xF1800000F1A, - 0xF3500000F36, - 0xF3700000F38, - 0xF3900000F3A, - 0xF7100000F7F, - 0xF8000000F85, - 0xF8600000F88, - 0xF8D00000F98, - 0xF9900000FBD, - 0xFC600000FC7, - 0x102D00001031, - 0x103200001038, - 0x10390000103B, - 0x103D0000103F, - 0x10580000105A, - 0x105E00001061, - 0x107100001075, - 0x108200001083, - 0x108500001087, - 0x108D0000108E, - 0x109D0000109E, - 0x135D00001360, - 0x171200001715, - 0x173200001734, - 0x175200001754, - 0x177200001774, - 0x17B4000017B6, - 0x17B7000017BE, - 0x17C6000017C7, - 0x17C9000017D4, - 0x17DD000017DE, - 0x180B0000180E, - 0x180F00001810, - 0x188500001887, - 0x18A9000018AA, - 0x192000001923, - 0x192700001929, - 0x193200001933, - 0x19390000193C, - 0x1A1700001A19, - 0x1A1B00001A1C, - 0x1A5600001A57, - 0x1A5800001A5F, - 0x1A6000001A61, - 0x1A6200001A63, - 0x1A6500001A6D, - 0x1A7300001A7D, - 0x1A7F00001A80, - 0x1AB000001ADE, - 0x1AE000001AEC, - 0x1B0000001B04, - 0x1B3400001B35, - 0x1B3600001B3B, - 0x1B3C00001B3D, - 0x1B4200001B43, - 0x1B6B00001B74, - 0x1B8000001B82, - 0x1BA200001BA6, - 0x1BA800001BAA, - 0x1BAB00001BAE, - 0x1BE600001BE7, - 0x1BE800001BEA, - 0x1BED00001BEE, - 0x1BEF00001BF2, - 0x1C2C00001C34, - 0x1C3600001C38, - 0x1CD000001CD3, - 0x1CD400001CE1, - 0x1CE200001CE9, - 0x1CED00001CEE, - 0x1CF400001CF5, - 0x1CF800001CFA, - 0x1DC000001E00, - 0x200B0000200C, - 0x200E00002010, - 0x202A0000202F, - 0x206000002065, - 0x206A00002070, - 0x20D0000020F1, - 0x2CEF00002CF2, - 0x2D7F00002D80, - 0x2DE000002E00, - 0x302A0000302E, - 0x30990000309B, - 0xA66F0000A673, - 0xA6740000A67E, - 0xA69E0000A6A0, - 0xA6F00000A6F2, - 0xA8020000A803, - 0xA8060000A807, - 0xA80B0000A80C, - 0xA8250000A827, - 0xA82C0000A82D, - 0xA8C40000A8C6, - 0xA8E00000A8F2, - 0xA8FF0000A900, - 0xA9260000A92E, - 0xA9470000A952, - 0xA9800000A983, - 0xA9B30000A9B4, - 0xA9B60000A9BA, - 0xA9BC0000A9BE, - 0xA9E50000A9E6, - 0xAA290000AA2F, - 0xAA310000AA33, - 0xAA350000AA37, - 0xAA430000AA44, - 0xAA4C0000AA4D, - 0xAA7C0000AA7D, - 0xAAB00000AAB1, - 0xAAB20000AAB5, - 0xAAB70000AAB9, - 0xAABE0000AAC0, - 0xAAC10000AAC2, - 0xAAEC0000AAEE, - 0xAAF60000AAF7, - 0xABE50000ABE6, - 0xABE80000ABE9, - 0xABED0000ABEE, - 0xFB1E0000FB1F, - 0xFE000000FE10, - 0xFE200000FE30, - 0xFEFF0000FF00, - 0xFFF90000FFFC, - 0x101FD000101FE, - 0x102E0000102E1, - 0x103760001037B, - 0x10A0100010A04, - 0x10A0500010A07, - 0x10A0C00010A10, - 0x10A3800010A3B, - 0x10A3F00010A40, - 0x10AE500010AE7, - 0x10D2400010D28, - 0x10D6900010D6E, - 0x10EAB00010EAD, - 0x10EFA00010F00, - 0x10F4600010F51, - 0x10F8200010F86, - 0x1100100011002, - 0x1103800011047, - 0x1107000011071, - 0x1107300011075, - 0x1107F00011082, - 0x110B3000110B7, - 0x110B9000110BB, - 0x110C2000110C3, - 0x1110000011103, - 0x111270001112C, - 0x1112D00011135, - 0x1117300011174, - 0x1118000011182, - 0x111B6000111BF, - 0x111C9000111CD, - 0x111CF000111D0, - 0x1122F00011232, - 0x1123400011235, - 0x1123600011238, - 0x1123E0001123F, - 0x1124100011242, - 0x112DF000112E0, - 0x112E3000112EB, - 0x1130000011302, - 0x1133B0001133D, - 0x1134000011341, - 0x113660001136D, - 0x1137000011375, - 0x113BB000113C1, - 0x113CE000113CF, - 0x113D0000113D1, - 0x113D2000113D3, - 0x113E1000113E3, - 0x1143800011440, - 0x1144200011445, - 0x1144600011447, - 0x1145E0001145F, - 0x114B3000114B9, - 0x114BA000114BB, - 0x114BF000114C1, - 0x114C2000114C4, - 0x115B2000115B6, - 0x115BC000115BE, - 0x115BF000115C1, - 0x115DC000115DE, - 0x116330001163B, - 0x1163D0001163E, - 0x1163F00011641, - 0x116AB000116AC, - 0x116AD000116AE, - 0x116B0000116B6, - 0x116B7000116B8, - 0x1171D0001171E, - 0x1171F00011720, - 0x1172200011726, - 0x117270001172C, - 0x1182F00011838, - 0x118390001183B, - 0x1193B0001193D, - 0x1193E0001193F, - 0x1194300011944, - 0x119D4000119D8, - 0x119DA000119DC, - 0x119E0000119E1, - 0x11A0100011A0B, - 0x11A3300011A39, - 0x11A3B00011A3F, - 0x11A4700011A48, - 0x11A5100011A57, - 0x11A5900011A5C, - 0x11A8A00011A97, - 0x11A9800011A9A, - 0x11B6000011B61, - 0x11B6200011B65, - 0x11B6600011B67, - 0x11C3000011C37, - 0x11C3800011C3E, - 0x11C3F00011C40, - 0x11C9200011CA8, - 0x11CAA00011CB1, - 0x11CB200011CB4, - 0x11CB500011CB7, - 0x11D3100011D37, - 0x11D3A00011D3B, - 0x11D3C00011D3E, - 0x11D3F00011D46, - 0x11D4700011D48, - 0x11D9000011D92, - 0x11D9500011D96, - 0x11D9700011D98, - 0x11EF300011EF5, - 0x11F0000011F02, - 0x11F3600011F3B, - 0x11F4000011F41, - 0x11F4200011F43, - 0x11F5A00011F5B, - 0x1343000013441, - 0x1344700013456, - 0x1611E0001612A, - 0x1612D00016130, - 0x16AF000016AF5, - 0x16B3000016B37, - 0x16F4F00016F50, - 0x16F8F00016F93, - 0x16FE400016FE5, - 0x1BC9D0001BC9F, - 0x1BCA00001BCA4, - 0x1CF000001CF2E, - 0x1CF300001CF47, - 0x1D1670001D16A, - 0x1D1730001D183, - 0x1D1850001D18C, - 0x1D1AA0001D1AE, - 0x1D2420001D245, - 0x1DA000001DA37, - 0x1DA3B0001DA6D, - 0x1DA750001DA76, - 0x1DA840001DA85, - 0x1DA9B0001DAA0, - 0x1DAA10001DAB0, - 0x1E0000001E007, - 0x1E0080001E019, - 0x1E01B0001E022, - 0x1E0230001E025, - 0x1E0260001E02B, - 0x1E08F0001E090, - 0x1E1300001E137, - 0x1E2AE0001E2AF, - 0x1E2EC0001E2F0, - 0x1E4EC0001E4F0, - 0x1E5EE0001E5F0, - 0x1E6E30001E6E4, - 0x1E6E60001E6E7, - 0x1E6EE0001E6F0, - 0x1E6F50001E6F6, - 0x1E8D00001E8D7, - 0x1E9440001E94C, - 0xE0001000E0002, - 0xE0020000E0080, - 0xE0100000E01F0, - ), -} - - -codepoint_classes = { - "PVALID": ( - 0x2D0000002E, - 0x300000003A, - 0x610000007B, - 0xDF000000F7, - 0xF800000100, - 0x10100000102, - 0x10300000104, - 0x10500000106, - 0x10700000108, - 0x1090000010A, - 0x10B0000010C, - 0x10D0000010E, - 0x10F00000110, - 0x11100000112, - 0x11300000114, - 0x11500000116, - 0x11700000118, - 0x1190000011A, - 0x11B0000011C, - 0x11D0000011E, - 0x11F00000120, - 0x12100000122, - 0x12300000124, - 0x12500000126, - 0x12700000128, - 0x1290000012A, - 0x12B0000012C, - 0x12D0000012E, - 0x12F00000130, - 0x13100000132, - 0x13500000136, - 0x13700000139, - 0x13A0000013B, - 0x13C0000013D, - 0x13E0000013F, - 0x14200000143, - 0x14400000145, - 0x14600000147, - 0x14800000149, - 0x14B0000014C, - 0x14D0000014E, - 0x14F00000150, - 0x15100000152, - 0x15300000154, - 0x15500000156, - 0x15700000158, - 0x1590000015A, - 0x15B0000015C, - 0x15D0000015E, - 0x15F00000160, - 0x16100000162, - 0x16300000164, - 0x16500000166, - 0x16700000168, - 0x1690000016A, - 0x16B0000016C, - 0x16D0000016E, - 0x16F00000170, - 0x17100000172, - 0x17300000174, - 0x17500000176, - 0x17700000178, - 0x17A0000017B, - 0x17C0000017D, - 0x17E0000017F, - 0x18000000181, - 0x18300000184, - 0x18500000186, - 0x18800000189, - 0x18C0000018E, - 0x19200000193, - 0x19500000196, - 0x1990000019C, - 0x19E0000019F, - 0x1A1000001A2, - 0x1A3000001A4, - 0x1A5000001A6, - 0x1A8000001A9, - 0x1AA000001AC, - 0x1AD000001AE, - 0x1B0000001B1, - 0x1B4000001B5, - 0x1B6000001B7, - 0x1B9000001BC, - 0x1BD000001C4, - 0x1CE000001CF, - 0x1D0000001D1, - 0x1D2000001D3, - 0x1D4000001D5, - 0x1D6000001D7, - 0x1D8000001D9, - 0x1DA000001DB, - 0x1DC000001DE, - 0x1DF000001E0, - 0x1E1000001E2, - 0x1E3000001E4, - 0x1E5000001E6, - 0x1E7000001E8, - 0x1E9000001EA, - 0x1EB000001EC, - 0x1ED000001EE, - 0x1EF000001F1, - 0x1F5000001F6, - 0x1F9000001FA, - 0x1FB000001FC, - 0x1FD000001FE, - 0x1FF00000200, - 0x20100000202, - 0x20300000204, - 0x20500000206, - 0x20700000208, - 0x2090000020A, - 0x20B0000020C, - 0x20D0000020E, - 0x20F00000210, - 0x21100000212, - 0x21300000214, - 0x21500000216, - 0x21700000218, - 0x2190000021A, - 0x21B0000021C, - 0x21D0000021E, - 0x21F00000220, - 0x22100000222, - 0x22300000224, - 0x22500000226, - 0x22700000228, - 0x2290000022A, - 0x22B0000022C, - 0x22D0000022E, - 0x22F00000230, - 0x23100000232, - 0x2330000023A, - 0x23C0000023D, - 0x23F00000241, - 0x24200000243, - 0x24700000248, - 0x2490000024A, - 0x24B0000024C, - 0x24D0000024E, - 0x24F000002B0, - 0x2B9000002C2, - 0x2C6000002D2, - 0x2EC000002ED, - 0x2EE000002EF, - 0x30000000340, - 0x34200000343, - 0x3460000034F, - 0x35000000370, - 0x37100000372, - 0x37300000374, - 0x37700000378, - 0x37B0000037E, - 0x39000000391, - 0x3AC000003CF, - 0x3D7000003D8, - 0x3D9000003DA, - 0x3DB000003DC, - 0x3DD000003DE, - 0x3DF000003E0, - 0x3E1000003E2, - 0x3E3000003E4, - 0x3E5000003E6, - 0x3E7000003E8, - 0x3E9000003EA, - 0x3EB000003EC, - 0x3ED000003EE, - 0x3EF000003F0, - 0x3F3000003F4, - 0x3F8000003F9, - 0x3FB000003FD, - 0x43000000460, - 0x46100000462, - 0x46300000464, - 0x46500000466, - 0x46700000468, - 0x4690000046A, - 0x46B0000046C, - 0x46D0000046E, - 0x46F00000470, - 0x47100000472, - 0x47300000474, - 0x47500000476, - 0x47700000478, - 0x4790000047A, - 0x47B0000047C, - 0x47D0000047E, - 0x47F00000480, - 0x48100000482, - 0x48300000488, - 0x48B0000048C, - 0x48D0000048E, - 0x48F00000490, - 0x49100000492, - 0x49300000494, - 0x49500000496, - 0x49700000498, - 0x4990000049A, - 0x49B0000049C, - 0x49D0000049E, - 0x49F000004A0, - 0x4A1000004A2, - 0x4A3000004A4, - 0x4A5000004A6, - 0x4A7000004A8, - 0x4A9000004AA, - 0x4AB000004AC, - 0x4AD000004AE, - 0x4AF000004B0, - 0x4B1000004B2, - 0x4B3000004B4, - 0x4B5000004B6, - 0x4B7000004B8, - 0x4B9000004BA, - 0x4BB000004BC, - 0x4BD000004BE, - 0x4BF000004C0, - 0x4C2000004C3, - 0x4C4000004C5, - 0x4C6000004C7, - 0x4C8000004C9, - 0x4CA000004CB, - 0x4CC000004CD, - 0x4CE000004D0, - 0x4D1000004D2, - 0x4D3000004D4, - 0x4D5000004D6, - 0x4D7000004D8, - 0x4D9000004DA, - 0x4DB000004DC, - 0x4DD000004DE, - 0x4DF000004E0, - 0x4E1000004E2, - 0x4E3000004E4, - 0x4E5000004E6, - 0x4E7000004E8, - 0x4E9000004EA, - 0x4EB000004EC, - 0x4ED000004EE, - 0x4EF000004F0, - 0x4F1000004F2, - 0x4F3000004F4, - 0x4F5000004F6, - 0x4F7000004F8, - 0x4F9000004FA, - 0x4FB000004FC, - 0x4FD000004FE, - 0x4FF00000500, - 0x50100000502, - 0x50300000504, - 0x50500000506, - 0x50700000508, - 0x5090000050A, - 0x50B0000050C, - 0x50D0000050E, - 0x50F00000510, - 0x51100000512, - 0x51300000514, - 0x51500000516, - 0x51700000518, - 0x5190000051A, - 0x51B0000051C, - 0x51D0000051E, - 0x51F00000520, - 0x52100000522, - 0x52300000524, - 0x52500000526, - 0x52700000528, - 0x5290000052A, - 0x52B0000052C, - 0x52D0000052E, - 0x52F00000530, - 0x5590000055A, - 0x56000000587, - 0x58800000589, - 0x591000005BE, - 0x5BF000005C0, - 0x5C1000005C3, - 0x5C4000005C6, - 0x5C7000005C8, - 0x5D0000005EB, - 0x5EF000005F3, - 0x6100000061B, - 0x62000000640, - 0x64100000660, - 0x66E00000675, - 0x679000006D4, - 0x6D5000006DD, - 0x6DF000006E9, - 0x6EA000006F0, - 0x6FA00000700, - 0x7100000074B, - 0x74D000007B2, - 0x7C0000007F6, - 0x7FD000007FE, - 0x8000000082E, - 0x8400000085C, - 0x8600000086B, - 0x87000000888, - 0x88900000890, - 0x897000008E2, - 0x8E300000958, - 0x96000000964, - 0x96600000970, - 0x97100000984, - 0x9850000098D, - 0x98F00000991, - 0x993000009A9, - 0x9AA000009B1, - 0x9B2000009B3, - 0x9B6000009BA, - 0x9BC000009C5, - 0x9C7000009C9, - 0x9CB000009CF, - 0x9D7000009D8, - 0x9E0000009E4, - 0x9E6000009F2, - 0x9FC000009FD, - 0x9FE000009FF, - 0xA0100000A04, - 0xA0500000A0B, - 0xA0F00000A11, - 0xA1300000A29, - 0xA2A00000A31, - 0xA3200000A33, - 0xA3500000A36, - 0xA3800000A3A, - 0xA3C00000A3D, - 0xA3E00000A43, - 0xA4700000A49, - 0xA4B00000A4E, - 0xA5100000A52, - 0xA5C00000A5D, - 0xA6600000A76, - 0xA8100000A84, - 0xA8500000A8E, - 0xA8F00000A92, - 0xA9300000AA9, - 0xAAA00000AB1, - 0xAB200000AB4, - 0xAB500000ABA, - 0xABC00000AC6, - 0xAC700000ACA, - 0xACB00000ACE, - 0xAD000000AD1, - 0xAE000000AE4, - 0xAE600000AF0, - 0xAF900000B00, - 0xB0100000B04, - 0xB0500000B0D, - 0xB0F00000B11, - 0xB1300000B29, - 0xB2A00000B31, - 0xB3200000B34, - 0xB3500000B3A, - 0xB3C00000B45, - 0xB4700000B49, - 0xB4B00000B4E, - 0xB5500000B58, - 0xB5F00000B64, - 0xB6600000B70, - 0xB7100000B72, - 0xB8200000B84, - 0xB8500000B8B, - 0xB8E00000B91, - 0xB9200000B96, - 0xB9900000B9B, - 0xB9C00000B9D, - 0xB9E00000BA0, - 0xBA300000BA5, - 0xBA800000BAB, - 0xBAE00000BBA, - 0xBBE00000BC3, - 0xBC600000BC9, - 0xBCA00000BCE, - 0xBD000000BD1, - 0xBD700000BD8, - 0xBE600000BF0, - 0xC0000000C0D, - 0xC0E00000C11, - 0xC1200000C29, - 0xC2A00000C3A, - 0xC3C00000C45, - 0xC4600000C49, - 0xC4A00000C4E, - 0xC5500000C57, - 0xC5800000C5B, - 0xC5C00000C5E, - 0xC6000000C64, - 0xC6600000C70, - 0xC8000000C84, - 0xC8500000C8D, - 0xC8E00000C91, - 0xC9200000CA9, - 0xCAA00000CB4, - 0xCB500000CBA, - 0xCBC00000CC5, - 0xCC600000CC9, - 0xCCA00000CCE, - 0xCD500000CD7, - 0xCDC00000CDF, - 0xCE000000CE4, - 0xCE600000CF0, - 0xCF100000CF4, - 0xD0000000D0D, - 0xD0E00000D11, - 0xD1200000D45, - 0xD4600000D49, - 0xD4A00000D4F, - 0xD5400000D58, - 0xD5F00000D64, - 0xD6600000D70, - 0xD7A00000D80, - 0xD8100000D84, - 0xD8500000D97, - 0xD9A00000DB2, - 0xDB300000DBC, - 0xDBD00000DBE, - 0xDC000000DC7, - 0xDCA00000DCB, - 0xDCF00000DD5, - 0xDD600000DD7, - 0xDD800000DE0, - 0xDE600000DF0, - 0xDF200000DF4, - 0xE0100000E33, - 0xE3400000E3B, - 0xE4000000E4F, - 0xE5000000E5A, - 0xE8100000E83, - 0xE8400000E85, - 0xE8600000E8B, - 0xE8C00000EA4, - 0xEA500000EA6, - 0xEA700000EB3, - 0xEB400000EBE, - 0xEC000000EC5, - 0xEC600000EC7, - 0xEC800000ECF, - 0xED000000EDA, - 0xEDE00000EE0, - 0xF0000000F01, - 0xF0B00000F0C, - 0xF1800000F1A, - 0xF2000000F2A, - 0xF3500000F36, - 0xF3700000F38, - 0xF3900000F3A, - 0xF3E00000F43, - 0xF4400000F48, - 0xF4900000F4D, - 0xF4E00000F52, - 0xF5300000F57, - 0xF5800000F5C, - 0xF5D00000F69, - 0xF6A00000F6D, - 0xF7100000F73, - 0xF7400000F75, - 0xF7A00000F81, - 0xF8200000F85, - 0xF8600000F93, - 0xF9400000F98, - 0xF9900000F9D, - 0xF9E00000FA2, - 0xFA300000FA7, - 0xFA800000FAC, - 0xFAD00000FB9, - 0xFBA00000FBD, - 0xFC600000FC7, - 0x10000000104A, - 0x10500000109E, - 0x10D0000010FB, - 0x10FD00001100, - 0x120000001249, - 0x124A0000124E, - 0x125000001257, - 0x125800001259, - 0x125A0000125E, - 0x126000001289, - 0x128A0000128E, - 0x1290000012B1, - 0x12B2000012B6, - 0x12B8000012BF, - 0x12C0000012C1, - 0x12C2000012C6, - 0x12C8000012D7, - 0x12D800001311, - 0x131200001316, - 0x13180000135B, - 0x135D00001360, - 0x138000001390, - 0x13A0000013F6, - 0x14010000166D, - 0x166F00001680, - 0x16810000169B, - 0x16A0000016EB, - 0x16F1000016F9, - 0x170000001716, - 0x171F00001735, - 0x174000001754, - 0x17600000176D, - 0x176E00001771, - 0x177200001774, - 0x1780000017B4, - 0x17B6000017D4, - 0x17D7000017D8, - 0x17DC000017DE, - 0x17E0000017EA, - 0x18100000181A, - 0x182000001879, - 0x1880000018AB, - 0x18B0000018F6, - 0x19000000191F, - 0x19200000192C, - 0x19300000193C, - 0x19460000196E, - 0x197000001975, - 0x1980000019AC, - 0x19B0000019CA, - 0x19D0000019DA, - 0x1A0000001A1C, - 0x1A2000001A5F, - 0x1A6000001A7D, - 0x1A7F00001A8A, - 0x1A9000001A9A, - 0x1AA700001AA8, - 0x1AB000001ABE, - 0x1ABF00001ADE, - 0x1AE000001AEC, - 0x1B0000001B4D, - 0x1B5000001B5A, - 0x1B6B00001B74, - 0x1B8000001BF4, - 0x1C0000001C38, - 0x1C4000001C4A, - 0x1C4D00001C7E, - 0x1C8A00001C8B, - 0x1CD000001CD3, - 0x1CD400001CFB, - 0x1D0000001D2C, - 0x1D2F00001D30, - 0x1D3B00001D3C, - 0x1D4E00001D4F, - 0x1D6B00001D78, - 0x1D7900001D9B, - 0x1DC000001E00, - 0x1E0100001E02, - 0x1E0300001E04, - 0x1E0500001E06, - 0x1E0700001E08, - 0x1E0900001E0A, - 0x1E0B00001E0C, - 0x1E0D00001E0E, - 0x1E0F00001E10, - 0x1E1100001E12, - 0x1E1300001E14, - 0x1E1500001E16, - 0x1E1700001E18, - 0x1E1900001E1A, - 0x1E1B00001E1C, - 0x1E1D00001E1E, - 0x1E1F00001E20, - 0x1E2100001E22, - 0x1E2300001E24, - 0x1E2500001E26, - 0x1E2700001E28, - 0x1E2900001E2A, - 0x1E2B00001E2C, - 0x1E2D00001E2E, - 0x1E2F00001E30, - 0x1E3100001E32, - 0x1E3300001E34, - 0x1E3500001E36, - 0x1E3700001E38, - 0x1E3900001E3A, - 0x1E3B00001E3C, - 0x1E3D00001E3E, - 0x1E3F00001E40, - 0x1E4100001E42, - 0x1E4300001E44, - 0x1E4500001E46, - 0x1E4700001E48, - 0x1E4900001E4A, - 0x1E4B00001E4C, - 0x1E4D00001E4E, - 0x1E4F00001E50, - 0x1E5100001E52, - 0x1E5300001E54, - 0x1E5500001E56, - 0x1E5700001E58, - 0x1E5900001E5A, - 0x1E5B00001E5C, - 0x1E5D00001E5E, - 0x1E5F00001E60, - 0x1E6100001E62, - 0x1E6300001E64, - 0x1E6500001E66, - 0x1E6700001E68, - 0x1E6900001E6A, - 0x1E6B00001E6C, - 0x1E6D00001E6E, - 0x1E6F00001E70, - 0x1E7100001E72, - 0x1E7300001E74, - 0x1E7500001E76, - 0x1E7700001E78, - 0x1E7900001E7A, - 0x1E7B00001E7C, - 0x1E7D00001E7E, - 0x1E7F00001E80, - 0x1E8100001E82, - 0x1E8300001E84, - 0x1E8500001E86, - 0x1E8700001E88, - 0x1E8900001E8A, - 0x1E8B00001E8C, - 0x1E8D00001E8E, - 0x1E8F00001E90, - 0x1E9100001E92, - 0x1E9300001E94, - 0x1E9500001E9A, - 0x1E9C00001E9E, - 0x1E9F00001EA0, - 0x1EA100001EA2, - 0x1EA300001EA4, - 0x1EA500001EA6, - 0x1EA700001EA8, - 0x1EA900001EAA, - 0x1EAB00001EAC, - 0x1EAD00001EAE, - 0x1EAF00001EB0, - 0x1EB100001EB2, - 0x1EB300001EB4, - 0x1EB500001EB6, - 0x1EB700001EB8, - 0x1EB900001EBA, - 0x1EBB00001EBC, - 0x1EBD00001EBE, - 0x1EBF00001EC0, - 0x1EC100001EC2, - 0x1EC300001EC4, - 0x1EC500001EC6, - 0x1EC700001EC8, - 0x1EC900001ECA, - 0x1ECB00001ECC, - 0x1ECD00001ECE, - 0x1ECF00001ED0, - 0x1ED100001ED2, - 0x1ED300001ED4, - 0x1ED500001ED6, - 0x1ED700001ED8, - 0x1ED900001EDA, - 0x1EDB00001EDC, - 0x1EDD00001EDE, - 0x1EDF00001EE0, - 0x1EE100001EE2, - 0x1EE300001EE4, - 0x1EE500001EE6, - 0x1EE700001EE8, - 0x1EE900001EEA, - 0x1EEB00001EEC, - 0x1EED00001EEE, - 0x1EEF00001EF0, - 0x1EF100001EF2, - 0x1EF300001EF4, - 0x1EF500001EF6, - 0x1EF700001EF8, - 0x1EF900001EFA, - 0x1EFB00001EFC, - 0x1EFD00001EFE, - 0x1EFF00001F08, - 0x1F1000001F16, - 0x1F2000001F28, - 0x1F3000001F38, - 0x1F4000001F46, - 0x1F5000001F58, - 0x1F6000001F68, - 0x1F7000001F71, - 0x1F7200001F73, - 0x1F7400001F75, - 0x1F7600001F77, - 0x1F7800001F79, - 0x1F7A00001F7B, - 0x1F7C00001F7D, - 0x1FB000001FB2, - 0x1FB600001FB7, - 0x1FC600001FC7, - 0x1FD000001FD3, - 0x1FD600001FD8, - 0x1FE000001FE3, - 0x1FE400001FE8, - 0x1FF600001FF7, - 0x214E0000214F, - 0x218400002185, - 0x2C3000002C60, - 0x2C6100002C62, - 0x2C6500002C67, - 0x2C6800002C69, - 0x2C6A00002C6B, - 0x2C6C00002C6D, - 0x2C7100002C72, - 0x2C7300002C75, - 0x2C7600002C7C, - 0x2C8100002C82, - 0x2C8300002C84, - 0x2C8500002C86, - 0x2C8700002C88, - 0x2C8900002C8A, - 0x2C8B00002C8C, - 0x2C8D00002C8E, - 0x2C8F00002C90, - 0x2C9100002C92, - 0x2C9300002C94, - 0x2C9500002C96, - 0x2C9700002C98, - 0x2C9900002C9A, - 0x2C9B00002C9C, - 0x2C9D00002C9E, - 0x2C9F00002CA0, - 0x2CA100002CA2, - 0x2CA300002CA4, - 0x2CA500002CA6, - 0x2CA700002CA8, - 0x2CA900002CAA, - 0x2CAB00002CAC, - 0x2CAD00002CAE, - 0x2CAF00002CB0, - 0x2CB100002CB2, - 0x2CB300002CB4, - 0x2CB500002CB6, - 0x2CB700002CB8, - 0x2CB900002CBA, - 0x2CBB00002CBC, - 0x2CBD00002CBE, - 0x2CBF00002CC0, - 0x2CC100002CC2, - 0x2CC300002CC4, - 0x2CC500002CC6, - 0x2CC700002CC8, - 0x2CC900002CCA, - 0x2CCB00002CCC, - 0x2CCD00002CCE, - 0x2CCF00002CD0, - 0x2CD100002CD2, - 0x2CD300002CD4, - 0x2CD500002CD6, - 0x2CD700002CD8, - 0x2CD900002CDA, - 0x2CDB00002CDC, - 0x2CDD00002CDE, - 0x2CDF00002CE0, - 0x2CE100002CE2, - 0x2CE300002CE5, - 0x2CEC00002CED, - 0x2CEE00002CF2, - 0x2CF300002CF4, - 0x2D0000002D26, - 0x2D2700002D28, - 0x2D2D00002D2E, - 0x2D3000002D68, - 0x2D7F00002D97, - 0x2DA000002DA7, - 0x2DA800002DAF, - 0x2DB000002DB7, - 0x2DB800002DBF, - 0x2DC000002DC7, - 0x2DC800002DCF, - 0x2DD000002DD7, - 0x2DD800002DDF, - 0x2DE000002E00, - 0x2E2F00002E30, - 0x300500003008, - 0x302A0000302E, - 0x303C0000303D, - 0x304100003097, - 0x30990000309B, - 0x309D0000309F, - 0x30A1000030FB, - 0x30FC000030FF, - 0x310500003130, - 0x31A0000031C0, - 0x31F000003200, - 0x340000004DC0, - 0x4E000000A48D, - 0xA4D00000A4FE, - 0xA5000000A60D, - 0xA6100000A62C, - 0xA6410000A642, - 0xA6430000A644, - 0xA6450000A646, - 0xA6470000A648, - 0xA6490000A64A, - 0xA64B0000A64C, - 0xA64D0000A64E, - 0xA64F0000A650, - 0xA6510000A652, - 0xA6530000A654, - 0xA6550000A656, - 0xA6570000A658, - 0xA6590000A65A, - 0xA65B0000A65C, - 0xA65D0000A65E, - 0xA65F0000A660, - 0xA6610000A662, - 0xA6630000A664, - 0xA6650000A666, - 0xA6670000A668, - 0xA6690000A66A, - 0xA66B0000A66C, - 0xA66D0000A670, - 0xA6740000A67E, - 0xA67F0000A680, - 0xA6810000A682, - 0xA6830000A684, - 0xA6850000A686, - 0xA6870000A688, - 0xA6890000A68A, - 0xA68B0000A68C, - 0xA68D0000A68E, - 0xA68F0000A690, - 0xA6910000A692, - 0xA6930000A694, - 0xA6950000A696, - 0xA6970000A698, - 0xA6990000A69A, - 0xA69B0000A69C, - 0xA69E0000A6E6, - 0xA6F00000A6F2, - 0xA7170000A720, - 0xA7230000A724, - 0xA7250000A726, - 0xA7270000A728, - 0xA7290000A72A, - 0xA72B0000A72C, - 0xA72D0000A72E, - 0xA72F0000A732, - 0xA7330000A734, - 0xA7350000A736, - 0xA7370000A738, - 0xA7390000A73A, - 0xA73B0000A73C, - 0xA73D0000A73E, - 0xA73F0000A740, - 0xA7410000A742, - 0xA7430000A744, - 0xA7450000A746, - 0xA7470000A748, - 0xA7490000A74A, - 0xA74B0000A74C, - 0xA74D0000A74E, - 0xA74F0000A750, - 0xA7510000A752, - 0xA7530000A754, - 0xA7550000A756, - 0xA7570000A758, - 0xA7590000A75A, - 0xA75B0000A75C, - 0xA75D0000A75E, - 0xA75F0000A760, - 0xA7610000A762, - 0xA7630000A764, - 0xA7650000A766, - 0xA7670000A768, - 0xA7690000A76A, - 0xA76B0000A76C, - 0xA76D0000A76E, - 0xA76F0000A770, - 0xA7710000A779, - 0xA77A0000A77B, - 0xA77C0000A77D, - 0xA77F0000A780, - 0xA7810000A782, - 0xA7830000A784, - 0xA7850000A786, - 0xA7870000A789, - 0xA78C0000A78D, - 0xA78E0000A790, - 0xA7910000A792, - 0xA7930000A796, - 0xA7970000A798, - 0xA7990000A79A, - 0xA79B0000A79C, - 0xA79D0000A79E, - 0xA79F0000A7A0, - 0xA7A10000A7A2, - 0xA7A30000A7A4, - 0xA7A50000A7A6, - 0xA7A70000A7A8, - 0xA7A90000A7AA, - 0xA7AF0000A7B0, - 0xA7B50000A7B6, - 0xA7B70000A7B8, - 0xA7B90000A7BA, - 0xA7BB0000A7BC, - 0xA7BD0000A7BE, - 0xA7BF0000A7C0, - 0xA7C10000A7C2, - 0xA7C30000A7C4, - 0xA7C80000A7C9, - 0xA7CA0000A7CB, - 0xA7CD0000A7CE, - 0xA7CF0000A7D0, - 0xA7D10000A7D2, - 0xA7D30000A7D4, - 0xA7D50000A7D6, - 0xA7D70000A7D8, - 0xA7D90000A7DA, - 0xA7DB0000A7DC, - 0xA7F60000A7F8, - 0xA7FA0000A828, - 0xA82C0000A82D, - 0xA8400000A874, - 0xA8800000A8C6, - 0xA8D00000A8DA, - 0xA8E00000A8F8, - 0xA8FB0000A8FC, - 0xA8FD0000A92E, - 0xA9300000A954, - 0xA9800000A9C1, - 0xA9CF0000A9DA, - 0xA9E00000A9FF, - 0xAA000000AA37, - 0xAA400000AA4E, - 0xAA500000AA5A, - 0xAA600000AA77, - 0xAA7A0000AAC3, - 0xAADB0000AADE, - 0xAAE00000AAF0, - 0xAAF20000AAF7, - 0xAB010000AB07, - 0xAB090000AB0F, - 0xAB110000AB17, - 0xAB200000AB27, - 0xAB280000AB2F, - 0xAB300000AB5B, - 0xAB600000AB69, - 0xABC00000ABEB, - 0xABEC0000ABEE, - 0xABF00000ABFA, - 0xAC000000D7A4, - 0xFA0E0000FA10, - 0xFA110000FA12, - 0xFA130000FA15, - 0xFA1F0000FA20, - 0xFA210000FA22, - 0xFA230000FA25, - 0xFA270000FA2A, - 0xFB1E0000FB1F, - 0xFE200000FE30, - 0xFE730000FE74, - 0x100000001000C, - 0x1000D00010027, - 0x100280001003B, - 0x1003C0001003E, - 0x1003F0001004E, - 0x100500001005E, - 0x10080000100FB, - 0x101FD000101FE, - 0x102800001029D, - 0x102A0000102D1, - 0x102E0000102E1, - 0x1030000010320, - 0x1032D00010341, - 0x103420001034A, - 0x103500001037B, - 0x103800001039E, - 0x103A0000103C4, - 0x103C8000103D0, - 0x104280001049E, - 0x104A0000104AA, - 0x104D8000104FC, - 0x1050000010528, - 0x1053000010564, - 0x10597000105A2, - 0x105A3000105B2, - 0x105B3000105BA, - 0x105BB000105BD, - 0x105C0000105F4, - 0x1060000010737, - 0x1074000010756, - 0x1076000010768, - 0x1078000010781, - 0x1080000010806, - 0x1080800010809, - 0x1080A00010836, - 0x1083700010839, - 0x1083C0001083D, - 0x1083F00010856, - 0x1086000010877, - 0x108800001089F, - 0x108E0000108F3, - 0x108F4000108F6, - 0x1090000010916, - 0x109200001093A, - 0x109400001095A, - 0x10980000109B8, - 0x109BE000109C0, - 0x10A0000010A04, - 0x10A0500010A07, - 0x10A0C00010A14, - 0x10A1500010A18, - 0x10A1900010A36, - 0x10A3800010A3B, - 0x10A3F00010A40, - 0x10A6000010A7D, - 0x10A8000010A9D, - 0x10AC000010AC8, - 0x10AC900010AE7, - 0x10B0000010B36, - 0x10B4000010B56, - 0x10B6000010B73, - 0x10B8000010B92, - 0x10C0000010C49, - 0x10CC000010CF3, - 0x10D0000010D28, - 0x10D3000010D3A, - 0x10D4000010D50, - 0x10D6900010D6E, - 0x10D6F00010D86, - 0x10E8000010EAA, - 0x10EAB00010EAD, - 0x10EB000010EB2, - 0x10EC200010EC8, - 0x10EFA00010F1D, - 0x10F2700010F28, - 0x10F3000010F51, - 0x10F7000010F86, - 0x10FB000010FC5, - 0x10FE000010FF7, - 0x1100000011047, - 0x1106600011076, - 0x1107F000110BB, - 0x110C2000110C3, - 0x110D0000110E9, - 0x110F0000110FA, - 0x1110000011135, - 0x1113600011140, - 0x1114400011148, - 0x1115000011174, - 0x1117600011177, - 0x11180000111C5, - 0x111C9000111CD, - 0x111CE000111DB, - 0x111DC000111DD, - 0x1120000011212, - 0x1121300011238, - 0x1123E00011242, - 0x1128000011287, - 0x1128800011289, - 0x1128A0001128E, - 0x1128F0001129E, - 0x1129F000112A9, - 0x112B0000112EB, - 0x112F0000112FA, - 0x1130000011304, - 0x113050001130D, - 0x1130F00011311, - 0x1131300011329, - 0x1132A00011331, - 0x1133200011334, - 0x113350001133A, - 0x1133B00011345, - 0x1134700011349, - 0x1134B0001134E, - 0x1135000011351, - 0x1135700011358, - 0x1135D00011364, - 0x113660001136D, - 0x1137000011375, - 0x113800001138A, - 0x1138B0001138C, - 0x1138E0001138F, - 0x11390000113B6, - 0x113B7000113C1, - 0x113C2000113C3, - 0x113C5000113C6, - 0x113C7000113CB, - 0x113CC000113D4, - 0x113E1000113E3, - 0x114000001144B, - 0x114500001145A, - 0x1145E00011462, - 0x11480000114C6, - 0x114C7000114C8, - 0x114D0000114DA, - 0x11580000115B6, - 0x115B8000115C1, - 0x115D8000115DE, - 0x1160000011641, - 0x1164400011645, - 0x116500001165A, - 0x11680000116B9, - 0x116C0000116CA, - 0x116D0000116E4, - 0x117000001171B, - 0x1171D0001172C, - 0x117300001173A, - 0x1174000011747, - 0x118000001183B, - 0x118C0000118EA, - 0x118FF00011907, - 0x119090001190A, - 0x1190C00011914, - 0x1191500011917, - 0x1191800011936, - 0x1193700011939, - 0x1193B00011944, - 0x119500001195A, - 0x119A0000119A8, - 0x119AA000119D8, - 0x119DA000119E2, - 0x119E3000119E5, - 0x11A0000011A3F, - 0x11A4700011A48, - 0x11A5000011A9A, - 0x11A9D00011A9E, - 0x11AB000011AF9, - 0x11B6000011B68, - 0x11BC000011BE1, - 0x11BF000011BFA, - 0x11C0000011C09, - 0x11C0A00011C37, - 0x11C3800011C41, - 0x11C5000011C5A, - 0x11C7200011C90, - 0x11C9200011CA8, - 0x11CA900011CB7, - 0x11D0000011D07, - 0x11D0800011D0A, - 0x11D0B00011D37, - 0x11D3A00011D3B, - 0x11D3C00011D3E, - 0x11D3F00011D48, - 0x11D5000011D5A, - 0x11D6000011D66, - 0x11D6700011D69, - 0x11D6A00011D8F, - 0x11D9000011D92, - 0x11D9300011D99, - 0x11DA000011DAA, - 0x11DB000011DDC, - 0x11DE000011DEA, - 0x11EE000011EF7, - 0x11F0000011F11, - 0x11F1200011F3B, - 0x11F3E00011F43, - 0x11F5000011F5B, - 0x11FB000011FB1, - 0x120000001239A, - 0x1248000012544, - 0x12F9000012FF1, - 0x1300000013430, - 0x1344000013456, - 0x13460000143FB, - 0x1440000014647, - 0x161000001613A, - 0x1680000016A39, - 0x16A4000016A5F, - 0x16A6000016A6A, - 0x16A7000016ABF, - 0x16AC000016ACA, - 0x16AD000016AEE, - 0x16AF000016AF5, - 0x16B0000016B37, - 0x16B4000016B44, - 0x16B5000016B5A, - 0x16B6300016B78, - 0x16B7D00016B90, - 0x16D4000016D6D, - 0x16D7000016D7A, - 0x16E6000016E80, - 0x16EBB00016ED4, - 0x16F0000016F4B, - 0x16F4F00016F88, - 0x16F8F00016FA0, - 0x16FE000016FE2, - 0x16FE300016FE5, - 0x16FF000016FF4, - 0x1700000018CD6, - 0x18CFF00018D1F, - 0x18D8000018DF3, - 0x1AFF00001AFF4, - 0x1AFF50001AFFC, - 0x1AFFD0001AFFF, - 0x1B0000001B123, - 0x1B1320001B133, - 0x1B1500001B153, - 0x1B1550001B156, - 0x1B1640001B168, - 0x1B1700001B2FC, - 0x1BC000001BC6B, - 0x1BC700001BC7D, - 0x1BC800001BC89, - 0x1BC900001BC9A, - 0x1BC9D0001BC9F, - 0x1CF000001CF2E, - 0x1CF300001CF47, - 0x1DA000001DA37, - 0x1DA3B0001DA6D, - 0x1DA750001DA76, - 0x1DA840001DA85, - 0x1DA9B0001DAA0, - 0x1DAA10001DAB0, - 0x1DF000001DF1F, - 0x1DF250001DF2B, - 0x1E0000001E007, - 0x1E0080001E019, - 0x1E01B0001E022, - 0x1E0230001E025, - 0x1E0260001E02B, - 0x1E08F0001E090, - 0x1E1000001E12D, - 0x1E1300001E13E, - 0x1E1400001E14A, - 0x1E14E0001E14F, - 0x1E2900001E2AF, - 0x1E2C00001E2FA, - 0x1E4D00001E4FA, - 0x1E5D00001E5FB, - 0x1E6C00001E6DF, - 0x1E6E00001E6F6, - 0x1E6FE0001E700, - 0x1E7E00001E7E7, - 0x1E7E80001E7EC, - 0x1E7ED0001E7EF, - 0x1E7F00001E7FF, - 0x1E8000001E8C5, - 0x1E8D00001E8D7, - 0x1E9220001E94C, - 0x1E9500001E95A, - 0x200000002A6E0, - 0x2A7000002B81E, - 0x2B8200002CEAE, - 0x2CEB00002EBE1, - 0x2EBF00002EE5E, - 0x300000003134B, - 0x313500003347A, - ), - "CONTEXTJ": (0x200C0000200E,), - "CONTEXTO": ( - 0xB7000000B8, - 0x37500000376, - 0x5F3000005F5, - 0x6600000066A, - 0x6F0000006FA, - 0x30FB000030FC, - ), -} diff --git a/bundle/python-cpu/Lib/site-packages/idna/intranges.py b/bundle/python-cpu/Lib/site-packages/idna/intranges.py deleted file mode 100644 index 19d77810caf332a1f310bd3b734dab7bbd82f64c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna/intranges.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Given a list of integers, made up of (hopefully) a small number of long runs -of consecutive integers, compute a representation of the form -((start1, end1), (start2, end2) ...). Then answer the question "was x present -in the original list?" in time O(log(# runs)). -""" - -import bisect - - -def intranges_from_list(list_: list[int]) -> tuple[int, ...]: - """Represent a list of integers as a sequence of ranges: - ((start_0, end_0), (start_1, end_1), ...), such that the original - integers are exactly those x such that start_i <= x < end_i for some i. - - Ranges are encoded as single integers (start << 32 | end), not as tuples. - """ - - sorted_list = sorted(list_) - ranges = [] - last_write = -1 - for i in range(len(sorted_list)): - if i + 1 < len(sorted_list) and sorted_list[i] == sorted_list[i + 1] - 1: - continue - current_range = sorted_list[last_write + 1 : i + 1] - ranges.append(_encode_range(current_range[0], current_range[-1] + 1)) - last_write = i - - return tuple(ranges) - - -def _encode_range(start: int, end: int) -> int: - return (start << 32) | end - - -def _decode_range(r: int) -> tuple[int, int]: - return (r >> 32), (r & ((1 << 32) - 1)) - - -def intranges_contain(int_: int, ranges: tuple[int, ...]) -> bool: - """Determine if `int_` falls into one of the ranges in `ranges`.""" - tuple_ = _encode_range(int_, 0) - pos = bisect.bisect_left(ranges, tuple_) - # we could be immediately ahead of a tuple (start, end) - # with start < int_ <= end - if pos > 0: - left, right = _decode_range(ranges[pos - 1]) - if left <= int_ < right: - return True - # or we could be immediately behind a tuple (int_, end) - if pos < len(ranges): - left, _ = _decode_range(ranges[pos]) - if left == int_: - return True - return False diff --git a/bundle/python-cpu/Lib/site-packages/idna/package_data.py b/bundle/python-cpu/Lib/site-packages/idna/package_data.py deleted file mode 100644 index 94e40398cda6322e7c348ce8c2c44b80a904e5c7..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna/package_data.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "3.18" diff --git a/bundle/python-cpu/Lib/site-packages/idna/py.typed b/bundle/python-cpu/Lib/site-packages/idna/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/idna/uts46data.py b/bundle/python-cpu/Lib/site-packages/idna/uts46data.py deleted file mode 100644 index f2d931fe5d23c043a47687c480cffffd069e7aa6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/idna/uts46data.py +++ /dev/null @@ -1,16896 +0,0 @@ -# This file is automatically generated by tools/idna-data - -from array import array -from typing import Optional - -"""IDNA Mapping Table from UTS46.""" - - -__version__ = "17.0.0" - -uts46_starts: "array[int]" = array( - "I", - ( - 0x0, - 0x1, - 0x2, - 0x3, - 0x4, - 0x5, - 0x6, - 0x7, - 0x8, - 0x9, - 0xA, - 0xB, - 0xC, - 0xD, - 0xE, - 0xF, - 0x10, - 0x11, - 0x12, - 0x13, - 0x14, - 0x15, - 0x16, - 0x17, - 0x18, - 0x19, - 0x1A, - 0x1B, - 0x1C, - 0x1D, - 0x1E, - 0x1F, - 0x20, - 0x21, - 0x22, - 0x23, - 0x24, - 0x25, - 0x26, - 0x27, - 0x28, - 0x29, - 0x2A, - 0x2B, - 0x2C, - 0x2D, - 0x2E, - 0x2F, - 0x30, - 0x31, - 0x32, - 0x33, - 0x34, - 0x35, - 0x36, - 0x37, - 0x38, - 0x39, - 0x3A, - 0x3B, - 0x3C, - 0x3D, - 0x3E, - 0x3F, - 0x40, - 0x41, - 0x42, - 0x43, - 0x44, - 0x45, - 0x46, - 0x47, - 0x48, - 0x49, - 0x4A, - 0x4B, - 0x4C, - 0x4D, - 0x4E, - 0x4F, - 0x50, - 0x51, - 0x52, - 0x53, - 0x54, - 0x55, - 0x56, - 0x57, - 0x58, - 0x59, - 0x5A, - 0x5B, - 0x5C, - 0x5D, - 0x5E, - 0x5F, - 0x60, - 0x61, - 0x62, - 0x63, - 0x64, - 0x65, - 0x66, - 0x67, - 0x68, - 0x69, - 0x6A, - 0x6B, - 0x6C, - 0x6D, - 0x6E, - 0x6F, - 0x70, - 0x71, - 0x72, - 0x73, - 0x74, - 0x75, - 0x76, - 0x77, - 0x78, - 0x79, - 0x7A, - 0x7B, - 0x7C, - 0x7D, - 0x7E, - 0x7F, - 0x80, - 0x81, - 0x82, - 0x83, - 0x84, - 0x85, - 0x86, - 0x87, - 0x88, - 0x89, - 0x8A, - 0x8B, - 0x8C, - 0x8D, - 0x8E, - 0x8F, - 0x90, - 0x91, - 0x92, - 0x93, - 0x94, - 0x95, - 0x96, - 0x97, - 0x98, - 0x99, - 0x9A, - 0x9B, - 0x9C, - 0x9D, - 0x9E, - 0x9F, - 0xA0, - 0xA1, - 0xA2, - 0xA3, - 0xA4, - 0xA5, - 0xA6, - 0xA7, - 0xA8, - 0xA9, - 0xAA, - 0xAB, - 0xAC, - 0xAD, - 0xAE, - 0xAF, - 0xB0, - 0xB1, - 0xB2, - 0xB3, - 0xB4, - 0xB5, - 0xB6, - 0xB7, - 0xB8, - 0xB9, - 0xBA, - 0xBB, - 0xBC, - 0xBD, - 0xBE, - 0xBF, - 0xC0, - 0xC1, - 0xC2, - 0xC3, - 0xC4, - 0xC5, - 0xC6, - 0xC7, - 0xC8, - 0xC9, - 0xCA, - 0xCB, - 0xCC, - 0xCD, - 0xCE, - 0xCF, - 0xD0, - 0xD1, - 0xD2, - 0xD3, - 0xD4, - 0xD5, - 0xD6, - 0xD7, - 0xD8, - 0xD9, - 0xDA, - 0xDB, - 0xDC, - 0xDD, - 0xDE, - 0xDF, - 0xE0, - 0xE1, - 0xE2, - 0xE3, - 0xE4, - 0xE5, - 0xE6, - 0xE7, - 0xE8, - 0xE9, - 0xEA, - 0xEB, - 0xEC, - 0xED, - 0xEE, - 0xEF, - 0xF0, - 0xF1, - 0xF2, - 0xF3, - 0xF4, - 0xF5, - 0xF6, - 0xF7, - 0xF8, - 0xF9, - 0xFA, - 0xFB, - 0xFC, - 0xFD, - 0xFE, - 0xFF, - 0x100, - 0x101, - 0x102, - 0x103, - 0x104, - 0x105, - 0x106, - 0x107, - 0x108, - 0x109, - 0x10A, - 0x10B, - 0x10C, - 0x10D, - 0x10E, - 0x10F, - 0x110, - 0x111, - 0x112, - 0x113, - 0x114, - 0x115, - 0x116, - 0x117, - 0x118, - 0x119, - 0x11A, - 0x11B, - 0x11C, - 0x11D, - 0x11E, - 0x11F, - 0x120, - 0x121, - 0x122, - 0x123, - 0x124, - 0x125, - 0x126, - 0x127, - 0x128, - 0x129, - 0x12A, - 0x12B, - 0x12C, - 0x12D, - 0x12E, - 0x12F, - 0x130, - 0x131, - 0x132, - 0x134, - 0x135, - 0x136, - 0x137, - 0x139, - 0x13A, - 0x13B, - 0x13C, - 0x13D, - 0x13E, - 0x13F, - 0x141, - 0x142, - 0x143, - 0x144, - 0x145, - 0x146, - 0x147, - 0x148, - 0x149, - 0x14A, - 0x14B, - 0x14C, - 0x14D, - 0x14E, - 0x14F, - 0x150, - 0x151, - 0x152, - 0x153, - 0x154, - 0x155, - 0x156, - 0x157, - 0x158, - 0x159, - 0x15A, - 0x15B, - 0x15C, - 0x15D, - 0x15E, - 0x15F, - 0x160, - 0x161, - 0x162, - 0x163, - 0x164, - 0x165, - 0x166, - 0x167, - 0x168, - 0x169, - 0x16A, - 0x16B, - 0x16C, - 0x16D, - 0x16E, - 0x16F, - 0x170, - 0x171, - 0x172, - 0x173, - 0x174, - 0x175, - 0x176, - 0x177, - 0x178, - 0x179, - 0x17A, - 0x17B, - 0x17C, - 0x17D, - 0x17E, - 0x17F, - 0x180, - 0x181, - 0x182, - 0x183, - 0x184, - 0x185, - 0x186, - 0x187, - 0x188, - 0x189, - 0x18A, - 0x18B, - 0x18C, - 0x18E, - 0x18F, - 0x190, - 0x191, - 0x192, - 0x193, - 0x194, - 0x195, - 0x196, - 0x197, - 0x198, - 0x199, - 0x19C, - 0x19D, - 0x19E, - 0x19F, - 0x1A0, - 0x1A1, - 0x1A2, - 0x1A3, - 0x1A4, - 0x1A5, - 0x1A6, - 0x1A7, - 0x1A8, - 0x1A9, - 0x1AA, - 0x1AC, - 0x1AD, - 0x1AE, - 0x1AF, - 0x1B0, - 0x1B1, - 0x1B2, - 0x1B3, - 0x1B4, - 0x1B5, - 0x1B6, - 0x1B7, - 0x1B8, - 0x1B9, - 0x1BC, - 0x1BD, - 0x1C4, - 0x1C7, - 0x1CA, - 0x1CD, - 0x1CE, - 0x1CF, - 0x1D0, - 0x1D1, - 0x1D2, - 0x1D3, - 0x1D4, - 0x1D5, - 0x1D6, - 0x1D7, - 0x1D8, - 0x1D9, - 0x1DA, - 0x1DB, - 0x1DC, - 0x1DE, - 0x1DF, - 0x1E0, - 0x1E1, - 0x1E2, - 0x1E3, - 0x1E4, - 0x1E5, - 0x1E6, - 0x1E7, - 0x1E8, - 0x1E9, - 0x1EA, - 0x1EB, - 0x1EC, - 0x1ED, - 0x1EE, - 0x1EF, - 0x1F1, - 0x1F4, - 0x1F5, - 0x1F6, - 0x1F7, - 0x1F8, - 0x1F9, - 0x1FA, - 0x1FB, - 0x1FC, - 0x1FD, - 0x1FE, - 0x1FF, - 0x200, - 0x201, - 0x202, - 0x203, - 0x204, - 0x205, - 0x206, - 0x207, - 0x208, - 0x209, - 0x20A, - 0x20B, - 0x20C, - 0x20D, - 0x20E, - 0x20F, - 0x210, - 0x211, - 0x212, - 0x213, - 0x214, - 0x215, - 0x216, - 0x217, - 0x218, - 0x219, - 0x21A, - 0x21B, - 0x21C, - 0x21D, - 0x21E, - 0x21F, - 0x220, - 0x221, - 0x222, - 0x223, - 0x224, - 0x225, - 0x226, - 0x227, - 0x228, - 0x229, - 0x22A, - 0x22B, - 0x22C, - 0x22D, - 0x22E, - 0x22F, - 0x230, - 0x231, - 0x232, - 0x233, - 0x23A, - 0x23B, - 0x23C, - 0x23D, - 0x23E, - 0x23F, - 0x241, - 0x242, - 0x243, - 0x244, - 0x245, - 0x246, - 0x247, - 0x248, - 0x249, - 0x24A, - 0x24B, - 0x24C, - 0x24D, - 0x24E, - 0x24F, - 0x2B0, - 0x2B1, - 0x2B2, - 0x2B3, - 0x2B4, - 0x2B5, - 0x2B6, - 0x2B7, - 0x2B8, - 0x2B9, - 0x2D8, - 0x2D9, - 0x2DA, - 0x2DB, - 0x2DC, - 0x2DD, - 0x2DE, - 0x2E0, - 0x2E1, - 0x2E2, - 0x2E3, - 0x2E4, - 0x2E5, - 0x340, - 0x341, - 0x342, - 0x343, - 0x344, - 0x345, - 0x346, - 0x34F, - 0x350, - 0x370, - 0x371, - 0x372, - 0x373, - 0x374, - 0x375, - 0x376, - 0x377, - 0x378, - 0x37A, - 0x37B, - 0x37E, - 0x37F, - 0x380, - 0x384, - 0x385, - 0x386, - 0x387, - 0x388, - 0x389, - 0x38A, - 0x38B, - 0x38C, - 0x38D, - 0x38E, - 0x38F, - 0x390, - 0x391, - 0x392, - 0x393, - 0x394, - 0x395, - 0x396, - 0x397, - 0x398, - 0x399, - 0x39A, - 0x39B, - 0x39C, - 0x39D, - 0x39E, - 0x39F, - 0x3A0, - 0x3A1, - 0x3A2, - 0x3A3, - 0x3A4, - 0x3A5, - 0x3A6, - 0x3A7, - 0x3A8, - 0x3A9, - 0x3AA, - 0x3AB, - 0x3AC, - 0x3C2, - 0x3C3, - 0x3CF, - 0x3D0, - 0x3D1, - 0x3D2, - 0x3D3, - 0x3D4, - 0x3D5, - 0x3D6, - 0x3D7, - 0x3D8, - 0x3D9, - 0x3DA, - 0x3DB, - 0x3DC, - 0x3DD, - 0x3DE, - 0x3DF, - 0x3E0, - 0x3E1, - 0x3E2, - 0x3E3, - 0x3E4, - 0x3E5, - 0x3E6, - 0x3E7, - 0x3E8, - 0x3E9, - 0x3EA, - 0x3EB, - 0x3EC, - 0x3ED, - 0x3EE, - 0x3EF, - 0x3F0, - 0x3F1, - 0x3F2, - 0x3F3, - 0x3F4, - 0x3F5, - 0x3F6, - 0x3F7, - 0x3F8, - 0x3F9, - 0x3FA, - 0x3FB, - 0x3FD, - 0x3FE, - 0x3FF, - 0x400, - 0x401, - 0x402, - 0x403, - 0x404, - 0x405, - 0x406, - 0x407, - 0x408, - 0x409, - 0x40A, - 0x40B, - 0x40C, - 0x40D, - 0x40E, - 0x40F, - 0x410, - 0x411, - 0x412, - 0x413, - 0x414, - 0x415, - 0x416, - 0x417, - 0x418, - 0x419, - 0x41A, - 0x41B, - 0x41C, - 0x41D, - 0x41E, - 0x41F, - 0x420, - 0x421, - 0x422, - 0x423, - 0x424, - 0x425, - 0x426, - 0x427, - 0x428, - 0x429, - 0x42A, - 0x42B, - 0x42C, - 0x42D, - 0x42E, - 0x42F, - 0x430, - 0x460, - 0x461, - 0x462, - 0x463, - 0x464, - 0x465, - 0x466, - 0x467, - 0x468, - 0x469, - 0x46A, - 0x46B, - 0x46C, - 0x46D, - 0x46E, - 0x46F, - 0x470, - 0x471, - 0x472, - 0x473, - 0x474, - 0x475, - 0x476, - 0x477, - 0x478, - 0x479, - 0x47A, - 0x47B, - 0x47C, - 0x47D, - 0x47E, - 0x47F, - 0x480, - 0x481, - 0x48A, - 0x48B, - 0x48C, - 0x48D, - 0x48E, - 0x48F, - 0x490, - 0x491, - 0x492, - 0x493, - 0x494, - 0x495, - 0x496, - 0x497, - 0x498, - 0x499, - 0x49A, - 0x49B, - 0x49C, - 0x49D, - 0x49E, - 0x49F, - 0x4A0, - 0x4A1, - 0x4A2, - 0x4A3, - 0x4A4, - 0x4A5, - 0x4A6, - 0x4A7, - 0x4A8, - 0x4A9, - 0x4AA, - 0x4AB, - 0x4AC, - 0x4AD, - 0x4AE, - 0x4AF, - 0x4B0, - 0x4B1, - 0x4B2, - 0x4B3, - 0x4B4, - 0x4B5, - 0x4B6, - 0x4B7, - 0x4B8, - 0x4B9, - 0x4BA, - 0x4BB, - 0x4BC, - 0x4BD, - 0x4BE, - 0x4BF, - 0x4C0, - 0x4C1, - 0x4C2, - 0x4C3, - 0x4C4, - 0x4C5, - 0x4C6, - 0x4C7, - 0x4C8, - 0x4C9, - 0x4CA, - 0x4CB, - 0x4CC, - 0x4CD, - 0x4CE, - 0x4D0, - 0x4D1, - 0x4D2, - 0x4D3, - 0x4D4, - 0x4D5, - 0x4D6, - 0x4D7, - 0x4D8, - 0x4D9, - 0x4DA, - 0x4DB, - 0x4DC, - 0x4DD, - 0x4DE, - 0x4DF, - 0x4E0, - 0x4E1, - 0x4E2, - 0x4E3, - 0x4E4, - 0x4E5, - 0x4E6, - 0x4E7, - 0x4E8, - 0x4E9, - 0x4EA, - 0x4EB, - 0x4EC, - 0x4ED, - 0x4EE, - 0x4EF, - 0x4F0, - 0x4F1, - 0x4F2, - 0x4F3, - 0x4F4, - 0x4F5, - 0x4F6, - 0x4F7, - 0x4F8, - 0x4F9, - 0x4FA, - 0x4FB, - 0x4FC, - 0x4FD, - 0x4FE, - 0x4FF, - 0x500, - 0x501, - 0x502, - 0x503, - 0x504, - 0x505, - 0x506, - 0x507, - 0x508, - 0x509, - 0x50A, - 0x50B, - 0x50C, - 0x50D, - 0x50E, - 0x50F, - 0x510, - 0x511, - 0x512, - 0x513, - 0x514, - 0x515, - 0x516, - 0x517, - 0x518, - 0x519, - 0x51A, - 0x51B, - 0x51C, - 0x51D, - 0x51E, - 0x51F, - 0x520, - 0x521, - 0x522, - 0x523, - 0x524, - 0x525, - 0x526, - 0x527, - 0x528, - 0x529, - 0x52A, - 0x52B, - 0x52C, - 0x52D, - 0x52E, - 0x52F, - 0x530, - 0x531, - 0x532, - 0x533, - 0x534, - 0x535, - 0x536, - 0x537, - 0x538, - 0x539, - 0x53A, - 0x53B, - 0x53C, - 0x53D, - 0x53E, - 0x53F, - 0x540, - 0x541, - 0x542, - 0x543, - 0x544, - 0x545, - 0x546, - 0x547, - 0x548, - 0x549, - 0x54A, - 0x54B, - 0x54C, - 0x54D, - 0x54E, - 0x54F, - 0x550, - 0x551, - 0x552, - 0x553, - 0x554, - 0x555, - 0x556, - 0x557, - 0x559, - 0x587, - 0x588, - 0x58B, - 0x58D, - 0x590, - 0x591, - 0x5C8, - 0x5D0, - 0x5EB, - 0x5EF, - 0x5F5, - 0x606, - 0x61C, - 0x61D, - 0x675, - 0x676, - 0x677, - 0x678, - 0x679, - 0x6DD, - 0x6DE, - 0x70E, - 0x710, - 0x74B, - 0x74D, - 0x7B2, - 0x7C0, - 0x7FB, - 0x7FD, - 0x82E, - 0x830, - 0x83F, - 0x840, - 0x85C, - 0x85E, - 0x85F, - 0x860, - 0x86B, - 0x870, - 0x890, - 0x897, - 0x8E2, - 0x8E3, - 0x958, - 0x959, - 0x95A, - 0x95B, - 0x95C, - 0x95D, - 0x95E, - 0x95F, - 0x960, - 0x984, - 0x985, - 0x98D, - 0x98F, - 0x991, - 0x993, - 0x9A9, - 0x9AA, - 0x9B1, - 0x9B2, - 0x9B3, - 0x9B6, - 0x9BA, - 0x9BC, - 0x9C5, - 0x9C7, - 0x9C9, - 0x9CB, - 0x9CF, - 0x9D7, - 0x9D8, - 0x9DC, - 0x9DD, - 0x9DE, - 0x9DF, - 0x9E0, - 0x9E4, - 0x9E6, - 0x9FF, - 0xA01, - 0xA04, - 0xA05, - 0xA0B, - 0xA0F, - 0xA11, - 0xA13, - 0xA29, - 0xA2A, - 0xA31, - 0xA32, - 0xA33, - 0xA34, - 0xA35, - 0xA36, - 0xA37, - 0xA38, - 0xA3A, - 0xA3C, - 0xA3D, - 0xA3E, - 0xA43, - 0xA47, - 0xA49, - 0xA4B, - 0xA4E, - 0xA51, - 0xA52, - 0xA59, - 0xA5A, - 0xA5B, - 0xA5C, - 0xA5D, - 0xA5E, - 0xA5F, - 0xA66, - 0xA77, - 0xA81, - 0xA84, - 0xA85, - 0xA8E, - 0xA8F, - 0xA92, - 0xA93, - 0xAA9, - 0xAAA, - 0xAB1, - 0xAB2, - 0xAB4, - 0xAB5, - 0xABA, - 0xABC, - 0xAC6, - 0xAC7, - 0xACA, - 0xACB, - 0xACE, - 0xAD0, - 0xAD1, - 0xAE0, - 0xAE4, - 0xAE6, - 0xAF2, - 0xAF9, - 0xB00, - 0xB01, - 0xB04, - 0xB05, - 0xB0D, - 0xB0F, - 0xB11, - 0xB13, - 0xB29, - 0xB2A, - 0xB31, - 0xB32, - 0xB34, - 0xB35, - 0xB3A, - 0xB3C, - 0xB45, - 0xB47, - 0xB49, - 0xB4B, - 0xB4E, - 0xB55, - 0xB58, - 0xB5C, - 0xB5D, - 0xB5E, - 0xB5F, - 0xB64, - 0xB66, - 0xB78, - 0xB82, - 0xB84, - 0xB85, - 0xB8B, - 0xB8E, - 0xB91, - 0xB92, - 0xB96, - 0xB99, - 0xB9B, - 0xB9C, - 0xB9D, - 0xB9E, - 0xBA0, - 0xBA3, - 0xBA5, - 0xBA8, - 0xBAB, - 0xBAE, - 0xBBA, - 0xBBE, - 0xBC3, - 0xBC6, - 0xBC9, - 0xBCA, - 0xBCE, - 0xBD0, - 0xBD1, - 0xBD7, - 0xBD8, - 0xBE6, - 0xBFB, - 0xC00, - 0xC0D, - 0xC0E, - 0xC11, - 0xC12, - 0xC29, - 0xC2A, - 0xC3A, - 0xC3C, - 0xC45, - 0xC46, - 0xC49, - 0xC4A, - 0xC4E, - 0xC55, - 0xC57, - 0xC58, - 0xC5B, - 0xC5C, - 0xC5E, - 0xC60, - 0xC64, - 0xC66, - 0xC70, - 0xC77, - 0xC8D, - 0xC8E, - 0xC91, - 0xC92, - 0xCA9, - 0xCAA, - 0xCB4, - 0xCB5, - 0xCBA, - 0xCBC, - 0xCC5, - 0xCC6, - 0xCC9, - 0xCCA, - 0xCCE, - 0xCD5, - 0xCD7, - 0xCDC, - 0xCDF, - 0xCE0, - 0xCE4, - 0xCE6, - 0xCF0, - 0xCF1, - 0xCF4, - 0xD00, - 0xD0D, - 0xD0E, - 0xD11, - 0xD12, - 0xD45, - 0xD46, - 0xD49, - 0xD4A, - 0xD50, - 0xD54, - 0xD64, - 0xD66, - 0xD80, - 0xD81, - 0xD84, - 0xD85, - 0xD97, - 0xD9A, - 0xDB2, - 0xDB3, - 0xDBC, - 0xDBD, - 0xDBE, - 0xDC0, - 0xDC7, - 0xDCA, - 0xDCB, - 0xDCF, - 0xDD5, - 0xDD6, - 0xDD7, - 0xDD8, - 0xDE0, - 0xDE6, - 0xDF0, - 0xDF2, - 0xDF5, - 0xE01, - 0xE33, - 0xE34, - 0xE3B, - 0xE3F, - 0xE5C, - 0xE81, - 0xE83, - 0xE84, - 0xE85, - 0xE86, - 0xE8B, - 0xE8C, - 0xEA4, - 0xEA5, - 0xEA6, - 0xEA7, - 0xEB3, - 0xEB4, - 0xEBE, - 0xEC0, - 0xEC5, - 0xEC6, - 0xEC7, - 0xEC8, - 0xECF, - 0xED0, - 0xEDA, - 0xEDC, - 0xEDD, - 0xEDE, - 0xEE0, - 0xF00, - 0xF0C, - 0xF0D, - 0xF43, - 0xF44, - 0xF48, - 0xF49, - 0xF4D, - 0xF4E, - 0xF52, - 0xF53, - 0xF57, - 0xF58, - 0xF5C, - 0xF5D, - 0xF69, - 0xF6A, - 0xF6D, - 0xF71, - 0xF73, - 0xF74, - 0xF75, - 0xF76, - 0xF77, - 0xF78, - 0xF79, - 0xF7A, - 0xF81, - 0xF82, - 0xF93, - 0xF94, - 0xF98, - 0xF99, - 0xF9D, - 0xF9E, - 0xFA2, - 0xFA3, - 0xFA7, - 0xFA8, - 0xFAC, - 0xFAD, - 0xFB9, - 0xFBA, - 0xFBD, - 0xFBE, - 0xFCD, - 0xFCE, - 0xFDB, - 0x1000, - 0x10A0, - 0x10A1, - 0x10A2, - 0x10A3, - 0x10A4, - 0x10A5, - 0x10A6, - 0x10A7, - 0x10A8, - 0x10A9, - 0x10AA, - 0x10AB, - 0x10AC, - 0x10AD, - 0x10AE, - 0x10AF, - 0x10B0, - 0x10B1, - 0x10B2, - 0x10B3, - 0x10B4, - 0x10B5, - 0x10B6, - 0x10B7, - 0x10B8, - 0x10B9, - 0x10BA, - 0x10BB, - 0x10BC, - 0x10BD, - 0x10BE, - 0x10BF, - 0x10C0, - 0x10C1, - 0x10C2, - 0x10C3, - 0x10C4, - 0x10C5, - 0x10C6, - 0x10C7, - 0x10C8, - 0x10CD, - 0x10CE, - 0x10D0, - 0x10FC, - 0x10FD, - 0x115F, - 0x1161, - 0x1249, - 0x124A, - 0x124E, - 0x1250, - 0x1257, - 0x1258, - 0x1259, - 0x125A, - 0x125E, - 0x1260, - 0x1289, - 0x128A, - 0x128E, - 0x1290, - 0x12B1, - 0x12B2, - 0x12B6, - 0x12B8, - 0x12BF, - 0x12C0, - 0x12C1, - 0x12C2, - 0x12C6, - 0x12C8, - 0x12D7, - 0x12D8, - 0x1311, - 0x1312, - 0x1316, - 0x1318, - 0x135B, - 0x135D, - 0x137D, - 0x1380, - 0x139A, - 0x13A0, - 0x13F6, - 0x13F8, - 0x13F9, - 0x13FA, - 0x13FB, - 0x13FC, - 0x13FD, - 0x13FE, - 0x1400, - 0x1680, - 0x1681, - 0x169D, - 0x16A0, - 0x16F9, - 0x1700, - 0x1716, - 0x171F, - 0x1737, - 0x1740, - 0x1754, - 0x1760, - 0x176D, - 0x176E, - 0x1771, - 0x1772, - 0x1774, - 0x1780, - 0x17B4, - 0x17B6, - 0x17DE, - 0x17E0, - 0x17EA, - 0x17F0, - 0x17FA, - 0x1800, - 0x180B, - 0x1810, - 0x181A, - 0x1820, - 0x1879, - 0x1880, - 0x18AB, - 0x18B0, - 0x18F6, - 0x1900, - 0x191F, - 0x1920, - 0x192C, - 0x1930, - 0x193C, - 0x1940, - 0x1941, - 0x1944, - 0x196E, - 0x1970, - 0x1975, - 0x1980, - 0x19AC, - 0x19B0, - 0x19CA, - 0x19D0, - 0x19DB, - 0x19DE, - 0x1A1C, - 0x1A1E, - 0x1A5F, - 0x1A60, - 0x1A7D, - 0x1A7F, - 0x1A8A, - 0x1A90, - 0x1A9A, - 0x1AA0, - 0x1AAE, - 0x1AB0, - 0x1ADE, - 0x1AE0, - 0x1AEC, - 0x1B00, - 0x1B4D, - 0x1B4E, - 0x1BF4, - 0x1BFC, - 0x1C38, - 0x1C3B, - 0x1C4A, - 0x1C4D, - 0x1C80, - 0x1C81, - 0x1C82, - 0x1C83, - 0x1C84, - 0x1C86, - 0x1C87, - 0x1C88, - 0x1C89, - 0x1C8A, - 0x1C8B, - 0x1C90, - 0x1C91, - 0x1C92, - 0x1C93, - 0x1C94, - 0x1C95, - 0x1C96, - 0x1C97, - 0x1C98, - 0x1C99, - 0x1C9A, - 0x1C9B, - 0x1C9C, - 0x1C9D, - 0x1C9E, - 0x1C9F, - 0x1CA0, - 0x1CA1, - 0x1CA2, - 0x1CA3, - 0x1CA4, - 0x1CA5, - 0x1CA6, - 0x1CA7, - 0x1CA8, - 0x1CA9, - 0x1CAA, - 0x1CAB, - 0x1CAC, - 0x1CAD, - 0x1CAE, - 0x1CAF, - 0x1CB0, - 0x1CB1, - 0x1CB2, - 0x1CB3, - 0x1CB4, - 0x1CB5, - 0x1CB6, - 0x1CB7, - 0x1CB8, - 0x1CB9, - 0x1CBA, - 0x1CBB, - 0x1CBD, - 0x1CBE, - 0x1CBF, - 0x1CC0, - 0x1CC8, - 0x1CD0, - 0x1CFB, - 0x1D00, - 0x1D2C, - 0x1D2D, - 0x1D2E, - 0x1D2F, - 0x1D30, - 0x1D31, - 0x1D32, - 0x1D33, - 0x1D34, - 0x1D35, - 0x1D36, - 0x1D37, - 0x1D38, - 0x1D39, - 0x1D3A, - 0x1D3B, - 0x1D3C, - 0x1D3D, - 0x1D3E, - 0x1D3F, - 0x1D40, - 0x1D41, - 0x1D42, - 0x1D43, - 0x1D44, - 0x1D45, - 0x1D46, - 0x1D47, - 0x1D48, - 0x1D49, - 0x1D4A, - 0x1D4B, - 0x1D4C, - 0x1D4D, - 0x1D4E, - 0x1D4F, - 0x1D50, - 0x1D51, - 0x1D52, - 0x1D53, - 0x1D54, - 0x1D55, - 0x1D56, - 0x1D57, - 0x1D58, - 0x1D59, - 0x1D5A, - 0x1D5B, - 0x1D5C, - 0x1D5D, - 0x1D5E, - 0x1D5F, - 0x1D60, - 0x1D61, - 0x1D62, - 0x1D63, - 0x1D64, - 0x1D65, - 0x1D66, - 0x1D67, - 0x1D68, - 0x1D69, - 0x1D6A, - 0x1D6B, - 0x1D78, - 0x1D79, - 0x1D9B, - 0x1D9C, - 0x1D9D, - 0x1D9E, - 0x1D9F, - 0x1DA0, - 0x1DA1, - 0x1DA2, - 0x1DA3, - 0x1DA4, - 0x1DA5, - 0x1DA6, - 0x1DA7, - 0x1DA8, - 0x1DA9, - 0x1DAA, - 0x1DAB, - 0x1DAC, - 0x1DAD, - 0x1DAE, - 0x1DAF, - 0x1DB0, - 0x1DB1, - 0x1DB2, - 0x1DB3, - 0x1DB4, - 0x1DB5, - 0x1DB6, - 0x1DB7, - 0x1DB8, - 0x1DB9, - 0x1DBA, - 0x1DBB, - 0x1DBC, - 0x1DBD, - 0x1DBE, - 0x1DBF, - 0x1DC0, - 0x1E00, - 0x1E01, - 0x1E02, - 0x1E03, - 0x1E04, - 0x1E05, - 0x1E06, - 0x1E07, - 0x1E08, - 0x1E09, - 0x1E0A, - 0x1E0B, - 0x1E0C, - 0x1E0D, - 0x1E0E, - 0x1E0F, - 0x1E10, - 0x1E11, - 0x1E12, - 0x1E13, - 0x1E14, - 0x1E15, - 0x1E16, - 0x1E17, - 0x1E18, - 0x1E19, - 0x1E1A, - 0x1E1B, - 0x1E1C, - 0x1E1D, - 0x1E1E, - 0x1E1F, - 0x1E20, - 0x1E21, - 0x1E22, - 0x1E23, - 0x1E24, - 0x1E25, - 0x1E26, - 0x1E27, - 0x1E28, - 0x1E29, - 0x1E2A, - 0x1E2B, - 0x1E2C, - 0x1E2D, - 0x1E2E, - 0x1E2F, - 0x1E30, - 0x1E31, - 0x1E32, - 0x1E33, - 0x1E34, - 0x1E35, - 0x1E36, - 0x1E37, - 0x1E38, - 0x1E39, - 0x1E3A, - 0x1E3B, - 0x1E3C, - 0x1E3D, - 0x1E3E, - 0x1E3F, - 0x1E40, - 0x1E41, - 0x1E42, - 0x1E43, - 0x1E44, - 0x1E45, - 0x1E46, - 0x1E47, - 0x1E48, - 0x1E49, - 0x1E4A, - 0x1E4B, - 0x1E4C, - 0x1E4D, - 0x1E4E, - 0x1E4F, - 0x1E50, - 0x1E51, - 0x1E52, - 0x1E53, - 0x1E54, - 0x1E55, - 0x1E56, - 0x1E57, - 0x1E58, - 0x1E59, - 0x1E5A, - 0x1E5B, - 0x1E5C, - 0x1E5D, - 0x1E5E, - 0x1E5F, - 0x1E60, - 0x1E61, - 0x1E62, - 0x1E63, - 0x1E64, - 0x1E65, - 0x1E66, - 0x1E67, - 0x1E68, - 0x1E69, - 0x1E6A, - 0x1E6B, - 0x1E6C, - 0x1E6D, - 0x1E6E, - 0x1E6F, - 0x1E70, - 0x1E71, - 0x1E72, - 0x1E73, - 0x1E74, - 0x1E75, - 0x1E76, - 0x1E77, - 0x1E78, - 0x1E79, - 0x1E7A, - 0x1E7B, - 0x1E7C, - 0x1E7D, - 0x1E7E, - 0x1E7F, - 0x1E80, - 0x1E81, - 0x1E82, - 0x1E83, - 0x1E84, - 0x1E85, - 0x1E86, - 0x1E87, - 0x1E88, - 0x1E89, - 0x1E8A, - 0x1E8B, - 0x1E8C, - 0x1E8D, - 0x1E8E, - 0x1E8F, - 0x1E90, - 0x1E91, - 0x1E92, - 0x1E93, - 0x1E94, - 0x1E95, - 0x1E9A, - 0x1E9B, - 0x1E9C, - 0x1E9E, - 0x1E9F, - 0x1EA0, - 0x1EA1, - 0x1EA2, - 0x1EA3, - 0x1EA4, - 0x1EA5, - 0x1EA6, - 0x1EA7, - 0x1EA8, - 0x1EA9, - 0x1EAA, - 0x1EAB, - 0x1EAC, - 0x1EAD, - 0x1EAE, - 0x1EAF, - 0x1EB0, - 0x1EB1, - 0x1EB2, - 0x1EB3, - 0x1EB4, - 0x1EB5, - 0x1EB6, - 0x1EB7, - 0x1EB8, - 0x1EB9, - 0x1EBA, - 0x1EBB, - 0x1EBC, - 0x1EBD, - 0x1EBE, - 0x1EBF, - 0x1EC0, - 0x1EC1, - 0x1EC2, - 0x1EC3, - 0x1EC4, - 0x1EC5, - 0x1EC6, - 0x1EC7, - 0x1EC8, - 0x1EC9, - 0x1ECA, - 0x1ECB, - 0x1ECC, - 0x1ECD, - 0x1ECE, - 0x1ECF, - 0x1ED0, - 0x1ED1, - 0x1ED2, - 0x1ED3, - 0x1ED4, - 0x1ED5, - 0x1ED6, - 0x1ED7, - 0x1ED8, - 0x1ED9, - 0x1EDA, - 0x1EDB, - 0x1EDC, - 0x1EDD, - 0x1EDE, - 0x1EDF, - 0x1EE0, - 0x1EE1, - 0x1EE2, - 0x1EE3, - 0x1EE4, - 0x1EE5, - 0x1EE6, - 0x1EE7, - 0x1EE8, - 0x1EE9, - 0x1EEA, - 0x1EEB, - 0x1EEC, - 0x1EED, - 0x1EEE, - 0x1EEF, - 0x1EF0, - 0x1EF1, - 0x1EF2, - 0x1EF3, - 0x1EF4, - 0x1EF5, - 0x1EF6, - 0x1EF7, - 0x1EF8, - 0x1EF9, - 0x1EFA, - 0x1EFB, - 0x1EFC, - 0x1EFD, - 0x1EFE, - 0x1EFF, - 0x1F08, - 0x1F09, - 0x1F0A, - 0x1F0B, - 0x1F0C, - 0x1F0D, - 0x1F0E, - 0x1F0F, - 0x1F10, - 0x1F16, - 0x1F18, - 0x1F19, - 0x1F1A, - 0x1F1B, - 0x1F1C, - 0x1F1D, - 0x1F1E, - 0x1F20, - 0x1F28, - 0x1F29, - 0x1F2A, - 0x1F2B, - 0x1F2C, - 0x1F2D, - 0x1F2E, - 0x1F2F, - 0x1F30, - 0x1F38, - 0x1F39, - 0x1F3A, - 0x1F3B, - 0x1F3C, - 0x1F3D, - 0x1F3E, - 0x1F3F, - 0x1F40, - 0x1F46, - 0x1F48, - 0x1F49, - 0x1F4A, - 0x1F4B, - 0x1F4C, - 0x1F4D, - 0x1F4E, - 0x1F50, - 0x1F58, - 0x1F59, - 0x1F5A, - 0x1F5B, - 0x1F5C, - 0x1F5D, - 0x1F5E, - 0x1F5F, - 0x1F60, - 0x1F68, - 0x1F69, - 0x1F6A, - 0x1F6B, - 0x1F6C, - 0x1F6D, - 0x1F6E, - 0x1F6F, - 0x1F70, - 0x1F71, - 0x1F72, - 0x1F73, - 0x1F74, - 0x1F75, - 0x1F76, - 0x1F77, - 0x1F78, - 0x1F79, - 0x1F7A, - 0x1F7B, - 0x1F7C, - 0x1F7D, - 0x1F7E, - 0x1F80, - 0x1F81, - 0x1F82, - 0x1F83, - 0x1F84, - 0x1F85, - 0x1F86, - 0x1F87, - 0x1F88, - 0x1F89, - 0x1F8A, - 0x1F8B, - 0x1F8C, - 0x1F8D, - 0x1F8E, - 0x1F8F, - 0x1F90, - 0x1F91, - 0x1F92, - 0x1F93, - 0x1F94, - 0x1F95, - 0x1F96, - 0x1F97, - 0x1F98, - 0x1F99, - 0x1F9A, - 0x1F9B, - 0x1F9C, - 0x1F9D, - 0x1F9E, - 0x1F9F, - 0x1FA0, - 0x1FA1, - 0x1FA2, - 0x1FA3, - 0x1FA4, - 0x1FA5, - 0x1FA6, - 0x1FA7, - 0x1FA8, - 0x1FA9, - 0x1FAA, - 0x1FAB, - 0x1FAC, - 0x1FAD, - 0x1FAE, - 0x1FAF, - 0x1FB0, - 0x1FB2, - 0x1FB3, - 0x1FB4, - 0x1FB5, - 0x1FB6, - 0x1FB7, - 0x1FB8, - 0x1FB9, - 0x1FBA, - 0x1FBB, - 0x1FBC, - 0x1FBD, - 0x1FBE, - 0x1FBF, - 0x1FC0, - 0x1FC1, - 0x1FC2, - 0x1FC3, - 0x1FC4, - 0x1FC5, - 0x1FC6, - 0x1FC7, - 0x1FC8, - 0x1FC9, - 0x1FCA, - 0x1FCB, - 0x1FCC, - 0x1FCD, - 0x1FCE, - 0x1FCF, - 0x1FD0, - 0x1FD3, - 0x1FD4, - 0x1FD6, - 0x1FD8, - 0x1FD9, - 0x1FDA, - 0x1FDB, - 0x1FDC, - 0x1FDD, - 0x1FDE, - 0x1FDF, - 0x1FE0, - 0x1FE3, - 0x1FE4, - 0x1FE8, - 0x1FE9, - 0x1FEA, - 0x1FEB, - 0x1FEC, - 0x1FED, - 0x1FEE, - 0x1FEF, - 0x1FF0, - 0x1FF2, - 0x1FF3, - 0x1FF4, - 0x1FF5, - 0x1FF6, - 0x1FF7, - 0x1FF8, - 0x1FF9, - 0x1FFA, - 0x1FFB, - 0x1FFC, - 0x1FFD, - 0x1FFE, - 0x1FFF, - 0x2000, - 0x200B, - 0x200C, - 0x200E, - 0x2010, - 0x2011, - 0x2012, - 0x2017, - 0x2018, - 0x2024, - 0x2027, - 0x2028, - 0x202F, - 0x2030, - 0x2033, - 0x2034, - 0x2035, - 0x2036, - 0x2037, - 0x2038, - 0x203C, - 0x203D, - 0x203E, - 0x203F, - 0x2047, - 0x2048, - 0x2049, - 0x204A, - 0x2057, - 0x2058, - 0x205F, - 0x2060, - 0x2065, - 0x206A, - 0x2070, - 0x2071, - 0x2072, - 0x2074, - 0x2075, - 0x2076, - 0x2077, - 0x2078, - 0x2079, - 0x207A, - 0x207B, - 0x207C, - 0x207D, - 0x207E, - 0x207F, - 0x2080, - 0x2081, - 0x2082, - 0x2083, - 0x2084, - 0x2085, - 0x2086, - 0x2087, - 0x2088, - 0x2089, - 0x208A, - 0x208B, - 0x208C, - 0x208D, - 0x208E, - 0x208F, - 0x2090, - 0x2091, - 0x2092, - 0x2093, - 0x2094, - 0x2095, - 0x2096, - 0x2097, - 0x2098, - 0x2099, - 0x209A, - 0x209B, - 0x209C, - 0x209D, - 0x20A0, - 0x20A8, - 0x20A9, - 0x20C2, - 0x20D0, - 0x20F1, - 0x2100, - 0x2101, - 0x2102, - 0x2103, - 0x2104, - 0x2105, - 0x2106, - 0x2107, - 0x2108, - 0x2109, - 0x210A, - 0x210B, - 0x210F, - 0x2110, - 0x2112, - 0x2114, - 0x2115, - 0x2116, - 0x2117, - 0x2119, - 0x211A, - 0x211B, - 0x211E, - 0x2120, - 0x2121, - 0x2122, - 0x2123, - 0x2124, - 0x2125, - 0x2126, - 0x2127, - 0x2128, - 0x2129, - 0x212A, - 0x212B, - 0x212C, - 0x212D, - 0x212E, - 0x212F, - 0x2131, - 0x2132, - 0x2133, - 0x2134, - 0x2135, - 0x2136, - 0x2137, - 0x2138, - 0x2139, - 0x213A, - 0x213B, - 0x213C, - 0x213D, - 0x213F, - 0x2140, - 0x2141, - 0x2145, - 0x2147, - 0x2148, - 0x2149, - 0x214A, - 0x2150, - 0x2151, - 0x2152, - 0x2153, - 0x2154, - 0x2155, - 0x2156, - 0x2157, - 0x2158, - 0x2159, - 0x215A, - 0x215B, - 0x215C, - 0x215D, - 0x215E, - 0x215F, - 0x2160, - 0x2161, - 0x2162, - 0x2163, - 0x2164, - 0x2165, - 0x2166, - 0x2167, - 0x2168, - 0x2169, - 0x216A, - 0x216B, - 0x216C, - 0x216D, - 0x216E, - 0x216F, - 0x2170, - 0x2171, - 0x2172, - 0x2173, - 0x2174, - 0x2175, - 0x2176, - 0x2177, - 0x2178, - 0x2179, - 0x217A, - 0x217B, - 0x217C, - 0x217D, - 0x217E, - 0x217F, - 0x2180, - 0x2183, - 0x2184, - 0x2189, - 0x218A, - 0x218C, - 0x2190, - 0x222C, - 0x222D, - 0x222E, - 0x222F, - 0x2230, - 0x2231, - 0x2329, - 0x232A, - 0x232B, - 0x242A, - 0x2440, - 0x244B, - 0x2460, - 0x2461, - 0x2462, - 0x2463, - 0x2464, - 0x2465, - 0x2466, - 0x2467, - 0x2468, - 0x2469, - 0x246A, - 0x246B, - 0x246C, - 0x246D, - 0x246E, - 0x246F, - 0x2470, - 0x2471, - 0x2472, - 0x2473, - 0x2474, - 0x2475, - 0x2476, - 0x2477, - 0x2478, - 0x2479, - 0x247A, - 0x247B, - 0x247C, - 0x247D, - 0x247E, - 0x247F, - 0x2480, - 0x2481, - 0x2482, - 0x2483, - 0x2484, - 0x2485, - 0x2486, - 0x2487, - 0x2488, - 0x249C, - 0x249D, - 0x249E, - 0x249F, - 0x24A0, - 0x24A1, - 0x24A2, - 0x24A3, - 0x24A4, - 0x24A5, - 0x24A6, - 0x24A7, - 0x24A8, - 0x24A9, - 0x24AA, - 0x24AB, - 0x24AC, - 0x24AD, - 0x24AE, - 0x24AF, - 0x24B0, - 0x24B1, - 0x24B2, - 0x24B3, - 0x24B4, - 0x24B5, - 0x24B6, - 0x24B7, - 0x24B8, - 0x24B9, - 0x24BA, - 0x24BB, - 0x24BC, - 0x24BD, - 0x24BE, - 0x24BF, - 0x24C0, - 0x24C1, - 0x24C2, - 0x24C3, - 0x24C4, - 0x24C5, - 0x24C6, - 0x24C7, - 0x24C8, - 0x24C9, - 0x24CA, - 0x24CB, - 0x24CC, - 0x24CD, - 0x24CE, - 0x24CF, - 0x24D0, - 0x24D1, - 0x24D2, - 0x24D3, - 0x24D4, - 0x24D5, - 0x24D6, - 0x24D7, - 0x24D8, - 0x24D9, - 0x24DA, - 0x24DB, - 0x24DC, - 0x24DD, - 0x24DE, - 0x24DF, - 0x24E0, - 0x24E1, - 0x24E2, - 0x24E3, - 0x24E4, - 0x24E5, - 0x24E6, - 0x24E7, - 0x24E8, - 0x24E9, - 0x24EA, - 0x24EB, - 0x2A0C, - 0x2A0D, - 0x2A74, - 0x2A75, - 0x2A76, - 0x2A77, - 0x2ADC, - 0x2ADD, - 0x2B74, - 0x2B76, - 0x2C00, - 0x2C01, - 0x2C02, - 0x2C03, - 0x2C04, - 0x2C05, - 0x2C06, - 0x2C07, - 0x2C08, - 0x2C09, - 0x2C0A, - 0x2C0B, - 0x2C0C, - 0x2C0D, - 0x2C0E, - 0x2C0F, - 0x2C10, - 0x2C11, - 0x2C12, - 0x2C13, - 0x2C14, - 0x2C15, - 0x2C16, - 0x2C17, - 0x2C18, - 0x2C19, - 0x2C1A, - 0x2C1B, - 0x2C1C, - 0x2C1D, - 0x2C1E, - 0x2C1F, - 0x2C20, - 0x2C21, - 0x2C22, - 0x2C23, - 0x2C24, - 0x2C25, - 0x2C26, - 0x2C27, - 0x2C28, - 0x2C29, - 0x2C2A, - 0x2C2B, - 0x2C2C, - 0x2C2D, - 0x2C2E, - 0x2C2F, - 0x2C30, - 0x2C60, - 0x2C61, - 0x2C62, - 0x2C63, - 0x2C64, - 0x2C65, - 0x2C67, - 0x2C68, - 0x2C69, - 0x2C6A, - 0x2C6B, - 0x2C6C, - 0x2C6D, - 0x2C6E, - 0x2C6F, - 0x2C70, - 0x2C71, - 0x2C72, - 0x2C73, - 0x2C75, - 0x2C76, - 0x2C7C, - 0x2C7D, - 0x2C7E, - 0x2C7F, - 0x2C80, - 0x2C81, - 0x2C82, - 0x2C83, - 0x2C84, - 0x2C85, - 0x2C86, - 0x2C87, - 0x2C88, - 0x2C89, - 0x2C8A, - 0x2C8B, - 0x2C8C, - 0x2C8D, - 0x2C8E, - 0x2C8F, - 0x2C90, - 0x2C91, - 0x2C92, - 0x2C93, - 0x2C94, - 0x2C95, - 0x2C96, - 0x2C97, - 0x2C98, - 0x2C99, - 0x2C9A, - 0x2C9B, - 0x2C9C, - 0x2C9D, - 0x2C9E, - 0x2C9F, - 0x2CA0, - 0x2CA1, - 0x2CA2, - 0x2CA3, - 0x2CA4, - 0x2CA5, - 0x2CA6, - 0x2CA7, - 0x2CA8, - 0x2CA9, - 0x2CAA, - 0x2CAB, - 0x2CAC, - 0x2CAD, - 0x2CAE, - 0x2CAF, - 0x2CB0, - 0x2CB1, - 0x2CB2, - 0x2CB3, - 0x2CB4, - 0x2CB5, - 0x2CB6, - 0x2CB7, - 0x2CB8, - 0x2CB9, - 0x2CBA, - 0x2CBB, - 0x2CBC, - 0x2CBD, - 0x2CBE, - 0x2CBF, - 0x2CC0, - 0x2CC1, - 0x2CC2, - 0x2CC3, - 0x2CC4, - 0x2CC5, - 0x2CC6, - 0x2CC7, - 0x2CC8, - 0x2CC9, - 0x2CCA, - 0x2CCB, - 0x2CCC, - 0x2CCD, - 0x2CCE, - 0x2CCF, - 0x2CD0, - 0x2CD1, - 0x2CD2, - 0x2CD3, - 0x2CD4, - 0x2CD5, - 0x2CD6, - 0x2CD7, - 0x2CD8, - 0x2CD9, - 0x2CDA, - 0x2CDB, - 0x2CDC, - 0x2CDD, - 0x2CDE, - 0x2CDF, - 0x2CE0, - 0x2CE1, - 0x2CE2, - 0x2CE3, - 0x2CEB, - 0x2CEC, - 0x2CED, - 0x2CEE, - 0x2CF2, - 0x2CF3, - 0x2CF4, - 0x2CF9, - 0x2D26, - 0x2D27, - 0x2D28, - 0x2D2D, - 0x2D2E, - 0x2D30, - 0x2D68, - 0x2D6F, - 0x2D70, - 0x2D71, - 0x2D7F, - 0x2D97, - 0x2DA0, - 0x2DA7, - 0x2DA8, - 0x2DAF, - 0x2DB0, - 0x2DB7, - 0x2DB8, - 0x2DBF, - 0x2DC0, - 0x2DC7, - 0x2DC8, - 0x2DCF, - 0x2DD0, - 0x2DD7, - 0x2DD8, - 0x2DDF, - 0x2DE0, - 0x2E5E, - 0x2E80, - 0x2E9A, - 0x2E9B, - 0x2E9F, - 0x2EA0, - 0x2EF3, - 0x2EF4, - 0x2F00, - 0x2F01, - 0x2F02, - 0x2F03, - 0x2F04, - 0x2F05, - 0x2F06, - 0x2F07, - 0x2F08, - 0x2F09, - 0x2F0A, - 0x2F0B, - 0x2F0C, - 0x2F0D, - 0x2F0E, - 0x2F0F, - 0x2F10, - 0x2F11, - 0x2F12, - 0x2F13, - 0x2F14, - 0x2F15, - 0x2F16, - 0x2F17, - 0x2F18, - 0x2F19, - 0x2F1A, - 0x2F1B, - 0x2F1C, - 0x2F1D, - 0x2F1E, - 0x2F1F, - 0x2F20, - 0x2F21, - 0x2F22, - 0x2F23, - 0x2F24, - 0x2F25, - 0x2F26, - 0x2F27, - 0x2F28, - 0x2F29, - 0x2F2A, - 0x2F2B, - 0x2F2C, - 0x2F2D, - 0x2F2E, - 0x2F2F, - 0x2F30, - 0x2F31, - 0x2F32, - 0x2F33, - 0x2F34, - 0x2F35, - 0x2F36, - 0x2F37, - 0x2F38, - 0x2F39, - 0x2F3A, - 0x2F3B, - 0x2F3C, - 0x2F3D, - 0x2F3E, - 0x2F3F, - 0x2F40, - 0x2F41, - 0x2F42, - 0x2F43, - 0x2F44, - 0x2F45, - 0x2F46, - 0x2F47, - 0x2F48, - 0x2F49, - 0x2F4A, - 0x2F4B, - 0x2F4C, - 0x2F4D, - 0x2F4E, - 0x2F4F, - 0x2F50, - 0x2F51, - 0x2F52, - 0x2F53, - 0x2F54, - 0x2F55, - 0x2F56, - 0x2F57, - 0x2F58, - 0x2F59, - 0x2F5A, - 0x2F5B, - 0x2F5C, - 0x2F5D, - 0x2F5E, - 0x2F5F, - 0x2F60, - 0x2F61, - 0x2F62, - 0x2F63, - 0x2F64, - 0x2F65, - 0x2F66, - 0x2F67, - 0x2F68, - 0x2F69, - 0x2F6A, - 0x2F6B, - 0x2F6C, - 0x2F6D, - 0x2F6E, - 0x2F6F, - 0x2F70, - 0x2F71, - 0x2F72, - 0x2F73, - 0x2F74, - 0x2F75, - 0x2F76, - 0x2F77, - 0x2F78, - 0x2F79, - 0x2F7A, - 0x2F7B, - 0x2F7C, - 0x2F7D, - 0x2F7E, - 0x2F7F, - 0x2F80, - 0x2F81, - 0x2F82, - 0x2F83, - 0x2F84, - 0x2F85, - 0x2F86, - 0x2F87, - 0x2F88, - 0x2F89, - 0x2F8A, - 0x2F8B, - 0x2F8C, - 0x2F8D, - 0x2F8E, - 0x2F8F, - 0x2F90, - 0x2F91, - 0x2F92, - 0x2F93, - 0x2F94, - 0x2F95, - 0x2F96, - 0x2F97, - 0x2F98, - 0x2F99, - 0x2F9A, - 0x2F9B, - 0x2F9C, - 0x2F9D, - 0x2F9E, - 0x2F9F, - 0x2FA0, - 0x2FA1, - 0x2FA2, - 0x2FA3, - 0x2FA4, - 0x2FA5, - 0x2FA6, - 0x2FA7, - 0x2FA8, - 0x2FA9, - 0x2FAA, - 0x2FAB, - 0x2FAC, - 0x2FAD, - 0x2FAE, - 0x2FAF, - 0x2FB0, - 0x2FB1, - 0x2FB2, - 0x2FB3, - 0x2FB4, - 0x2FB5, - 0x2FB6, - 0x2FB7, - 0x2FB8, - 0x2FB9, - 0x2FBA, - 0x2FBB, - 0x2FBC, - 0x2FBD, - 0x2FBE, - 0x2FBF, - 0x2FC0, - 0x2FC1, - 0x2FC2, - 0x2FC3, - 0x2FC4, - 0x2FC5, - 0x2FC6, - 0x2FC7, - 0x2FC8, - 0x2FC9, - 0x2FCA, - 0x2FCB, - 0x2FCC, - 0x2FCD, - 0x2FCE, - 0x2FCF, - 0x2FD0, - 0x2FD1, - 0x2FD2, - 0x2FD3, - 0x2FD4, - 0x2FD5, - 0x2FD6, - 0x3000, - 0x3001, - 0x3002, - 0x3003, - 0x3036, - 0x3037, - 0x3038, - 0x3039, - 0x303A, - 0x303B, - 0x3040, - 0x3041, - 0x3097, - 0x3099, - 0x309B, - 0x309C, - 0x309D, - 0x309F, - 0x30A0, - 0x30FF, - 0x3100, - 0x3105, - 0x3130, - 0x3131, - 0x3132, - 0x3133, - 0x3134, - 0x3135, - 0x3136, - 0x3137, - 0x3138, - 0x3139, - 0x313A, - 0x313B, - 0x313C, - 0x313D, - 0x313E, - 0x313F, - 0x3140, - 0x3141, - 0x3142, - 0x3143, - 0x3144, - 0x3145, - 0x3146, - 0x3147, - 0x3148, - 0x3149, - 0x314A, - 0x314B, - 0x314C, - 0x314D, - 0x314E, - 0x314F, - 0x3150, - 0x3151, - 0x3152, - 0x3153, - 0x3154, - 0x3155, - 0x3156, - 0x3157, - 0x3158, - 0x3159, - 0x315A, - 0x315B, - 0x315C, - 0x315D, - 0x315E, - 0x315F, - 0x3160, - 0x3161, - 0x3162, - 0x3163, - 0x3164, - 0x3165, - 0x3166, - 0x3167, - 0x3168, - 0x3169, - 0x316A, - 0x316B, - 0x316C, - 0x316D, - 0x316E, - 0x316F, - 0x3170, - 0x3171, - 0x3172, - 0x3173, - 0x3174, - 0x3175, - 0x3176, - 0x3177, - 0x3178, - 0x3179, - 0x317A, - 0x317B, - 0x317C, - 0x317D, - 0x317E, - 0x317F, - 0x3180, - 0x3181, - 0x3182, - 0x3183, - 0x3184, - 0x3185, - 0x3186, - 0x3187, - 0x3188, - 0x3189, - 0x318A, - 0x318B, - 0x318C, - 0x318D, - 0x318E, - 0x318F, - 0x3190, - 0x3192, - 0x3193, - 0x3194, - 0x3195, - 0x3196, - 0x3197, - 0x3198, - 0x3199, - 0x319A, - 0x319B, - 0x319C, - 0x319D, - 0x319E, - 0x319F, - 0x31A0, - 0x31E6, - 0x31F0, - 0x3200, - 0x3201, - 0x3202, - 0x3203, - 0x3204, - 0x3205, - 0x3206, - 0x3207, - 0x3208, - 0x3209, - 0x320A, - 0x320B, - 0x320C, - 0x320D, - 0x320E, - 0x320F, - 0x3210, - 0x3211, - 0x3212, - 0x3213, - 0x3214, - 0x3215, - 0x3216, - 0x3217, - 0x3218, - 0x3219, - 0x321A, - 0x321B, - 0x321C, - 0x321D, - 0x321E, - 0x321F, - 0x3220, - 0x3221, - 0x3222, - 0x3223, - 0x3224, - 0x3225, - 0x3226, - 0x3227, - 0x3228, - 0x3229, - 0x322A, - 0x322B, - 0x322C, - 0x322D, - 0x322E, - 0x322F, - 0x3230, - 0x3231, - 0x3232, - 0x3233, - 0x3234, - 0x3235, - 0x3236, - 0x3237, - 0x3238, - 0x3239, - 0x323A, - 0x323B, - 0x323C, - 0x323D, - 0x323E, - 0x323F, - 0x3240, - 0x3241, - 0x3242, - 0x3243, - 0x3244, - 0x3245, - 0x3246, - 0x3247, - 0x3248, - 0x3250, - 0x3251, - 0x3252, - 0x3253, - 0x3254, - 0x3255, - 0x3256, - 0x3257, - 0x3258, - 0x3259, - 0x325A, - 0x325B, - 0x325C, - 0x325D, - 0x325E, - 0x325F, - 0x3260, - 0x3261, - 0x3262, - 0x3263, - 0x3264, - 0x3265, - 0x3266, - 0x3267, - 0x3268, - 0x3269, - 0x326A, - 0x326B, - 0x326C, - 0x326D, - 0x326E, - 0x326F, - 0x3270, - 0x3271, - 0x3272, - 0x3273, - 0x3274, - 0x3275, - 0x3276, - 0x3277, - 0x3278, - 0x3279, - 0x327A, - 0x327B, - 0x327C, - 0x327D, - 0x327E, - 0x327F, - 0x3280, - 0x3281, - 0x3282, - 0x3283, - 0x3284, - 0x3285, - 0x3286, - 0x3287, - 0x3288, - 0x3289, - 0x328A, - 0x328B, - 0x328C, - 0x328D, - 0x328E, - 0x328F, - 0x3290, - 0x3291, - 0x3292, - 0x3293, - 0x3294, - 0x3295, - 0x3296, - 0x3297, - 0x3298, - 0x3299, - 0x329A, - 0x329B, - 0x329C, - 0x329D, - 0x329E, - 0x329F, - 0x32A0, - 0x32A1, - 0x32A2, - 0x32A3, - 0x32A4, - 0x32A5, - 0x32A6, - 0x32A7, - 0x32A8, - 0x32A9, - 0x32AA, - 0x32AB, - 0x32AC, - 0x32AD, - 0x32AE, - 0x32AF, - 0x32B0, - 0x32B1, - 0x32B2, - 0x32B3, - 0x32B4, - 0x32B5, - 0x32B6, - 0x32B7, - 0x32B8, - 0x32B9, - 0x32BA, - 0x32BB, - 0x32BC, - 0x32BD, - 0x32BE, - 0x32BF, - 0x32C0, - 0x32C1, - 0x32C2, - 0x32C3, - 0x32C4, - 0x32C5, - 0x32C6, - 0x32C7, - 0x32C8, - 0x32C9, - 0x32CA, - 0x32CB, - 0x32CC, - 0x32CD, - 0x32CE, - 0x32CF, - 0x32D0, - 0x32D1, - 0x32D2, - 0x32D3, - 0x32D4, - 0x32D5, - 0x32D6, - 0x32D7, - 0x32D8, - 0x32D9, - 0x32DA, - 0x32DB, - 0x32DC, - 0x32DD, - 0x32DE, - 0x32DF, - 0x32E0, - 0x32E1, - 0x32E2, - 0x32E3, - 0x32E4, - 0x32E5, - 0x32E6, - 0x32E7, - 0x32E8, - 0x32E9, - 0x32EA, - 0x32EB, - 0x32EC, - 0x32ED, - 0x32EE, - 0x32EF, - 0x32F0, - 0x32F1, - 0x32F2, - 0x32F3, - 0x32F4, - 0x32F5, - 0x32F6, - 0x32F7, - 0x32F8, - 0x32F9, - 0x32FA, - 0x32FB, - 0x32FC, - 0x32FD, - 0x32FE, - 0x32FF, - 0x3300, - 0x3301, - 0x3302, - 0x3303, - 0x3304, - 0x3305, - 0x3306, - 0x3307, - 0x3308, - 0x3309, - 0x330A, - 0x330B, - 0x330C, - 0x330D, - 0x330E, - 0x330F, - 0x3310, - 0x3311, - 0x3312, - 0x3313, - 0x3314, - 0x3315, - 0x3316, - 0x3317, - 0x3318, - 0x3319, - 0x331A, - 0x331B, - 0x331C, - 0x331D, - 0x331E, - 0x331F, - 0x3320, - 0x3321, - 0x3322, - 0x3323, - 0x3324, - 0x3325, - 0x3326, - 0x3327, - 0x3328, - 0x3329, - 0x332A, - 0x332B, - 0x332C, - 0x332D, - 0x332E, - 0x332F, - 0x3330, - 0x3331, - 0x3332, - 0x3333, - 0x3334, - 0x3335, - 0x3336, - 0x3337, - 0x3338, - 0x3339, - 0x333A, - 0x333B, - 0x333C, - 0x333D, - 0x333E, - 0x333F, - 0x3340, - 0x3341, - 0x3342, - 0x3343, - 0x3344, - 0x3345, - 0x3346, - 0x3347, - 0x3348, - 0x3349, - 0x334A, - 0x334B, - 0x334C, - 0x334D, - 0x334E, - 0x334F, - 0x3350, - 0x3351, - 0x3352, - 0x3353, - 0x3354, - 0x3355, - 0x3356, - 0x3357, - 0x3358, - 0x3359, - 0x335A, - 0x335B, - 0x335C, - 0x335D, - 0x335E, - 0x335F, - 0x3360, - 0x3361, - 0x3362, - 0x3363, - 0x3364, - 0x3365, - 0x3366, - 0x3367, - 0x3368, - 0x3369, - 0x336A, - 0x336B, - 0x336C, - 0x336D, - 0x336E, - 0x336F, - 0x3370, - 0x3371, - 0x3372, - 0x3373, - 0x3374, - 0x3375, - 0x3376, - 0x3377, - 0x3378, - 0x3379, - 0x337A, - 0x337B, - 0x337C, - 0x337D, - 0x337E, - 0x337F, - 0x3380, - 0x3381, - 0x3382, - 0x3383, - 0x3384, - 0x3385, - 0x3386, - 0x3387, - 0x3388, - 0x3389, - 0x338A, - 0x338B, - 0x338C, - 0x338D, - 0x338E, - 0x338F, - 0x3390, - 0x3391, - 0x3392, - 0x3393, - 0x3394, - 0x3395, - 0x3396, - 0x3397, - 0x3398, - 0x3399, - 0x339A, - 0x339B, - 0x339C, - 0x339D, - 0x339E, - 0x339F, - 0x33A0, - 0x33A1, - 0x33A2, - 0x33A3, - 0x33A4, - 0x33A5, - 0x33A6, - 0x33A7, - 0x33A8, - 0x33A9, - 0x33AA, - 0x33AB, - 0x33AC, - 0x33AD, - 0x33AE, - 0x33AF, - 0x33B0, - 0x33B1, - 0x33B2, - 0x33B3, - 0x33B4, - 0x33B5, - 0x33B6, - 0x33B7, - 0x33B8, - 0x33B9, - 0x33BA, - 0x33BB, - 0x33BC, - 0x33BD, - 0x33BE, - 0x33BF, - 0x33C0, - 0x33C1, - 0x33C2, - 0x33C3, - 0x33C4, - 0x33C5, - 0x33C6, - 0x33C7, - 0x33C8, - 0x33C9, - 0x33CA, - 0x33CB, - 0x33CC, - 0x33CD, - 0x33CE, - 0x33CF, - 0x33D0, - 0x33D1, - 0x33D2, - 0x33D3, - 0x33D4, - 0x33D5, - 0x33D6, - 0x33D7, - 0x33D8, - 0x33D9, - 0x33DA, - 0x33DB, - 0x33DC, - 0x33DD, - 0x33DE, - 0x33DF, - 0x33E0, - 0x33E1, - 0x33E2, - 0x33E3, - 0x33E4, - 0x33E5, - 0x33E6, - 0x33E7, - 0x33E8, - 0x33E9, - 0x33EA, - 0x33EB, - 0x33EC, - 0x33ED, - 0x33EE, - 0x33EF, - 0x33F0, - 0x33F1, - 0x33F2, - 0x33F3, - 0x33F4, - 0x33F5, - 0x33F6, - 0x33F7, - 0x33F8, - 0x33F9, - 0x33FA, - 0x33FB, - 0x33FC, - 0x33FD, - 0x33FE, - 0x33FF, - 0x3400, - 0xA48D, - 0xA490, - 0xA4C7, - 0xA4D0, - 0xA62C, - 0xA640, - 0xA641, - 0xA642, - 0xA643, - 0xA644, - 0xA645, - 0xA646, - 0xA647, - 0xA648, - 0xA649, - 0xA64A, - 0xA64B, - 0xA64C, - 0xA64D, - 0xA64E, - 0xA64F, - 0xA650, - 0xA651, - 0xA652, - 0xA653, - 0xA654, - 0xA655, - 0xA656, - 0xA657, - 0xA658, - 0xA659, - 0xA65A, - 0xA65B, - 0xA65C, - 0xA65D, - 0xA65E, - 0xA65F, - 0xA660, - 0xA661, - 0xA662, - 0xA663, - 0xA664, - 0xA665, - 0xA666, - 0xA667, - 0xA668, - 0xA669, - 0xA66A, - 0xA66B, - 0xA66C, - 0xA66D, - 0xA680, - 0xA681, - 0xA682, - 0xA683, - 0xA684, - 0xA685, - 0xA686, - 0xA687, - 0xA688, - 0xA689, - 0xA68A, - 0xA68B, - 0xA68C, - 0xA68D, - 0xA68E, - 0xA68F, - 0xA690, - 0xA691, - 0xA692, - 0xA693, - 0xA694, - 0xA695, - 0xA696, - 0xA697, - 0xA698, - 0xA699, - 0xA69A, - 0xA69B, - 0xA69C, - 0xA69D, - 0xA69E, - 0xA6F8, - 0xA700, - 0xA722, - 0xA723, - 0xA724, - 0xA725, - 0xA726, - 0xA727, - 0xA728, - 0xA729, - 0xA72A, - 0xA72B, - 0xA72C, - 0xA72D, - 0xA72E, - 0xA72F, - 0xA732, - 0xA733, - 0xA734, - 0xA735, - 0xA736, - 0xA737, - 0xA738, - 0xA739, - 0xA73A, - 0xA73B, - 0xA73C, - 0xA73D, - 0xA73E, - 0xA73F, - 0xA740, - 0xA741, - 0xA742, - 0xA743, - 0xA744, - 0xA745, - 0xA746, - 0xA747, - 0xA748, - 0xA749, - 0xA74A, - 0xA74B, - 0xA74C, - 0xA74D, - 0xA74E, - 0xA74F, - 0xA750, - 0xA751, - 0xA752, - 0xA753, - 0xA754, - 0xA755, - 0xA756, - 0xA757, - 0xA758, - 0xA759, - 0xA75A, - 0xA75B, - 0xA75C, - 0xA75D, - 0xA75E, - 0xA75F, - 0xA760, - 0xA761, - 0xA762, - 0xA763, - 0xA764, - 0xA765, - 0xA766, - 0xA767, - 0xA768, - 0xA769, - 0xA76A, - 0xA76B, - 0xA76C, - 0xA76D, - 0xA76E, - 0xA76F, - 0xA770, - 0xA771, - 0xA779, - 0xA77A, - 0xA77B, - 0xA77C, - 0xA77D, - 0xA77E, - 0xA77F, - 0xA780, - 0xA781, - 0xA782, - 0xA783, - 0xA784, - 0xA785, - 0xA786, - 0xA787, - 0xA78B, - 0xA78C, - 0xA78D, - 0xA78E, - 0xA790, - 0xA791, - 0xA792, - 0xA793, - 0xA796, - 0xA797, - 0xA798, - 0xA799, - 0xA79A, - 0xA79B, - 0xA79C, - 0xA79D, - 0xA79E, - 0xA79F, - 0xA7A0, - 0xA7A1, - 0xA7A2, - 0xA7A3, - 0xA7A4, - 0xA7A5, - 0xA7A6, - 0xA7A7, - 0xA7A8, - 0xA7A9, - 0xA7AA, - 0xA7AB, - 0xA7AC, - 0xA7AD, - 0xA7AE, - 0xA7AF, - 0xA7B0, - 0xA7B1, - 0xA7B2, - 0xA7B3, - 0xA7B4, - 0xA7B5, - 0xA7B6, - 0xA7B7, - 0xA7B8, - 0xA7B9, - 0xA7BA, - 0xA7BB, - 0xA7BC, - 0xA7BD, - 0xA7BE, - 0xA7BF, - 0xA7C0, - 0xA7C1, - 0xA7C2, - 0xA7C3, - 0xA7C4, - 0xA7C5, - 0xA7C6, - 0xA7C7, - 0xA7C8, - 0xA7C9, - 0xA7CA, - 0xA7CB, - 0xA7CC, - 0xA7CD, - 0xA7CE, - 0xA7CF, - 0xA7D0, - 0xA7D1, - 0xA7D2, - 0xA7D3, - 0xA7D4, - 0xA7D5, - 0xA7D6, - 0xA7D7, - 0xA7D8, - 0xA7D9, - 0xA7DA, - 0xA7DB, - 0xA7DC, - 0xA7DD, - 0xA7F1, - 0xA7F2, - 0xA7F3, - 0xA7F4, - 0xA7F5, - 0xA7F6, - 0xA7F8, - 0xA7F9, - 0xA7FA, - 0xA82D, - 0xA830, - 0xA83A, - 0xA840, - 0xA878, - 0xA880, - 0xA8C6, - 0xA8CE, - 0xA8DA, - 0xA8E0, - 0xA954, - 0xA95F, - 0xA97D, - 0xA980, - 0xA9CE, - 0xA9CF, - 0xA9DA, - 0xA9DE, - 0xA9FF, - 0xAA00, - 0xAA37, - 0xAA40, - 0xAA4E, - 0xAA50, - 0xAA5A, - 0xAA5C, - 0xAAC3, - 0xAADB, - 0xAAF7, - 0xAB01, - 0xAB07, - 0xAB09, - 0xAB0F, - 0xAB11, - 0xAB17, - 0xAB20, - 0xAB27, - 0xAB28, - 0xAB2F, - 0xAB30, - 0xAB5C, - 0xAB5D, - 0xAB5E, - 0xAB5F, - 0xAB60, - 0xAB69, - 0xAB6A, - 0xAB6C, - 0xAB70, - 0xAB71, - 0xAB72, - 0xAB73, - 0xAB74, - 0xAB75, - 0xAB76, - 0xAB77, - 0xAB78, - 0xAB79, - 0xAB7A, - 0xAB7B, - 0xAB7C, - 0xAB7D, - 0xAB7E, - 0xAB7F, - 0xAB80, - 0xAB81, - 0xAB82, - 0xAB83, - 0xAB84, - 0xAB85, - 0xAB86, - 0xAB87, - 0xAB88, - 0xAB89, - 0xAB8A, - 0xAB8B, - 0xAB8C, - 0xAB8D, - 0xAB8E, - 0xAB8F, - 0xAB90, - 0xAB91, - 0xAB92, - 0xAB93, - 0xAB94, - 0xAB95, - 0xAB96, - 0xAB97, - 0xAB98, - 0xAB99, - 0xAB9A, - 0xAB9B, - 0xAB9C, - 0xAB9D, - 0xAB9E, - 0xAB9F, - 0xABA0, - 0xABA1, - 0xABA2, - 0xABA3, - 0xABA4, - 0xABA5, - 0xABA6, - 0xABA7, - 0xABA8, - 0xABA9, - 0xABAA, - 0xABAB, - 0xABAC, - 0xABAD, - 0xABAE, - 0xABAF, - 0xABB0, - 0xABB1, - 0xABB2, - 0xABB3, - 0xABB4, - 0xABB5, - 0xABB6, - 0xABB7, - 0xABB8, - 0xABB9, - 0xABBA, - 0xABBB, - 0xABBC, - 0xABBD, - 0xABBE, - 0xABBF, - 0xABC0, - 0xABEE, - 0xABF0, - 0xABFA, - 0xAC00, - 0xD7A4, - 0xD7B0, - 0xD7C7, - 0xD7CB, - 0xD7FC, - 0xF900, - 0xF901, - 0xF902, - 0xF903, - 0xF904, - 0xF905, - 0xF906, - 0xF907, - 0xF909, - 0xF90A, - 0xF90B, - 0xF90C, - 0xF90D, - 0xF90E, - 0xF90F, - 0xF910, - 0xF911, - 0xF912, - 0xF913, - 0xF914, - 0xF915, - 0xF916, - 0xF917, - 0xF918, - 0xF919, - 0xF91A, - 0xF91B, - 0xF91C, - 0xF91D, - 0xF91E, - 0xF91F, - 0xF920, - 0xF921, - 0xF922, - 0xF923, - 0xF924, - 0xF925, - 0xF926, - 0xF927, - 0xF928, - 0xF929, - 0xF92A, - 0xF92B, - 0xF92C, - 0xF92D, - 0xF92E, - 0xF92F, - 0xF930, - 0xF931, - 0xF932, - 0xF933, - 0xF934, - 0xF935, - 0xF936, - 0xF937, - 0xF938, - 0xF939, - 0xF93A, - 0xF93B, - 0xF93C, - 0xF93D, - 0xF93E, - 0xF93F, - 0xF940, - 0xF941, - 0xF942, - 0xF943, - 0xF944, - 0xF945, - 0xF946, - 0xF947, - 0xF948, - 0xF949, - 0xF94A, - 0xF94B, - 0xF94C, - 0xF94D, - 0xF94E, - 0xF94F, - 0xF950, - 0xF951, - 0xF952, - 0xF953, - 0xF954, - 0xF955, - 0xF956, - 0xF957, - 0xF958, - 0xF959, - 0xF95A, - 0xF95B, - 0xF95C, - 0xF95D, - 0xF95E, - 0xF95F, - 0xF960, - 0xF961, - 0xF962, - 0xF963, - 0xF964, - 0xF965, - 0xF966, - 0xF967, - 0xF968, - 0xF969, - 0xF96A, - 0xF96B, - 0xF96C, - 0xF96D, - 0xF96E, - 0xF96F, - 0xF970, - 0xF971, - 0xF972, - 0xF973, - 0xF974, - 0xF975, - 0xF976, - 0xF977, - 0xF978, - 0xF979, - 0xF97A, - 0xF97B, - 0xF97C, - 0xF97D, - 0xF97E, - 0xF97F, - 0xF980, - 0xF981, - 0xF982, - 0xF983, - 0xF984, - 0xF985, - 0xF986, - 0xF987, - 0xF988, - 0xF989, - 0xF98A, - 0xF98B, - 0xF98C, - 0xF98D, - 0xF98E, - 0xF98F, - 0xF990, - 0xF991, - 0xF992, - 0xF993, - 0xF994, - 0xF995, - 0xF996, - 0xF997, - 0xF998, - 0xF999, - 0xF99A, - 0xF99B, - 0xF99C, - 0xF99D, - 0xF99E, - 0xF99F, - 0xF9A0, - 0xF9A1, - 0xF9A2, - 0xF9A3, - 0xF9A4, - 0xF9A5, - 0xF9A6, - 0xF9A7, - 0xF9A8, - 0xF9A9, - 0xF9AA, - 0xF9AB, - 0xF9AC, - 0xF9AD, - 0xF9AE, - 0xF9AF, - 0xF9B0, - 0xF9B1, - 0xF9B2, - 0xF9B3, - 0xF9B4, - 0xF9B5, - 0xF9B6, - 0xF9B7, - 0xF9B8, - 0xF9B9, - 0xF9BA, - 0xF9BB, - 0xF9BC, - 0xF9BD, - 0xF9BE, - 0xF9BF, - 0xF9C0, - 0xF9C1, - 0xF9C2, - 0xF9C3, - 0xF9C4, - 0xF9C5, - 0xF9C6, - 0xF9C7, - 0xF9C8, - 0xF9C9, - 0xF9CA, - 0xF9CB, - 0xF9CC, - 0xF9CD, - 0xF9CE, - 0xF9CF, - 0xF9D0, - 0xF9D1, - 0xF9D2, - 0xF9D3, - 0xF9D4, - 0xF9D5, - 0xF9D6, - 0xF9D7, - 0xF9D8, - 0xF9D9, - 0xF9DA, - 0xF9DB, - 0xF9DC, - 0xF9DD, - 0xF9DE, - 0xF9DF, - 0xF9E0, - 0xF9E1, - 0xF9E2, - 0xF9E3, - 0xF9E4, - 0xF9E5, - 0xF9E6, - 0xF9E7, - 0xF9E8, - 0xF9E9, - 0xF9EA, - 0xF9EB, - 0xF9EC, - 0xF9ED, - 0xF9EE, - 0xF9EF, - 0xF9F0, - 0xF9F1, - 0xF9F2, - 0xF9F3, - 0xF9F4, - 0xF9F5, - 0xF9F6, - 0xF9F7, - 0xF9F8, - 0xF9F9, - 0xF9FA, - 0xF9FB, - 0xF9FC, - 0xF9FD, - 0xF9FE, - 0xF9FF, - 0xFA00, - 0xFA01, - 0xFA02, - 0xFA03, - 0xFA04, - 0xFA05, - 0xFA06, - 0xFA07, - 0xFA08, - 0xFA09, - 0xFA0A, - 0xFA0B, - 0xFA0C, - 0xFA0D, - 0xFA0E, - 0xFA10, - 0xFA11, - 0xFA12, - 0xFA13, - 0xFA15, - 0xFA16, - 0xFA17, - 0xFA18, - 0xFA19, - 0xFA1A, - 0xFA1B, - 0xFA1C, - 0xFA1D, - 0xFA1E, - 0xFA1F, - 0xFA20, - 0xFA21, - 0xFA22, - 0xFA23, - 0xFA25, - 0xFA26, - 0xFA27, - 0xFA2A, - 0xFA2B, - 0xFA2C, - 0xFA2D, - 0xFA2E, - 0xFA2F, - 0xFA30, - 0xFA31, - 0xFA32, - 0xFA33, - 0xFA34, - 0xFA35, - 0xFA36, - 0xFA37, - 0xFA38, - 0xFA39, - 0xFA3A, - 0xFA3B, - 0xFA3C, - 0xFA3D, - 0xFA3E, - 0xFA3F, - 0xFA40, - 0xFA41, - 0xFA42, - 0xFA43, - 0xFA44, - 0xFA45, - 0xFA46, - 0xFA47, - 0xFA48, - 0xFA49, - 0xFA4A, - 0xFA4B, - 0xFA4C, - 0xFA4D, - 0xFA4E, - 0xFA4F, - 0xFA50, - 0xFA51, - 0xFA52, - 0xFA53, - 0xFA54, - 0xFA55, - 0xFA56, - 0xFA57, - 0xFA58, - 0xFA59, - 0xFA5A, - 0xFA5B, - 0xFA5C, - 0xFA5D, - 0xFA5F, - 0xFA60, - 0xFA61, - 0xFA62, - 0xFA63, - 0xFA64, - 0xFA65, - 0xFA66, - 0xFA67, - 0xFA68, - 0xFA69, - 0xFA6A, - 0xFA6B, - 0xFA6C, - 0xFA6D, - 0xFA6E, - 0xFA70, - 0xFA71, - 0xFA72, - 0xFA73, - 0xFA74, - 0xFA75, - 0xFA76, - 0xFA77, - 0xFA78, - 0xFA79, - 0xFA7A, - 0xFA7B, - 0xFA7C, - 0xFA7D, - 0xFA7E, - 0xFA7F, - 0xFA80, - 0xFA81, - 0xFA82, - 0xFA83, - 0xFA84, - 0xFA85, - 0xFA86, - 0xFA87, - 0xFA88, - 0xFA89, - 0xFA8A, - 0xFA8B, - 0xFA8C, - 0xFA8D, - 0xFA8E, - 0xFA8F, - 0xFA90, - 0xFA91, - 0xFA92, - 0xFA93, - 0xFA94, - 0xFA95, - 0xFA96, - 0xFA97, - 0xFA98, - 0xFA99, - 0xFA9A, - 0xFA9B, - 0xFA9C, - 0xFA9D, - 0xFA9E, - 0xFA9F, - 0xFAA0, - 0xFAA1, - 0xFAA2, - 0xFAA3, - 0xFAA4, - 0xFAA5, - 0xFAA6, - 0xFAA7, - 0xFAA8, - 0xFAA9, - 0xFAAA, - 0xFAAB, - 0xFAAC, - 0xFAAD, - 0xFAAE, - 0xFAAF, - 0xFAB0, - 0xFAB1, - 0xFAB2, - 0xFAB3, - 0xFAB4, - 0xFAB5, - 0xFAB6, - 0xFAB7, - 0xFAB8, - 0xFAB9, - 0xFABA, - 0xFABB, - 0xFABC, - 0xFABD, - 0xFABE, - 0xFABF, - 0xFAC0, - 0xFAC1, - 0xFAC2, - 0xFAC3, - 0xFAC4, - 0xFAC5, - 0xFAC6, - 0xFAC7, - 0xFAC8, - 0xFAC9, - 0xFACA, - 0xFACB, - 0xFACC, - 0xFACD, - 0xFACE, - 0xFACF, - 0xFAD0, - 0xFAD1, - 0xFAD2, - 0xFAD3, - 0xFAD4, - 0xFAD5, - 0xFAD6, - 0xFAD7, - 0xFAD8, - 0xFAD9, - 0xFADA, - 0xFB00, - 0xFB01, - 0xFB02, - 0xFB03, - 0xFB04, - 0xFB05, - 0xFB07, - 0xFB13, - 0xFB14, - 0xFB15, - 0xFB16, - 0xFB17, - 0xFB18, - 0xFB1D, - 0xFB1E, - 0xFB1F, - 0xFB20, - 0xFB21, - 0xFB22, - 0xFB23, - 0xFB24, - 0xFB25, - 0xFB26, - 0xFB27, - 0xFB28, - 0xFB29, - 0xFB2A, - 0xFB2B, - 0xFB2C, - 0xFB2D, - 0xFB2E, - 0xFB2F, - 0xFB30, - 0xFB31, - 0xFB32, - 0xFB33, - 0xFB34, - 0xFB35, - 0xFB36, - 0xFB37, - 0xFB38, - 0xFB39, - 0xFB3A, - 0xFB3B, - 0xFB3C, - 0xFB3D, - 0xFB3E, - 0xFB3F, - 0xFB40, - 0xFB41, - 0xFB42, - 0xFB43, - 0xFB44, - 0xFB45, - 0xFB46, - 0xFB47, - 0xFB48, - 0xFB49, - 0xFB4A, - 0xFB4B, - 0xFB4C, - 0xFB4D, - 0xFB4E, - 0xFB4F, - 0xFB50, - 0xFB52, - 0xFB56, - 0xFB5A, - 0xFB5E, - 0xFB62, - 0xFB66, - 0xFB6A, - 0xFB6E, - 0xFB72, - 0xFB76, - 0xFB7A, - 0xFB7E, - 0xFB82, - 0xFB84, - 0xFB86, - 0xFB88, - 0xFB8A, - 0xFB8C, - 0xFB8E, - 0xFB92, - 0xFB96, - 0xFB9A, - 0xFB9E, - 0xFBA0, - 0xFBA4, - 0xFBA6, - 0xFBAA, - 0xFBAE, - 0xFBB0, - 0xFBB2, - 0xFBD3, - 0xFBD7, - 0xFBD9, - 0xFBDB, - 0xFBDD, - 0xFBDE, - 0xFBE0, - 0xFBE2, - 0xFBE4, - 0xFBE8, - 0xFBEA, - 0xFBEC, - 0xFBEE, - 0xFBF0, - 0xFBF2, - 0xFBF4, - 0xFBF6, - 0xFBF9, - 0xFBFC, - 0xFC00, - 0xFC01, - 0xFC02, - 0xFC03, - 0xFC04, - 0xFC05, - 0xFC06, - 0xFC07, - 0xFC08, - 0xFC09, - 0xFC0A, - 0xFC0B, - 0xFC0C, - 0xFC0D, - 0xFC0E, - 0xFC0F, - 0xFC10, - 0xFC11, - 0xFC12, - 0xFC13, - 0xFC14, - 0xFC15, - 0xFC16, - 0xFC17, - 0xFC18, - 0xFC19, - 0xFC1A, - 0xFC1B, - 0xFC1C, - 0xFC1D, - 0xFC1E, - 0xFC1F, - 0xFC20, - 0xFC21, - 0xFC22, - 0xFC23, - 0xFC24, - 0xFC25, - 0xFC26, - 0xFC27, - 0xFC28, - 0xFC29, - 0xFC2A, - 0xFC2B, - 0xFC2C, - 0xFC2D, - 0xFC2E, - 0xFC2F, - 0xFC30, - 0xFC31, - 0xFC32, - 0xFC33, - 0xFC34, - 0xFC35, - 0xFC36, - 0xFC37, - 0xFC38, - 0xFC39, - 0xFC3A, - 0xFC3B, - 0xFC3C, - 0xFC3D, - 0xFC3E, - 0xFC3F, - 0xFC40, - 0xFC41, - 0xFC42, - 0xFC43, - 0xFC44, - 0xFC45, - 0xFC46, - 0xFC47, - 0xFC48, - 0xFC49, - 0xFC4A, - 0xFC4B, - 0xFC4C, - 0xFC4D, - 0xFC4E, - 0xFC4F, - 0xFC50, - 0xFC51, - 0xFC52, - 0xFC53, - 0xFC54, - 0xFC55, - 0xFC56, - 0xFC57, - 0xFC58, - 0xFC59, - 0xFC5A, - 0xFC5B, - 0xFC5C, - 0xFC5D, - 0xFC5E, - 0xFC5F, - 0xFC60, - 0xFC61, - 0xFC62, - 0xFC63, - 0xFC64, - 0xFC65, - 0xFC66, - 0xFC67, - 0xFC68, - 0xFC69, - 0xFC6A, - 0xFC6B, - 0xFC6C, - 0xFC6D, - 0xFC6E, - 0xFC6F, - 0xFC70, - 0xFC71, - 0xFC72, - 0xFC73, - 0xFC74, - 0xFC75, - 0xFC76, - 0xFC77, - 0xFC78, - 0xFC79, - 0xFC7A, - 0xFC7B, - 0xFC7C, - 0xFC7D, - 0xFC7E, - 0xFC7F, - 0xFC80, - 0xFC81, - 0xFC82, - 0xFC83, - 0xFC84, - 0xFC85, - 0xFC86, - 0xFC87, - 0xFC88, - 0xFC89, - 0xFC8A, - 0xFC8B, - 0xFC8C, - 0xFC8D, - 0xFC8E, - 0xFC8F, - 0xFC90, - 0xFC91, - 0xFC92, - 0xFC93, - 0xFC94, - 0xFC95, - 0xFC96, - 0xFC97, - 0xFC98, - 0xFC99, - 0xFC9A, - 0xFC9B, - 0xFC9C, - 0xFC9D, - 0xFC9E, - 0xFC9F, - 0xFCA0, - 0xFCA1, - 0xFCA2, - 0xFCA3, - 0xFCA4, - 0xFCA5, - 0xFCA6, - 0xFCA7, - 0xFCA8, - 0xFCA9, - 0xFCAA, - 0xFCAB, - 0xFCAC, - 0xFCAD, - 0xFCAE, - 0xFCAF, - 0xFCB0, - 0xFCB1, - 0xFCB2, - 0xFCB3, - 0xFCB4, - 0xFCB5, - 0xFCB6, - 0xFCB7, - 0xFCB8, - 0xFCB9, - 0xFCBA, - 0xFCBB, - 0xFCBC, - 0xFCBD, - 0xFCBE, - 0xFCBF, - 0xFCC0, - 0xFCC1, - 0xFCC2, - 0xFCC3, - 0xFCC4, - 0xFCC5, - 0xFCC6, - 0xFCC7, - 0xFCC8, - 0xFCC9, - 0xFCCA, - 0xFCCB, - 0xFCCC, - 0xFCCD, - 0xFCCE, - 0xFCCF, - 0xFCD0, - 0xFCD1, - 0xFCD2, - 0xFCD3, - 0xFCD4, - 0xFCD5, - 0xFCD6, - 0xFCD7, - 0xFCD8, - 0xFCD9, - 0xFCDA, - 0xFCDB, - 0xFCDC, - 0xFCDD, - 0xFCDE, - 0xFCDF, - 0xFCE0, - 0xFCE1, - 0xFCE2, - 0xFCE3, - 0xFCE4, - 0xFCE5, - 0xFCE6, - 0xFCE7, - 0xFCE8, - 0xFCE9, - 0xFCEA, - 0xFCEB, - 0xFCEC, - 0xFCED, - 0xFCEE, - 0xFCEF, - 0xFCF0, - 0xFCF1, - 0xFCF2, - 0xFCF3, - 0xFCF4, - 0xFCF5, - 0xFCF6, - 0xFCF7, - 0xFCF8, - 0xFCF9, - 0xFCFA, - 0xFCFB, - 0xFCFC, - 0xFCFD, - 0xFCFE, - 0xFCFF, - 0xFD00, - 0xFD01, - 0xFD02, - 0xFD03, - 0xFD04, - 0xFD05, - 0xFD06, - 0xFD07, - 0xFD08, - 0xFD09, - 0xFD0A, - 0xFD0B, - 0xFD0C, - 0xFD0D, - 0xFD0E, - 0xFD0F, - 0xFD10, - 0xFD11, - 0xFD12, - 0xFD13, - 0xFD14, - 0xFD15, - 0xFD16, - 0xFD17, - 0xFD18, - 0xFD19, - 0xFD1A, - 0xFD1B, - 0xFD1C, - 0xFD1D, - 0xFD1E, - 0xFD1F, - 0xFD20, - 0xFD21, - 0xFD22, - 0xFD23, - 0xFD24, - 0xFD25, - 0xFD26, - 0xFD27, - 0xFD28, - 0xFD29, - 0xFD2A, - 0xFD2B, - 0xFD2C, - 0xFD2D, - 0xFD2E, - 0xFD2F, - 0xFD30, - 0xFD31, - 0xFD32, - 0xFD33, - 0xFD34, - 0xFD35, - 0xFD36, - 0xFD37, - 0xFD38, - 0xFD39, - 0xFD3A, - 0xFD3B, - 0xFD3C, - 0xFD3E, - 0xFD50, - 0xFD51, - 0xFD53, - 0xFD54, - 0xFD55, - 0xFD56, - 0xFD57, - 0xFD58, - 0xFD5A, - 0xFD5B, - 0xFD5C, - 0xFD5D, - 0xFD5E, - 0xFD5F, - 0xFD61, - 0xFD62, - 0xFD64, - 0xFD66, - 0xFD67, - 0xFD69, - 0xFD6A, - 0xFD6C, - 0xFD6E, - 0xFD6F, - 0xFD71, - 0xFD73, - 0xFD74, - 0xFD75, - 0xFD76, - 0xFD78, - 0xFD79, - 0xFD7A, - 0xFD7B, - 0xFD7C, - 0xFD7E, - 0xFD7F, - 0xFD80, - 0xFD81, - 0xFD82, - 0xFD83, - 0xFD85, - 0xFD87, - 0xFD89, - 0xFD8A, - 0xFD8B, - 0xFD8C, - 0xFD8D, - 0xFD8E, - 0xFD8F, - 0xFD90, - 0xFD92, - 0xFD93, - 0xFD94, - 0xFD95, - 0xFD96, - 0xFD97, - 0xFD99, - 0xFD9A, - 0xFD9B, - 0xFD9C, - 0xFD9E, - 0xFD9F, - 0xFDA0, - 0xFDA1, - 0xFDA2, - 0xFDA3, - 0xFDA4, - 0xFDA5, - 0xFDA6, - 0xFDA7, - 0xFDA8, - 0xFDA9, - 0xFDAA, - 0xFDAB, - 0xFDAC, - 0xFDAD, - 0xFDAE, - 0xFDAF, - 0xFDB0, - 0xFDB1, - 0xFDB2, - 0xFDB3, - 0xFDB4, - 0xFDB5, - 0xFDB6, - 0xFDB7, - 0xFDB8, - 0xFDB9, - 0xFDBA, - 0xFDBB, - 0xFDBC, - 0xFDBD, - 0xFDBE, - 0xFDBF, - 0xFDC0, - 0xFDC1, - 0xFDC2, - 0xFDC3, - 0xFDC4, - 0xFDC5, - 0xFDC6, - 0xFDC7, - 0xFDC8, - 0xFDD0, - 0xFDF0, - 0xFDF1, - 0xFDF2, - 0xFDF3, - 0xFDF4, - 0xFDF5, - 0xFDF6, - 0xFDF7, - 0xFDF8, - 0xFDF9, - 0xFDFA, - 0xFDFB, - 0xFDFC, - 0xFDFD, - 0xFE00, - 0xFE10, - 0xFE11, - 0xFE12, - 0xFE13, - 0xFE14, - 0xFE15, - 0xFE16, - 0xFE17, - 0xFE18, - 0xFE19, - 0xFE20, - 0xFE30, - 0xFE31, - 0xFE32, - 0xFE33, - 0xFE35, - 0xFE36, - 0xFE37, - 0xFE38, - 0xFE39, - 0xFE3A, - 0xFE3B, - 0xFE3C, - 0xFE3D, - 0xFE3E, - 0xFE3F, - 0xFE40, - 0xFE41, - 0xFE42, - 0xFE43, - 0xFE44, - 0xFE45, - 0xFE47, - 0xFE48, - 0xFE49, - 0xFE4D, - 0xFE50, - 0xFE51, - 0xFE52, - 0xFE54, - 0xFE55, - 0xFE56, - 0xFE57, - 0xFE58, - 0xFE59, - 0xFE5A, - 0xFE5B, - 0xFE5C, - 0xFE5D, - 0xFE5E, - 0xFE5F, - 0xFE60, - 0xFE61, - 0xFE62, - 0xFE63, - 0xFE64, - 0xFE65, - 0xFE66, - 0xFE67, - 0xFE68, - 0xFE69, - 0xFE6A, - 0xFE6B, - 0xFE6C, - 0xFE70, - 0xFE71, - 0xFE72, - 0xFE73, - 0xFE74, - 0xFE75, - 0xFE76, - 0xFE77, - 0xFE78, - 0xFE79, - 0xFE7A, - 0xFE7B, - 0xFE7C, - 0xFE7D, - 0xFE7E, - 0xFE7F, - 0xFE80, - 0xFE81, - 0xFE83, - 0xFE85, - 0xFE87, - 0xFE89, - 0xFE8D, - 0xFE8F, - 0xFE93, - 0xFE95, - 0xFE99, - 0xFE9D, - 0xFEA1, - 0xFEA5, - 0xFEA9, - 0xFEAB, - 0xFEAD, - 0xFEAF, - 0xFEB1, - 0xFEB5, - 0xFEB9, - 0xFEBD, - 0xFEC1, - 0xFEC5, - 0xFEC9, - 0xFECD, - 0xFED1, - 0xFED5, - 0xFED9, - 0xFEDD, - 0xFEE1, - 0xFEE5, - 0xFEE9, - 0xFEED, - 0xFEEF, - 0xFEF1, - 0xFEF5, - 0xFEF7, - 0xFEF9, - 0xFEFB, - 0xFEFD, - 0xFEFF, - 0xFF00, - 0xFF01, - 0xFF02, - 0xFF03, - 0xFF04, - 0xFF05, - 0xFF06, - 0xFF07, - 0xFF08, - 0xFF09, - 0xFF0A, - 0xFF0B, - 0xFF0C, - 0xFF0D, - 0xFF0E, - 0xFF0F, - 0xFF10, - 0xFF11, - 0xFF12, - 0xFF13, - 0xFF14, - 0xFF15, - 0xFF16, - 0xFF17, - 0xFF18, - 0xFF19, - 0xFF1A, - 0xFF1B, - 0xFF1C, - 0xFF1D, - 0xFF1E, - 0xFF1F, - 0xFF20, - 0xFF21, - 0xFF22, - 0xFF23, - 0xFF24, - 0xFF25, - 0xFF26, - 0xFF27, - 0xFF28, - 0xFF29, - 0xFF2A, - 0xFF2B, - 0xFF2C, - 0xFF2D, - 0xFF2E, - 0xFF2F, - 0xFF30, - 0xFF31, - 0xFF32, - 0xFF33, - 0xFF34, - 0xFF35, - 0xFF36, - 0xFF37, - 0xFF38, - 0xFF39, - 0xFF3A, - 0xFF3B, - 0xFF3C, - 0xFF3D, - 0xFF3E, - 0xFF3F, - 0xFF40, - 0xFF41, - 0xFF42, - 0xFF43, - 0xFF44, - 0xFF45, - 0xFF46, - 0xFF47, - 0xFF48, - 0xFF49, - 0xFF4A, - 0xFF4B, - 0xFF4C, - 0xFF4D, - 0xFF4E, - 0xFF4F, - 0xFF50, - 0xFF51, - 0xFF52, - 0xFF53, - 0xFF54, - 0xFF55, - 0xFF56, - 0xFF57, - 0xFF58, - 0xFF59, - 0xFF5A, - 0xFF5B, - 0xFF5C, - 0xFF5D, - 0xFF5E, - 0xFF5F, - 0xFF60, - 0xFF61, - 0xFF62, - 0xFF63, - 0xFF64, - 0xFF65, - 0xFF66, - 0xFF67, - 0xFF68, - 0xFF69, - 0xFF6A, - 0xFF6B, - 0xFF6C, - 0xFF6D, - 0xFF6E, - 0xFF6F, - 0xFF70, - 0xFF71, - 0xFF72, - 0xFF73, - 0xFF74, - 0xFF75, - 0xFF76, - 0xFF77, - 0xFF78, - 0xFF79, - 0xFF7A, - 0xFF7B, - 0xFF7C, - 0xFF7D, - 0xFF7E, - 0xFF7F, - 0xFF80, - 0xFF81, - 0xFF82, - 0xFF83, - 0xFF84, - 0xFF85, - 0xFF86, - 0xFF87, - 0xFF88, - 0xFF89, - 0xFF8A, - 0xFF8B, - 0xFF8C, - 0xFF8D, - 0xFF8E, - 0xFF8F, - 0xFF90, - 0xFF91, - 0xFF92, - 0xFF93, - 0xFF94, - 0xFF95, - 0xFF96, - 0xFF97, - 0xFF98, - 0xFF99, - 0xFF9A, - 0xFF9B, - 0xFF9C, - 0xFF9D, - 0xFF9E, - 0xFF9F, - 0xFFA0, - 0xFFA1, - 0xFFA2, - 0xFFA3, - 0xFFA4, - 0xFFA5, - 0xFFA6, - 0xFFA7, - 0xFFA8, - 0xFFA9, - 0xFFAA, - 0xFFAB, - 0xFFAC, - 0xFFAD, - 0xFFAE, - 0xFFAF, - 0xFFB0, - 0xFFB1, - 0xFFB2, - 0xFFB3, - 0xFFB4, - 0xFFB5, - 0xFFB6, - 0xFFB7, - 0xFFB8, - 0xFFB9, - 0xFFBA, - 0xFFBB, - 0xFFBC, - 0xFFBD, - 0xFFBE, - 0xFFBF, - 0xFFC2, - 0xFFC3, - 0xFFC4, - 0xFFC5, - 0xFFC6, - 0xFFC7, - 0xFFC8, - 0xFFCA, - 0xFFCB, - 0xFFCC, - 0xFFCD, - 0xFFCE, - 0xFFCF, - 0xFFD0, - 0xFFD2, - 0xFFD3, - 0xFFD4, - 0xFFD5, - 0xFFD6, - 0xFFD7, - 0xFFD8, - 0xFFDA, - 0xFFDB, - 0xFFDC, - 0xFFDD, - 0xFFE0, - 0xFFE1, - 0xFFE2, - 0xFFE3, - 0xFFE4, - 0xFFE5, - 0xFFE6, - 0xFFE7, - 0xFFE8, - 0xFFE9, - 0xFFEA, - 0xFFEB, - 0xFFEC, - 0xFFED, - 0xFFEE, - 0xFFEF, - 0x10000, - 0x1000C, - 0x1000D, - 0x10027, - 0x10028, - 0x1003B, - 0x1003C, - 0x1003E, - 0x1003F, - 0x1004E, - 0x10050, - 0x1005E, - 0x10080, - 0x100FB, - 0x10100, - 0x10103, - 0x10107, - 0x10134, - 0x10137, - 0x1018F, - 0x10190, - 0x1019D, - 0x101A0, - 0x101A1, - 0x101D0, - 0x101FE, - 0x10280, - 0x1029D, - 0x102A0, - 0x102D1, - 0x102E0, - 0x102FC, - 0x10300, - 0x10324, - 0x1032D, - 0x1034B, - 0x10350, - 0x1037B, - 0x10380, - 0x1039E, - 0x1039F, - 0x103C4, - 0x103C8, - 0x103D6, - 0x10400, - 0x10401, - 0x10402, - 0x10403, - 0x10404, - 0x10405, - 0x10406, - 0x10407, - 0x10408, - 0x10409, - 0x1040A, - 0x1040B, - 0x1040C, - 0x1040D, - 0x1040E, - 0x1040F, - 0x10410, - 0x10411, - 0x10412, - 0x10413, - 0x10414, - 0x10415, - 0x10416, - 0x10417, - 0x10418, - 0x10419, - 0x1041A, - 0x1041B, - 0x1041C, - 0x1041D, - 0x1041E, - 0x1041F, - 0x10420, - 0x10421, - 0x10422, - 0x10423, - 0x10424, - 0x10425, - 0x10426, - 0x10427, - 0x10428, - 0x1049E, - 0x104A0, - 0x104AA, - 0x104B0, - 0x104B1, - 0x104B2, - 0x104B3, - 0x104B4, - 0x104B5, - 0x104B6, - 0x104B7, - 0x104B8, - 0x104B9, - 0x104BA, - 0x104BB, - 0x104BC, - 0x104BD, - 0x104BE, - 0x104BF, - 0x104C0, - 0x104C1, - 0x104C2, - 0x104C3, - 0x104C4, - 0x104C5, - 0x104C6, - 0x104C7, - 0x104C8, - 0x104C9, - 0x104CA, - 0x104CB, - 0x104CC, - 0x104CD, - 0x104CE, - 0x104CF, - 0x104D0, - 0x104D1, - 0x104D2, - 0x104D3, - 0x104D4, - 0x104D8, - 0x104FC, - 0x10500, - 0x10528, - 0x10530, - 0x10564, - 0x1056F, - 0x10570, - 0x10571, - 0x10572, - 0x10573, - 0x10574, - 0x10575, - 0x10576, - 0x10577, - 0x10578, - 0x10579, - 0x1057A, - 0x1057B, - 0x1057C, - 0x1057D, - 0x1057E, - 0x1057F, - 0x10580, - 0x10581, - 0x10582, - 0x10583, - 0x10584, - 0x10585, - 0x10586, - 0x10587, - 0x10588, - 0x10589, - 0x1058A, - 0x1058B, - 0x1058C, - 0x1058D, - 0x1058E, - 0x1058F, - 0x10590, - 0x10591, - 0x10592, - 0x10593, - 0x10594, - 0x10595, - 0x10596, - 0x10597, - 0x105A2, - 0x105A3, - 0x105B2, - 0x105B3, - 0x105BA, - 0x105BB, - 0x105BD, - 0x105C0, - 0x105F4, - 0x10600, - 0x10737, - 0x10740, - 0x10756, - 0x10760, - 0x10768, - 0x10780, - 0x10781, - 0x10782, - 0x10783, - 0x10784, - 0x10785, - 0x10786, - 0x10787, - 0x10788, - 0x10789, - 0x1078A, - 0x1078B, - 0x1078C, - 0x1078D, - 0x1078E, - 0x1078F, - 0x10790, - 0x10791, - 0x10792, - 0x10793, - 0x10794, - 0x10795, - 0x10796, - 0x10797, - 0x10798, - 0x10799, - 0x1079A, - 0x1079B, - 0x1079C, - 0x1079D, - 0x1079E, - 0x1079F, - 0x107A0, - 0x107A1, - 0x107A2, - 0x107A3, - 0x107A4, - 0x107A5, - 0x107A6, - 0x107A7, - 0x107A8, - 0x107A9, - 0x107AA, - 0x107AB, - 0x107AC, - 0x107AD, - 0x107AE, - 0x107AF, - 0x107B0, - 0x107B1, - 0x107B2, - 0x107B3, - 0x107B4, - 0x107B5, - 0x107B6, - 0x107B7, - 0x107B8, - 0x107B9, - 0x107BA, - 0x107BB, - 0x10800, - 0x10806, - 0x10808, - 0x10809, - 0x1080A, - 0x10836, - 0x10837, - 0x10839, - 0x1083C, - 0x1083D, - 0x1083F, - 0x10856, - 0x10857, - 0x1089F, - 0x108A7, - 0x108B0, - 0x108E0, - 0x108F3, - 0x108F4, - 0x108F6, - 0x108FB, - 0x1091C, - 0x1091F, - 0x1093A, - 0x1093F, - 0x1095A, - 0x10980, - 0x109B8, - 0x109BC, - 0x109D0, - 0x109D2, - 0x10A04, - 0x10A05, - 0x10A07, - 0x10A0C, - 0x10A14, - 0x10A15, - 0x10A18, - 0x10A19, - 0x10A36, - 0x10A38, - 0x10A3B, - 0x10A3F, - 0x10A49, - 0x10A50, - 0x10A59, - 0x10A60, - 0x10AA0, - 0x10AC0, - 0x10AE7, - 0x10AEB, - 0x10AF7, - 0x10B00, - 0x10B36, - 0x10B39, - 0x10B56, - 0x10B58, - 0x10B73, - 0x10B78, - 0x10B92, - 0x10B99, - 0x10B9D, - 0x10BA9, - 0x10BB0, - 0x10C00, - 0x10C49, - 0x10C80, - 0x10C81, - 0x10C82, - 0x10C83, - 0x10C84, - 0x10C85, - 0x10C86, - 0x10C87, - 0x10C88, - 0x10C89, - 0x10C8A, - 0x10C8B, - 0x10C8C, - 0x10C8D, - 0x10C8E, - 0x10C8F, - 0x10C90, - 0x10C91, - 0x10C92, - 0x10C93, - 0x10C94, - 0x10C95, - 0x10C96, - 0x10C97, - 0x10C98, - 0x10C99, - 0x10C9A, - 0x10C9B, - 0x10C9C, - 0x10C9D, - 0x10C9E, - 0x10C9F, - 0x10CA0, - 0x10CA1, - 0x10CA2, - 0x10CA3, - 0x10CA4, - 0x10CA5, - 0x10CA6, - 0x10CA7, - 0x10CA8, - 0x10CA9, - 0x10CAA, - 0x10CAB, - 0x10CAC, - 0x10CAD, - 0x10CAE, - 0x10CAF, - 0x10CB0, - 0x10CB1, - 0x10CB2, - 0x10CB3, - 0x10CC0, - 0x10CF3, - 0x10CFA, - 0x10D28, - 0x10D30, - 0x10D3A, - 0x10D40, - 0x10D50, - 0x10D51, - 0x10D52, - 0x10D53, - 0x10D54, - 0x10D55, - 0x10D56, - 0x10D57, - 0x10D58, - 0x10D59, - 0x10D5A, - 0x10D5B, - 0x10D5C, - 0x10D5D, - 0x10D5E, - 0x10D5F, - 0x10D60, - 0x10D61, - 0x10D62, - 0x10D63, - 0x10D64, - 0x10D65, - 0x10D66, - 0x10D69, - 0x10D86, - 0x10D8E, - 0x10D90, - 0x10E60, - 0x10E7F, - 0x10E80, - 0x10EAA, - 0x10EAB, - 0x10EAE, - 0x10EB0, - 0x10EB2, - 0x10EC2, - 0x10EC8, - 0x10ED0, - 0x10ED9, - 0x10EFA, - 0x10F28, - 0x10F30, - 0x10F5A, - 0x10F70, - 0x10F8A, - 0x10FB0, - 0x10FCC, - 0x10FE0, - 0x10FF7, - 0x11000, - 0x1104E, - 0x11052, - 0x11076, - 0x1107F, - 0x110BD, - 0x110BE, - 0x110C3, - 0x110D0, - 0x110E9, - 0x110F0, - 0x110FA, - 0x11100, - 0x11135, - 0x11136, - 0x11148, - 0x11150, - 0x11177, - 0x11180, - 0x111E0, - 0x111E1, - 0x111F5, - 0x11200, - 0x11212, - 0x11213, - 0x11242, - 0x11280, - 0x11287, - 0x11288, - 0x11289, - 0x1128A, - 0x1128E, - 0x1128F, - 0x1129E, - 0x1129F, - 0x112AA, - 0x112B0, - 0x112EB, - 0x112F0, - 0x112FA, - 0x11300, - 0x11304, - 0x11305, - 0x1130D, - 0x1130F, - 0x11311, - 0x11313, - 0x11329, - 0x1132A, - 0x11331, - 0x11332, - 0x11334, - 0x11335, - 0x1133A, - 0x1133B, - 0x11345, - 0x11347, - 0x11349, - 0x1134B, - 0x1134E, - 0x11350, - 0x11351, - 0x11357, - 0x11358, - 0x1135D, - 0x11364, - 0x11366, - 0x1136D, - 0x11370, - 0x11375, - 0x11380, - 0x1138A, - 0x1138B, - 0x1138C, - 0x1138E, - 0x1138F, - 0x11390, - 0x113B6, - 0x113B7, - 0x113C1, - 0x113C2, - 0x113C3, - 0x113C5, - 0x113C6, - 0x113C7, - 0x113CB, - 0x113CC, - 0x113D6, - 0x113D7, - 0x113D9, - 0x113E1, - 0x113E3, - 0x11400, - 0x1145C, - 0x1145D, - 0x11462, - 0x11480, - 0x114C8, - 0x114D0, - 0x114DA, - 0x11580, - 0x115B6, - 0x115B8, - 0x115DE, - 0x11600, - 0x11645, - 0x11650, - 0x1165A, - 0x11660, - 0x1166D, - 0x11680, - 0x116BA, - 0x116C0, - 0x116CA, - 0x116D0, - 0x116E4, - 0x11700, - 0x1171B, - 0x1171D, - 0x1172C, - 0x11730, - 0x11747, - 0x11800, - 0x1183C, - 0x118A0, - 0x118A1, - 0x118A2, - 0x118A3, - 0x118A4, - 0x118A5, - 0x118A6, - 0x118A7, - 0x118A8, - 0x118A9, - 0x118AA, - 0x118AB, - 0x118AC, - 0x118AD, - 0x118AE, - 0x118AF, - 0x118B0, - 0x118B1, - 0x118B2, - 0x118B3, - 0x118B4, - 0x118B5, - 0x118B6, - 0x118B7, - 0x118B8, - 0x118B9, - 0x118BA, - 0x118BB, - 0x118BC, - 0x118BD, - 0x118BE, - 0x118BF, - 0x118C0, - 0x118F3, - 0x118FF, - 0x11907, - 0x11909, - 0x1190A, - 0x1190C, - 0x11914, - 0x11915, - 0x11917, - 0x11918, - 0x11936, - 0x11937, - 0x11939, - 0x1193B, - 0x11947, - 0x11950, - 0x1195A, - 0x119A0, - 0x119A8, - 0x119AA, - 0x119D8, - 0x119DA, - 0x119E5, - 0x11A00, - 0x11A48, - 0x11A50, - 0x11AA3, - 0x11AB0, - 0x11AF9, - 0x11B00, - 0x11B0A, - 0x11B60, - 0x11B68, - 0x11BC0, - 0x11BE2, - 0x11BF0, - 0x11BFA, - 0x11C00, - 0x11C09, - 0x11C0A, - 0x11C37, - 0x11C38, - 0x11C46, - 0x11C50, - 0x11C6D, - 0x11C70, - 0x11C90, - 0x11C92, - 0x11CA8, - 0x11CA9, - 0x11CB7, - 0x11D00, - 0x11D07, - 0x11D08, - 0x11D0A, - 0x11D0B, - 0x11D37, - 0x11D3A, - 0x11D3B, - 0x11D3C, - 0x11D3E, - 0x11D3F, - 0x11D48, - 0x11D50, - 0x11D5A, - 0x11D60, - 0x11D66, - 0x11D67, - 0x11D69, - 0x11D6A, - 0x11D8F, - 0x11D90, - 0x11D92, - 0x11D93, - 0x11D99, - 0x11DA0, - 0x11DAA, - 0x11DB0, - 0x11DDC, - 0x11DE0, - 0x11DEA, - 0x11EE0, - 0x11EF9, - 0x11F00, - 0x11F11, - 0x11F12, - 0x11F3B, - 0x11F3E, - 0x11F5B, - 0x11FB0, - 0x11FB1, - 0x11FC0, - 0x11FF2, - 0x11FFF, - 0x1239A, - 0x12400, - 0x1246F, - 0x12470, - 0x12475, - 0x12480, - 0x12544, - 0x12F90, - 0x12FF3, - 0x13000, - 0x13430, - 0x13440, - 0x13456, - 0x13460, - 0x143FB, - 0x14400, - 0x14647, - 0x16100, - 0x1613A, - 0x16800, - 0x16A39, - 0x16A40, - 0x16A5F, - 0x16A60, - 0x16A6A, - 0x16A6E, - 0x16ABF, - 0x16AC0, - 0x16ACA, - 0x16AD0, - 0x16AEE, - 0x16AF0, - 0x16AF6, - 0x16B00, - 0x16B46, - 0x16B50, - 0x16B5A, - 0x16B5B, - 0x16B62, - 0x16B63, - 0x16B78, - 0x16B7D, - 0x16B90, - 0x16D40, - 0x16D7A, - 0x16E40, - 0x16E41, - 0x16E42, - 0x16E43, - 0x16E44, - 0x16E45, - 0x16E46, - 0x16E47, - 0x16E48, - 0x16E49, - 0x16E4A, - 0x16E4B, - 0x16E4C, - 0x16E4D, - 0x16E4E, - 0x16E4F, - 0x16E50, - 0x16E51, - 0x16E52, - 0x16E53, - 0x16E54, - 0x16E55, - 0x16E56, - 0x16E57, - 0x16E58, - 0x16E59, - 0x16E5A, - 0x16E5B, - 0x16E5C, - 0x16E5D, - 0x16E5E, - 0x16E5F, - 0x16E60, - 0x16E9B, - 0x16EA0, - 0x16EA1, - 0x16EA2, - 0x16EA3, - 0x16EA4, - 0x16EA5, - 0x16EA6, - 0x16EA7, - 0x16EA8, - 0x16EA9, - 0x16EAA, - 0x16EAB, - 0x16EAC, - 0x16EAD, - 0x16EAE, - 0x16EAF, - 0x16EB0, - 0x16EB1, - 0x16EB2, - 0x16EB3, - 0x16EB4, - 0x16EB5, - 0x16EB6, - 0x16EB7, - 0x16EB8, - 0x16EB9, - 0x16EBB, - 0x16ED4, - 0x16F00, - 0x16F4B, - 0x16F4F, - 0x16F88, - 0x16F8F, - 0x16FA0, - 0x16FE0, - 0x16FE5, - 0x16FF0, - 0x16FF7, - 0x17000, - 0x18CD6, - 0x18CFF, - 0x18D1F, - 0x18D80, - 0x18DF3, - 0x1AFF0, - 0x1AFF4, - 0x1AFF5, - 0x1AFFC, - 0x1AFFD, - 0x1AFFF, - 0x1B000, - 0x1B123, - 0x1B132, - 0x1B133, - 0x1B150, - 0x1B153, - 0x1B155, - 0x1B156, - 0x1B164, - 0x1B168, - 0x1B170, - 0x1B2FC, - 0x1BC00, - 0x1BC6B, - 0x1BC70, - 0x1BC7D, - 0x1BC80, - 0x1BC89, - 0x1BC90, - 0x1BC9A, - 0x1BC9C, - 0x1BCA0, - 0x1BCA4, - 0x1CC00, - 0x1CCD6, - 0x1CCD7, - 0x1CCD8, - 0x1CCD9, - 0x1CCDA, - 0x1CCDB, - 0x1CCDC, - 0x1CCDD, - 0x1CCDE, - 0x1CCDF, - 0x1CCE0, - 0x1CCE1, - 0x1CCE2, - 0x1CCE3, - 0x1CCE4, - 0x1CCE5, - 0x1CCE6, - 0x1CCE7, - 0x1CCE8, - 0x1CCE9, - 0x1CCEA, - 0x1CCEB, - 0x1CCEC, - 0x1CCED, - 0x1CCEE, - 0x1CCEF, - 0x1CCF0, - 0x1CCF1, - 0x1CCF2, - 0x1CCF3, - 0x1CCF4, - 0x1CCF5, - 0x1CCF6, - 0x1CCF7, - 0x1CCF8, - 0x1CCF9, - 0x1CCFA, - 0x1CCFD, - 0x1CD00, - 0x1CEB4, - 0x1CEBA, - 0x1CED1, - 0x1CEE0, - 0x1CEF1, - 0x1CF00, - 0x1CF2E, - 0x1CF30, - 0x1CF47, - 0x1CF50, - 0x1CFC4, - 0x1D000, - 0x1D0F6, - 0x1D100, - 0x1D127, - 0x1D129, - 0x1D15E, - 0x1D15F, - 0x1D160, - 0x1D161, - 0x1D162, - 0x1D163, - 0x1D164, - 0x1D165, - 0x1D173, - 0x1D17B, - 0x1D1BB, - 0x1D1BC, - 0x1D1BD, - 0x1D1BE, - 0x1D1BF, - 0x1D1C0, - 0x1D1C1, - 0x1D1EB, - 0x1D200, - 0x1D246, - 0x1D2C0, - 0x1D2D4, - 0x1D2E0, - 0x1D2F4, - 0x1D300, - 0x1D357, - 0x1D360, - 0x1D379, - 0x1D400, - 0x1D401, - 0x1D402, - 0x1D403, - 0x1D404, - 0x1D405, - 0x1D406, - 0x1D407, - 0x1D408, - 0x1D409, - 0x1D40A, - 0x1D40B, - 0x1D40C, - 0x1D40D, - 0x1D40E, - 0x1D40F, - 0x1D410, - 0x1D411, - 0x1D412, - 0x1D413, - 0x1D414, - 0x1D415, - 0x1D416, - 0x1D417, - 0x1D418, - 0x1D419, - 0x1D41A, - 0x1D41B, - 0x1D41C, - 0x1D41D, - 0x1D41E, - 0x1D41F, - 0x1D420, - 0x1D421, - 0x1D422, - 0x1D423, - 0x1D424, - 0x1D425, - 0x1D426, - 0x1D427, - 0x1D428, - 0x1D429, - 0x1D42A, - 0x1D42B, - 0x1D42C, - 0x1D42D, - 0x1D42E, - 0x1D42F, - 0x1D430, - 0x1D431, - 0x1D432, - 0x1D433, - 0x1D434, - 0x1D435, - 0x1D436, - 0x1D437, - 0x1D438, - 0x1D439, - 0x1D43A, - 0x1D43B, - 0x1D43C, - 0x1D43D, - 0x1D43E, - 0x1D43F, - 0x1D440, - 0x1D441, - 0x1D442, - 0x1D443, - 0x1D444, - 0x1D445, - 0x1D446, - 0x1D447, - 0x1D448, - 0x1D449, - 0x1D44A, - 0x1D44B, - 0x1D44C, - 0x1D44D, - 0x1D44E, - 0x1D44F, - 0x1D450, - 0x1D451, - 0x1D452, - 0x1D453, - 0x1D454, - 0x1D455, - 0x1D456, - 0x1D457, - 0x1D458, - 0x1D459, - 0x1D45A, - 0x1D45B, - 0x1D45C, - 0x1D45D, - 0x1D45E, - 0x1D45F, - 0x1D460, - 0x1D461, - 0x1D462, - 0x1D463, - 0x1D464, - 0x1D465, - 0x1D466, - 0x1D467, - 0x1D468, - 0x1D469, - 0x1D46A, - 0x1D46B, - 0x1D46C, - 0x1D46D, - 0x1D46E, - 0x1D46F, - 0x1D470, - 0x1D471, - 0x1D472, - 0x1D473, - 0x1D474, - 0x1D475, - 0x1D476, - 0x1D477, - 0x1D478, - 0x1D479, - 0x1D47A, - 0x1D47B, - 0x1D47C, - 0x1D47D, - 0x1D47E, - 0x1D47F, - 0x1D480, - 0x1D481, - 0x1D482, - 0x1D483, - 0x1D484, - 0x1D485, - 0x1D486, - 0x1D487, - 0x1D488, - 0x1D489, - 0x1D48A, - 0x1D48B, - 0x1D48C, - 0x1D48D, - 0x1D48E, - 0x1D48F, - 0x1D490, - 0x1D491, - 0x1D492, - 0x1D493, - 0x1D494, - 0x1D495, - 0x1D496, - 0x1D497, - 0x1D498, - 0x1D499, - 0x1D49A, - 0x1D49B, - 0x1D49C, - 0x1D49D, - 0x1D49E, - 0x1D49F, - 0x1D4A0, - 0x1D4A2, - 0x1D4A3, - 0x1D4A5, - 0x1D4A6, - 0x1D4A7, - 0x1D4A9, - 0x1D4AA, - 0x1D4AB, - 0x1D4AC, - 0x1D4AD, - 0x1D4AE, - 0x1D4AF, - 0x1D4B0, - 0x1D4B1, - 0x1D4B2, - 0x1D4B3, - 0x1D4B4, - 0x1D4B5, - 0x1D4B6, - 0x1D4B7, - 0x1D4B8, - 0x1D4B9, - 0x1D4BA, - 0x1D4BB, - 0x1D4BC, - 0x1D4BD, - 0x1D4BE, - 0x1D4BF, - 0x1D4C0, - 0x1D4C1, - 0x1D4C2, - 0x1D4C3, - 0x1D4C4, - 0x1D4C5, - 0x1D4C6, - 0x1D4C7, - 0x1D4C8, - 0x1D4C9, - 0x1D4CA, - 0x1D4CB, - 0x1D4CC, - 0x1D4CD, - 0x1D4CE, - 0x1D4CF, - 0x1D4D0, - 0x1D4D1, - 0x1D4D2, - 0x1D4D3, - 0x1D4D4, - 0x1D4D5, - 0x1D4D6, - 0x1D4D7, - 0x1D4D8, - 0x1D4D9, - 0x1D4DA, - 0x1D4DB, - 0x1D4DC, - 0x1D4DD, - 0x1D4DE, - 0x1D4DF, - 0x1D4E0, - 0x1D4E1, - 0x1D4E2, - 0x1D4E3, - 0x1D4E4, - 0x1D4E5, - 0x1D4E6, - 0x1D4E7, - 0x1D4E8, - 0x1D4E9, - 0x1D4EA, - 0x1D4EB, - 0x1D4EC, - 0x1D4ED, - 0x1D4EE, - 0x1D4EF, - 0x1D4F0, - 0x1D4F1, - 0x1D4F2, - 0x1D4F3, - 0x1D4F4, - 0x1D4F5, - 0x1D4F6, - 0x1D4F7, - 0x1D4F8, - 0x1D4F9, - 0x1D4FA, - 0x1D4FB, - 0x1D4FC, - 0x1D4FD, - 0x1D4FE, - 0x1D4FF, - 0x1D500, - 0x1D501, - 0x1D502, - 0x1D503, - 0x1D504, - 0x1D505, - 0x1D506, - 0x1D507, - 0x1D508, - 0x1D509, - 0x1D50A, - 0x1D50B, - 0x1D50D, - 0x1D50E, - 0x1D50F, - 0x1D510, - 0x1D511, - 0x1D512, - 0x1D513, - 0x1D514, - 0x1D515, - 0x1D516, - 0x1D517, - 0x1D518, - 0x1D519, - 0x1D51A, - 0x1D51B, - 0x1D51C, - 0x1D51D, - 0x1D51E, - 0x1D51F, - 0x1D520, - 0x1D521, - 0x1D522, - 0x1D523, - 0x1D524, - 0x1D525, - 0x1D526, - 0x1D527, - 0x1D528, - 0x1D529, - 0x1D52A, - 0x1D52B, - 0x1D52C, - 0x1D52D, - 0x1D52E, - 0x1D52F, - 0x1D530, - 0x1D531, - 0x1D532, - 0x1D533, - 0x1D534, - 0x1D535, - 0x1D536, - 0x1D537, - 0x1D538, - 0x1D539, - 0x1D53A, - 0x1D53B, - 0x1D53C, - 0x1D53D, - 0x1D53E, - 0x1D53F, - 0x1D540, - 0x1D541, - 0x1D542, - 0x1D543, - 0x1D544, - 0x1D545, - 0x1D546, - 0x1D547, - 0x1D54A, - 0x1D54B, - 0x1D54C, - 0x1D54D, - 0x1D54E, - 0x1D54F, - 0x1D550, - 0x1D551, - 0x1D552, - 0x1D553, - 0x1D554, - 0x1D555, - 0x1D556, - 0x1D557, - 0x1D558, - 0x1D559, - 0x1D55A, - 0x1D55B, - 0x1D55C, - 0x1D55D, - 0x1D55E, - 0x1D55F, - 0x1D560, - 0x1D561, - 0x1D562, - 0x1D563, - 0x1D564, - 0x1D565, - 0x1D566, - 0x1D567, - 0x1D568, - 0x1D569, - 0x1D56A, - 0x1D56B, - 0x1D56C, - 0x1D56D, - 0x1D56E, - 0x1D56F, - 0x1D570, - 0x1D571, - 0x1D572, - 0x1D573, - 0x1D574, - 0x1D575, - 0x1D576, - 0x1D577, - 0x1D578, - 0x1D579, - 0x1D57A, - 0x1D57B, - 0x1D57C, - 0x1D57D, - 0x1D57E, - 0x1D57F, - 0x1D580, - 0x1D581, - 0x1D582, - 0x1D583, - 0x1D584, - 0x1D585, - 0x1D586, - 0x1D587, - 0x1D588, - 0x1D589, - 0x1D58A, - 0x1D58B, - 0x1D58C, - 0x1D58D, - 0x1D58E, - 0x1D58F, - 0x1D590, - 0x1D591, - 0x1D592, - 0x1D593, - 0x1D594, - 0x1D595, - 0x1D596, - 0x1D597, - 0x1D598, - 0x1D599, - 0x1D59A, - 0x1D59B, - 0x1D59C, - 0x1D59D, - 0x1D59E, - 0x1D59F, - 0x1D5A0, - 0x1D5A1, - 0x1D5A2, - 0x1D5A3, - 0x1D5A4, - 0x1D5A5, - 0x1D5A6, - 0x1D5A7, - 0x1D5A8, - 0x1D5A9, - 0x1D5AA, - 0x1D5AB, - 0x1D5AC, - 0x1D5AD, - 0x1D5AE, - 0x1D5AF, - 0x1D5B0, - 0x1D5B1, - 0x1D5B2, - 0x1D5B3, - 0x1D5B4, - 0x1D5B5, - 0x1D5B6, - 0x1D5B7, - 0x1D5B8, - 0x1D5B9, - 0x1D5BA, - 0x1D5BB, - 0x1D5BC, - 0x1D5BD, - 0x1D5BE, - 0x1D5BF, - 0x1D5C0, - 0x1D5C1, - 0x1D5C2, - 0x1D5C3, - 0x1D5C4, - 0x1D5C5, - 0x1D5C6, - 0x1D5C7, - 0x1D5C8, - 0x1D5C9, - 0x1D5CA, - 0x1D5CB, - 0x1D5CC, - 0x1D5CD, - 0x1D5CE, - 0x1D5CF, - 0x1D5D0, - 0x1D5D1, - 0x1D5D2, - 0x1D5D3, - 0x1D5D4, - 0x1D5D5, - 0x1D5D6, - 0x1D5D7, - 0x1D5D8, - 0x1D5D9, - 0x1D5DA, - 0x1D5DB, - 0x1D5DC, - 0x1D5DD, - 0x1D5DE, - 0x1D5DF, - 0x1D5E0, - 0x1D5E1, - 0x1D5E2, - 0x1D5E3, - 0x1D5E4, - 0x1D5E5, - 0x1D5E6, - 0x1D5E7, - 0x1D5E8, - 0x1D5E9, - 0x1D5EA, - 0x1D5EB, - 0x1D5EC, - 0x1D5ED, - 0x1D5EE, - 0x1D5EF, - 0x1D5F0, - 0x1D5F1, - 0x1D5F2, - 0x1D5F3, - 0x1D5F4, - 0x1D5F5, - 0x1D5F6, - 0x1D5F7, - 0x1D5F8, - 0x1D5F9, - 0x1D5FA, - 0x1D5FB, - 0x1D5FC, - 0x1D5FD, - 0x1D5FE, - 0x1D5FF, - 0x1D600, - 0x1D601, - 0x1D602, - 0x1D603, - 0x1D604, - 0x1D605, - 0x1D606, - 0x1D607, - 0x1D608, - 0x1D609, - 0x1D60A, - 0x1D60B, - 0x1D60C, - 0x1D60D, - 0x1D60E, - 0x1D60F, - 0x1D610, - 0x1D611, - 0x1D612, - 0x1D613, - 0x1D614, - 0x1D615, - 0x1D616, - 0x1D617, - 0x1D618, - 0x1D619, - 0x1D61A, - 0x1D61B, - 0x1D61C, - 0x1D61D, - 0x1D61E, - 0x1D61F, - 0x1D620, - 0x1D621, - 0x1D622, - 0x1D623, - 0x1D624, - 0x1D625, - 0x1D626, - 0x1D627, - 0x1D628, - 0x1D629, - 0x1D62A, - 0x1D62B, - 0x1D62C, - 0x1D62D, - 0x1D62E, - 0x1D62F, - 0x1D630, - 0x1D631, - 0x1D632, - 0x1D633, - 0x1D634, - 0x1D635, - 0x1D636, - 0x1D637, - 0x1D638, - 0x1D639, - 0x1D63A, - 0x1D63B, - 0x1D63C, - 0x1D63D, - 0x1D63E, - 0x1D63F, - 0x1D640, - 0x1D641, - 0x1D642, - 0x1D643, - 0x1D644, - 0x1D645, - 0x1D646, - 0x1D647, - 0x1D648, - 0x1D649, - 0x1D64A, - 0x1D64B, - 0x1D64C, - 0x1D64D, - 0x1D64E, - 0x1D64F, - 0x1D650, - 0x1D651, - 0x1D652, - 0x1D653, - 0x1D654, - 0x1D655, - 0x1D656, - 0x1D657, - 0x1D658, - 0x1D659, - 0x1D65A, - 0x1D65B, - 0x1D65C, - 0x1D65D, - 0x1D65E, - 0x1D65F, - 0x1D660, - 0x1D661, - 0x1D662, - 0x1D663, - 0x1D664, - 0x1D665, - 0x1D666, - 0x1D667, - 0x1D668, - 0x1D669, - 0x1D66A, - 0x1D66B, - 0x1D66C, - 0x1D66D, - 0x1D66E, - 0x1D66F, - 0x1D670, - 0x1D671, - 0x1D672, - 0x1D673, - 0x1D674, - 0x1D675, - 0x1D676, - 0x1D677, - 0x1D678, - 0x1D679, - 0x1D67A, - 0x1D67B, - 0x1D67C, - 0x1D67D, - 0x1D67E, - 0x1D67F, - 0x1D680, - 0x1D681, - 0x1D682, - 0x1D683, - 0x1D684, - 0x1D685, - 0x1D686, - 0x1D687, - 0x1D688, - 0x1D689, - 0x1D68A, - 0x1D68B, - 0x1D68C, - 0x1D68D, - 0x1D68E, - 0x1D68F, - 0x1D690, - 0x1D691, - 0x1D692, - 0x1D693, - 0x1D694, - 0x1D695, - 0x1D696, - 0x1D697, - 0x1D698, - 0x1D699, - 0x1D69A, - 0x1D69B, - 0x1D69C, - 0x1D69D, - 0x1D69E, - 0x1D69F, - 0x1D6A0, - 0x1D6A1, - 0x1D6A2, - 0x1D6A3, - 0x1D6A4, - 0x1D6A5, - 0x1D6A6, - 0x1D6A8, - 0x1D6A9, - 0x1D6AA, - 0x1D6AB, - 0x1D6AC, - 0x1D6AD, - 0x1D6AE, - 0x1D6AF, - 0x1D6B0, - 0x1D6B1, - 0x1D6B2, - 0x1D6B3, - 0x1D6B4, - 0x1D6B5, - 0x1D6B6, - 0x1D6B7, - 0x1D6B8, - 0x1D6B9, - 0x1D6BA, - 0x1D6BB, - 0x1D6BC, - 0x1D6BD, - 0x1D6BE, - 0x1D6BF, - 0x1D6C0, - 0x1D6C1, - 0x1D6C2, - 0x1D6C3, - 0x1D6C4, - 0x1D6C5, - 0x1D6C6, - 0x1D6C7, - 0x1D6C8, - 0x1D6C9, - 0x1D6CA, - 0x1D6CB, - 0x1D6CC, - 0x1D6CD, - 0x1D6CE, - 0x1D6CF, - 0x1D6D0, - 0x1D6D1, - 0x1D6D2, - 0x1D6D3, - 0x1D6D5, - 0x1D6D6, - 0x1D6D7, - 0x1D6D8, - 0x1D6D9, - 0x1D6DA, - 0x1D6DB, - 0x1D6DC, - 0x1D6DD, - 0x1D6DE, - 0x1D6DF, - 0x1D6E0, - 0x1D6E1, - 0x1D6E2, - 0x1D6E3, - 0x1D6E4, - 0x1D6E5, - 0x1D6E6, - 0x1D6E7, - 0x1D6E8, - 0x1D6E9, - 0x1D6EA, - 0x1D6EB, - 0x1D6EC, - 0x1D6ED, - 0x1D6EE, - 0x1D6EF, - 0x1D6F0, - 0x1D6F1, - 0x1D6F2, - 0x1D6F3, - 0x1D6F4, - 0x1D6F5, - 0x1D6F6, - 0x1D6F7, - 0x1D6F8, - 0x1D6F9, - 0x1D6FA, - 0x1D6FB, - 0x1D6FC, - 0x1D6FD, - 0x1D6FE, - 0x1D6FF, - 0x1D700, - 0x1D701, - 0x1D702, - 0x1D703, - 0x1D704, - 0x1D705, - 0x1D706, - 0x1D707, - 0x1D708, - 0x1D709, - 0x1D70A, - 0x1D70B, - 0x1D70C, - 0x1D70D, - 0x1D70F, - 0x1D710, - 0x1D711, - 0x1D712, - 0x1D713, - 0x1D714, - 0x1D715, - 0x1D716, - 0x1D717, - 0x1D718, - 0x1D719, - 0x1D71A, - 0x1D71B, - 0x1D71C, - 0x1D71D, - 0x1D71E, - 0x1D71F, - 0x1D720, - 0x1D721, - 0x1D722, - 0x1D723, - 0x1D724, - 0x1D725, - 0x1D726, - 0x1D727, - 0x1D728, - 0x1D729, - 0x1D72A, - 0x1D72B, - 0x1D72C, - 0x1D72D, - 0x1D72E, - 0x1D72F, - 0x1D730, - 0x1D731, - 0x1D732, - 0x1D733, - 0x1D734, - 0x1D735, - 0x1D736, - 0x1D737, - 0x1D738, - 0x1D739, - 0x1D73A, - 0x1D73B, - 0x1D73C, - 0x1D73D, - 0x1D73E, - 0x1D73F, - 0x1D740, - 0x1D741, - 0x1D742, - 0x1D743, - 0x1D744, - 0x1D745, - 0x1D746, - 0x1D747, - 0x1D749, - 0x1D74A, - 0x1D74B, - 0x1D74C, - 0x1D74D, - 0x1D74E, - 0x1D74F, - 0x1D750, - 0x1D751, - 0x1D752, - 0x1D753, - 0x1D754, - 0x1D755, - 0x1D756, - 0x1D757, - 0x1D758, - 0x1D759, - 0x1D75A, - 0x1D75B, - 0x1D75C, - 0x1D75D, - 0x1D75E, - 0x1D75F, - 0x1D760, - 0x1D761, - 0x1D762, - 0x1D763, - 0x1D764, - 0x1D765, - 0x1D766, - 0x1D767, - 0x1D768, - 0x1D769, - 0x1D76A, - 0x1D76B, - 0x1D76C, - 0x1D76D, - 0x1D76E, - 0x1D76F, - 0x1D770, - 0x1D771, - 0x1D772, - 0x1D773, - 0x1D774, - 0x1D775, - 0x1D776, - 0x1D777, - 0x1D778, - 0x1D779, - 0x1D77A, - 0x1D77B, - 0x1D77C, - 0x1D77D, - 0x1D77E, - 0x1D77F, - 0x1D780, - 0x1D781, - 0x1D783, - 0x1D784, - 0x1D785, - 0x1D786, - 0x1D787, - 0x1D788, - 0x1D789, - 0x1D78A, - 0x1D78B, - 0x1D78C, - 0x1D78D, - 0x1D78E, - 0x1D78F, - 0x1D790, - 0x1D791, - 0x1D792, - 0x1D793, - 0x1D794, - 0x1D795, - 0x1D796, - 0x1D797, - 0x1D798, - 0x1D799, - 0x1D79A, - 0x1D79B, - 0x1D79C, - 0x1D79D, - 0x1D79E, - 0x1D79F, - 0x1D7A0, - 0x1D7A1, - 0x1D7A2, - 0x1D7A3, - 0x1D7A4, - 0x1D7A5, - 0x1D7A6, - 0x1D7A7, - 0x1D7A8, - 0x1D7A9, - 0x1D7AA, - 0x1D7AB, - 0x1D7AC, - 0x1D7AD, - 0x1D7AE, - 0x1D7AF, - 0x1D7B0, - 0x1D7B1, - 0x1D7B2, - 0x1D7B3, - 0x1D7B4, - 0x1D7B5, - 0x1D7B6, - 0x1D7B7, - 0x1D7B8, - 0x1D7B9, - 0x1D7BA, - 0x1D7BB, - 0x1D7BD, - 0x1D7BE, - 0x1D7BF, - 0x1D7C0, - 0x1D7C1, - 0x1D7C2, - 0x1D7C3, - 0x1D7C4, - 0x1D7C5, - 0x1D7C6, - 0x1D7C7, - 0x1D7C8, - 0x1D7C9, - 0x1D7CA, - 0x1D7CC, - 0x1D7CE, - 0x1D7CF, - 0x1D7D0, - 0x1D7D1, - 0x1D7D2, - 0x1D7D3, - 0x1D7D4, - 0x1D7D5, - 0x1D7D6, - 0x1D7D7, - 0x1D7D8, - 0x1D7D9, - 0x1D7DA, - 0x1D7DB, - 0x1D7DC, - 0x1D7DD, - 0x1D7DE, - 0x1D7DF, - 0x1D7E0, - 0x1D7E1, - 0x1D7E2, - 0x1D7E3, - 0x1D7E4, - 0x1D7E5, - 0x1D7E6, - 0x1D7E7, - 0x1D7E8, - 0x1D7E9, - 0x1D7EA, - 0x1D7EB, - 0x1D7EC, - 0x1D7ED, - 0x1D7EE, - 0x1D7EF, - 0x1D7F0, - 0x1D7F1, - 0x1D7F2, - 0x1D7F3, - 0x1D7F4, - 0x1D7F5, - 0x1D7F6, - 0x1D7F7, - 0x1D7F8, - 0x1D7F9, - 0x1D7FA, - 0x1D7FB, - 0x1D7FC, - 0x1D7FD, - 0x1D7FE, - 0x1D7FF, - 0x1D800, - 0x1DA8C, - 0x1DA9B, - 0x1DAA0, - 0x1DAA1, - 0x1DAB0, - 0x1DF00, - 0x1DF1F, - 0x1DF25, - 0x1DF2B, - 0x1E000, - 0x1E007, - 0x1E008, - 0x1E019, - 0x1E01B, - 0x1E022, - 0x1E023, - 0x1E025, - 0x1E026, - 0x1E02B, - 0x1E030, - 0x1E031, - 0x1E032, - 0x1E033, - 0x1E034, - 0x1E035, - 0x1E036, - 0x1E037, - 0x1E038, - 0x1E039, - 0x1E03A, - 0x1E03B, - 0x1E03C, - 0x1E03D, - 0x1E03E, - 0x1E03F, - 0x1E040, - 0x1E041, - 0x1E042, - 0x1E043, - 0x1E044, - 0x1E045, - 0x1E046, - 0x1E047, - 0x1E048, - 0x1E049, - 0x1E04A, - 0x1E04B, - 0x1E04C, - 0x1E04D, - 0x1E04E, - 0x1E04F, - 0x1E050, - 0x1E051, - 0x1E052, - 0x1E053, - 0x1E054, - 0x1E055, - 0x1E056, - 0x1E057, - 0x1E058, - 0x1E059, - 0x1E05A, - 0x1E05B, - 0x1E05C, - 0x1E05D, - 0x1E05E, - 0x1E05F, - 0x1E060, - 0x1E061, - 0x1E062, - 0x1E063, - 0x1E064, - 0x1E065, - 0x1E066, - 0x1E067, - 0x1E068, - 0x1E069, - 0x1E06A, - 0x1E06B, - 0x1E06C, - 0x1E06D, - 0x1E06E, - 0x1E08F, - 0x1E090, - 0x1E100, - 0x1E12D, - 0x1E130, - 0x1E13E, - 0x1E140, - 0x1E14A, - 0x1E14E, - 0x1E150, - 0x1E290, - 0x1E2AF, - 0x1E2C0, - 0x1E2FA, - 0x1E2FF, - 0x1E300, - 0x1E4D0, - 0x1E4FA, - 0x1E5D0, - 0x1E5FB, - 0x1E5FF, - 0x1E600, - 0x1E6C0, - 0x1E6DF, - 0x1E6E0, - 0x1E6F6, - 0x1E6FE, - 0x1E700, - 0x1E7E0, - 0x1E7E7, - 0x1E7E8, - 0x1E7EC, - 0x1E7ED, - 0x1E7EF, - 0x1E7F0, - 0x1E7FF, - 0x1E800, - 0x1E8C5, - 0x1E8C7, - 0x1E8D7, - 0x1E900, - 0x1E901, - 0x1E902, - 0x1E903, - 0x1E904, - 0x1E905, - 0x1E906, - 0x1E907, - 0x1E908, - 0x1E909, - 0x1E90A, - 0x1E90B, - 0x1E90C, - 0x1E90D, - 0x1E90E, - 0x1E90F, - 0x1E910, - 0x1E911, - 0x1E912, - 0x1E913, - 0x1E914, - 0x1E915, - 0x1E916, - 0x1E917, - 0x1E918, - 0x1E919, - 0x1E91A, - 0x1E91B, - 0x1E91C, - 0x1E91D, - 0x1E91E, - 0x1E91F, - 0x1E920, - 0x1E921, - 0x1E922, - 0x1E94C, - 0x1E950, - 0x1E95A, - 0x1E95E, - 0x1E960, - 0x1EC71, - 0x1ECB5, - 0x1ED01, - 0x1ED3E, - 0x1EE00, - 0x1EE01, - 0x1EE02, - 0x1EE03, - 0x1EE04, - 0x1EE05, - 0x1EE06, - 0x1EE07, - 0x1EE08, - 0x1EE09, - 0x1EE0A, - 0x1EE0B, - 0x1EE0C, - 0x1EE0D, - 0x1EE0E, - 0x1EE0F, - 0x1EE10, - 0x1EE11, - 0x1EE12, - 0x1EE13, - 0x1EE14, - 0x1EE15, - 0x1EE16, - 0x1EE17, - 0x1EE18, - 0x1EE19, - 0x1EE1A, - 0x1EE1B, - 0x1EE1C, - 0x1EE1D, - 0x1EE1E, - 0x1EE1F, - 0x1EE20, - 0x1EE21, - 0x1EE22, - 0x1EE23, - 0x1EE24, - 0x1EE25, - 0x1EE27, - 0x1EE28, - 0x1EE29, - 0x1EE2A, - 0x1EE2B, - 0x1EE2C, - 0x1EE2D, - 0x1EE2E, - 0x1EE2F, - 0x1EE30, - 0x1EE31, - 0x1EE32, - 0x1EE33, - 0x1EE34, - 0x1EE35, - 0x1EE36, - 0x1EE37, - 0x1EE38, - 0x1EE39, - 0x1EE3A, - 0x1EE3B, - 0x1EE3C, - 0x1EE42, - 0x1EE43, - 0x1EE47, - 0x1EE48, - 0x1EE49, - 0x1EE4A, - 0x1EE4B, - 0x1EE4C, - 0x1EE4D, - 0x1EE4E, - 0x1EE4F, - 0x1EE50, - 0x1EE51, - 0x1EE52, - 0x1EE53, - 0x1EE54, - 0x1EE55, - 0x1EE57, - 0x1EE58, - 0x1EE59, - 0x1EE5A, - 0x1EE5B, - 0x1EE5C, - 0x1EE5D, - 0x1EE5E, - 0x1EE5F, - 0x1EE60, - 0x1EE61, - 0x1EE62, - 0x1EE63, - 0x1EE64, - 0x1EE65, - 0x1EE67, - 0x1EE68, - 0x1EE69, - 0x1EE6A, - 0x1EE6B, - 0x1EE6C, - 0x1EE6D, - 0x1EE6E, - 0x1EE6F, - 0x1EE70, - 0x1EE71, - 0x1EE72, - 0x1EE73, - 0x1EE74, - 0x1EE75, - 0x1EE76, - 0x1EE77, - 0x1EE78, - 0x1EE79, - 0x1EE7A, - 0x1EE7B, - 0x1EE7C, - 0x1EE7D, - 0x1EE7E, - 0x1EE7F, - 0x1EE80, - 0x1EE81, - 0x1EE82, - 0x1EE83, - 0x1EE84, - 0x1EE85, - 0x1EE86, - 0x1EE87, - 0x1EE88, - 0x1EE89, - 0x1EE8A, - 0x1EE8B, - 0x1EE8C, - 0x1EE8D, - 0x1EE8E, - 0x1EE8F, - 0x1EE90, - 0x1EE91, - 0x1EE92, - 0x1EE93, - 0x1EE94, - 0x1EE95, - 0x1EE96, - 0x1EE97, - 0x1EE98, - 0x1EE99, - 0x1EE9A, - 0x1EE9B, - 0x1EE9C, - 0x1EEA1, - 0x1EEA2, - 0x1EEA3, - 0x1EEA4, - 0x1EEA5, - 0x1EEA6, - 0x1EEA7, - 0x1EEA8, - 0x1EEA9, - 0x1EEAA, - 0x1EEAB, - 0x1EEAC, - 0x1EEAD, - 0x1EEAE, - 0x1EEAF, - 0x1EEB0, - 0x1EEB1, - 0x1EEB2, - 0x1EEB3, - 0x1EEB4, - 0x1EEB5, - 0x1EEB6, - 0x1EEB7, - 0x1EEB8, - 0x1EEB9, - 0x1EEBA, - 0x1EEBB, - 0x1EEBC, - 0x1EEF0, - 0x1EEF2, - 0x1F000, - 0x1F02C, - 0x1F030, - 0x1F094, - 0x1F0A0, - 0x1F0AF, - 0x1F0B1, - 0x1F0C0, - 0x1F0C1, - 0x1F0D0, - 0x1F0D1, - 0x1F0F6, - 0x1F101, - 0x1F102, - 0x1F103, - 0x1F104, - 0x1F105, - 0x1F106, - 0x1F107, - 0x1F108, - 0x1F109, - 0x1F10A, - 0x1F10B, - 0x1F110, - 0x1F111, - 0x1F112, - 0x1F113, - 0x1F114, - 0x1F115, - 0x1F116, - 0x1F117, - 0x1F118, - 0x1F119, - 0x1F11A, - 0x1F11B, - 0x1F11C, - 0x1F11D, - 0x1F11E, - 0x1F11F, - 0x1F120, - 0x1F121, - 0x1F122, - 0x1F123, - 0x1F124, - 0x1F125, - 0x1F126, - 0x1F127, - 0x1F128, - 0x1F129, - 0x1F12A, - 0x1F12B, - 0x1F12C, - 0x1F12D, - 0x1F12E, - 0x1F12F, - 0x1F130, - 0x1F131, - 0x1F132, - 0x1F133, - 0x1F134, - 0x1F135, - 0x1F136, - 0x1F137, - 0x1F138, - 0x1F139, - 0x1F13A, - 0x1F13B, - 0x1F13C, - 0x1F13D, - 0x1F13E, - 0x1F13F, - 0x1F140, - 0x1F141, - 0x1F142, - 0x1F143, - 0x1F144, - 0x1F145, - 0x1F146, - 0x1F147, - 0x1F148, - 0x1F149, - 0x1F14A, - 0x1F14B, - 0x1F14C, - 0x1F14D, - 0x1F14E, - 0x1F14F, - 0x1F150, - 0x1F16A, - 0x1F16B, - 0x1F16C, - 0x1F16D, - 0x1F190, - 0x1F191, - 0x1F1AE, - 0x1F1E6, - 0x1F200, - 0x1F201, - 0x1F202, - 0x1F203, - 0x1F210, - 0x1F211, - 0x1F212, - 0x1F213, - 0x1F214, - 0x1F215, - 0x1F216, - 0x1F217, - 0x1F218, - 0x1F219, - 0x1F21A, - 0x1F21B, - 0x1F21C, - 0x1F21D, - 0x1F21E, - 0x1F21F, - 0x1F220, - 0x1F221, - 0x1F222, - 0x1F223, - 0x1F224, - 0x1F225, - 0x1F226, - 0x1F227, - 0x1F228, - 0x1F229, - 0x1F22A, - 0x1F22B, - 0x1F22C, - 0x1F22D, - 0x1F22E, - 0x1F22F, - 0x1F230, - 0x1F231, - 0x1F232, - 0x1F233, - 0x1F234, - 0x1F235, - 0x1F236, - 0x1F237, - 0x1F238, - 0x1F239, - 0x1F23A, - 0x1F23B, - 0x1F23C, - 0x1F240, - 0x1F241, - 0x1F242, - 0x1F243, - 0x1F244, - 0x1F245, - 0x1F246, - 0x1F247, - 0x1F248, - 0x1F249, - 0x1F250, - 0x1F251, - 0x1F252, - 0x1F260, - 0x1F266, - 0x1F300, - 0x1F6D9, - 0x1F6DC, - 0x1F6ED, - 0x1F6F0, - 0x1F6FD, - 0x1F700, - 0x1F7DA, - 0x1F7E0, - 0x1F7EC, - 0x1F7F0, - 0x1F7F1, - 0x1F800, - 0x1F80C, - 0x1F810, - 0x1F848, - 0x1F850, - 0x1F85A, - 0x1F860, - 0x1F888, - 0x1F890, - 0x1F8AE, - 0x1F8B0, - 0x1F8BC, - 0x1F8C0, - 0x1F8C2, - 0x1F8D0, - 0x1F8D9, - 0x1F900, - 0x1FA58, - 0x1FA60, - 0x1FA6E, - 0x1FA70, - 0x1FA7D, - 0x1FA80, - 0x1FA8B, - 0x1FA8E, - 0x1FAC7, - 0x1FAC8, - 0x1FAC9, - 0x1FACD, - 0x1FADD, - 0x1FADF, - 0x1FAEB, - 0x1FAEF, - 0x1FAF9, - 0x1FB00, - 0x1FB93, - 0x1FB94, - 0x1FBF0, - 0x1FBF1, - 0x1FBF2, - 0x1FBF3, - 0x1FBF4, - 0x1FBF5, - 0x1FBF6, - 0x1FBF7, - 0x1FBF8, - 0x1FBF9, - 0x1FBFA, - 0x1FBFB, - 0x20000, - 0x2A6E0, - 0x2A700, - 0x2B81E, - 0x2B820, - 0x2CEAE, - 0x2CEB0, - 0x2EBE1, - 0x2EBF0, - 0x2EE5E, - 0x2F800, - 0x2F801, - 0x2F802, - 0x2F803, - 0x2F804, - 0x2F805, - 0x2F806, - 0x2F807, - 0x2F808, - 0x2F809, - 0x2F80A, - 0x2F80B, - 0x2F80C, - 0x2F80D, - 0x2F80E, - 0x2F80F, - 0x2F810, - 0x2F811, - 0x2F812, - 0x2F813, - 0x2F814, - 0x2F815, - 0x2F816, - 0x2F817, - 0x2F818, - 0x2F819, - 0x2F81A, - 0x2F81B, - 0x2F81C, - 0x2F81D, - 0x2F81E, - 0x2F81F, - 0x2F820, - 0x2F821, - 0x2F822, - 0x2F823, - 0x2F824, - 0x2F825, - 0x2F826, - 0x2F827, - 0x2F828, - 0x2F829, - 0x2F82A, - 0x2F82B, - 0x2F82C, - 0x2F82D, - 0x2F82E, - 0x2F82F, - 0x2F830, - 0x2F831, - 0x2F834, - 0x2F835, - 0x2F836, - 0x2F837, - 0x2F838, - 0x2F839, - 0x2F83A, - 0x2F83B, - 0x2F83C, - 0x2F83D, - 0x2F83E, - 0x2F83F, - 0x2F840, - 0x2F841, - 0x2F842, - 0x2F843, - 0x2F844, - 0x2F845, - 0x2F847, - 0x2F848, - 0x2F849, - 0x2F84A, - 0x2F84B, - 0x2F84C, - 0x2F84D, - 0x2F84E, - 0x2F84F, - 0x2F850, - 0x2F851, - 0x2F852, - 0x2F853, - 0x2F854, - 0x2F855, - 0x2F856, - 0x2F857, - 0x2F858, - 0x2F859, - 0x2F85A, - 0x2F85B, - 0x2F85C, - 0x2F85D, - 0x2F85E, - 0x2F85F, - 0x2F860, - 0x2F861, - 0x2F862, - 0x2F863, - 0x2F864, - 0x2F865, - 0x2F866, - 0x2F867, - 0x2F868, - 0x2F869, - 0x2F86A, - 0x2F86C, - 0x2F86D, - 0x2F86E, - 0x2F86F, - 0x2F870, - 0x2F871, - 0x2F872, - 0x2F873, - 0x2F874, - 0x2F875, - 0x2F876, - 0x2F877, - 0x2F878, - 0x2F879, - 0x2F87A, - 0x2F87B, - 0x2F87C, - 0x2F87D, - 0x2F87E, - 0x2F87F, - 0x2F880, - 0x2F881, - 0x2F882, - 0x2F883, - 0x2F884, - 0x2F885, - 0x2F886, - 0x2F887, - 0x2F888, - 0x2F889, - 0x2F88A, - 0x2F88B, - 0x2F88C, - 0x2F88D, - 0x2F88E, - 0x2F88F, - 0x2F890, - 0x2F891, - 0x2F893, - 0x2F894, - 0x2F896, - 0x2F897, - 0x2F898, - 0x2F899, - 0x2F89A, - 0x2F89B, - 0x2F89C, - 0x2F89D, - 0x2F89E, - 0x2F89F, - 0x2F8A0, - 0x2F8A1, - 0x2F8A2, - 0x2F8A3, - 0x2F8A4, - 0x2F8A5, - 0x2F8A6, - 0x2F8A7, - 0x2F8A8, - 0x2F8A9, - 0x2F8AA, - 0x2F8AB, - 0x2F8AC, - 0x2F8AD, - 0x2F8AE, - 0x2F8AF, - 0x2F8B0, - 0x2F8B1, - 0x2F8B2, - 0x2F8B3, - 0x2F8B4, - 0x2F8B5, - 0x2F8B6, - 0x2F8B7, - 0x2F8B8, - 0x2F8B9, - 0x2F8BA, - 0x2F8BB, - 0x2F8BC, - 0x2F8BD, - 0x2F8BE, - 0x2F8BF, - 0x2F8C0, - 0x2F8C1, - 0x2F8C2, - 0x2F8C3, - 0x2F8C4, - 0x2F8C5, - 0x2F8C6, - 0x2F8C7, - 0x2F8C8, - 0x2F8C9, - 0x2F8CA, - 0x2F8CB, - 0x2F8CC, - 0x2F8CD, - 0x2F8CE, - 0x2F8CF, - 0x2F8D0, - 0x2F8D1, - 0x2F8D2, - 0x2F8D3, - 0x2F8D4, - 0x2F8D5, - 0x2F8D6, - 0x2F8D7, - 0x2F8D8, - 0x2F8D9, - 0x2F8DA, - 0x2F8DB, - 0x2F8DC, - 0x2F8DD, - 0x2F8DE, - 0x2F8DF, - 0x2F8E0, - 0x2F8E1, - 0x2F8E2, - 0x2F8E3, - 0x2F8E4, - 0x2F8E5, - 0x2F8E6, - 0x2F8E7, - 0x2F8E8, - 0x2F8E9, - 0x2F8EA, - 0x2F8EB, - 0x2F8EC, - 0x2F8ED, - 0x2F8EE, - 0x2F8EF, - 0x2F8F0, - 0x2F8F1, - 0x2F8F2, - 0x2F8F3, - 0x2F8F4, - 0x2F8F5, - 0x2F8F6, - 0x2F8F7, - 0x2F8F8, - 0x2F8F9, - 0x2F8FA, - 0x2F8FB, - 0x2F8FC, - 0x2F8FD, - 0x2F8FE, - 0x2F8FF, - 0x2F900, - 0x2F901, - 0x2F902, - 0x2F903, - 0x2F904, - 0x2F905, - 0x2F906, - 0x2F907, - 0x2F908, - 0x2F909, - 0x2F90A, - 0x2F90B, - 0x2F90C, - 0x2F90D, - 0x2F90E, - 0x2F90F, - 0x2F910, - 0x2F911, - 0x2F912, - 0x2F913, - 0x2F914, - 0x2F915, - 0x2F916, - 0x2F917, - 0x2F918, - 0x2F919, - 0x2F91A, - 0x2F91B, - 0x2F91C, - 0x2F91D, - 0x2F91E, - 0x2F91F, - 0x2F920, - 0x2F921, - 0x2F922, - 0x2F923, - 0x2F924, - 0x2F925, - 0x2F926, - 0x2F927, - 0x2F928, - 0x2F929, - 0x2F92A, - 0x2F92B, - 0x2F92C, - 0x2F92E, - 0x2F92F, - 0x2F930, - 0x2F931, - 0x2F932, - 0x2F933, - 0x2F934, - 0x2F935, - 0x2F936, - 0x2F937, - 0x2F938, - 0x2F939, - 0x2F93A, - 0x2F93B, - 0x2F93C, - 0x2F93D, - 0x2F93E, - 0x2F93F, - 0x2F940, - 0x2F941, - 0x2F942, - 0x2F943, - 0x2F944, - 0x2F945, - 0x2F946, - 0x2F948, - 0x2F949, - 0x2F94A, - 0x2F94B, - 0x2F94C, - 0x2F94D, - 0x2F94E, - 0x2F94F, - 0x2F950, - 0x2F951, - 0x2F952, - 0x2F953, - 0x2F954, - 0x2F955, - 0x2F956, - 0x2F957, - 0x2F958, - 0x2F959, - 0x2F95A, - 0x2F95B, - 0x2F95C, - 0x2F95D, - 0x2F95F, - 0x2F960, - 0x2F961, - 0x2F962, - 0x2F963, - 0x2F964, - 0x2F965, - 0x2F966, - 0x2F967, - 0x2F968, - 0x2F969, - 0x2F96A, - 0x2F96B, - 0x2F96C, - 0x2F96D, - 0x2F96E, - 0x2F96F, - 0x2F970, - 0x2F971, - 0x2F972, - 0x2F973, - 0x2F974, - 0x2F975, - 0x2F976, - 0x2F977, - 0x2F978, - 0x2F979, - 0x2F97A, - 0x2F97B, - 0x2F97C, - 0x2F97D, - 0x2F97E, - 0x2F97F, - 0x2F980, - 0x2F981, - 0x2F982, - 0x2F983, - 0x2F984, - 0x2F985, - 0x2F986, - 0x2F987, - 0x2F988, - 0x2F989, - 0x2F98A, - 0x2F98B, - 0x2F98C, - 0x2F98D, - 0x2F98E, - 0x2F98F, - 0x2F990, - 0x2F991, - 0x2F992, - 0x2F993, - 0x2F994, - 0x2F995, - 0x2F996, - 0x2F997, - 0x2F998, - 0x2F999, - 0x2F99A, - 0x2F99B, - 0x2F99C, - 0x2F99D, - 0x2F99E, - 0x2F99F, - 0x2F9A0, - 0x2F9A1, - 0x2F9A2, - 0x2F9A3, - 0x2F9A4, - 0x2F9A5, - 0x2F9A6, - 0x2F9A7, - 0x2F9A8, - 0x2F9A9, - 0x2F9AA, - 0x2F9AB, - 0x2F9AC, - 0x2F9AD, - 0x2F9AE, - 0x2F9AF, - 0x2F9B0, - 0x2F9B1, - 0x2F9B2, - 0x2F9B3, - 0x2F9B4, - 0x2F9B5, - 0x2F9B6, - 0x2F9B7, - 0x2F9B8, - 0x2F9B9, - 0x2F9BA, - 0x2F9BB, - 0x2F9BC, - 0x2F9BD, - 0x2F9BE, - 0x2F9BF, - 0x2F9C0, - 0x2F9C1, - 0x2F9C2, - 0x2F9C3, - 0x2F9C4, - 0x2F9C5, - 0x2F9C6, - 0x2F9C7, - 0x2F9C8, - 0x2F9C9, - 0x2F9CA, - 0x2F9CB, - 0x2F9CC, - 0x2F9CD, - 0x2F9CE, - 0x2F9CF, - 0x2F9D0, - 0x2F9D1, - 0x2F9D2, - 0x2F9D3, - 0x2F9D4, - 0x2F9D5, - 0x2F9D6, - 0x2F9D7, - 0x2F9D8, - 0x2F9D9, - 0x2F9DA, - 0x2F9DB, - 0x2F9DC, - 0x2F9DD, - 0x2F9DE, - 0x2F9DF, - 0x2F9E0, - 0x2F9E1, - 0x2F9E2, - 0x2F9E3, - 0x2F9E4, - 0x2F9E5, - 0x2F9E6, - 0x2F9E7, - 0x2F9E8, - 0x2F9E9, - 0x2F9EA, - 0x2F9EB, - 0x2F9EC, - 0x2F9ED, - 0x2F9EE, - 0x2F9EF, - 0x2F9F0, - 0x2F9F1, - 0x2F9F2, - 0x2F9F3, - 0x2F9F4, - 0x2F9F5, - 0x2F9F6, - 0x2F9F7, - 0x2F9F8, - 0x2F9F9, - 0x2F9FA, - 0x2F9FB, - 0x2F9FC, - 0x2F9FD, - 0x2F9FE, - 0x2FA00, - 0x2FA01, - 0x2FA02, - 0x2FA03, - 0x2FA04, - 0x2FA05, - 0x2FA06, - 0x2FA07, - 0x2FA08, - 0x2FA09, - 0x2FA0A, - 0x2FA0B, - 0x2FA0C, - 0x2FA0D, - 0x2FA0E, - 0x2FA0F, - 0x2FA10, - 0x2FA11, - 0x2FA12, - 0x2FA13, - 0x2FA14, - 0x2FA15, - 0x2FA16, - 0x2FA17, - 0x2FA18, - 0x2FA19, - 0x2FA1A, - 0x2FA1B, - 0x2FA1C, - 0x2FA1D, - 0x2FA1E, - 0x30000, - 0x3134B, - 0x31350, - 0x3347A, - 0xE0100, - 0xE01F0, - ), -) - -uts46_statuses: bytes = ( - b"VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" - b"VMMMMMMMMMMMMMMMMMMMMMMMMMMVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" - b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXMVVVVVVVMVMVVIVMVVMMMMVVMMMVMMMV" - b"MMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMDVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVMVMVMVMMV" - b"MVMVMVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVMVMVMM" - b"VMVMMVMMMVMMMMVMMVMMMVMMVMMVMVMVMMVMVMVMMVMMMVMVMMVMVMMMMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMMVMMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" - b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMMVMVMMMMVMVMVMVMVMMMMMMMMMVMMMMMM" - b"VMMMMMVMMVMMMVIVMVMVMVMVXMVMMXMMMMMMMXMXMMVMMMMMMMMMMMMMMMMMXMMM" - b"MMMMMMVDVMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMVMVMMMVMMVMVMMVMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" - b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVXMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMXVMVXVXVXVXVXVXVMMMMVXVXVXVXVXVXVXVXVXVX" - b"VXVXVMMMMMMMMVXVXVXVXVXVXVXVXVXVXVXMMXMVXVXVXVXVXVXVXVMXVMXVXVXV" - b"XVXVXVXMMMVXMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" - b"XVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" - b"XVMVXVXVXVXVXVXVXVMVXVXVXVXVXMMVXVMVMVXVMVMVMVMVMVXVMVMMMMMVMVMV" - b"XVMVMVMVMVMVXVXVXVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMXMXVMV" - b"IVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMMMMMMXVXVXVXVXVXVXVXVXVX" - b"VIVXVXVXVIVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVMMM" - b"MMMMMMVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMVXVXVMMMV" - b"MMMMMMMMMMMVMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVM" - b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" - b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMMMMMMMVXMMMMMMXVMMMMMMMMVMMMMMM" - b"MMVXMMMMMMXVXMXMXMXMVMMMMMMMMVMVMVMVMVMVMVMXMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMXVMMMMMMMMMMMMMMXVMMMMMMMMMVMXVM" - b"MMMXMMMVMVMMMMMMMMXMMMXVMMMMMMMMXMIDXVMVMVXVXMVMMVMMVMVMVMMMVMVM" - b"IXIMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMXVMVXVXMMMMVMMMVM" - b"MMMMMVMMVMMMVMMMVMVMVMVMMMMVMMMMMMMMMMVMMMMMVMMMMVMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVXVMMVMMVMMVXVXMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMMMVMVXVMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMMMVMVMVMVMMMMVMVMVMM" - b"MMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVXVXVXVXVXMVXVXVXVXVX" - b"VXVXVXVXVXVXVXVMVMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMVMVMVMMMVXVXVMMVMVMXV" - b"XMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMIMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVMMMMMMMMMMMMMMVXVMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMMMMMMMMM" - b"MMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXMVMVMVMVMVMVMVM" - b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVXV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMMMMVM" - b"MMMMVMVMVMVMVMVMVMVMMMMVMVMMVMVMVMVMVMVMVMVMXMMMMMVMMVXVXVXVXVXV" - b"XVXVXVXVXVXVXVXVXVXVXVXVXVXVXVMMMMVMVXMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVX" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMVMVMVMMMMMMMMMMVMVMVMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMXMMMMMMXMMMMMXMVMMMMMMMMMMMMMMMMMMMMMMMMXMMMMM" - b"XMXMMXMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXMMMMMMMMMMMMMVIMMXMMM" - b"MMMXVXMMMMMMMMMMMMMMMMMMMVMMMMMMXMMMMMMMMMMMMMMMMMMMXMMMMXMMMVMX" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXIXMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMIMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMXMMMMM" - b"MXMMMMMMXMMMXMMMMMMMXMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" - b"XVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVMMMMMMMMMMMXMMMMMMMMMMMMMMMXMMM" - b"MMMMXMMXVXVXVXVXVXVXVXVXVMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMXMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMXVXVXVXVMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMVXMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVIXVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVX" - b"VXVXVXVXVXVXVMMMMMMMVIVMMMMMMVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMXMMXMXMMXMMMMXMMMMMMMMMMMMXMXMMMMMMMXMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMMMMXMMMMM" - b"MMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMXMXMMMMMMMXMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVX" - b"VXMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMXMXMXMMMMMMMMMMXMMMMXMXMXMX" - b"MXMXMXMMMXMMXMXMXMXMXMXMXMMXMXMMMMXMMMMMMMXMMMMXMMMMXMXMMMMMMMMM" - b"MXMMMMMMMMMMMMMMMMMXMMMXMMMMMXMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXMM" - b"MMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMVMMMVMVXVMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMXMMMMMMMMMXMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVMMMMMMMMMMVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXIX" -) - -uts46_replacements: tuple[Optional[str], ...] = ( - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - " ", - None, - None, - None, - None, - None, - None, - None, - " ̈", - None, - "a", - None, - None, - None, - None, - " ̄", - None, - None, - "2", - "3", - " ́", - "μ", - None, - None, - " ̧", - "1", - "o", - None, - "1⁄4", - "1⁄2", - "3⁄4", - None, - "à", - "á", - "â", - "ã", - "ä", - "å", - "æ", - "ç", - "è", - "é", - "ê", - "ë", - "ì", - "í", - "î", - "ï", - "ð", - "ñ", - "ò", - "ó", - "ô", - "õ", - "ö", - None, - "ø", - "ù", - "ú", - "û", - "ü", - "ý", - "þ", - "ss", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ā", - None, - "ă", - None, - "ą", - None, - "ć", - None, - "ĉ", - None, - "ċ", - None, - "č", - None, - "ď", - None, - "đ", - None, - "ē", - None, - "ĕ", - None, - "ė", - None, - "ę", - None, - "ě", - None, - "ĝ", - None, - "ğ", - None, - "ġ", - None, - "ģ", - None, - "ĥ", - None, - "ħ", - None, - "ĩ", - None, - "ī", - None, - "ĭ", - None, - "į", - None, - "i̇", - None, - "ij", - "ĵ", - None, - "ķ", - None, - "ĺ", - None, - "ļ", - None, - "ľ", - None, - "l·", - "ł", - None, - "ń", - None, - "ņ", - None, - "ň", - None, - "ʼn", - "ŋ", - None, - "ō", - None, - "ŏ", - None, - "ő", - None, - "œ", - None, - "ŕ", - None, - "ŗ", - None, - "ř", - None, - "ś", - None, - "ŝ", - None, - "ş", - None, - "š", - None, - "ţ", - None, - "ť", - None, - "ŧ", - None, - "ũ", - None, - "ū", - None, - "ŭ", - None, - "ů", - None, - "ű", - None, - "ų", - None, - "ŵ", - None, - "ŷ", - None, - "ÿ", - "ź", - None, - "ż", - None, - "ž", - None, - "s", - None, - "ɓ", - "ƃ", - None, - "ƅ", - None, - "ɔ", - "ƈ", - None, - "ɖ", - "ɗ", - "ƌ", - None, - "ǝ", - "ə", - "ɛ", - "ƒ", - None, - "ɠ", - "ɣ", - None, - "ɩ", - "ɨ", - "ƙ", - None, - "ɯ", - "ɲ", - None, - "ɵ", - "ơ", - None, - "ƣ", - None, - "ƥ", - None, - "ʀ", - "ƨ", - None, - "ʃ", - None, - "ƭ", - None, - "ʈ", - "ư", - None, - "ʊ", - "ʋ", - "ƴ", - None, - "ƶ", - None, - "ʒ", - "ƹ", - None, - "ƽ", - None, - "dž", - "lj", - "nj", - "ǎ", - None, - "ǐ", - None, - "ǒ", - None, - "ǔ", - None, - "ǖ", - None, - "ǘ", - None, - "ǚ", - None, - "ǜ", - None, - "ǟ", - None, - "ǡ", - None, - "ǣ", - None, - "ǥ", - None, - "ǧ", - None, - "ǩ", - None, - "ǫ", - None, - "ǭ", - None, - "ǯ", - None, - "dz", - "ǵ", - None, - "ƕ", - "ƿ", - "ǹ", - None, - "ǻ", - None, - "ǽ", - None, - "ǿ", - None, - "ȁ", - None, - "ȃ", - None, - "ȅ", - None, - "ȇ", - None, - "ȉ", - None, - "ȋ", - None, - "ȍ", - None, - "ȏ", - None, - "ȑ", - None, - "ȓ", - None, - "ȕ", - None, - "ȗ", - None, - "ș", - None, - "ț", - None, - "ȝ", - None, - "ȟ", - None, - "ƞ", - None, - "ȣ", - None, - "ȥ", - None, - "ȧ", - None, - "ȩ", - None, - "ȫ", - None, - "ȭ", - None, - "ȯ", - None, - "ȱ", - None, - "ȳ", - None, - "ⱥ", - "ȼ", - None, - "ƚ", - "ⱦ", - None, - "ɂ", - None, - "ƀ", - "ʉ", - "ʌ", - "ɇ", - None, - "ɉ", - None, - "ɋ", - None, - "ɍ", - None, - "ɏ", - None, - "h", - "ɦ", - "j", - "r", - "ɹ", - "ɻ", - "ʁ", - "w", - "y", - None, - " ̆", - " ̇", - " ̊", - " ̨", - " ̃", - " ̋", - None, - "ɣ", - "l", - "s", - "x", - "ʕ", - None, - "̀", - "́", - None, - "̓", - "̈́", - "ι", - None, - None, - None, - "ͱ", - None, - "ͳ", - None, - "ʹ", - None, - "ͷ", - None, - None, - " ι", - None, - ";", - "ϳ", - None, - " ́", - " ̈́", - "ά", - "·", - "έ", - "ή", - "ί", - None, - "ό", - None, - "ύ", - "ώ", - None, - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - None, - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "ϊ", - "ϋ", - None, - "σ", - None, - "ϗ", - "β", - "θ", - "υ", - "ύ", - "ϋ", - "φ", - "π", - None, - "ϙ", - None, - "ϛ", - None, - "ϝ", - None, - "ϟ", - None, - "ϡ", - None, - "ϣ", - None, - "ϥ", - None, - "ϧ", - None, - "ϩ", - None, - "ϫ", - None, - "ϭ", - None, - "ϯ", - None, - "κ", - "ρ", - "σ", - None, - "θ", - "ε", - None, - "ϸ", - None, - "σ", - "ϻ", - None, - "ͻ", - "ͼ", - "ͽ", - "ѐ", - "ё", - "ђ", - "ѓ", - "є", - "ѕ", - "і", - "ї", - "ј", - "љ", - "њ", - "ћ", - "ќ", - "ѝ", - "ў", - "џ", - "а", - "б", - "в", - "г", - "д", - "е", - "ж", - "з", - "и", - "й", - "к", - "л", - "м", - "н", - "о", - "п", - "р", - "с", - "т", - "у", - "ф", - "х", - "ц", - "ч", - "ш", - "щ", - "ъ", - "ы", - "ь", - "э", - "ю", - "я", - None, - "ѡ", - None, - "ѣ", - None, - "ѥ", - None, - "ѧ", - None, - "ѩ", - None, - "ѫ", - None, - "ѭ", - None, - "ѯ", - None, - "ѱ", - None, - "ѳ", - None, - "ѵ", - None, - "ѷ", - None, - "ѹ", - None, - "ѻ", - None, - "ѽ", - None, - "ѿ", - None, - "ҁ", - None, - "ҋ", - None, - "ҍ", - None, - "ҏ", - None, - "ґ", - None, - "ғ", - None, - "ҕ", - None, - "җ", - None, - "ҙ", - None, - "қ", - None, - "ҝ", - None, - "ҟ", - None, - "ҡ", - None, - "ң", - None, - "ҥ", - None, - "ҧ", - None, - "ҩ", - None, - "ҫ", - None, - "ҭ", - None, - "ү", - None, - "ұ", - None, - "ҳ", - None, - "ҵ", - None, - "ҷ", - None, - "ҹ", - None, - "һ", - None, - "ҽ", - None, - "ҿ", - None, - "ӏ", - "ӂ", - None, - "ӄ", - None, - "ӆ", - None, - "ӈ", - None, - "ӊ", - None, - "ӌ", - None, - "ӎ", - None, - "ӑ", - None, - "ӓ", - None, - "ӕ", - None, - "ӗ", - None, - "ә", - None, - "ӛ", - None, - "ӝ", - None, - "ӟ", - None, - "ӡ", - None, - "ӣ", - None, - "ӥ", - None, - "ӧ", - None, - "ө", - None, - "ӫ", - None, - "ӭ", - None, - "ӯ", - None, - "ӱ", - None, - "ӳ", - None, - "ӵ", - None, - "ӷ", - None, - "ӹ", - None, - "ӻ", - None, - "ӽ", - None, - "ӿ", - None, - "ԁ", - None, - "ԃ", - None, - "ԅ", - None, - "ԇ", - None, - "ԉ", - None, - "ԋ", - None, - "ԍ", - None, - "ԏ", - None, - "ԑ", - None, - "ԓ", - None, - "ԕ", - None, - "ԗ", - None, - "ԙ", - None, - "ԛ", - None, - "ԝ", - None, - "ԟ", - None, - "ԡ", - None, - "ԣ", - None, - "ԥ", - None, - "ԧ", - None, - "ԩ", - None, - "ԫ", - None, - "ԭ", - None, - "ԯ", - None, - None, - "ա", - "բ", - "գ", - "դ", - "ե", - "զ", - "է", - "ը", - "թ", - "ժ", - "ի", - "լ", - "խ", - "ծ", - "կ", - "հ", - "ձ", - "ղ", - "ճ", - "մ", - "յ", - "ն", - "շ", - "ո", - "չ", - "պ", - "ջ", - "ռ", - "ս", - "վ", - "տ", - "ր", - "ց", - "ւ", - "փ", - "ք", - "օ", - "ֆ", - None, - None, - "եւ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "اٴ", - "وٴ", - "ۇٴ", - "يٴ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "क़", - "ख़", - "ग़", - "ज़", - "ड़", - "ढ़", - "फ़", - "य़", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ড়", - "ঢ়", - None, - "য়", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ਲ਼", - None, - None, - "ਸ਼", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ਖ਼", - "ਗ਼", - "ਜ਼", - None, - None, - "ਫ਼", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ଡ଼", - "ଢ଼", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ํา", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ໍາ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ຫນ", - "ຫມ", - None, - None, - None, - "་", - None, - "གྷ", - None, - None, - None, - "ཌྷ", - None, - "དྷ", - None, - "བྷ", - None, - "ཛྷ", - None, - "ཀྵ", - None, - None, - None, - "ཱི", - None, - "ཱུ", - "ྲྀ", - "ྲཱྀ", - "ླྀ", - "ླཱྀ", - None, - "ཱྀ", - None, - "ྒྷ", - None, - None, - None, - "ྜྷ", - None, - "ྡྷ", - None, - "ྦྷ", - None, - "ྫྷ", - None, - "ྐྵ", - None, - None, - None, - None, - None, - None, - None, - "ⴀ", - "ⴁ", - "ⴂ", - "ⴃ", - "ⴄ", - "ⴅ", - "ⴆ", - "ⴇ", - "ⴈ", - "ⴉ", - "ⴊ", - "ⴋ", - "ⴌ", - "ⴍ", - "ⴎ", - "ⴏ", - "ⴐ", - "ⴑ", - "ⴒ", - "ⴓ", - "ⴔ", - "ⴕ", - "ⴖ", - "ⴗ", - "ⴘ", - "ⴙ", - "ⴚ", - "ⴛ", - "ⴜ", - "ⴝ", - "ⴞ", - "ⴟ", - "ⴠ", - "ⴡ", - "ⴢ", - "ⴣ", - "ⴤ", - "ⴥ", - None, - "ⴧ", - None, - "ⴭ", - None, - None, - "ნ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "Ᏸ", - "Ᏹ", - "Ᏺ", - "Ᏻ", - "Ᏼ", - "Ᏽ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "в", - "д", - "о", - "с", - "т", - "ъ", - "ѣ", - "ꙋ", - "\u1c8a", - None, - None, - "ა", - "ბ", - "გ", - "დ", - "ე", - "ვ", - "ზ", - "თ", - "ი", - "კ", - "ლ", - "მ", - "ნ", - "ო", - "პ", - "ჟ", - "რ", - "ს", - "ტ", - "უ", - "ფ", - "ქ", - "ღ", - "ყ", - "შ", - "ჩ", - "ც", - "ძ", - "წ", - "ჭ", - "ხ", - "ჯ", - "ჰ", - "ჱ", - "ჲ", - "ჳ", - "ჴ", - "ჵ", - "ჶ", - "ჷ", - "ჸ", - "ჹ", - "ჺ", - None, - "ჽ", - "ჾ", - "ჿ", - None, - None, - None, - None, - None, - "a", - "æ", - "b", - None, - "d", - "e", - "ǝ", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - None, - "o", - "ȣ", - "p", - "r", - "t", - "u", - "w", - "a", - "ɐ", - "ɑ", - "ᴂ", - "b", - "d", - "e", - "ə", - "ɛ", - "ɜ", - "g", - None, - "k", - "m", - "ŋ", - "o", - "ɔ", - "ᴖ", - "ᴗ", - "p", - "t", - "u", - "ᴝ", - "ɯ", - "v", - "ᴥ", - "β", - "γ", - "δ", - "φ", - "χ", - "i", - "r", - "u", - "v", - "β", - "γ", - "ρ", - "φ", - "χ", - None, - "н", - None, - "ɒ", - "c", - "ɕ", - "ð", - "ɜ", - "f", - "ɟ", - "ɡ", - "ɥ", - "ɨ", - "ɩ", - "ɪ", - "ᵻ", - "ʝ", - "ɭ", - "ᶅ", - "ʟ", - "ɱ", - "ɰ", - "ɲ", - "ɳ", - "ɴ", - "ɵ", - "ɸ", - "ʂ", - "ʃ", - "ƫ", - "ʉ", - "ʊ", - "ᴜ", - "ʋ", - "ʌ", - "z", - "ʐ", - "ʑ", - "ʒ", - "θ", - None, - "ḁ", - None, - "ḃ", - None, - "ḅ", - None, - "ḇ", - None, - "ḉ", - None, - "ḋ", - None, - "ḍ", - None, - "ḏ", - None, - "ḑ", - None, - "ḓ", - None, - "ḕ", - None, - "ḗ", - None, - "ḙ", - None, - "ḛ", - None, - "ḝ", - None, - "ḟ", - None, - "ḡ", - None, - "ḣ", - None, - "ḥ", - None, - "ḧ", - None, - "ḩ", - None, - "ḫ", - None, - "ḭ", - None, - "ḯ", - None, - "ḱ", - None, - "ḳ", - None, - "ḵ", - None, - "ḷ", - None, - "ḹ", - None, - "ḻ", - None, - "ḽ", - None, - "ḿ", - None, - "ṁ", - None, - "ṃ", - None, - "ṅ", - None, - "ṇ", - None, - "ṉ", - None, - "ṋ", - None, - "ṍ", - None, - "ṏ", - None, - "ṑ", - None, - "ṓ", - None, - "ṕ", - None, - "ṗ", - None, - "ṙ", - None, - "ṛ", - None, - "ṝ", - None, - "ṟ", - None, - "ṡ", - None, - "ṣ", - None, - "ṥ", - None, - "ṧ", - None, - "ṩ", - None, - "ṫ", - None, - "ṭ", - None, - "ṯ", - None, - "ṱ", - None, - "ṳ", - None, - "ṵ", - None, - "ṷ", - None, - "ṹ", - None, - "ṻ", - None, - "ṽ", - None, - "ṿ", - None, - "ẁ", - None, - "ẃ", - None, - "ẅ", - None, - "ẇ", - None, - "ẉ", - None, - "ẋ", - None, - "ẍ", - None, - "ẏ", - None, - "ẑ", - None, - "ẓ", - None, - "ẕ", - None, - "aʾ", - "ṡ", - None, - "ß", - None, - "ạ", - None, - "ả", - None, - "ấ", - None, - "ầ", - None, - "ẩ", - None, - "ẫ", - None, - "ậ", - None, - "ắ", - None, - "ằ", - None, - "ẳ", - None, - "ẵ", - None, - "ặ", - None, - "ẹ", - None, - "ẻ", - None, - "ẽ", - None, - "ế", - None, - "ề", - None, - "ể", - None, - "ễ", - None, - "ệ", - None, - "ỉ", - None, - "ị", - None, - "ọ", - None, - "ỏ", - None, - "ố", - None, - "ồ", - None, - "ổ", - None, - "ỗ", - None, - "ộ", - None, - "ớ", - None, - "ờ", - None, - "ở", - None, - "ỡ", - None, - "ợ", - None, - "ụ", - None, - "ủ", - None, - "ứ", - None, - "ừ", - None, - "ử", - None, - "ữ", - None, - "ự", - None, - "ỳ", - None, - "ỵ", - None, - "ỷ", - None, - "ỹ", - None, - "ỻ", - None, - "ỽ", - None, - "ỿ", - None, - "ἀ", - "ἁ", - "ἂ", - "ἃ", - "ἄ", - "ἅ", - "ἆ", - "ἇ", - None, - None, - "ἐ", - "ἑ", - "ἒ", - "ἓ", - "ἔ", - "ἕ", - None, - None, - "ἠ", - "ἡ", - "ἢ", - "ἣ", - "ἤ", - "ἥ", - "ἦ", - "ἧ", - None, - "ἰ", - "ἱ", - "ἲ", - "ἳ", - "ἴ", - "ἵ", - "ἶ", - "ἷ", - None, - None, - "ὀ", - "ὁ", - "ὂ", - "ὃ", - "ὄ", - "ὅ", - None, - None, - None, - "ὑ", - None, - "ὓ", - None, - "ὕ", - None, - "ὗ", - None, - "ὠ", - "ὡ", - "ὢ", - "ὣ", - "ὤ", - "ὥ", - "ὦ", - "ὧ", - None, - "ά", - None, - "έ", - None, - "ή", - None, - "ί", - None, - "ό", - None, - "ύ", - None, - "ώ", - None, - "ἀι", - "ἁι", - "ἂι", - "ἃι", - "ἄι", - "ἅι", - "ἆι", - "ἇι", - "ἀι", - "ἁι", - "ἂι", - "ἃι", - "ἄι", - "ἅι", - "ἆι", - "ἇι", - "ἠι", - "ἡι", - "ἢι", - "ἣι", - "ἤι", - "ἥι", - "ἦι", - "ἧι", - "ἠι", - "ἡι", - "ἢι", - "ἣι", - "ἤι", - "ἥι", - "ἦι", - "ἧι", - "ὠι", - "ὡι", - "ὢι", - "ὣι", - "ὤι", - "ὥι", - "ὦι", - "ὧι", - "ὠι", - "ὡι", - "ὢι", - "ὣι", - "ὤι", - "ὥι", - "ὦι", - "ὧι", - None, - "ὰι", - "αι", - "άι", - None, - None, - "ᾶι", - "ᾰ", - "ᾱ", - "ὰ", - "ά", - "αι", - " ̓", - "ι", - " ̓", - " ͂", - " ̈͂", - "ὴι", - "ηι", - "ήι", - None, - None, - "ῆι", - "ὲ", - "έ", - "ὴ", - "ή", - "ηι", - " ̓̀", - " ̓́", - " ̓͂", - None, - "ΐ", - None, - None, - "ῐ", - "ῑ", - "ὶ", - "ί", - None, - " ̔̀", - " ̔́", - " ̔͂", - None, - "ΰ", - None, - "ῠ", - "ῡ", - "ὺ", - "ύ", - "ῥ", - " ̈̀", - " ̈́", - "`", - None, - "ὼι", - "ωι", - "ώι", - None, - None, - "ῶι", - "ὸ", - "ό", - "ὼ", - "ώ", - "ωι", - " ́", - " ̔", - None, - " ", - None, - "", - None, - None, - "‐", - None, - " ̳", - None, - None, - None, - None, - " ", - None, - "′′", - "′′′", - None, - "‵‵", - "‵‵‵", - None, - "!!", - None, - " ̅", - None, - "??", - "?!", - "!?", - None, - "′′′′", - None, - " ", - None, - None, - None, - "0", - "i", - None, - "4", - "5", - "6", - "7", - "8", - "9", - "+", - "−", - "=", - "(", - ")", - "n", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "+", - "−", - "=", - "(", - ")", - None, - "a", - "e", - "o", - "x", - "ə", - "h", - "k", - "l", - "m", - "n", - "p", - "s", - "t", - None, - None, - "rs", - None, - None, - None, - None, - "a/c", - "a/s", - "c", - "°c", - None, - "c/o", - "c/u", - "ɛ", - None, - "°f", - "g", - "h", - "ħ", - "i", - "l", - None, - "n", - "no", - None, - "p", - "q", - "r", - None, - "sm", - "tel", - "tm", - None, - "z", - None, - "ω", - None, - "z", - None, - "k", - "å", - "b", - "c", - None, - "e", - "f", - "ⅎ", - "m", - "o", - "א", - "ב", - "ג", - "ד", - "i", - None, - "fax", - "π", - "γ", - "π", - "∑", - None, - "d", - "e", - "i", - "j", - None, - "1⁄7", - "1⁄9", - "1⁄10", - "1⁄3", - "2⁄3", - "1⁄5", - "2⁄5", - "3⁄5", - "4⁄5", - "1⁄6", - "5⁄6", - "1⁄8", - "3⁄8", - "5⁄8", - "7⁄8", - "1⁄", - "i", - "ii", - "iii", - "iv", - "v", - "vi", - "vii", - "viii", - "ix", - "x", - "xi", - "xii", - "l", - "c", - "d", - "m", - "i", - "ii", - "iii", - "iv", - "v", - "vi", - "vii", - "viii", - "ix", - "x", - "xi", - "xii", - "l", - "c", - "d", - "m", - None, - "ↄ", - None, - "0⁄3", - None, - None, - None, - "∫∫", - "∫∫∫", - None, - "∮∮", - "∮∮∮", - None, - "〈", - "〉", - None, - None, - None, - None, - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "10", - "11", - "12", - "13", - "14", - "15", - "16", - "17", - "18", - "19", - "20", - "(1)", - "(2)", - "(3)", - "(4)", - "(5)", - "(6)", - "(7)", - "(8)", - "(9)", - "(10)", - "(11)", - "(12)", - "(13)", - "(14)", - "(15)", - "(16)", - "(17)", - "(18)", - "(19)", - "(20)", - None, - "(a)", - "(b)", - "(c)", - "(d)", - "(e)", - "(f)", - "(g)", - "(h)", - "(i)", - "(j)", - "(k)", - "(l)", - "(m)", - "(n)", - "(o)", - "(p)", - "(q)", - "(r)", - "(s)", - "(t)", - "(u)", - "(v)", - "(w)", - "(x)", - "(y)", - "(z)", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "0", - None, - "∫∫∫∫", - None, - "::=", - "==", - "===", - None, - "⫝̸", - None, - None, - None, - "ⰰ", - "ⰱ", - "ⰲ", - "ⰳ", - "ⰴ", - "ⰵ", - "ⰶ", - "ⰷ", - "ⰸ", - "ⰹ", - "ⰺ", - "ⰻ", - "ⰼ", - "ⰽ", - "ⰾ", - "ⰿ", - "ⱀ", - "ⱁ", - "ⱂ", - "ⱃ", - "ⱄ", - "ⱅ", - "ⱆ", - "ⱇ", - "ⱈ", - "ⱉ", - "ⱊ", - "ⱋ", - "ⱌ", - "ⱍ", - "ⱎ", - "ⱏ", - "ⱐ", - "ⱑ", - "ⱒ", - "ⱓ", - "ⱔ", - "ⱕ", - "ⱖ", - "ⱗ", - "ⱘ", - "ⱙ", - "ⱚ", - "ⱛ", - "ⱜ", - "ⱝ", - "ⱞ", - "ⱟ", - None, - "ⱡ", - None, - "ɫ", - "ᵽ", - "ɽ", - None, - "ⱨ", - None, - "ⱪ", - None, - "ⱬ", - None, - "ɑ", - "ɱ", - "ɐ", - "ɒ", - None, - "ⱳ", - None, - "ⱶ", - None, - "j", - "v", - "ȿ", - "ɀ", - "ⲁ", - None, - "ⲃ", - None, - "ⲅ", - None, - "ⲇ", - None, - "ⲉ", - None, - "ⲋ", - None, - "ⲍ", - None, - "ⲏ", - None, - "ⲑ", - None, - "ⲓ", - None, - "ⲕ", - None, - "ⲗ", - None, - "ⲙ", - None, - "ⲛ", - None, - "ⲝ", - None, - "ⲟ", - None, - "ⲡ", - None, - "ⲣ", - None, - "ⲥ", - None, - "ⲧ", - None, - "ⲩ", - None, - "ⲫ", - None, - "ⲭ", - None, - "ⲯ", - None, - "ⲱ", - None, - "ⲳ", - None, - "ⲵ", - None, - "ⲷ", - None, - "ⲹ", - None, - "ⲻ", - None, - "ⲽ", - None, - "ⲿ", - None, - "ⳁ", - None, - "ⳃ", - None, - "ⳅ", - None, - "ⳇ", - None, - "ⳉ", - None, - "ⳋ", - None, - "ⳍ", - None, - "ⳏ", - None, - "ⳑ", - None, - "ⳓ", - None, - "ⳕ", - None, - "ⳗ", - None, - "ⳙ", - None, - "ⳛ", - None, - "ⳝ", - None, - "ⳟ", - None, - "ⳡ", - None, - "ⳣ", - None, - "ⳬ", - None, - "ⳮ", - None, - "ⳳ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ⵡ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "母", - None, - "龟", - None, - "一", - "丨", - "丶", - "丿", - "乙", - "亅", - "二", - "亠", - "人", - "儿", - "入", - "八", - "冂", - "冖", - "冫", - "几", - "凵", - "刀", - "力", - "勹", - "匕", - "匚", - "匸", - "十", - "卜", - "卩", - "厂", - "厶", - "又", - "口", - "囗", - "土", - "士", - "夂", - "夊", - "夕", - "大", - "女", - "子", - "宀", - "寸", - "小", - "尢", - "尸", - "屮", - "山", - "巛", - "工", - "己", - "巾", - "干", - "幺", - "广", - "廴", - "廾", - "弋", - "弓", - "彐", - "彡", - "彳", - "心", - "戈", - "戶", - "手", - "支", - "攴", - "文", - "斗", - "斤", - "方", - "无", - "日", - "曰", - "月", - "木", - "欠", - "止", - "歹", - "殳", - "毋", - "比", - "毛", - "氏", - "气", - "水", - "火", - "爪", - "父", - "爻", - "爿", - "片", - "牙", - "牛", - "犬", - "玄", - "玉", - "瓜", - "瓦", - "甘", - "生", - "用", - "田", - "疋", - "疒", - "癶", - "白", - "皮", - "皿", - "目", - "矛", - "矢", - "石", - "示", - "禸", - "禾", - "穴", - "立", - "竹", - "米", - "糸", - "缶", - "网", - "羊", - "羽", - "老", - "而", - "耒", - "耳", - "聿", - "肉", - "臣", - "自", - "至", - "臼", - "舌", - "舛", - "舟", - "艮", - "色", - "艸", - "虍", - "虫", - "血", - "行", - "衣", - "襾", - "見", - "角", - "言", - "谷", - "豆", - "豕", - "豸", - "貝", - "赤", - "走", - "足", - "身", - "車", - "辛", - "辰", - "辵", - "邑", - "酉", - "釆", - "里", - "金", - "長", - "門", - "阜", - "隶", - "隹", - "雨", - "靑", - "非", - "面", - "革", - "韋", - "韭", - "音", - "頁", - "風", - "飛", - "食", - "首", - "香", - "馬", - "骨", - "高", - "髟", - "鬥", - "鬯", - "鬲", - "鬼", - "魚", - "鳥", - "鹵", - "鹿", - "麥", - "麻", - "黃", - "黍", - "黑", - "黹", - "黽", - "鼎", - "鼓", - "鼠", - "鼻", - "齊", - "齒", - "龍", - "龜", - "龠", - None, - " ", - None, - ".", - None, - "〒", - None, - "十", - "卄", - "卅", - None, - None, - None, - None, - None, - " ゙", - " ゚", - None, - "より", - None, - "コト", - None, - None, - None, - "ᄀ", - "ᄁ", - "ᆪ", - "ᄂ", - "ᆬ", - "ᆭ", - "ᄃ", - "ᄄ", - "ᄅ", - "ᆰ", - "ᆱ", - "ᆲ", - "ᆳ", - "ᆴ", - "ᆵ", - "ᄚ", - "ᄆ", - "ᄇ", - "ᄈ", - "ᄡ", - "ᄉ", - "ᄊ", - "ᄋ", - "ᄌ", - "ᄍ", - "ᄎ", - "ᄏ", - "ᄐ", - "ᄑ", - "ᄒ", - "ᅡ", - "ᅢ", - "ᅣ", - "ᅤ", - "ᅥ", - "ᅦ", - "ᅧ", - "ᅨ", - "ᅩ", - "ᅪ", - "ᅫ", - "ᅬ", - "ᅭ", - "ᅮ", - "ᅯ", - "ᅰ", - "ᅱ", - "ᅲ", - "ᅳ", - "ᅴ", - "ᅵ", - None, - "ᄔ", - "ᄕ", - "ᇇ", - "ᇈ", - "ᇌ", - "ᇎ", - "ᇓ", - "ᇗ", - "ᇙ", - "ᄜ", - "ᇝ", - "ᇟ", - "ᄝ", - "ᄞ", - "ᄠ", - "ᄢ", - "ᄣ", - "ᄧ", - "ᄩ", - "ᄫ", - "ᄬ", - "ᄭ", - "ᄮ", - "ᄯ", - "ᄲ", - "ᄶ", - "ᅀ", - "ᅇ", - "ᅌ", - "ᇱ", - "ᇲ", - "ᅗ", - "ᅘ", - "ᅙ", - "ᆄ", - "ᆅ", - "ᆈ", - "ᆑ", - "ᆒ", - "ᆔ", - "ᆞ", - "ᆡ", - None, - None, - "一", - "二", - "三", - "四", - "上", - "中", - "下", - "甲", - "乙", - "丙", - "丁", - "天", - "地", - "人", - None, - None, - None, - "(ᄀ)", - "(ᄂ)", - "(ᄃ)", - "(ᄅ)", - "(ᄆ)", - "(ᄇ)", - "(ᄉ)", - "(ᄋ)", - "(ᄌ)", - "(ᄎ)", - "(ᄏ)", - "(ᄐ)", - "(ᄑ)", - "(ᄒ)", - "(가)", - "(나)", - "(다)", - "(라)", - "(마)", - "(바)", - "(사)", - "(아)", - "(자)", - "(차)", - "(카)", - "(타)", - "(파)", - "(하)", - "(주)", - "(오전)", - "(오후)", - None, - "(一)", - "(二)", - "(三)", - "(四)", - "(五)", - "(六)", - "(七)", - "(八)", - "(九)", - "(十)", - "(月)", - "(火)", - "(水)", - "(木)", - "(金)", - "(土)", - "(日)", - "(株)", - "(有)", - "(社)", - "(名)", - "(特)", - "(財)", - "(祝)", - "(労)", - "(代)", - "(呼)", - "(学)", - "(監)", - "(企)", - "(資)", - "(協)", - "(祭)", - "(休)", - "(自)", - "(至)", - "問", - "幼", - "文", - "箏", - None, - "pte", - "21", - "22", - "23", - "24", - "25", - "26", - "27", - "28", - "29", - "30", - "31", - "32", - "33", - "34", - "35", - "ᄀ", - "ᄂ", - "ᄃ", - "ᄅ", - "ᄆ", - "ᄇ", - "ᄉ", - "ᄋ", - "ᄌ", - "ᄎ", - "ᄏ", - "ᄐ", - "ᄑ", - "ᄒ", - "가", - "나", - "다", - "라", - "마", - "바", - "사", - "아", - "자", - "차", - "카", - "타", - "파", - "하", - "참고", - "주의", - "우", - None, - "一", - "二", - "三", - "四", - "五", - "六", - "七", - "八", - "九", - "十", - "月", - "火", - "水", - "木", - "金", - "土", - "日", - "株", - "有", - "社", - "名", - "特", - "財", - "祝", - "労", - "秘", - "男", - "女", - "適", - "優", - "印", - "注", - "項", - "休", - "写", - "正", - "上", - "中", - "下", - "左", - "右", - "医", - "宗", - "学", - "監", - "企", - "資", - "協", - "夜", - "36", - "37", - "38", - "39", - "40", - "41", - "42", - "43", - "44", - "45", - "46", - "47", - "48", - "49", - "50", - "1月", - "2月", - "3月", - "4月", - "5月", - "6月", - "7月", - "8月", - "9月", - "10月", - "11月", - "12月", - "hg", - "erg", - "ev", - "ltd", - "ア", - "イ", - "ウ", - "エ", - "オ", - "カ", - "キ", - "ク", - "ケ", - "コ", - "サ", - "シ", - "ス", - "セ", - "ソ", - "タ", - "チ", - "ツ", - "テ", - "ト", - "ナ", - "ニ", - "ヌ", - "ネ", - "ノ", - "ハ", - "ヒ", - "フ", - "ヘ", - "ホ", - "マ", - "ミ", - "ム", - "メ", - "モ", - "ヤ", - "ユ", - "ヨ", - "ラ", - "リ", - "ル", - "レ", - "ロ", - "ワ", - "ヰ", - "ヱ", - "ヲ", - "令和", - "アパート", - "アルファ", - "アンペア", - "アール", - "イニング", - "インチ", - "ウォン", - "エスクード", - "エーカー", - "オンス", - "オーム", - "カイリ", - "カラット", - "カロリー", - "ガロン", - "ガンマ", - "ギガ", - "ギニー", - "キュリー", - "ギルダー", - "キロ", - "キログラム", - "キロメートル", - "キロワット", - "グラム", - "グラムトン", - "クルゼイロ", - "クローネ", - "ケース", - "コルナ", - "コーポ", - "サイクル", - "サンチーム", - "シリング", - "センチ", - "セント", - "ダース", - "デシ", - "ドル", - "トン", - "ナノ", - "ノット", - "ハイツ", - "パーセント", - "パーツ", - "バーレル", - "ピアストル", - "ピクル", - "ピコ", - "ビル", - "ファラッド", - "フィート", - "ブッシェル", - "フラン", - "ヘクタール", - "ペソ", - "ペニヒ", - "ヘルツ", - "ペンス", - "ページ", - "ベータ", - "ポイント", - "ボルト", - "ホン", - "ポンド", - "ホール", - "ホーン", - "マイクロ", - "マイル", - "マッハ", - "マルク", - "マンション", - "ミクロン", - "ミリ", - "ミリバール", - "メガ", - "メガトン", - "メートル", - "ヤード", - "ヤール", - "ユアン", - "リットル", - "リラ", - "ルピー", - "ルーブル", - "レム", - "レントゲン", - "ワット", - "0点", - "1点", - "2点", - "3点", - "4点", - "5点", - "6点", - "7点", - "8点", - "9点", - "10点", - "11点", - "12点", - "13点", - "14点", - "15点", - "16点", - "17点", - "18点", - "19点", - "20点", - "21点", - "22点", - "23点", - "24点", - "hpa", - "da", - "au", - "bar", - "ov", - "pc", - "dm", - "dm2", - "dm3", - "iu", - "平成", - "昭和", - "大正", - "明治", - "株式会社", - "pa", - "na", - "μa", - "ma", - "ka", - "kb", - "mb", - "gb", - "cal", - "kcal", - "pf", - "nf", - "μf", - "μg", - "mg", - "kg", - "hz", - "khz", - "mhz", - "ghz", - "thz", - "μl", - "ml", - "dl", - "kl", - "fm", - "nm", - "μm", - "mm", - "cm", - "km", - "mm2", - "cm2", - "m2", - "km2", - "mm3", - "cm3", - "m3", - "km3", - "m∕s", - "m∕s2", - "pa", - "kpa", - "mpa", - "gpa", - "rad", - "rad∕s", - "rad∕s2", - "ps", - "ns", - "μs", - "ms", - "pv", - "nv", - "μv", - "mv", - "kv", - "mv", - "pw", - "nw", - "μw", - "mw", - "kw", - "mw", - "kω", - "mω", - None, - "bq", - "cc", - "cd", - "c∕kg", - None, - "db", - "gy", - "ha", - "hp", - "in", - "kk", - "km", - "kt", - "lm", - "ln", - "log", - "lx", - "mb", - "mil", - "mol", - "ph", - None, - "ppm", - "pr", - "sr", - "sv", - "wb", - "v∕m", - "a∕m", - "1日", - "2日", - "3日", - "4日", - "5日", - "6日", - "7日", - "8日", - "9日", - "10日", - "11日", - "12日", - "13日", - "14日", - "15日", - "16日", - "17日", - "18日", - "19日", - "20日", - "21日", - "22日", - "23日", - "24日", - "25日", - "26日", - "27日", - "28日", - "29日", - "30日", - "31日", - "gal", - None, - None, - None, - None, - None, - None, - "ꙁ", - None, - "ꙃ", - None, - "ꙅ", - None, - "ꙇ", - None, - "ꙉ", - None, - "ꙋ", - None, - "ꙍ", - None, - "ꙏ", - None, - "ꙑ", - None, - "ꙓ", - None, - "ꙕ", - None, - "ꙗ", - None, - "ꙙ", - None, - "ꙛ", - None, - "ꙝ", - None, - "ꙟ", - None, - "ꙡ", - None, - "ꙣ", - None, - "ꙥ", - None, - "ꙧ", - None, - "ꙩ", - None, - "ꙫ", - None, - "ꙭ", - None, - "ꚁ", - None, - "ꚃ", - None, - "ꚅ", - None, - "ꚇ", - None, - "ꚉ", - None, - "ꚋ", - None, - "ꚍ", - None, - "ꚏ", - None, - "ꚑ", - None, - "ꚓ", - None, - "ꚕ", - None, - "ꚗ", - None, - "ꚙ", - None, - "ꚛ", - None, - "ъ", - "ь", - None, - None, - None, - "ꜣ", - None, - "ꜥ", - None, - "ꜧ", - None, - "ꜩ", - None, - "ꜫ", - None, - "ꜭ", - None, - "ꜯ", - None, - "ꜳ", - None, - "ꜵ", - None, - "ꜷ", - None, - "ꜹ", - None, - "ꜻ", - None, - "ꜽ", - None, - "ꜿ", - None, - "ꝁ", - None, - "ꝃ", - None, - "ꝅ", - None, - "ꝇ", - None, - "ꝉ", - None, - "ꝋ", - None, - "ꝍ", - None, - "ꝏ", - None, - "ꝑ", - None, - "ꝓ", - None, - "ꝕ", - None, - "ꝗ", - None, - "ꝙ", - None, - "ꝛ", - None, - "ꝝ", - None, - "ꝟ", - None, - "ꝡ", - None, - "ꝣ", - None, - "ꝥ", - None, - "ꝧ", - None, - "ꝩ", - None, - "ꝫ", - None, - "ꝭ", - None, - "ꝯ", - None, - "ꝯ", - None, - "ꝺ", - None, - "ꝼ", - None, - "ᵹ", - "ꝿ", - None, - "ꞁ", - None, - "ꞃ", - None, - "ꞅ", - None, - "ꞇ", - None, - "ꞌ", - None, - "ɥ", - None, - "ꞑ", - None, - "ꞓ", - None, - "ꞗ", - None, - "ꞙ", - None, - "ꞛ", - None, - "ꞝ", - None, - "ꞟ", - None, - "ꞡ", - None, - "ꞣ", - None, - "ꞥ", - None, - "ꞧ", - None, - "ꞩ", - None, - "ɦ", - "ɜ", - "ɡ", - "ɬ", - "ɪ", - None, - "ʞ", - "ʇ", - "ʝ", - "ꭓ", - "ꞵ", - None, - "ꞷ", - None, - "ꞹ", - None, - "ꞻ", - None, - "ꞽ", - None, - "ꞿ", - None, - "ꟁ", - None, - "ꟃ", - None, - "ꞔ", - "ʂ", - "ᶎ", - "ꟈ", - None, - "ꟊ", - None, - "ɤ", - "\ua7cd", - None, - "\ua7cf", - None, - "ꟑ", - None, - "ꟓ", - None, - "ꟕ", - None, - "ꟗ", - None, - "ꟙ", - None, - "\ua7db", - None, - "ƛ", - None, - "s", - "c", - "f", - "q", - "ꟶ", - None, - "ħ", - "œ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ꜧ", - "ꬷ", - "ɫ", - "ꭒ", - None, - "ʍ", - None, - None, - "Ꭰ", - "Ꭱ", - "Ꭲ", - "Ꭳ", - "Ꭴ", - "Ꭵ", - "Ꭶ", - "Ꭷ", - "Ꭸ", - "Ꭹ", - "Ꭺ", - "Ꭻ", - "Ꭼ", - "Ꭽ", - "Ꭾ", - "Ꭿ", - "Ꮀ", - "Ꮁ", - "Ꮂ", - "Ꮃ", - "Ꮄ", - "Ꮅ", - "Ꮆ", - "Ꮇ", - "Ꮈ", - "Ꮉ", - "Ꮊ", - "Ꮋ", - "Ꮌ", - "Ꮍ", - "Ꮎ", - "Ꮏ", - "Ꮐ", - "Ꮑ", - "Ꮒ", - "Ꮓ", - "Ꮔ", - "Ꮕ", - "Ꮖ", - "Ꮗ", - "Ꮘ", - "Ꮙ", - "Ꮚ", - "Ꮛ", - "Ꮜ", - "Ꮝ", - "Ꮞ", - "Ꮟ", - "Ꮠ", - "Ꮡ", - "Ꮢ", - "Ꮣ", - "Ꮤ", - "Ꮥ", - "Ꮦ", - "Ꮧ", - "Ꮨ", - "Ꮩ", - "Ꮪ", - "Ꮫ", - "Ꮬ", - "Ꮭ", - "Ꮮ", - "Ꮯ", - "Ꮰ", - "Ꮱ", - "Ꮲ", - "Ꮳ", - "Ꮴ", - "Ꮵ", - "Ꮶ", - "Ꮷ", - "Ꮸ", - "Ꮹ", - "Ꮺ", - "Ꮻ", - "Ꮼ", - "Ꮽ", - "Ꮾ", - "Ꮿ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "豈", - "更", - "車", - "賈", - "滑", - "串", - "句", - "龜", - "契", - "金", - "喇", - "奈", - "懶", - "癩", - "羅", - "蘿", - "螺", - "裸", - "邏", - "樂", - "洛", - "烙", - "珞", - "落", - "酪", - "駱", - "亂", - "卵", - "欄", - "爛", - "蘭", - "鸞", - "嵐", - "濫", - "藍", - "襤", - "拉", - "臘", - "蠟", - "廊", - "朗", - "浪", - "狼", - "郎", - "來", - "冷", - "勞", - "擄", - "櫓", - "爐", - "盧", - "老", - "蘆", - "虜", - "路", - "露", - "魯", - "鷺", - "碌", - "祿", - "綠", - "菉", - "錄", - "鹿", - "論", - "壟", - "弄", - "籠", - "聾", - "牢", - "磊", - "賂", - "雷", - "壘", - "屢", - "樓", - "淚", - "漏", - "累", - "縷", - "陋", - "勒", - "肋", - "凜", - "凌", - "稜", - "綾", - "菱", - "陵", - "讀", - "拏", - "樂", - "諾", - "丹", - "寧", - "怒", - "率", - "異", - "北", - "磻", - "便", - "復", - "不", - "泌", - "數", - "索", - "參", - "塞", - "省", - "葉", - "說", - "殺", - "辰", - "沈", - "拾", - "若", - "掠", - "略", - "亮", - "兩", - "凉", - "梁", - "糧", - "良", - "諒", - "量", - "勵", - "呂", - "女", - "廬", - "旅", - "濾", - "礪", - "閭", - "驪", - "麗", - "黎", - "力", - "曆", - "歷", - "轢", - "年", - "憐", - "戀", - "撚", - "漣", - "煉", - "璉", - "秊", - "練", - "聯", - "輦", - "蓮", - "連", - "鍊", - "列", - "劣", - "咽", - "烈", - "裂", - "說", - "廉", - "念", - "捻", - "殮", - "簾", - "獵", - "令", - "囹", - "寧", - "嶺", - "怜", - "玲", - "瑩", - "羚", - "聆", - "鈴", - "零", - "靈", - "領", - "例", - "禮", - "醴", - "隸", - "惡", - "了", - "僚", - "寮", - "尿", - "料", - "樂", - "燎", - "療", - "蓼", - "遼", - "龍", - "暈", - "阮", - "劉", - "杻", - "柳", - "流", - "溜", - "琉", - "留", - "硫", - "紐", - "類", - "六", - "戮", - "陸", - "倫", - "崙", - "淪", - "輪", - "律", - "慄", - "栗", - "率", - "隆", - "利", - "吏", - "履", - "易", - "李", - "梨", - "泥", - "理", - "痢", - "罹", - "裏", - "裡", - "里", - "離", - "匿", - "溺", - "吝", - "燐", - "璘", - "藺", - "隣", - "鱗", - "麟", - "林", - "淋", - "臨", - "立", - "笠", - "粒", - "狀", - "炙", - "識", - "什", - "茶", - "刺", - "切", - "度", - "拓", - "糖", - "宅", - "洞", - "暴", - "輻", - "行", - "降", - "見", - "廓", - "兀", - "嗀", - None, - "塚", - None, - "晴", - None, - "凞", - "猪", - "益", - "礼", - "神", - "祥", - "福", - "靖", - "精", - "羽", - None, - "蘒", - None, - "諸", - None, - "逸", - "都", - None, - "飯", - "飼", - "館", - "鶴", - "郞", - "隷", - "侮", - "僧", - "免", - "勉", - "勤", - "卑", - "喝", - "嘆", - "器", - "塀", - "墨", - "層", - "屮", - "悔", - "慨", - "憎", - "懲", - "敏", - "既", - "暑", - "梅", - "海", - "渚", - "漢", - "煮", - "爫", - "琢", - "碑", - "社", - "祉", - "祈", - "祐", - "祖", - "祝", - "禍", - "禎", - "穀", - "突", - "節", - "練", - "縉", - "繁", - "署", - "者", - "臭", - "艹", - "著", - "褐", - "視", - "謁", - "謹", - "賓", - "贈", - "辶", - "逸", - "難", - "響", - "頻", - "恵", - "𤋮", - "舘", - None, - "並", - "况", - "全", - "侀", - "充", - "冀", - "勇", - "勺", - "喝", - "啕", - "喙", - "嗢", - "塚", - "墳", - "奄", - "奔", - "婢", - "嬨", - "廒", - "廙", - "彩", - "徭", - "惘", - "慎", - "愈", - "憎", - "慠", - "懲", - "戴", - "揄", - "搜", - "摒", - "敖", - "晴", - "朗", - "望", - "杖", - "歹", - "殺", - "流", - "滛", - "滋", - "漢", - "瀞", - "煮", - "瞧", - "爵", - "犯", - "猪", - "瑱", - "甆", - "画", - "瘝", - "瘟", - "益", - "盛", - "直", - "睊", - "着", - "磌", - "窱", - "節", - "类", - "絛", - "練", - "缾", - "者", - "荒", - "華", - "蝹", - "襁", - "覆", - "視", - "調", - "諸", - "請", - "謁", - "諾", - "諭", - "謹", - "變", - "贈", - "輸", - "遲", - "醙", - "鉶", - "陼", - "難", - "靖", - "韛", - "響", - "頋", - "頻", - "鬒", - "龜", - "𢡊", - "𢡄", - "𣏕", - "㮝", - "䀘", - "䀹", - "𥉉", - "𥳐", - "𧻓", - "齃", - "龎", - None, - "ff", - "fi", - "fl", - "ffi", - "ffl", - "st", - None, - "մն", - "մե", - "մի", - "վն", - "մխ", - None, - "יִ", - None, - "ײַ", - "ע", - "א", - "ד", - "ה", - "כ", - "ל", - "ם", - "ר", - "ת", - "+", - "שׁ", - "שׂ", - "שּׁ", - "שּׂ", - "אַ", - "אָ", - "אּ", - "בּ", - "גּ", - "דּ", - "הּ", - "וּ", - "זּ", - None, - "טּ", - "יּ", - "ךּ", - "כּ", - "לּ", - None, - "מּ", - None, - "נּ", - "סּ", - None, - "ףּ", - "פּ", - None, - "צּ", - "קּ", - "רּ", - "שּ", - "תּ", - "וֹ", - "בֿ", - "כֿ", - "פֿ", - "אל", - "ٱ", - "ٻ", - "پ", - "ڀ", - "ٺ", - "ٿ", - "ٹ", - "ڤ", - "ڦ", - "ڄ", - "ڃ", - "چ", - "ڇ", - "ڍ", - "ڌ", - "ڎ", - "ڈ", - "ژ", - "ڑ", - "ک", - "گ", - "ڳ", - "ڱ", - "ں", - "ڻ", - "ۀ", - "ہ", - "ھ", - "ے", - "ۓ", - None, - "ڭ", - "ۇ", - "ۆ", - "ۈ", - "ۇٴ", - "ۋ", - "ۅ", - "ۉ", - "ې", - "ى", - "ئا", - "ئە", - "ئو", - "ئۇ", - "ئۆ", - "ئۈ", - "ئې", - "ئى", - "ی", - "ئج", - "ئح", - "ئم", - "ئى", - "ئي", - "بج", - "بح", - "بخ", - "بم", - "بى", - "بي", - "تج", - "تح", - "تخ", - "تم", - "تى", - "تي", - "ثج", - "ثم", - "ثى", - "ثي", - "جح", - "جم", - "حج", - "حم", - "خج", - "خح", - "خم", - "سج", - "سح", - "سخ", - "سم", - "صح", - "صم", - "ضج", - "ضح", - "ضخ", - "ضم", - "طح", - "طم", - "ظم", - "عج", - "عم", - "غج", - "غم", - "فج", - "فح", - "فخ", - "فم", - "فى", - "في", - "قح", - "قم", - "قى", - "قي", - "كا", - "كج", - "كح", - "كخ", - "كل", - "كم", - "كى", - "كي", - "لج", - "لح", - "لخ", - "لم", - "لى", - "لي", - "مج", - "مح", - "مخ", - "مم", - "مى", - "مي", - "نج", - "نح", - "نخ", - "نم", - "نى", - "ني", - "هج", - "هم", - "هى", - "هي", - "يج", - "يح", - "يخ", - "يم", - "يى", - "يي", - "ذٰ", - "رٰ", - "ىٰ", - " ٌّ", - " ٍّ", - " َّ", - " ُّ", - " ِّ", - " ّٰ", - "ئر", - "ئز", - "ئم", - "ئن", - "ئى", - "ئي", - "بر", - "بز", - "بم", - "بن", - "بى", - "بي", - "تر", - "تز", - "تم", - "تن", - "تى", - "تي", - "ثر", - "ثز", - "ثم", - "ثن", - "ثى", - "ثي", - "فى", - "في", - "قى", - "قي", - "كا", - "كل", - "كم", - "كى", - "كي", - "لم", - "لى", - "لي", - "ما", - "مم", - "نر", - "نز", - "نم", - "نن", - "نى", - "ني", - "ىٰ", - "ير", - "يز", - "يم", - "ين", - "يى", - "يي", - "ئج", - "ئح", - "ئخ", - "ئم", - "ئه", - "بج", - "بح", - "بخ", - "بم", - "به", - "تج", - "تح", - "تخ", - "تم", - "ته", - "ثم", - "جح", - "جم", - "حج", - "حم", - "خج", - "خم", - "سج", - "سح", - "سخ", - "سم", - "صح", - "صخ", - "صم", - "ضج", - "ضح", - "ضخ", - "ضم", - "طح", - "ظم", - "عج", - "عم", - "غج", - "غم", - "فج", - "فح", - "فخ", - "فم", - "قح", - "قم", - "كج", - "كح", - "كخ", - "كل", - "كم", - "لج", - "لح", - "لخ", - "لم", - "له", - "مج", - "مح", - "مخ", - "مم", - "نج", - "نح", - "نخ", - "نم", - "نه", - "هج", - "هم", - "هٰ", - "يج", - "يح", - "يخ", - "يم", - "يه", - "ئم", - "ئه", - "بم", - "به", - "تم", - "ته", - "ثم", - "ثه", - "سم", - "سه", - "شم", - "شه", - "كل", - "كم", - "لم", - "نم", - "نه", - "يم", - "يه", - "ـَّ", - "ـُّ", - "ـِّ", - "طى", - "طي", - "عى", - "عي", - "غى", - "غي", - "سى", - "سي", - "شى", - "شي", - "حى", - "حي", - "جى", - "جي", - "خى", - "خي", - "صى", - "صي", - "ضى", - "ضي", - "شج", - "شح", - "شخ", - "شم", - "شر", - "سر", - "صر", - "ضر", - "طى", - "طي", - "عى", - "عي", - "غى", - "غي", - "سى", - "سي", - "شى", - "شي", - "حى", - "حي", - "جى", - "جي", - "خى", - "خي", - "صى", - "صي", - "ضى", - "ضي", - "شج", - "شح", - "شخ", - "شم", - "شر", - "سر", - "صر", - "ضر", - "شج", - "شح", - "شخ", - "شم", - "سه", - "شه", - "طم", - "سج", - "سح", - "سخ", - "شج", - "شح", - "شخ", - "طم", - "ظم", - "اً", - None, - "تجم", - "تحج", - "تحم", - "تخم", - "تمج", - "تمح", - "تمخ", - "جمح", - "حمي", - "حمى", - "سحج", - "سجح", - "سجى", - "سمح", - "سمج", - "سمم", - "صحح", - "صمم", - "شحم", - "شجي", - "شمخ", - "شمم", - "ضحى", - "ضخم", - "طمح", - "طمم", - "طمي", - "عجم", - "عمم", - "عمى", - "غمم", - "غمي", - "غمى", - "فخم", - "قمح", - "قمم", - "لحم", - "لحي", - "لحى", - "لجج", - "لخم", - "لمح", - "محج", - "محم", - "محي", - "مجح", - "مجم", - "مخج", - "مخم", - None, - "مجخ", - "همج", - "همم", - "نحم", - "نحى", - "نجم", - "نجى", - "نمي", - "نمى", - "يمم", - "بخي", - "تجي", - "تجى", - "تخي", - "تخى", - "تمي", - "تمى", - "جمي", - "جحى", - "جمى", - "سخى", - "صحي", - "شحي", - "ضحي", - "لجي", - "لمي", - "يحي", - "يجي", - "يمي", - "ممي", - "قمي", - "نحي", - "قمح", - "لحم", - "عمي", - "كمي", - "نجح", - "مخي", - "لجم", - "كمم", - "لجم", - "نجح", - "جحي", - "حجي", - "مجي", - "فمي", - "بحي", - "كمم", - "عجم", - "صمم", - "سخي", - "نجي", - None, - None, - "صلے", - "قلے", - "الله", - "اكبر", - "محمد", - "صلعم", - "رسول", - "عليه", - "وسلم", - "صلى", - "صلى الله عليه وسلم", - "جل جلاله", - "ریال", - None, - None, - ",", - "、", - None, - ":", - ";", - "!", - "?", - "〖", - "〗", - None, - None, - None, - "—", - "–", - "_", - "(", - ")", - "{", - "}", - "〔", - "〕", - "【", - "】", - "《", - "》", - "〈", - "〉", - "「", - "」", - "『", - "』", - None, - "[", - "]", - " ̅", - "_", - ",", - "、", - None, - ";", - ":", - "?", - "!", - "—", - "(", - ")", - "{", - "}", - "〔", - "〕", - "#", - "&", - "*", - "+", - "-", - "<", - ">", - "=", - None, - "\\", - "$", - "%", - "@", - None, - " ً", - "ـً", - " ٌ", - None, - " ٍ", - None, - " َ", - "ـَ", - " ُ", - "ـُ", - " ِ", - "ـِ", - " ّ", - "ـّ", - " ْ", - "ـْ", - "ء", - "آ", - "أ", - "ؤ", - "إ", - "ئ", - "ا", - "ب", - "ة", - "ت", - "ث", - "ج", - "ح", - "خ", - "د", - "ذ", - "ر", - "ز", - "س", - "ش", - "ص", - "ض", - "ط", - "ظ", - "ع", - "غ", - "ف", - "ق", - "ك", - "ل", - "م", - "ن", - "ه", - "و", - "ى", - "ي", - "لآ", - "لأ", - "لإ", - "لا", - None, - None, - None, - "!", - '"', - "#", - "$", - "%", - "&", - "'", - "(", - ")", - "*", - "+", - ",", - "-", - ".", - "/", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - ":", - ";", - "<", - "=", - ">", - "?", - "@", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "[", - "\\", - "]", - "^", - "_", - "`", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "{", - "|", - "}", - "~", - "⦅", - "⦆", - ".", - "「", - "」", - "、", - "・", - "ヲ", - "ァ", - "ィ", - "ゥ", - "ェ", - "ォ", - "ャ", - "ュ", - "ョ", - "ッ", - "ー", - "ア", - "イ", - "ウ", - "エ", - "オ", - "カ", - "キ", - "ク", - "ケ", - "コ", - "サ", - "シ", - "ス", - "セ", - "ソ", - "タ", - "チ", - "ツ", - "テ", - "ト", - "ナ", - "ニ", - "ヌ", - "ネ", - "ノ", - "ハ", - "ヒ", - "フ", - "ヘ", - "ホ", - "マ", - "ミ", - "ム", - "メ", - "モ", - "ヤ", - "ユ", - "ヨ", - "ラ", - "リ", - "ル", - "レ", - "ロ", - "ワ", - "ン", - "゙", - "゚", - None, - "ᄀ", - "ᄁ", - "ᆪ", - "ᄂ", - "ᆬ", - "ᆭ", - "ᄃ", - "ᄄ", - "ᄅ", - "ᆰ", - "ᆱ", - "ᆲ", - "ᆳ", - "ᆴ", - "ᆵ", - "ᄚ", - "ᄆ", - "ᄇ", - "ᄈ", - "ᄡ", - "ᄉ", - "ᄊ", - "ᄋ", - "ᄌ", - "ᄍ", - "ᄎ", - "ᄏ", - "ᄐ", - "ᄑ", - "ᄒ", - None, - "ᅡ", - "ᅢ", - "ᅣ", - "ᅤ", - "ᅥ", - "ᅦ", - None, - "ᅧ", - "ᅨ", - "ᅩ", - "ᅪ", - "ᅫ", - "ᅬ", - None, - "ᅭ", - "ᅮ", - "ᅯ", - "ᅰ", - "ᅱ", - "ᅲ", - None, - "ᅳ", - "ᅴ", - "ᅵ", - None, - "¢", - "£", - "¬", - " ̄", - "¦", - "¥", - "₩", - None, - "│", - "←", - "↑", - "→", - "↓", - "■", - "○", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𐐨", - "𐐩", - "𐐪", - "𐐫", - "𐐬", - "𐐭", - "𐐮", - "𐐯", - "𐐰", - "𐐱", - "𐐲", - "𐐳", - "𐐴", - "𐐵", - "𐐶", - "𐐷", - "𐐸", - "𐐹", - "𐐺", - "𐐻", - "𐐼", - "𐐽", - "𐐾", - "𐐿", - "𐑀", - "𐑁", - "𐑂", - "𐑃", - "𐑄", - "𐑅", - "𐑆", - "𐑇", - "𐑈", - "𐑉", - "𐑊", - "𐑋", - "𐑌", - "𐑍", - "𐑎", - "𐑏", - None, - None, - None, - None, - "𐓘", - "𐓙", - "𐓚", - "𐓛", - "𐓜", - "𐓝", - "𐓞", - "𐓟", - "𐓠", - "𐓡", - "𐓢", - "𐓣", - "𐓤", - "𐓥", - "𐓦", - "𐓧", - "𐓨", - "𐓩", - "𐓪", - "𐓫", - "𐓬", - "𐓭", - "𐓮", - "𐓯", - "𐓰", - "𐓱", - "𐓲", - "𐓳", - "𐓴", - "𐓵", - "𐓶", - "𐓷", - "𐓸", - "𐓹", - "𐓺", - "𐓻", - None, - None, - None, - None, - None, - None, - None, - None, - "𐖗", - "𐖘", - "𐖙", - "𐖚", - "𐖛", - "𐖜", - "𐖝", - "𐖞", - "𐖟", - "𐖠", - "𐖡", - None, - "𐖣", - "𐖤", - "𐖥", - "𐖦", - "𐖧", - "𐖨", - "𐖩", - "𐖪", - "𐖫", - "𐖬", - "𐖭", - "𐖮", - "𐖯", - "𐖰", - "𐖱", - None, - "𐖳", - "𐖴", - "𐖵", - "𐖶", - "𐖷", - "𐖸", - "𐖹", - None, - "𐖻", - "𐖼", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ː", - "ˑ", - "æ", - "ʙ", - "ɓ", - None, - "ʣ", - "ꭦ", - "ʥ", - "ʤ", - "ɖ", - "ɗ", - "ᶑ", - "ɘ", - "ɞ", - "ʩ", - "ɤ", - "ɢ", - "ɠ", - "ʛ", - "ħ", - "ʜ", - "ɧ", - "ʄ", - "ʪ", - "ʫ", - "ɬ", - "𝼄", - "ꞎ", - "ɮ", - "𝼅", - "ʎ", - "𝼆", - "ø", - "ɶ", - "ɷ", - "q", - "ɺ", - "𝼈", - "ɽ", - "ɾ", - "ʀ", - "ʨ", - "ʦ", - "ꭧ", - "ʧ", - "ʈ", - "ⱱ", - None, - "ʏ", - "ʡ", - "ʢ", - "ʘ", - "ǀ", - "ǁ", - "ǂ", - "𝼊", - "𝼞", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𐳀", - "𐳁", - "𐳂", - "𐳃", - "𐳄", - "𐳅", - "𐳆", - "𐳇", - "𐳈", - "𐳉", - "𐳊", - "𐳋", - "𐳌", - "𐳍", - "𐳎", - "𐳏", - "𐳐", - "𐳑", - "𐳒", - "𐳓", - "𐳔", - "𐳕", - "𐳖", - "𐳗", - "𐳘", - "𐳙", - "𐳚", - "𐳛", - "𐳜", - "𐳝", - "𐳞", - "𐳟", - "𐳠", - "𐳡", - "𐳢", - "𐳣", - "𐳤", - "𐳥", - "𐳦", - "𐳧", - "𐳨", - "𐳩", - "𐳪", - "𐳫", - "𐳬", - "𐳭", - "𐳮", - "𐳯", - "𐳰", - "𐳱", - "𐳲", - None, - None, - None, - None, - None, - None, - None, - None, - "\U00010d70", - "\U00010d71", - "\U00010d72", - "\U00010d73", - "\U00010d74", - "\U00010d75", - "\U00010d76", - "\U00010d77", - "\U00010d78", - "\U00010d79", - "\U00010d7a", - "\U00010d7b", - "\U00010d7c", - "\U00010d7d", - "\U00010d7e", - "\U00010d7f", - "\U00010d80", - "\U00010d81", - "\U00010d82", - "\U00010d83", - "\U00010d84", - "\U00010d85", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𑣀", - "𑣁", - "𑣂", - "𑣃", - "𑣄", - "𑣅", - "𑣆", - "𑣇", - "𑣈", - "𑣉", - "𑣊", - "𑣋", - "𑣌", - "𑣍", - "𑣎", - "𑣏", - "𑣐", - "𑣑", - "𑣒", - "𑣓", - "𑣔", - "𑣕", - "𑣖", - "𑣗", - "𑣘", - "𑣙", - "𑣚", - "𑣛", - "𑣜", - "𑣝", - "𑣞", - "𑣟", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𖹠", - "𖹡", - "𖹢", - "𖹣", - "𖹤", - "𖹥", - "𖹦", - "𖹧", - "𖹨", - "𖹩", - "𖹪", - "𖹫", - "𖹬", - "𖹭", - "𖹮", - "𖹯", - "𖹰", - "𖹱", - "𖹲", - "𖹳", - "𖹴", - "𖹵", - "𖹶", - "𖹷", - "𖹸", - "𖹹", - "𖹺", - "𖹻", - "𖹼", - "𖹽", - "𖹾", - "𖹿", - None, - None, - "\U00016ebb", - "\U00016ebc", - "\U00016ebd", - "\U00016ebe", - "\U00016ebf", - "\U00016ec0", - "\U00016ec1", - "\U00016ec2", - "\U00016ec3", - "\U00016ec4", - "\U00016ec5", - "\U00016ec6", - "\U00016ec7", - "\U00016ec8", - "\U00016ec9", - "\U00016eca", - "\U00016ecb", - "\U00016ecc", - "\U00016ecd", - "\U00016ece", - "\U00016ecf", - "\U00016ed0", - "\U00016ed1", - "\U00016ed2", - "\U00016ed3", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𝅗𝅥", - "𝅘𝅥", - "𝅘𝅥𝅮", - "𝅘𝅥𝅯", - "𝅘𝅥𝅰", - "𝅘𝅥𝅱", - "𝅘𝅥𝅲", - None, - None, - None, - "𝆹𝅥", - "𝆺𝅥", - "𝆹𝅥𝅮", - "𝆺𝅥𝅮", - "𝆹𝅥𝅯", - "𝆺𝅥𝅯", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - None, - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - None, - "c", - "d", - None, - "g", - None, - "j", - "k", - None, - "n", - "o", - "p", - "q", - None, - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - None, - "f", - None, - "h", - "i", - "j", - "k", - "l", - "m", - "n", - None, - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - None, - "d", - "e", - "f", - "g", - None, - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - None, - "s", - "t", - "u", - "v", - "w", - "x", - "y", - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - None, - "d", - "e", - "f", - "g", - None, - "i", - "j", - "k", - "l", - "m", - None, - "o", - None, - "s", - "t", - "u", - "v", - "w", - "x", - "y", - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "ı", - "ȷ", - None, - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "θ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∇", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∂", - "ε", - "θ", - "κ", - "φ", - "ρ", - "π", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "θ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∇", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∂", - "ε", - "θ", - "κ", - "φ", - "ρ", - "π", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "θ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∇", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∂", - "ε", - "θ", - "κ", - "φ", - "ρ", - "π", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "θ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∇", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∂", - "ε", - "θ", - "κ", - "φ", - "ρ", - "π", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "θ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∇", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∂", - "ε", - "θ", - "κ", - "φ", - "ρ", - "π", - "ϝ", - None, - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "а", - "б", - "в", - "г", - "д", - "е", - "ж", - "з", - "и", - "к", - "л", - "м", - "о", - "п", - "р", - "с", - "т", - "у", - "ф", - "х", - "ц", - "ч", - "ш", - "ы", - "э", - "ю", - "ꚉ", - "ә", - "і", - "ј", - "ө", - "ү", - "ӏ", - "а", - "б", - "в", - "г", - "д", - "е", - "ж", - "з", - "и", - "к", - "л", - "о", - "п", - "с", - "у", - "ф", - "х", - "ц", - "ч", - "ш", - "ъ", - "ы", - "ґ", - "і", - "ѕ", - "џ", - "ҫ", - "ꙑ", - "ұ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𞤢", - "𞤣", - "𞤤", - "𞤥", - "𞤦", - "𞤧", - "𞤨", - "𞤩", - "𞤪", - "𞤫", - "𞤬", - "𞤭", - "𞤮", - "𞤯", - "𞤰", - "𞤱", - "𞤲", - "𞤳", - "𞤴", - "𞤵", - "𞤶", - "𞤷", - "𞤸", - "𞤹", - "𞤺", - "𞤻", - "𞤼", - "𞤽", - "𞤾", - "𞤿", - "𞥀", - "𞥁", - "𞥂", - "𞥃", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ا", - "ب", - "ج", - "د", - None, - "و", - "ز", - "ح", - "ط", - "ي", - "ك", - "ل", - "م", - "ن", - "س", - "ع", - "ف", - "ص", - "ق", - "ر", - "ش", - "ت", - "ث", - "خ", - "ذ", - "ض", - "ظ", - "غ", - "ٮ", - "ں", - "ڡ", - "ٯ", - None, - "ب", - "ج", - None, - "ه", - None, - "ح", - None, - "ي", - "ك", - "ل", - "م", - "ن", - "س", - "ع", - "ف", - "ص", - "ق", - None, - "ش", - "ت", - "ث", - "خ", - None, - "ض", - None, - "غ", - None, - "ج", - None, - "ح", - None, - "ي", - None, - "ل", - None, - "ن", - "س", - "ع", - None, - "ص", - "ق", - None, - "ش", - None, - "خ", - None, - "ض", - None, - "غ", - None, - "ں", - None, - "ٯ", - None, - "ب", - "ج", - None, - "ه", - None, - "ح", - "ط", - "ي", - "ك", - None, - "م", - "ن", - "س", - "ع", - "ف", - "ص", - "ق", - None, - "ش", - "ت", - "ث", - "خ", - None, - "ض", - "ظ", - "غ", - "ٮ", - None, - "ڡ", - None, - "ا", - "ب", - "ج", - "د", - "ه", - "و", - "ز", - "ح", - "ط", - "ي", - None, - "ل", - "م", - "ن", - "س", - "ع", - "ف", - "ص", - "ق", - "ر", - "ش", - "ت", - "ث", - "خ", - "ذ", - "ض", - "ظ", - "غ", - None, - "ب", - "ج", - "د", - None, - "و", - "ز", - "ح", - "ط", - "ي", - None, - "ل", - "م", - "ن", - "س", - "ع", - "ف", - "ص", - "ق", - "ر", - "ش", - "ت", - "ث", - "خ", - "ذ", - "ض", - "ظ", - "غ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "0,", - "1,", - "2,", - "3,", - "4,", - "5,", - "6,", - "7,", - "8,", - "9,", - None, - "(a)", - "(b)", - "(c)", - "(d)", - "(e)", - "(f)", - "(g)", - "(h)", - "(i)", - "(j)", - "(k)", - "(l)", - "(m)", - "(n)", - "(o)", - "(p)", - "(q)", - "(r)", - "(s)", - "(t)", - "(u)", - "(v)", - "(w)", - "(x)", - "(y)", - "(z)", - "〔s〕", - "c", - "r", - "cd", - "wz", - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "hv", - "mv", - "sd", - "ss", - "ppv", - "wc", - None, - "mc", - "md", - "mr", - None, - "dj", - None, - None, - None, - "ほか", - "ココ", - "サ", - None, - "手", - "字", - "双", - "デ", - "二", - "多", - "解", - "天", - "交", - "映", - "無", - "料", - "前", - "後", - "再", - "新", - "初", - "終", - "生", - "販", - "声", - "吹", - "演", - "投", - "捕", - "一", - "三", - "遊", - "左", - "中", - "右", - "指", - "走", - "打", - "禁", - "空", - "合", - "満", - "有", - "月", - "申", - "割", - "営", - "配", - None, - "〔本〕", - "〔三〕", - "〔二〕", - "〔安〕", - "〔点〕", - "〔打〕", - "〔盗〕", - "〔勝〕", - "〔敗〕", - None, - "得", - "可", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "丽", - "丸", - "乁", - "𠄢", - "你", - "侮", - "侻", - "倂", - "偺", - "備", - "僧", - "像", - "㒞", - "𠘺", - "免", - "兔", - "兤", - "具", - "𠔜", - "㒹", - "內", - "再", - "𠕋", - "冗", - "冤", - "仌", - "冬", - "况", - "𩇟", - "凵", - "刃", - "㓟", - "刻", - "剆", - "割", - "剷", - "㔕", - "勇", - "勉", - "勤", - "勺", - "包", - "匆", - "北", - "卉", - "卑", - "博", - "即", - "卽", - "卿", - "𠨬", - "灰", - "及", - "叟", - "𠭣", - "叫", - "叱", - "吆", - "咞", - "吸", - "呈", - "周", - "咢", - "哶", - "唐", - "啓", - "啣", - "善", - "喙", - "喫", - "喳", - "嗂", - "圖", - "嘆", - "圗", - "噑", - "噴", - "切", - "壮", - "城", - "埴", - "堍", - "型", - "堲", - "報", - "墬", - "𡓤", - "売", - "壷", - "夆", - "多", - "夢", - "奢", - "𡚨", - "𡛪", - "姬", - "娛", - "娧", - "姘", - "婦", - "㛮", - "㛼", - "嬈", - "嬾", - "𡧈", - "寃", - "寘", - "寧", - "寳", - "𡬘", - "寿", - "将", - "当", - "尢", - "㞁", - "屠", - "屮", - "峀", - "岍", - "𡷤", - "嵃", - "𡷦", - "嵮", - "嵫", - "嵼", - "巡", - "巢", - "㠯", - "巽", - "帨", - "帽", - "幩", - "㡢", - "𢆃", - "㡼", - "庰", - "庳", - "庶", - "廊", - "𪎒", - "廾", - "𢌱", - "舁", - "弢", - "㣇", - "𣊸", - "𦇚", - "形", - "彫", - "㣣", - "徚", - "忍", - "志", - "忹", - "悁", - "㤺", - "㤜", - "悔", - "𢛔", - "惇", - "慈", - "慌", - "慎", - "慌", - "慺", - "憎", - "憲", - "憤", - "憯", - "懞", - "懲", - "懶", - "成", - "戛", - "扝", - "抱", - "拔", - "捐", - "𢬌", - "挽", - "拼", - "捨", - "掃", - "揤", - "𢯱", - "搢", - "揅", - "掩", - "㨮", - "摩", - "摾", - "撝", - "摷", - "㩬", - "敏", - "敬", - "𣀊", - "旣", - "書", - "晉", - "㬙", - "暑", - "㬈", - "㫤", - "冒", - "冕", - "最", - "暜", - "肭", - "䏙", - "朗", - "望", - "朡", - "杞", - "杓", - "𣏃", - "㭉", - "柺", - "枅", - "桒", - "梅", - "𣑭", - "梎", - "栟", - "椔", - "㮝", - "楂", - "榣", - "槪", - "檨", - "𣚣", - "櫛", - "㰘", - "次", - "𣢧", - "歔", - "㱎", - "歲", - "殟", - "殺", - "殻", - "𣪍", - "𡴋", - "𣫺", - "汎", - "𣲼", - "沿", - "泍", - "汧", - "洖", - "派", - "海", - "流", - "浩", - "浸", - "涅", - "𣴞", - "洴", - "港", - "湮", - "㴳", - "滋", - "滇", - "𣻑", - "淹", - "潮", - "𣽞", - "𣾎", - "濆", - "瀹", - "瀞", - "瀛", - "㶖", - "灊", - "災", - "灷", - "炭", - "𠔥", - "煅", - "𤉣", - "熜", - "𤎫", - "爨", - "爵", - "牐", - "𤘈", - "犀", - "犕", - "𤜵", - "𤠔", - "獺", - "王", - "㺬", - "玥", - "㺸", - "瑇", - "瑜", - "瑱", - "璅", - "瓊", - "㼛", - "甤", - "𤰶", - "甾", - "𤲒", - "異", - "𢆟", - "瘐", - "𤾡", - "𤾸", - "𥁄", - "㿼", - "䀈", - "直", - "𥃳", - "𥃲", - "𥄙", - "𥄳", - "眞", - "真", - "睊", - "䀹", - "瞋", - "䁆", - "䂖", - "𥐝", - "硎", - "碌", - "磌", - "䃣", - "𥘦", - "祖", - "𥚚", - "𥛅", - "福", - "秫", - "䄯", - "穀", - "穊", - "穏", - "𥥼", - "𥪧", - "竮", - "䈂", - "𥮫", - "篆", - "築", - "䈧", - "𥲀", - "糒", - "䊠", - "糨", - "糣", - "紀", - "𥾆", - "絣", - "䌁", - "緇", - "縂", - "繅", - "䌴", - "𦈨", - "𦉇", - "䍙", - "𦋙", - "罺", - "𦌾", - "羕", - "翺", - "者", - "𦓚", - "𦔣", - "聠", - "𦖨", - "聰", - "𣍟", - "䏕", - "育", - "脃", - "䐋", - "脾", - "媵", - "𦞧", - "𦞵", - "𣎓", - "𣎜", - "舁", - "舄", - "辞", - "䑫", - "芑", - "芋", - "芝", - "劳", - "花", - "芳", - "芽", - "苦", - "𦬼", - "若", - "茝", - "荣", - "莭", - "茣", - "莽", - "菧", - "著", - "荓", - "菊", - "菌", - "菜", - "𦰶", - "𦵫", - "𦳕", - "䔫", - "蓱", - "蓳", - "蔖", - "𧏊", - "蕤", - "𦼬", - "䕝", - "䕡", - "𦾱", - "𧃒", - "䕫", - "虐", - "虜", - "虧", - "虩", - "蚩", - "蚈", - "蜎", - "蛢", - "蝹", - "蜨", - "蝫", - "螆", - "䗗", - "蟡", - "蠁", - "䗹", - "衠", - "衣", - "𧙧", - "裗", - "裞", - "䘵", - "裺", - "㒻", - "𧢮", - "𧥦", - "䚾", - "䛇", - "誠", - "諭", - "變", - "豕", - "𧲨", - "貫", - "賁", - "贛", - "起", - "𧼯", - "𠠄", - "跋", - "趼", - "跰", - "𠣞", - "軔", - "輸", - "𨗒", - "𨗭", - "邔", - "郱", - "鄑", - "𨜮", - "鄛", - "鈸", - "鋗", - "鋘", - "鉼", - "鏹", - "鐕", - "𨯺", - "開", - "䦕", - "閷", - "𨵷", - "䧦", - "雃", - "嶲", - "霣", - "𩅅", - "𩈚", - "䩮", - "䩶", - "韠", - "𩐊", - "䪲", - "𩒖", - "頋", - "頩", - "𩖶", - "飢", - "䬳", - "餩", - "馧", - "駂", - "駾", - "䯎", - "𩬰", - "鬒", - "鱀", - "鳽", - "䳎", - "䳭", - "鵧", - "𪃎", - "䳸", - "𪄅", - "𪈎", - "𪊑", - "麻", - "䵖", - "黹", - "黾", - "鼅", - "鼏", - "鼖", - "鼻", - "𪘀", - None, - None, - None, - None, - None, - None, - None, -) diff --git a/bundle/python-cpu/Lib/site-packages/isympy.py b/bundle/python-cpu/Lib/site-packages/isympy.py deleted file mode 100644 index 50e9bc78d08904b8c177105ee90d984ea4b01d20..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/isympy.py +++ /dev/null @@ -1,342 +0,0 @@ -""" -Python shell for SymPy. - -This is just a normal Python shell (IPython shell if you have the -IPython package installed), that executes the following commands for -the user: - - >>> from __future__ import division - >>> from sympy import * - >>> x, y, z, t = symbols('x y z t') - >>> k, m, n = symbols('k m n', integer=True) - >>> f, g, h = symbols('f g h', cls=Function) - >>> init_printing() - -So starting 'isympy' is equivalent to starting Python (or IPython) and -executing the above commands by hand. It is intended for easy and quick -experimentation with SymPy. isympy is a good way to use SymPy as an -interactive calculator. If you have IPython and Matplotlib installed, then -interactive plotting is enabled by default. - -COMMAND LINE OPTIONS --------------------- - --c CONSOLE, --console=CONSOLE - - Use the specified shell (Python or IPython) shell as the console - backend instead of the default one (IPython if present, Python - otherwise), e.g.: - - $isympy -c python - - CONSOLE must be one of 'ipython' or 'python' - --p PRETTY, --pretty PRETTY - - Setup pretty-printing in SymPy. When pretty-printing is enabled, - expressions can be printed with Unicode or ASCII. The default is - to use pretty-printing (with Unicode if the terminal supports it). - When this option is 'no', expressions will not be pretty-printed - and ASCII will be used: - - $isympy -p no - - PRETTY must be one of 'unicode', 'ascii', or 'no' - --t TYPES, --types=TYPES - - Setup the ground types for the polys. By default, gmpy ground types - are used if gmpy2 or gmpy is installed, otherwise it falls back to python - ground types, which are a little bit slower. You can manually - choose python ground types even if gmpy is installed (e.g., for - testing purposes): - - $isympy -t python - - TYPES must be one of 'gmpy', 'gmpy1' or 'python' - - Note that the ground type gmpy1 is primarily intended for testing; it - forces the use of gmpy version 1 even if gmpy2 is available. - - This is the same as setting the environment variable - SYMPY_GROUND_TYPES to the given ground type (e.g., - SYMPY_GROUND_TYPES='gmpy') - - The ground types can be determined interactively from the variable - sympy.polys.domains.GROUND_TYPES. - --o ORDER, --order ORDER - - Setup the ordering of terms for printing. The default is lex, which - orders terms lexicographically (e.g., x**2 + x + 1). You can choose - other orderings, such as rev-lex, which will use reverse - lexicographic ordering (e.g., 1 + x + x**2): - - $isympy -o rev-lex - - ORDER must be one of 'lex', 'rev-lex', 'grlex', 'rev-grlex', - 'grevlex', 'rev-grevlex', 'old', or 'none'. - - Note that for very large expressions, ORDER='none' may speed up - printing considerably but the terms will have no canonical order. - --q, --quiet - - Print only Python's and SymPy's versions to stdout at startup. - --d, --doctest - - Use the same format that should be used for doctests. This is - equivalent to -c python -p no. - --C, --no-cache - - Disable the caching mechanism. Disabling the cache may slow certain - operations down considerably. This is useful for testing the cache, - or for benchmarking, as the cache can result in deceptive timings. - - This is equivalent to setting the environment variable - SYMPY_USE_CACHE to 'no'. - --a, --auto-symbols (requires at least IPython 0.11) - - Automatically create missing symbols. Normally, typing a name of a - Symbol that has not been instantiated first would raise NameError, - but with this option enabled, any undefined name will be - automatically created as a Symbol. - - Note that this is intended only for interactive, calculator style - usage. In a script that uses SymPy, Symbols should be instantiated - at the top, so that it's clear what they are. - - This will not override any names that are already defined, which - includes the single character letters represented by the mnemonic - QCOSINE (see the "Gotchas and Pitfalls" document in the - documentation). You can delete existing names by executing "del - name". If a name is defined, typing "'name' in dir()" will return True. - - The Symbols that are created using this have default assumptions. - If you want to place assumptions on symbols, you should create them - using symbols() or var(). - - Finally, this only works in the top level namespace. So, for - example, if you define a function in isympy with an undefined - Symbol, it will not work. - - See also the -i and -I options. - --i, --int-to-Integer (requires at least IPython 0.11) - - Automatically wrap int literals with Integer. This makes it so that - things like 1/2 will come out as Rational(1, 2), rather than 0.5. This - works by preprocessing the source and wrapping all int literals with - Integer. Note that this will not change the behavior of int literals - assigned to variables, and it also won't change the behavior of functions - that return int literals. - - If you want an int, you can wrap the literal in int(), e.g. int(3)/int(2) - gives 1.5 (with division imported from __future__). - --I, --interactive (requires at least IPython 0.11) - - This is equivalent to --auto-symbols --int-to-Integer. Future options - designed for ease of interactive use may be added to this. - --D, --debug - - Enable debugging output. This is the same as setting the - environment variable SYMPY_DEBUG to 'True'. The debug status is set - in the variable SYMPY_DEBUG within isympy. - --- IPython options - - Additionally you can pass command line options directly to the IPython - interpreter (the standard Python shell is not supported). However you - need to add the '--' separator between two types of options, e.g the - startup banner option and the colors option. You need to enter the - options as required by the version of IPython that you are using, too: - - in IPython 0.11, - - $isympy -q -- --colors=NoColor - - or older versions of IPython, - - $isympy -q -- -colors NoColor - -See also isympy --help. -""" - -import os -import sys - -# DO NOT IMPORT SYMPY HERE! Or the setting of the sympy environment variables -# by the command line will break. - -def main() -> None: - from argparse import ArgumentParser, RawDescriptionHelpFormatter - - VERSION = None - if '--version' in sys.argv: - # We cannot import sympy before this is run, because flags like -C and - # -t set environment variables that must be set before SymPy is - # imported. The only thing we need to import it for is to get the - # version, which only matters with the --version flag. - import sympy - VERSION = sympy.__version__ - - usage = 'isympy [options] -- [ipython options]' - parser = ArgumentParser( - usage=usage, - description=__doc__, - formatter_class=RawDescriptionHelpFormatter, - ) - - parser.add_argument('--version', action='version', version=VERSION) - - parser.add_argument( - '-c', '--console', - dest='console', - action='store', - default=None, - choices=['ipython', 'python'], - metavar='CONSOLE', - help='select type of interactive session: ipython | python; defaults ' - 'to ipython if IPython is installed, otherwise python') - - parser.add_argument( - '-p', '--pretty', - dest='pretty', - action='store', - default=None, - metavar='PRETTY', - choices=['unicode', 'ascii', 'no'], - help='setup pretty printing: unicode | ascii | no; defaults to ' - 'unicode printing if the terminal supports it, otherwise ascii') - - parser.add_argument( - '-t', '--types', - dest='types', - action='store', - default=None, - metavar='TYPES', - choices=['gmpy', 'gmpy1', 'python'], - help='setup ground types: gmpy | gmpy1 | python; defaults to gmpy if gmpy2 ' - 'or gmpy is installed, otherwise python') - - parser.add_argument( - '-o', '--order', - dest='order', - action='store', - default=None, - metavar='ORDER', - choices=['lex', 'grlex', 'grevlex', 'rev-lex', 'rev-grlex', 'rev-grevlex', 'old', 'none'], - help='setup ordering of terms: [rev-]lex | [rev-]grlex | [rev-]grevlex | old | none; defaults to lex') - - parser.add_argument( - '-q', '--quiet', - dest='quiet', - action='store_true', - default=False, - help='print only version information at startup') - - parser.add_argument( - '-d', '--doctest', - dest='doctest', - action='store_true', - default=False, - help='use the doctest format for output (you can just copy and paste it)') - - parser.add_argument( - '-C', '--no-cache', - dest='cache', - action='store_false', - default=True, - help='disable caching mechanism') - - parser.add_argument( - '-a', '--auto-symbols', - dest='auto_symbols', - action='store_true', - default=False, - help='automatically construct missing symbols') - - parser.add_argument( - '-i', '--int-to-Integer', - dest='auto_int_to_Integer', - action='store_true', - default=False, - help="automatically wrap int literals with Integer") - - parser.add_argument( - '-I', '--interactive', - dest='interactive', - action='store_true', - default=False, - help="equivalent to -a -i") - - parser.add_argument( - '-D', '--debug', - dest='debug', - action='store_true', - default=False, - help='enable debugging output') - - (options, ipy_args) = parser.parse_known_args() - if '--' in ipy_args: - ipy_args.remove('--') - - if not options.cache: - os.environ['SYMPY_USE_CACHE'] = 'no' - - if options.types: - os.environ['SYMPY_GROUND_TYPES'] = options.types - - if options.debug: - os.environ['SYMPY_DEBUG'] = str(options.debug) - - if options.doctest: - options.pretty = 'no' - options.console = 'python' - - session = options.console - - if session is not None: - ipython = session == 'ipython' - else: - try: - import IPython - ipython = True - except ImportError: - if not options.quiet: - from sympy.interactive.session import no_ipython - print(no_ipython) - ipython = False - - args = { - 'pretty_print': True, - 'use_unicode': None, - 'use_latex': None, - 'order': None, - 'argv': ipy_args, - } - - if options.pretty == 'unicode': - args['use_unicode'] = True - elif options.pretty == 'ascii': - args['use_unicode'] = False - elif options.pretty == 'no': - args['pretty_print'] = False - - if options.order is not None: - args['order'] = options.order - - args['quiet'] = options.quiet - args['auto_symbols'] = options.auto_symbols or options.interactive - args['auto_int_to_Integer'] = options.auto_int_to_Integer or options.interactive - - from sympy.interactive import init_session - init_session(ipython, **args) - -if __name__ == "__main__": - main() diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/INSTALLER deleted file mode 100644 index 5c69047b2eb8235994febeeae1da4a82365a240a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -uv \ No newline at end of file diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/LICENSE.txt b/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/LICENSE.txt deleted file mode 100644 index 7b190ca6712aa09eede3e6de79f68d7fa29072da..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/LICENSE.txt +++ /dev/null @@ -1,28 +0,0 @@ -Copyright 2011 Pallets - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/METADATA deleted file mode 100644 index ddf54648499557c652181f6126362ffd5751c273..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/METADATA +++ /dev/null @@ -1,60 +0,0 @@ -Metadata-Version: 2.1 -Name: itsdangerous -Version: 2.2.0 -Summary: Safely pass data to untrusted environments and back. -Maintainer-email: Pallets -Requires-Python: >=3.8 -Description-Content-Type: text/markdown -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: BSD License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Typing :: Typed -Project-URL: Changes, https://itsdangerous.palletsprojects.com/changes/ -Project-URL: Chat, https://discord.gg/pallets -Project-URL: Documentation, https://itsdangerous.palletsprojects.com/ -Project-URL: Donate, https://palletsprojects.com/donate -Project-URL: Source, https://github.com/pallets/itsdangerous/ - -# ItsDangerous - -... so better sign this - -Various helpers to pass data to untrusted environments and to get it -back safe and sound. Data is cryptographically signed to ensure that a -token has not been tampered with. - -It's possible to customize how data is serialized. Data is compressed as -needed. A timestamp can be added and verified automatically while -loading a token. - - -## A Simple Example - -Here's how you could generate a token for transmitting a user's id and -name between web requests. - -```python -from itsdangerous import URLSafeSerializer -auth_s = URLSafeSerializer("secret key", "auth") -token = auth_s.dumps({"id": 5, "name": "itsdangerous"}) - -print(token) -# eyJpZCI6NSwibmFtZSI6Iml0c2Rhbmdlcm91cyJ9.6YP6T0BaO67XP--9UzTrmurXSmg - -data = auth_s.loads(token) -print(data["name"]) -# itsdangerous -``` - - -## Donate - -The Pallets organization develops and supports ItsDangerous and other -popular packages. In order to grow the community of contributors and -users, and allow the maintainers to devote more time to the projects, -[please donate today][]. - -[please donate today]: https://palletsprojects.com/donate - diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/RECORD deleted file mode 100644 index a00c3834538354f7fb17b6292f87e5ca93fba19a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/RECORD +++ /dev/null @@ -1,15 +0,0 @@ -itsdangerous-2.2.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 -itsdangerous-2.2.0.dist-info/LICENSE.txt,sha256=Y68JiRtr6K0aQlLtQ68PTvun_JSOIoNnvtfzxa4LCdc,1475 -itsdangerous-2.2.0.dist-info/METADATA,sha256=0rk0-1ZwihuU5DnwJVwPWoEI4yWOyCexih3JyZHblhE,1924 -itsdangerous-2.2.0.dist-info/RECORD,, -itsdangerous-2.2.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -itsdangerous-2.2.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81 -itsdangerous/__init__.py,sha256=4SK75sCe29xbRgQE1ZQtMHnKUuZYAf3bSpZOrff1IAY,1427 -itsdangerous/_json.py,sha256=wPQGmge2yZ9328EHKF6gadGeyGYCJQKxtU-iLKE6UnA,473 -itsdangerous/encoding.py,sha256=wwTz5q_3zLcaAdunk6_vSoStwGqYWe307Zl_U87aRFM,1409 -itsdangerous/exc.py,sha256=Rr3exo0MRFEcPZltwecyK16VV1bE2K9_F1-d-ljcUn4,3201 -itsdangerous/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -itsdangerous/serializer.py,sha256=PmdwADLqkSyQLZ0jOKAgDsAW4k_H0TlA71Ei3z0C5aI,15601 -itsdangerous/signer.py,sha256=YO0CV7NBvHA6j549REHJFUjUojw2pHqwcUpQnU7yNYQ,9647 -itsdangerous/timed.py,sha256=6RvDMqNumGMxf0-HlpaZdN9PUQQmRvrQGplKhxuivUs,8083 -itsdangerous/url_safe.py,sha256=az4e5fXi_vs-YbWj8YZwn4wiVKfeD--GEKRT5Ueu4P4,2505 diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/REQUESTED b/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/REQUESTED deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/WHEEL deleted file mode 100644 index 3b5e64b5e6c4a210201d1676a891fd57b15cda99..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous-2.2.0.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: flit 3.9.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous/__init__.py b/bundle/python-cpu/Lib/site-packages/itsdangerous/__init__.py deleted file mode 100644 index ea55256ebd6feed68d8f1c5afedd141df6c7e766..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import annotations - -import typing as t - -from .encoding import base64_decode as base64_decode -from .encoding import base64_encode as base64_encode -from .encoding import want_bytes as want_bytes -from .exc import BadData as BadData -from .exc import BadHeader as BadHeader -from .exc import BadPayload as BadPayload -from .exc import BadSignature as BadSignature -from .exc import BadTimeSignature as BadTimeSignature -from .exc import SignatureExpired as SignatureExpired -from .serializer import Serializer as Serializer -from .signer import HMACAlgorithm as HMACAlgorithm -from .signer import NoneAlgorithm as NoneAlgorithm -from .signer import Signer as Signer -from .timed import TimedSerializer as TimedSerializer -from .timed import TimestampSigner as TimestampSigner -from .url_safe import URLSafeSerializer as URLSafeSerializer -from .url_safe import URLSafeTimedSerializer as URLSafeTimedSerializer - - -def __getattr__(name: str) -> t.Any: - if name == "__version__": - import importlib.metadata - import warnings - - warnings.warn( - "The '__version__' attribute is deprecated and will be removed in" - " ItsDangerous 2.3. Use feature detection or" - " 'importlib.metadata.version(\"itsdangerous\")' instead.", - DeprecationWarning, - stacklevel=2, - ) - return importlib.metadata.version("itsdangerous") - - raise AttributeError(name) diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous/_json.py b/bundle/python-cpu/Lib/site-packages/itsdangerous/_json.py deleted file mode 100644 index fc23feaaff690c477901dc50cc52dd18086051c5..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous/_json.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -import json as _json -import typing as t - - -class _CompactJSON: - """Wrapper around json module that strips whitespace.""" - - @staticmethod - def loads(payload: str | bytes) -> t.Any: - return _json.loads(payload) - - @staticmethod - def dumps(obj: t.Any, **kwargs: t.Any) -> str: - kwargs.setdefault("ensure_ascii", False) - kwargs.setdefault("separators", (",", ":")) - return _json.dumps(obj, **kwargs) diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous/encoding.py b/bundle/python-cpu/Lib/site-packages/itsdangerous/encoding.py deleted file mode 100644 index f5ca80f905c9bb29b993c437a89174e668704658..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous/encoding.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -import base64 -import string -import struct -import typing as t - -from .exc import BadData - - -def want_bytes( - s: str | bytes, encoding: str = "utf-8", errors: str = "strict" -) -> bytes: - if isinstance(s, str): - s = s.encode(encoding, errors) - - return s - - -def base64_encode(string: str | bytes) -> bytes: - """Base64 encode a string of bytes or text. The resulting bytes are - safe to use in URLs. - """ - string = want_bytes(string) - return base64.urlsafe_b64encode(string).rstrip(b"=") - - -def base64_decode(string: str | bytes) -> bytes: - """Base64 decode a URL-safe string of bytes or text. The result is - bytes. - """ - string = want_bytes(string, encoding="ascii", errors="ignore") - string += b"=" * (-len(string) % 4) - - try: - return base64.urlsafe_b64decode(string) - except (TypeError, ValueError) as e: - raise BadData("Invalid base64-encoded data") from e - - -# The alphabet used by base64.urlsafe_* -_base64_alphabet = f"{string.ascii_letters}{string.digits}-_=".encode("ascii") - -_int64_struct = struct.Struct(">Q") -_int_to_bytes = _int64_struct.pack -_bytes_to_int = t.cast("t.Callable[[bytes], tuple[int]]", _int64_struct.unpack) - - -def int_to_bytes(num: int) -> bytes: - return _int_to_bytes(num).lstrip(b"\x00") - - -def bytes_to_int(bytestr: bytes) -> int: - return _bytes_to_int(bytestr.rjust(8, b"\x00"))[0] diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous/exc.py b/bundle/python-cpu/Lib/site-packages/itsdangerous/exc.py deleted file mode 100644 index a75adcd52762458db556111d8b90415e75df7277..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous/exc.py +++ /dev/null @@ -1,106 +0,0 @@ -from __future__ import annotations - -import typing as t -from datetime import datetime - - -class BadData(Exception): - """Raised if bad data of any sort was encountered. This is the base - for all exceptions that ItsDangerous defines. - - .. versionadded:: 0.15 - """ - - def __init__(self, message: str): - super().__init__(message) - self.message = message - - def __str__(self) -> str: - return self.message - - -class BadSignature(BadData): - """Raised if a signature does not match.""" - - def __init__(self, message: str, payload: t.Any | None = None): - super().__init__(message) - - #: The payload that failed the signature test. In some - #: situations you might still want to inspect this, even if - #: you know it was tampered with. - #: - #: .. versionadded:: 0.14 - self.payload: t.Any | None = payload - - -class BadTimeSignature(BadSignature): - """Raised if a time-based signature is invalid. This is a subclass - of :class:`BadSignature`. - """ - - def __init__( - self, - message: str, - payload: t.Any | None = None, - date_signed: datetime | None = None, - ): - super().__init__(message, payload) - - #: If the signature expired this exposes the date of when the - #: signature was created. This can be helpful in order to - #: tell the user how long a link has been gone stale. - #: - #: .. versionchanged:: 2.0 - #: The datetime value is timezone-aware rather than naive. - #: - #: .. versionadded:: 0.14 - self.date_signed = date_signed - - -class SignatureExpired(BadTimeSignature): - """Raised if a signature timestamp is older than ``max_age``. This - is a subclass of :exc:`BadTimeSignature`. - """ - - -class BadHeader(BadSignature): - """Raised if a signed header is invalid in some form. This only - happens for serializers that have a header that goes with the - signature. - - .. versionadded:: 0.24 - """ - - def __init__( - self, - message: str, - payload: t.Any | None = None, - header: t.Any | None = None, - original_error: Exception | None = None, - ): - super().__init__(message, payload) - - #: If the header is actually available but just malformed it - #: might be stored here. - self.header: t.Any | None = header - - #: If available, the error that indicates why the payload was - #: not valid. This might be ``None``. - self.original_error: Exception | None = original_error - - -class BadPayload(BadData): - """Raised if a payload is invalid. This could happen if the payload - is loaded despite an invalid signature, or if there is a mismatch - between the serializer and deserializer. The original exception - that occurred during loading is stored on as :attr:`original_error`. - - .. versionadded:: 0.15 - """ - - def __init__(self, message: str, original_error: Exception | None = None): - super().__init__(message) - - #: If available, the error that indicates why the payload was - #: not valid. This might be ``None``. - self.original_error: Exception | None = original_error diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous/py.typed b/bundle/python-cpu/Lib/site-packages/itsdangerous/py.typed deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous/serializer.py b/bundle/python-cpu/Lib/site-packages/itsdangerous/serializer.py deleted file mode 100644 index 5ddf3871d8898c90fabd5bddd8299404e6e7e2f3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous/serializer.py +++ /dev/null @@ -1,406 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import json -import typing as t - -from .encoding import want_bytes -from .exc import BadPayload -from .exc import BadSignature -from .signer import _make_keys_list -from .signer import Signer - -if t.TYPE_CHECKING: - import typing_extensions as te - - # This should be either be str or bytes. To avoid having to specify the - # bound type, it falls back to a union if structural matching fails. - _TSerialized = te.TypeVar( - "_TSerialized", bound=t.Union[str, bytes], default=t.Union[str, bytes] - ) -else: - # Still available at runtime on Python < 3.13, but without the default. - _TSerialized = t.TypeVar("_TSerialized", bound=t.Union[str, bytes]) - - -class _PDataSerializer(t.Protocol[_TSerialized]): - def loads(self, payload: _TSerialized, /) -> t.Any: ... - # A signature with additional arguments is not handled correctly by type - # checkers right now, so an overload is used below for serializers that - # don't match this strict protocol. - def dumps(self, obj: t.Any, /) -> _TSerialized: ... - - -# Use TypeIs once it's available in typing_extensions or 3.13. -def is_text_serializer( - serializer: _PDataSerializer[t.Any], -) -> te.TypeGuard[_PDataSerializer[str]]: - """Checks whether a serializer generates text or binary.""" - return isinstance(serializer.dumps({}), str) - - -class Serializer(t.Generic[_TSerialized]): - """A serializer wraps a :class:`~itsdangerous.signer.Signer` to - enable serializing and securely signing data other than bytes. It - can unsign to verify that the data hasn't been changed. - - The serializer provides :meth:`dumps` and :meth:`loads`, similar to - :mod:`json`, and by default uses :mod:`json` internally to serialize - the data to bytes. - - The secret key should be a random string of ``bytes`` and should not - be saved to code or version control. Different salts should be used - to distinguish signing in different contexts. See :doc:`/concepts` - for information about the security of the secret key and salt. - - :param secret_key: The secret key to sign and verify with. Can be a - list of keys, oldest to newest, to support key rotation. - :param salt: Extra key to combine with ``secret_key`` to distinguish - signatures in different contexts. - :param serializer: An object that provides ``dumps`` and ``loads`` - methods for serializing data to a string. Defaults to - :attr:`default_serializer`, which defaults to :mod:`json`. - :param serializer_kwargs: Keyword arguments to pass when calling - ``serializer.dumps``. - :param signer: A ``Signer`` class to instantiate when signing data. - Defaults to :attr:`default_signer`, which defaults to - :class:`~itsdangerous.signer.Signer`. - :param signer_kwargs: Keyword arguments to pass when instantiating - the ``Signer`` class. - :param fallback_signers: List of signer parameters to try when - unsigning with the default signer fails. Each item can be a dict - of ``signer_kwargs``, a ``Signer`` class, or a tuple of - ``(signer, signer_kwargs)``. Defaults to - :attr:`default_fallback_signers`. - - .. versionchanged:: 2.0 - Added support for key rotation by passing a list to - ``secret_key``. - - .. versionchanged:: 2.0 - Removed the default SHA-512 fallback signer from - ``default_fallback_signers``. - - .. versionchanged:: 1.1 - Added support for ``fallback_signers`` and configured a default - SHA-512 fallback. This fallback is for users who used the yanked - 1.0.0 release which defaulted to SHA-512. - - .. versionchanged:: 0.14 - The ``signer`` and ``signer_kwargs`` parameters were added to - the constructor. - """ - - #: The default serialization module to use to serialize data to a - #: string internally. The default is :mod:`json`, but can be changed - #: to any object that provides ``dumps`` and ``loads`` methods. - default_serializer: _PDataSerializer[t.Any] = json - - #: The default ``Signer`` class to instantiate when signing data. - #: The default is :class:`itsdangerous.signer.Signer`. - default_signer: type[Signer] = Signer - - #: The default fallback signers to try when unsigning fails. - default_fallback_signers: list[ - dict[str, t.Any] | tuple[type[Signer], dict[str, t.Any]] | type[Signer] - ] = [] - - # Serializer[str] if no data serializer is provided, or if it returns str. - @t.overload - def __init__( - self: Serializer[str], - secret_key: str | bytes | cabc.Iterable[str] | cabc.Iterable[bytes], - salt: str | bytes | None = b"itsdangerous", - serializer: None | _PDataSerializer[str] = None, - serializer_kwargs: dict[str, t.Any] | None = None, - signer: type[Signer] | None = None, - signer_kwargs: dict[str, t.Any] | None = None, - fallback_signers: list[ - dict[str, t.Any] | tuple[type[Signer], dict[str, t.Any]] | type[Signer] - ] - | None = None, - ): ... - - # Serializer[bytes] with a bytes data serializer positional argument. - @t.overload - def __init__( - self: Serializer[bytes], - secret_key: str | bytes | cabc.Iterable[str] | cabc.Iterable[bytes], - salt: str | bytes | None, - serializer: _PDataSerializer[bytes], - serializer_kwargs: dict[str, t.Any] | None = None, - signer: type[Signer] | None = None, - signer_kwargs: dict[str, t.Any] | None = None, - fallback_signers: list[ - dict[str, t.Any] | tuple[type[Signer], dict[str, t.Any]] | type[Signer] - ] - | None = None, - ): ... - - # Serializer[bytes] with a bytes data serializer keyword argument. - @t.overload - def __init__( - self: Serializer[bytes], - secret_key: str | bytes | cabc.Iterable[str] | cabc.Iterable[bytes], - salt: str | bytes | None = b"itsdangerous", - *, - serializer: _PDataSerializer[bytes], - serializer_kwargs: dict[str, t.Any] | None = None, - signer: type[Signer] | None = None, - signer_kwargs: dict[str, t.Any] | None = None, - fallback_signers: list[ - dict[str, t.Any] | tuple[type[Signer], dict[str, t.Any]] | type[Signer] - ] - | None = None, - ): ... - - # Fall back with a positional argument. If the strict signature of - # _PDataSerializer doesn't match, fall back to a union, requiring the user - # to specify the type. - @t.overload - def __init__( - self, - secret_key: str | bytes | cabc.Iterable[str] | cabc.Iterable[bytes], - salt: str | bytes | None, - serializer: t.Any, - serializer_kwargs: dict[str, t.Any] | None = None, - signer: type[Signer] | None = None, - signer_kwargs: dict[str, t.Any] | None = None, - fallback_signers: list[ - dict[str, t.Any] | tuple[type[Signer], dict[str, t.Any]] | type[Signer] - ] - | None = None, - ): ... - - # Fall back with a keyword argument. - @t.overload - def __init__( - self, - secret_key: str | bytes | cabc.Iterable[str] | cabc.Iterable[bytes], - salt: str | bytes | None = b"itsdangerous", - *, - serializer: t.Any, - serializer_kwargs: dict[str, t.Any] | None = None, - signer: type[Signer] | None = None, - signer_kwargs: dict[str, t.Any] | None = None, - fallback_signers: list[ - dict[str, t.Any] | tuple[type[Signer], dict[str, t.Any]] | type[Signer] - ] - | None = None, - ): ... - - def __init__( - self, - secret_key: str | bytes | cabc.Iterable[str] | cabc.Iterable[bytes], - salt: str | bytes | None = b"itsdangerous", - serializer: t.Any | None = None, - serializer_kwargs: dict[str, t.Any] | None = None, - signer: type[Signer] | None = None, - signer_kwargs: dict[str, t.Any] | None = None, - fallback_signers: list[ - dict[str, t.Any] | tuple[type[Signer], dict[str, t.Any]] | type[Signer] - ] - | None = None, - ): - #: The list of secret keys to try for verifying signatures, from - #: oldest to newest. The newest (last) key is used for signing. - #: - #: This allows a key rotation system to keep a list of allowed - #: keys and remove expired ones. - self.secret_keys: list[bytes] = _make_keys_list(secret_key) - - if salt is not None: - salt = want_bytes(salt) - # if salt is None then the signer's default is used - - self.salt = salt - - if serializer is None: - serializer = self.default_serializer - - self.serializer: _PDataSerializer[_TSerialized] = serializer - self.is_text_serializer: bool = is_text_serializer(serializer) - - if signer is None: - signer = self.default_signer - - self.signer: type[Signer] = signer - self.signer_kwargs: dict[str, t.Any] = signer_kwargs or {} - - if fallback_signers is None: - fallback_signers = list(self.default_fallback_signers) - - self.fallback_signers: list[ - dict[str, t.Any] | tuple[type[Signer], dict[str, t.Any]] | type[Signer] - ] = fallback_signers - self.serializer_kwargs: dict[str, t.Any] = serializer_kwargs or {} - - @property - def secret_key(self) -> bytes: - """The newest (last) entry in the :attr:`secret_keys` list. This - is for compatibility from before key rotation support was added. - """ - return self.secret_keys[-1] - - def load_payload( - self, payload: bytes, serializer: _PDataSerializer[t.Any] | None = None - ) -> t.Any: - """Loads the encoded object. This function raises - :class:`.BadPayload` if the payload is not valid. The - ``serializer`` parameter can be used to override the serializer - stored on the class. The encoded ``payload`` should always be - bytes. - """ - if serializer is None: - use_serializer = self.serializer - is_text = self.is_text_serializer - else: - use_serializer = serializer - is_text = is_text_serializer(serializer) - - try: - if is_text: - return use_serializer.loads(payload.decode("utf-8")) # type: ignore[arg-type] - - return use_serializer.loads(payload) # type: ignore[arg-type] - except Exception as e: - raise BadPayload( - "Could not load the payload because an exception" - " occurred on unserializing the data.", - original_error=e, - ) from e - - def dump_payload(self, obj: t.Any) -> bytes: - """Dumps the encoded object. The return value is always bytes. - If the internal serializer returns text, the value will be - encoded as UTF-8. - """ - return want_bytes(self.serializer.dumps(obj, **self.serializer_kwargs)) - - def make_signer(self, salt: str | bytes | None = None) -> Signer: - """Creates a new instance of the signer to be used. The default - implementation uses the :class:`.Signer` base class. - """ - if salt is None: - salt = self.salt - - return self.signer(self.secret_keys, salt=salt, **self.signer_kwargs) - - def iter_unsigners(self, salt: str | bytes | None = None) -> cabc.Iterator[Signer]: - """Iterates over all signers to be tried for unsigning. Starts - with the configured signer, then constructs each signer - specified in ``fallback_signers``. - """ - if salt is None: - salt = self.salt - - yield self.make_signer(salt) - - for fallback in self.fallback_signers: - if isinstance(fallback, dict): - kwargs = fallback - fallback = self.signer - elif isinstance(fallback, tuple): - fallback, kwargs = fallback - else: - kwargs = self.signer_kwargs - - for secret_key in self.secret_keys: - yield fallback(secret_key, salt=salt, **kwargs) - - def dumps(self, obj: t.Any, salt: str | bytes | None = None) -> _TSerialized: - """Returns a signed string serialized with the internal - serializer. The return value can be either a byte or unicode - string depending on the format of the internal serializer. - """ - payload = want_bytes(self.dump_payload(obj)) - rv = self.make_signer(salt).sign(payload) - - if self.is_text_serializer: - return rv.decode("utf-8") # type: ignore[return-value] - - return rv # type: ignore[return-value] - - def dump(self, obj: t.Any, f: t.IO[t.Any], salt: str | bytes | None = None) -> None: - """Like :meth:`dumps` but dumps into a file. The file handle has - to be compatible with what the internal serializer expects. - """ - f.write(self.dumps(obj, salt)) - - def loads( - self, s: str | bytes, salt: str | bytes | None = None, **kwargs: t.Any - ) -> t.Any: - """Reverse of :meth:`dumps`. Raises :exc:`.BadSignature` if the - signature validation fails. - """ - s = want_bytes(s) - last_exception = None - - for signer in self.iter_unsigners(salt): - try: - return self.load_payload(signer.unsign(s)) - except BadSignature as err: - last_exception = err - - raise t.cast(BadSignature, last_exception) - - def load(self, f: t.IO[t.Any], salt: str | bytes | None = None) -> t.Any: - """Like :meth:`loads` but loads from a file.""" - return self.loads(f.read(), salt) - - def loads_unsafe( - self, s: str | bytes, salt: str | bytes | None = None - ) -> tuple[bool, t.Any]: - """Like :meth:`loads` but without verifying the signature. This - is potentially very dangerous to use depending on how your - serializer works. The return value is ``(signature_valid, - payload)`` instead of just the payload. The first item will be a - boolean that indicates if the signature is valid. This function - never fails. - - Use it for debugging only and if you know that your serializer - module is not exploitable (for example, do not use it with a - pickle serializer). - - .. versionadded:: 0.15 - """ - return self._loads_unsafe_impl(s, salt) - - def _loads_unsafe_impl( - self, - s: str | bytes, - salt: str | bytes | None, - load_kwargs: dict[str, t.Any] | None = None, - load_payload_kwargs: dict[str, t.Any] | None = None, - ) -> tuple[bool, t.Any]: - """Low level helper function to implement :meth:`loads_unsafe` - in serializer subclasses. - """ - if load_kwargs is None: - load_kwargs = {} - - try: - return True, self.loads(s, salt=salt, **load_kwargs) - except BadSignature as e: - if e.payload is None: - return False, None - - if load_payload_kwargs is None: - load_payload_kwargs = {} - - try: - return ( - False, - self.load_payload(e.payload, **load_payload_kwargs), - ) - except BadPayload: - return False, None - - def load_unsafe( - self, f: t.IO[t.Any], salt: str | bytes | None = None - ) -> tuple[bool, t.Any]: - """Like :meth:`loads_unsafe` but loads from a file. - - .. versionadded:: 0.15 - """ - return self.loads_unsafe(f.read(), salt=salt) diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous/signer.py b/bundle/python-cpu/Lib/site-packages/itsdangerous/signer.py deleted file mode 100644 index e324dc03da90d9002200b68088f501df62777cd6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous/signer.py +++ /dev/null @@ -1,266 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import hashlib -import hmac -import typing as t - -from .encoding import _base64_alphabet -from .encoding import base64_decode -from .encoding import base64_encode -from .encoding import want_bytes -from .exc import BadSignature - - -class SigningAlgorithm: - """Subclasses must implement :meth:`get_signature` to provide - signature generation functionality. - """ - - def get_signature(self, key: bytes, value: bytes) -> bytes: - """Returns the signature for the given key and value.""" - raise NotImplementedError() - - def verify_signature(self, key: bytes, value: bytes, sig: bytes) -> bool: - """Verifies the given signature matches the expected - signature. - """ - return hmac.compare_digest(sig, self.get_signature(key, value)) - - -class NoneAlgorithm(SigningAlgorithm): - """Provides an algorithm that does not perform any signing and - returns an empty signature. - """ - - def get_signature(self, key: bytes, value: bytes) -> bytes: - return b"" - - -def _lazy_sha1(string: bytes = b"") -> t.Any: - """Don't access ``hashlib.sha1`` until runtime. FIPS builds may not include - SHA-1, in which case the import and use as a default would fail before the - developer can configure something else. - """ - return hashlib.sha1(string) - - -class HMACAlgorithm(SigningAlgorithm): - """Provides signature generation using HMACs.""" - - #: The digest method to use with the MAC algorithm. This defaults to - #: SHA1, but can be changed to any other function in the hashlib - #: module. - default_digest_method: t.Any = staticmethod(_lazy_sha1) - - def __init__(self, digest_method: t.Any = None): - if digest_method is None: - digest_method = self.default_digest_method - - self.digest_method: t.Any = digest_method - - def get_signature(self, key: bytes, value: bytes) -> bytes: - mac = hmac.new(key, msg=value, digestmod=self.digest_method) - return mac.digest() - - -def _make_keys_list( - secret_key: str | bytes | cabc.Iterable[str] | cabc.Iterable[bytes], -) -> list[bytes]: - if isinstance(secret_key, (str, bytes)): - return [want_bytes(secret_key)] - - return [want_bytes(s) for s in secret_key] # pyright: ignore - - -class Signer: - """A signer securely signs bytes, then unsigns them to verify that - the value hasn't been changed. - - The secret key should be a random string of ``bytes`` and should not - be saved to code or version control. Different salts should be used - to distinguish signing in different contexts. See :doc:`/concepts` - for information about the security of the secret key and salt. - - :param secret_key: The secret key to sign and verify with. Can be a - list of keys, oldest to newest, to support key rotation. - :param salt: Extra key to combine with ``secret_key`` to distinguish - signatures in different contexts. - :param sep: Separator between the signature and value. - :param key_derivation: How to derive the signing key from the secret - key and salt. Possible values are ``concat``, ``django-concat``, - or ``hmac``. Defaults to :attr:`default_key_derivation`, which - defaults to ``django-concat``. - :param digest_method: Hash function to use when generating the HMAC - signature. Defaults to :attr:`default_digest_method`, which - defaults to :func:`hashlib.sha1`. Note that the security of the - hash alone doesn't apply when used intermediately in HMAC. - :param algorithm: A :class:`SigningAlgorithm` instance to use - instead of building a default :class:`HMACAlgorithm` with the - ``digest_method``. - - .. versionchanged:: 2.0 - Added support for key rotation by passing a list to - ``secret_key``. - - .. versionchanged:: 0.18 - ``algorithm`` was added as an argument to the class constructor. - - .. versionchanged:: 0.14 - ``key_derivation`` and ``digest_method`` were added as arguments - to the class constructor. - """ - - #: The default digest method to use for the signer. The default is - #: :func:`hashlib.sha1`, but can be changed to any :mod:`hashlib` or - #: compatible object. Note that the security of the hash alone - #: doesn't apply when used intermediately in HMAC. - #: - #: .. versionadded:: 0.14 - default_digest_method: t.Any = staticmethod(_lazy_sha1) - - #: The default scheme to use to derive the signing key from the - #: secret key and salt. The default is ``django-concat``. Possible - #: values are ``concat``, ``django-concat``, and ``hmac``. - #: - #: .. versionadded:: 0.14 - default_key_derivation: str = "django-concat" - - def __init__( - self, - secret_key: str | bytes | cabc.Iterable[str] | cabc.Iterable[bytes], - salt: str | bytes | None = b"itsdangerous.Signer", - sep: str | bytes = b".", - key_derivation: str | None = None, - digest_method: t.Any | None = None, - algorithm: SigningAlgorithm | None = None, - ): - #: The list of secret keys to try for verifying signatures, from - #: oldest to newest. The newest (last) key is used for signing. - #: - #: This allows a key rotation system to keep a list of allowed - #: keys and remove expired ones. - self.secret_keys: list[bytes] = _make_keys_list(secret_key) - self.sep: bytes = want_bytes(sep) - - if self.sep in _base64_alphabet: - raise ValueError( - "The given separator cannot be used because it may be" - " contained in the signature itself. ASCII letters," - " digits, and '-_=' must not be used." - ) - - if salt is not None: - salt = want_bytes(salt) - else: - salt = b"itsdangerous.Signer" - - self.salt = salt - - if key_derivation is None: - key_derivation = self.default_key_derivation - - self.key_derivation: str = key_derivation - - if digest_method is None: - digest_method = self.default_digest_method - - self.digest_method: t.Any = digest_method - - if algorithm is None: - algorithm = HMACAlgorithm(self.digest_method) - - self.algorithm: SigningAlgorithm = algorithm - - @property - def secret_key(self) -> bytes: - """The newest (last) entry in the :attr:`secret_keys` list. This - is for compatibility from before key rotation support was added. - """ - return self.secret_keys[-1] - - def derive_key(self, secret_key: str | bytes | None = None) -> bytes: - """This method is called to derive the key. The default key - derivation choices can be overridden here. Key derivation is not - intended to be used as a security method to make a complex key - out of a short password. Instead you should use large random - secret keys. - - :param secret_key: A specific secret key to derive from. - Defaults to the last item in :attr:`secret_keys`. - - .. versionchanged:: 2.0 - Added the ``secret_key`` parameter. - """ - if secret_key is None: - secret_key = self.secret_keys[-1] - else: - secret_key = want_bytes(secret_key) - - if self.key_derivation == "concat": - return t.cast(bytes, self.digest_method(self.salt + secret_key).digest()) - elif self.key_derivation == "django-concat": - return t.cast( - bytes, self.digest_method(self.salt + b"signer" + secret_key).digest() - ) - elif self.key_derivation == "hmac": - mac = hmac.new(secret_key, digestmod=self.digest_method) - mac.update(self.salt) - return mac.digest() - elif self.key_derivation == "none": - return secret_key - else: - raise TypeError("Unknown key derivation method") - - def get_signature(self, value: str | bytes) -> bytes: - """Returns the signature for the given value.""" - value = want_bytes(value) - key = self.derive_key() - sig = self.algorithm.get_signature(key, value) - return base64_encode(sig) - - def sign(self, value: str | bytes) -> bytes: - """Signs the given string.""" - value = want_bytes(value) - return value + self.sep + self.get_signature(value) - - def verify_signature(self, value: str | bytes, sig: str | bytes) -> bool: - """Verifies the signature for the given value.""" - try: - sig = base64_decode(sig) - except Exception: - return False - - value = want_bytes(value) - - for secret_key in reversed(self.secret_keys): - key = self.derive_key(secret_key) - - if self.algorithm.verify_signature(key, value, sig): - return True - - return False - - def unsign(self, signed_value: str | bytes) -> bytes: - """Unsigns the given string.""" - signed_value = want_bytes(signed_value) - - if self.sep not in signed_value: - raise BadSignature(f"No {self.sep!r} found in value") - - value, sig = signed_value.rsplit(self.sep, 1) - - if self.verify_signature(value, sig): - return value - - raise BadSignature(f"Signature {sig!r} does not match", payload=value) - - def validate(self, signed_value: str | bytes) -> bool: - """Only validates the given signed value. Returns ``True`` if - the signature exists and is valid. - """ - try: - self.unsign(signed_value) - return True - except BadSignature: - return False diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous/timed.py b/bundle/python-cpu/Lib/site-packages/itsdangerous/timed.py deleted file mode 100644 index 73843755d90979f750976705c5eaeeb3b98f178c..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous/timed.py +++ /dev/null @@ -1,228 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import time -import typing as t -from datetime import datetime -from datetime import timezone - -from .encoding import base64_decode -from .encoding import base64_encode -from .encoding import bytes_to_int -from .encoding import int_to_bytes -from .encoding import want_bytes -from .exc import BadSignature -from .exc import BadTimeSignature -from .exc import SignatureExpired -from .serializer import _TSerialized -from .serializer import Serializer -from .signer import Signer - - -class TimestampSigner(Signer): - """Works like the regular :class:`.Signer` but also records the time - of the signing and can be used to expire signatures. The - :meth:`unsign` method can raise :exc:`.SignatureExpired` if the - unsigning failed because the signature is expired. - """ - - def get_timestamp(self) -> int: - """Returns the current timestamp. The function must return an - integer. - """ - return int(time.time()) - - def timestamp_to_datetime(self, ts: int) -> datetime: - """Convert the timestamp from :meth:`get_timestamp` into an - aware :class`datetime.datetime` in UTC. - - .. versionchanged:: 2.0 - The timestamp is returned as a timezone-aware ``datetime`` - in UTC rather than a naive ``datetime`` assumed to be UTC. - """ - return datetime.fromtimestamp(ts, tz=timezone.utc) - - def sign(self, value: str | bytes) -> bytes: - """Signs the given string and also attaches time information.""" - value = want_bytes(value) - timestamp = base64_encode(int_to_bytes(self.get_timestamp())) - sep = want_bytes(self.sep) - value = value + sep + timestamp - return value + sep + self.get_signature(value) - - # Ignore overlapping signatures check, return_timestamp is the only - # parameter that affects the return type. - - @t.overload - def unsign( # type: ignore[overload-overlap] - self, - signed_value: str | bytes, - max_age: int | None = None, - return_timestamp: t.Literal[False] = False, - ) -> bytes: ... - - @t.overload - def unsign( - self, - signed_value: str | bytes, - max_age: int | None = None, - return_timestamp: t.Literal[True] = True, - ) -> tuple[bytes, datetime]: ... - - def unsign( - self, - signed_value: str | bytes, - max_age: int | None = None, - return_timestamp: bool = False, - ) -> tuple[bytes, datetime] | bytes: - """Works like the regular :meth:`.Signer.unsign` but can also - validate the time. See the base docstring of the class for - the general behavior. If ``return_timestamp`` is ``True`` the - timestamp of the signature will be returned as an aware - :class:`datetime.datetime` object in UTC. - - .. versionchanged:: 2.0 - The timestamp is returned as a timezone-aware ``datetime`` - in UTC rather than a naive ``datetime`` assumed to be UTC. - """ - try: - result = super().unsign(signed_value) - sig_error = None - except BadSignature as e: - sig_error = e - result = e.payload or b"" - - sep = want_bytes(self.sep) - - # If there is no timestamp in the result there is something - # seriously wrong. In case there was a signature error, we raise - # that one directly, otherwise we have a weird situation in - # which we shouldn't have come except someone uses a time-based - # serializer on non-timestamp data, so catch that. - if sep not in result: - if sig_error: - raise sig_error - - raise BadTimeSignature("timestamp missing", payload=result) - - value, ts_bytes = result.rsplit(sep, 1) - ts_int: int | None = None - ts_dt: datetime | None = None - - try: - ts_int = bytes_to_int(base64_decode(ts_bytes)) - except Exception: - pass - - # Signature is *not* okay. Raise a proper error now that we have - # split the value and the timestamp. - if sig_error is not None: - if ts_int is not None: - try: - ts_dt = self.timestamp_to_datetime(ts_int) - except (ValueError, OSError, OverflowError) as exc: - # Windows raises OSError - # 32-bit raises OverflowError - raise BadTimeSignature( - "Malformed timestamp", payload=value - ) from exc - - raise BadTimeSignature(str(sig_error), payload=value, date_signed=ts_dt) - - # Signature was okay but the timestamp is actually not there or - # malformed. Should not happen, but we handle it anyway. - if ts_int is None: - raise BadTimeSignature("Malformed timestamp", payload=value) - - # Check timestamp is not older than max_age - if max_age is not None: - age = self.get_timestamp() - ts_int - - if age > max_age: - raise SignatureExpired( - f"Signature age {age} > {max_age} seconds", - payload=value, - date_signed=self.timestamp_to_datetime(ts_int), - ) - - if age < 0: - raise SignatureExpired( - f"Signature age {age} < 0 seconds", - payload=value, - date_signed=self.timestamp_to_datetime(ts_int), - ) - - if return_timestamp: - return value, self.timestamp_to_datetime(ts_int) - - return value - - def validate(self, signed_value: str | bytes, max_age: int | None = None) -> bool: - """Only validates the given signed value. Returns ``True`` if - the signature exists and is valid.""" - try: - self.unsign(signed_value, max_age=max_age) - return True - except BadSignature: - return False - - -class TimedSerializer(Serializer[_TSerialized]): - """Uses :class:`TimestampSigner` instead of the default - :class:`.Signer`. - """ - - default_signer: type[TimestampSigner] = TimestampSigner - - def iter_unsigners( - self, salt: str | bytes | None = None - ) -> cabc.Iterator[TimestampSigner]: - return t.cast("cabc.Iterator[TimestampSigner]", super().iter_unsigners(salt)) - - # TODO: Signature is incompatible because parameters were added - # before salt. - - def loads( # type: ignore[override] - self, - s: str | bytes, - max_age: int | None = None, - return_timestamp: bool = False, - salt: str | bytes | None = None, - ) -> t.Any: - """Reverse of :meth:`dumps`, raises :exc:`.BadSignature` if the - signature validation fails. If a ``max_age`` is provided it will - ensure the signature is not older than that time in seconds. In - case the signature is outdated, :exc:`.SignatureExpired` is - raised. All arguments are forwarded to the signer's - :meth:`~TimestampSigner.unsign` method. - """ - s = want_bytes(s) - last_exception = None - - for signer in self.iter_unsigners(salt): - try: - base64d, timestamp = signer.unsign( - s, max_age=max_age, return_timestamp=True - ) - payload = self.load_payload(base64d) - - if return_timestamp: - return payload, timestamp - - return payload - except SignatureExpired: - # The signature was unsigned successfully but was - # expired. Do not try the next signer. - raise - except BadSignature as err: - last_exception = err - - raise t.cast(BadSignature, last_exception) - - def loads_unsafe( # type: ignore[override] - self, - s: str | bytes, - max_age: int | None = None, - salt: str | bytes | None = None, - ) -> tuple[bool, t.Any]: - return self._loads_unsafe_impl(s, salt, load_kwargs={"max_age": max_age}) diff --git a/bundle/python-cpu/Lib/site-packages/itsdangerous/url_safe.py b/bundle/python-cpu/Lib/site-packages/itsdangerous/url_safe.py deleted file mode 100644 index 56a0793315a19f24c8385c017a26ec8a9d4c23c3..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/itsdangerous/url_safe.py +++ /dev/null @@ -1,83 +0,0 @@ -from __future__ import annotations - -import typing as t -import zlib - -from ._json import _CompactJSON -from .encoding import base64_decode -from .encoding import base64_encode -from .exc import BadPayload -from .serializer import _PDataSerializer -from .serializer import Serializer -from .timed import TimedSerializer - - -class URLSafeSerializerMixin(Serializer[str]): - """Mixed in with a regular serializer it will attempt to zlib - compress the string to make it shorter if necessary. It will also - base64 encode the string so that it can safely be placed in a URL. - """ - - default_serializer: _PDataSerializer[str] = _CompactJSON - - def load_payload( - self, - payload: bytes, - *args: t.Any, - serializer: t.Any | None = None, - **kwargs: t.Any, - ) -> t.Any: - decompress = False - - if payload.startswith(b"."): - payload = payload[1:] - decompress = True - - try: - json = base64_decode(payload) - except Exception as e: - raise BadPayload( - "Could not base64 decode the payload because of an exception", - original_error=e, - ) from e - - if decompress: - try: - json = zlib.decompress(json) - except Exception as e: - raise BadPayload( - "Could not zlib decompress the payload before decoding the payload", - original_error=e, - ) from e - - return super().load_payload(json, *args, **kwargs) - - def dump_payload(self, obj: t.Any) -> bytes: - json = super().dump_payload(obj) - is_compressed = False - compressed = zlib.compress(json) - - if len(compressed) < (len(json) - 1): - json = compressed - is_compressed = True - - base64d = base64_encode(json) - - if is_compressed: - base64d = b"." + base64d - - return base64d - - -class URLSafeSerializer(URLSafeSerializerMixin, Serializer[str]): - """Works like :class:`.Serializer` but dumps and loads into a URL - safe string consisting of the upper and lowercase character of the - alphabet as well as ``'_'``, ``'-'`` and ``'.'``. - """ - - -class URLSafeTimedSerializer(URLSafeSerializerMixin, TimedSerializer[str]): - """Works like :class:`.TimedSerializer` but dumps and loads into a - URL safe string consisting of the upper and lowercase character of - the alphabet as well as ``'_'``, ``'-'`` and ``'.'``. - """ diff --git a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/INSTALLER b/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/INSTALLER deleted file mode 100644 index a1b589e38a32041e49332e5e81c2d363dc418d68..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/METADATA b/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/METADATA deleted file mode 100644 index ffef2ff3bfa0c42b6e6e3eefda700391d181c9a0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/METADATA +++ /dev/null @@ -1,84 +0,0 @@ -Metadata-Version: 2.4 -Name: Jinja2 -Version: 3.1.6 -Summary: A very fast and expressive template engine. -Maintainer-email: Pallets -Requires-Python: >=3.7 -Description-Content-Type: text/markdown -Classifier: Development Status :: 5 - Production/Stable -Classifier: Environment :: Web Environment -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: BSD License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content -Classifier: Topic :: Text Processing :: Markup :: HTML -Classifier: Typing :: Typed -License-File: LICENSE.txt -Requires-Dist: MarkupSafe>=2.0 -Requires-Dist: Babel>=2.7 ; extra == "i18n" -Project-URL: Changes, https://jinja.palletsprojects.com/changes/ -Project-URL: Chat, https://discord.gg/pallets -Project-URL: Documentation, https://jinja.palletsprojects.com/ -Project-URL: Donate, https://palletsprojects.com/donate -Project-URL: Source, https://github.com/pallets/jinja/ -Provides-Extra: i18n - -# Jinja - -Jinja is a fast, expressive, extensible templating engine. Special -placeholders in the template allow writing code similar to Python -syntax. Then the template is passed data to render the final document. - -It includes: - -- Template inheritance and inclusion. -- Define and import macros within templates. -- HTML templates can use autoescaping to prevent XSS from untrusted - user input. -- A sandboxed environment can safely render untrusted templates. -- AsyncIO support for generating templates and calling async - functions. -- I18N support with Babel. -- Templates are compiled to optimized Python code just-in-time and - cached, or can be compiled ahead-of-time. -- Exceptions point to the correct line in templates to make debugging - easier. -- Extensible filters, tests, functions, and even syntax. - -Jinja's philosophy is that while application logic belongs in Python if -possible, it shouldn't make the template designer's job difficult by -restricting functionality too much. - - -## In A Nutshell - -```jinja -{% extends "base.html" %} -{% block title %}Members{% endblock %} -{% block content %} - -{% endblock %} -``` - -## Donate - -The Pallets organization develops and supports Jinja and other popular -packages. In order to grow the community of contributors and users, and -allow the maintainers to devote more time to the projects, [please -donate today][]. - -[please donate today]: https://palletsprojects.com/donate - -## Contributing - -See our [detailed contributing documentation][contrib] for many ways to -contribute, including reporting issues, requesting features, asking or answering -questions, and making PRs. - -[contrib]: https://palletsprojects.com/contributing/ - diff --git a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/RECORD b/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/RECORD deleted file mode 100644 index 20d4fee9c2e506ae021da8a8d2805745375c2ee6..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/RECORD +++ /dev/null @@ -1,57 +0,0 @@ -jinja2-3.1.6.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -jinja2-3.1.6.dist-info/METADATA,sha256=aMVUj7Z8QTKhOJjZsx7FDGvqKr3ZFdkh8hQ1XDpkmcg,2871 -jinja2-3.1.6.dist-info/RECORD,, -jinja2-3.1.6.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82 -jinja2-3.1.6.dist-info/entry_points.txt,sha256=OL85gYU1eD8cuPlikifFngXpeBjaxl6rIJ8KkC_3r-I,58 -jinja2-3.1.6.dist-info/licenses/LICENSE.txt,sha256=O0nc7kEF6ze6wQ-vG-JgQI_oXSUrjp3y4JefweCUQ3s,1475 -jinja2/__init__.py,sha256=xxepO9i7DHsqkQrgBEduLtfoz2QCuT6_gbL4XSN1hbU,1928 -jinja2/__pycache__/__init__.cpython-310.pyc,, -jinja2/__pycache__/_identifier.cpython-310.pyc,, -jinja2/__pycache__/async_utils.cpython-310.pyc,, -jinja2/__pycache__/bccache.cpython-310.pyc,, -jinja2/__pycache__/compiler.cpython-310.pyc,, -jinja2/__pycache__/constants.cpython-310.pyc,, -jinja2/__pycache__/debug.cpython-310.pyc,, -jinja2/__pycache__/defaults.cpython-310.pyc,, -jinja2/__pycache__/environment.cpython-310.pyc,, -jinja2/__pycache__/exceptions.cpython-310.pyc,, -jinja2/__pycache__/ext.cpython-310.pyc,, -jinja2/__pycache__/filters.cpython-310.pyc,, -jinja2/__pycache__/idtracking.cpython-310.pyc,, -jinja2/__pycache__/lexer.cpython-310.pyc,, -jinja2/__pycache__/loaders.cpython-310.pyc,, -jinja2/__pycache__/meta.cpython-310.pyc,, -jinja2/__pycache__/nativetypes.cpython-310.pyc,, -jinja2/__pycache__/nodes.cpython-310.pyc,, -jinja2/__pycache__/optimizer.cpython-310.pyc,, -jinja2/__pycache__/parser.cpython-310.pyc,, -jinja2/__pycache__/runtime.cpython-310.pyc,, -jinja2/__pycache__/sandbox.cpython-310.pyc,, -jinja2/__pycache__/tests.cpython-310.pyc,, -jinja2/__pycache__/utils.cpython-310.pyc,, -jinja2/__pycache__/visitor.cpython-310.pyc,, -jinja2/_identifier.py,sha256=_zYctNKzRqlk_murTNlzrju1FFJL7Va_Ijqqd7ii2lU,1958 -jinja2/async_utils.py,sha256=vK-PdsuorOMnWSnEkT3iUJRIkTnYgO2T6MnGxDgHI5o,2834 -jinja2/bccache.py,sha256=gh0qs9rulnXo0PhX5jTJy2UHzI8wFnQ63o_vw7nhzRg,14061 -jinja2/compiler.py,sha256=9RpCQl5X88BHllJiPsHPh295Hh0uApvwFJNQuutULeM,74131 -jinja2/constants.py,sha256=GMoFydBF_kdpaRKPoM5cl5MviquVRLVyZtfp5-16jg0,1433 -jinja2/debug.py,sha256=CnHqCDHd-BVGvti_8ZsTolnXNhA3ECsY-6n_2pwU8Hw,6297 -jinja2/defaults.py,sha256=boBcSw78h-lp20YbaXSJsqkAI2uN_mD_TtCydpeq5wU,1267 -jinja2/environment.py,sha256=9nhrP7Ch-NbGX00wvyr4yy-uhNHq2OCc60ggGrni_fk,61513 -jinja2/exceptions.py,sha256=ioHeHrWwCWNaXX1inHmHVblvc4haO7AXsjCp3GfWvx0,5071 -jinja2/ext.py,sha256=5PF5eHfh8mXAIxXHHRB2xXbXohi8pE3nHSOxa66uS7E,31875 -jinja2/filters.py,sha256=PQ_Egd9n9jSgtnGQYyF4K5j2nYwhUIulhPnyimkdr-k,55212 -jinja2/idtracking.py,sha256=-ll5lIp73pML3ErUYiIJj7tdmWxcH_IlDv3yA_hiZYo,10555 -jinja2/lexer.py,sha256=LYiYio6br-Tep9nPcupWXsPEtjluw3p1mU-lNBVRUfk,29786 -jinja2/loaders.py,sha256=wIrnxjvcbqh5VwW28NSkfotiDq8qNCxIOSFbGUiSLB4,24055 -jinja2/meta.py,sha256=OTDPkaFvU2Hgvx-6akz7154F8BIWaRmvJcBFvwopHww,4397 -jinja2/nativetypes.py,sha256=7GIGALVJgdyL80oZJdQUaUfwSt5q2lSSZbXt0dNf_M4,4210 -jinja2/nodes.py,sha256=m1Duzcr6qhZI8JQ6VyJgUNinjAf5bQzijSmDnMsvUx8,34579 -jinja2/optimizer.py,sha256=rJnCRlQ7pZsEEmMhsQDgC_pKyDHxP5TPS6zVPGsgcu8,1651 -jinja2/parser.py,sha256=lLOFy3sEmHc5IaEHRiH1sQVnId2moUQzhyeJZTtdY30,40383 -jinja2/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -jinja2/runtime.py,sha256=gDk-GvdriJXqgsGbHgrcKTP0Yp6zPXzhzrIpCFH3jAU,34249 -jinja2/sandbox.py,sha256=Mw2aitlY2I8la7FYhcX2YG9BtUYcLnD0Gh3d29cDWrY,15009 -jinja2/tests.py,sha256=VLsBhVFnWg-PxSBz1MhRnNWgP1ovXk3neO1FLQMeC9Q,5926 -jinja2/utils.py,sha256=rRp3o9e7ZKS4fyrWRbELyLcpuGVTFcnooaOa1qx_FIk,24129 -jinja2/visitor.py,sha256=EcnL1PIwf_4RVCOMxsRNuR8AXHbS1qfAdMOE2ngKJz4,3557 diff --git a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/WHEEL b/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/WHEEL deleted file mode 100644 index 23d2d7e9a5d381ef8a375db09f82052144d1fd96..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: flit 3.11.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/entry_points.txt b/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/entry_points.txt deleted file mode 100644 index abc3eae3b3bc573957cf7401711948799b3465c0..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/entry_points.txt +++ /dev/null @@ -1,3 +0,0 @@ -[babel.extractors] -jinja2=jinja2.ext:babel_extract[i18n] - diff --git a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/licenses/LICENSE.txt b/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/licenses/LICENSE.txt deleted file mode 100644 index c37cae49ec77ad6ebb25568c1605f1fee5313cfb..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2-3.1.6.dist-info/licenses/LICENSE.txt +++ /dev/null @@ -1,28 +0,0 @@ -Copyright 2007 Pallets - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bundle/python-cpu/Lib/site-packages/jinja2/__init__.py b/bundle/python-cpu/Lib/site-packages/jinja2/__init__.py deleted file mode 100644 index 1a423a3eac16ac550e8e4008d7f5d79401b50e0f..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Jinja is a template engine written in pure Python. It provides a -non-XML syntax that supports inline expressions and an optional -sandboxed environment. -""" - -from .bccache import BytecodeCache as BytecodeCache -from .bccache import FileSystemBytecodeCache as FileSystemBytecodeCache -from .bccache import MemcachedBytecodeCache as MemcachedBytecodeCache -from .environment import Environment as Environment -from .environment import Template as Template -from .exceptions import TemplateAssertionError as TemplateAssertionError -from .exceptions import TemplateError as TemplateError -from .exceptions import TemplateNotFound as TemplateNotFound -from .exceptions import TemplateRuntimeError as TemplateRuntimeError -from .exceptions import TemplatesNotFound as TemplatesNotFound -from .exceptions import TemplateSyntaxError as TemplateSyntaxError -from .exceptions import UndefinedError as UndefinedError -from .loaders import BaseLoader as BaseLoader -from .loaders import ChoiceLoader as ChoiceLoader -from .loaders import DictLoader as DictLoader -from .loaders import FileSystemLoader as FileSystemLoader -from .loaders import FunctionLoader as FunctionLoader -from .loaders import ModuleLoader as ModuleLoader -from .loaders import PackageLoader as PackageLoader -from .loaders import PrefixLoader as PrefixLoader -from .runtime import ChainableUndefined as ChainableUndefined -from .runtime import DebugUndefined as DebugUndefined -from .runtime import make_logging_undefined as make_logging_undefined -from .runtime import StrictUndefined as StrictUndefined -from .runtime import Undefined as Undefined -from .utils import clear_caches as clear_caches -from .utils import is_undefined as is_undefined -from .utils import pass_context as pass_context -from .utils import pass_environment as pass_environment -from .utils import pass_eval_context as pass_eval_context -from .utils import select_autoescape as select_autoescape - -__version__ = "3.1.6" diff --git a/bundle/python-cpu/Lib/site-packages/jinja2/_identifier.py b/bundle/python-cpu/Lib/site-packages/jinja2/_identifier.py deleted file mode 100644 index 928c1503c7d414a8a86bbf5a82c68d42cb089bd2..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2/_identifier.py +++ /dev/null @@ -1,6 +0,0 @@ -import re - -# generated by scripts/generate_identifier_pattern.py -pattern = re.compile( - r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߽߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛࣓-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣ৾ਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣૺ-૿ଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఄా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഀ-ഃ഻഼ാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳷-᳹᷀-᷹᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꣿꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏_𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𐴤-𐽆𐴧-𐽐𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑄴𑅅𑅆𑅳𑆀-𑆂𑆳-𑇀𑇉-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌻𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑑞𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑠬-𑠺𑨁-𑨊𑨳-𑨹𑨻-𑨾𑩇𑩑-𑩛𑪊-𑪙𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𑴱-𑴶𑴺𑴼𑴽𑴿-𑵅𑵇𑶊-𑶎𑶐𑶑𑶓-𑶗𑻳-𑻶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+" # noqa: B950 -) diff --git a/bundle/python-cpu/Lib/site-packages/jinja2/async_utils.py b/bundle/python-cpu/Lib/site-packages/jinja2/async_utils.py deleted file mode 100644 index f0c140205c50a3df9863ce1ab610b0c62a483f1b..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2/async_utils.py +++ /dev/null @@ -1,99 +0,0 @@ -import inspect -import typing as t -from functools import WRAPPER_ASSIGNMENTS -from functools import wraps - -from .utils import _PassArg -from .utils import pass_eval_context - -if t.TYPE_CHECKING: - import typing_extensions as te - -V = t.TypeVar("V") - - -def async_variant(normal_func): # type: ignore - def decorator(async_func): # type: ignore - pass_arg = _PassArg.from_obj(normal_func) - need_eval_context = pass_arg is None - - if pass_arg is _PassArg.environment: - - def is_async(args: t.Any) -> bool: - return t.cast(bool, args[0].is_async) - - else: - - def is_async(args: t.Any) -> bool: - return t.cast(bool, args[0].environment.is_async) - - # Take the doc and annotations from the sync function, but the - # name from the async function. Pallets-Sphinx-Themes - # build_function_directive expects __wrapped__ to point to the - # sync function. - async_func_attrs = ("__module__", "__name__", "__qualname__") - normal_func_attrs = tuple(set(WRAPPER_ASSIGNMENTS).difference(async_func_attrs)) - - @wraps(normal_func, assigned=normal_func_attrs) - @wraps(async_func, assigned=async_func_attrs, updated=()) - def wrapper(*args, **kwargs): # type: ignore - b = is_async(args) - - if need_eval_context: - args = args[1:] - - if b: - return async_func(*args, **kwargs) - - return normal_func(*args, **kwargs) - - if need_eval_context: - wrapper = pass_eval_context(wrapper) - - wrapper.jinja_async_variant = True # type: ignore[attr-defined] - return wrapper - - return decorator - - -_common_primitives = {int, float, bool, str, list, dict, tuple, type(None)} - - -async def auto_await(value: t.Union[t.Awaitable["V"], "V"]) -> "V": - # Avoid a costly call to isawaitable - if type(value) in _common_primitives: - return t.cast("V", value) - - if inspect.isawaitable(value): - return await t.cast("t.Awaitable[V]", value) - - return value - - -class _IteratorToAsyncIterator(t.Generic[V]): - def __init__(self, iterator: "t.Iterator[V]"): - self._iterator = iterator - - def __aiter__(self) -> "te.Self": - return self - - async def __anext__(self) -> V: - try: - return next(self._iterator) - except StopIteration as e: - raise StopAsyncIteration(e.value) from e - - -def auto_aiter( - iterable: "t.Union[t.AsyncIterable[V], t.Iterable[V]]", -) -> "t.AsyncIterator[V]": - if hasattr(iterable, "__aiter__"): - return iterable.__aiter__() - else: - return _IteratorToAsyncIterator(iter(iterable)) - - -async def auto_to_list( - value: "t.Union[t.AsyncIterable[V], t.Iterable[V]]", -) -> t.List["V"]: - return [x async for x in auto_aiter(value)] diff --git a/bundle/python-cpu/Lib/site-packages/jinja2/bccache.py b/bundle/python-cpu/Lib/site-packages/jinja2/bccache.py deleted file mode 100644 index ada8b099ff251ea9c6da4c42e1383f37e359f06a..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2/bccache.py +++ /dev/null @@ -1,408 +0,0 @@ -"""The optional bytecode cache system. This is useful if you have very -complex template situations and the compilation of all those templates -slows down your application too much. - -Situations where this is useful are often forking web applications that -are initialized on the first request. -""" - -import errno -import fnmatch -import marshal -import os -import pickle -import stat -import sys -import tempfile -import typing as t -from hashlib import sha1 -from io import BytesIO -from types import CodeType - -if t.TYPE_CHECKING: - import typing_extensions as te - - from .environment import Environment - - class _MemcachedClient(te.Protocol): - def get(self, key: str) -> bytes: ... - - def set( - self, key: str, value: bytes, timeout: t.Optional[int] = None - ) -> None: ... - - -bc_version = 5 -# Magic bytes to identify Jinja bytecode cache files. Contains the -# Python major and minor version to avoid loading incompatible bytecode -# if a project upgrades its Python version. -bc_magic = ( - b"j2" - + pickle.dumps(bc_version, 2) - + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2) -) - - -class Bucket: - """Buckets are used to store the bytecode for one template. It's created - and initialized by the bytecode cache and passed to the loading functions. - - The buckets get an internal checksum from the cache assigned and use this - to automatically reject outdated cache material. Individual bytecode - cache subclasses don't have to care about cache invalidation. - """ - - def __init__(self, environment: "Environment", key: str, checksum: str) -> None: - self.environment = environment - self.key = key - self.checksum = checksum - self.reset() - - def reset(self) -> None: - """Resets the bucket (unloads the bytecode).""" - self.code: t.Optional[CodeType] = None - - def load_bytecode(self, f: t.BinaryIO) -> None: - """Loads bytecode from a file or file like object.""" - # make sure the magic header is correct - magic = f.read(len(bc_magic)) - if magic != bc_magic: - self.reset() - return - # the source code of the file changed, we need to reload - checksum = pickle.load(f) - if self.checksum != checksum: - self.reset() - return - # if marshal_load fails then we need to reload - try: - self.code = marshal.load(f) - except (EOFError, ValueError, TypeError): - self.reset() - return - - def write_bytecode(self, f: t.IO[bytes]) -> None: - """Dump the bytecode into the file or file like object passed.""" - if self.code is None: - raise TypeError("can't write empty bucket") - f.write(bc_magic) - pickle.dump(self.checksum, f, 2) - marshal.dump(self.code, f) - - def bytecode_from_string(self, string: bytes) -> None: - """Load bytecode from bytes.""" - self.load_bytecode(BytesIO(string)) - - def bytecode_to_string(self) -> bytes: - """Return the bytecode as bytes.""" - out = BytesIO() - self.write_bytecode(out) - return out.getvalue() - - -class BytecodeCache: - """To implement your own bytecode cache you have to subclass this class - and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of - these methods are passed a :class:`~jinja2.bccache.Bucket`. - - A very basic bytecode cache that saves the bytecode on the file system:: - - from os import path - - class MyCache(BytecodeCache): - - def __init__(self, directory): - self.directory = directory - - def load_bytecode(self, bucket): - filename = path.join(self.directory, bucket.key) - if path.exists(filename): - with open(filename, 'rb') as f: - bucket.load_bytecode(f) - - def dump_bytecode(self, bucket): - filename = path.join(self.directory, bucket.key) - with open(filename, 'wb') as f: - bucket.write_bytecode(f) - - A more advanced version of a filesystem based bytecode cache is part of - Jinja. - """ - - def load_bytecode(self, bucket: Bucket) -> None: - """Subclasses have to override this method to load bytecode into a - bucket. If they are not able to find code in the cache for the - bucket, it must not do anything. - """ - raise NotImplementedError() - - def dump_bytecode(self, bucket: Bucket) -> None: - """Subclasses have to override this method to write the bytecode - from a bucket back to the cache. If it unable to do so it must not - fail silently but raise an exception. - """ - raise NotImplementedError() - - def clear(self) -> None: - """Clears the cache. This method is not used by Jinja but should be - implemented to allow applications to clear the bytecode cache used - by a particular environment. - """ - - def get_cache_key( - self, name: str, filename: t.Optional[t.Union[str]] = None - ) -> str: - """Returns the unique hash key for this template name.""" - hash = sha1(name.encode("utf-8")) - - if filename is not None: - hash.update(f"|{filename}".encode()) - - return hash.hexdigest() - - def get_source_checksum(self, source: str) -> str: - """Returns a checksum for the source.""" - return sha1(source.encode("utf-8")).hexdigest() - - def get_bucket( - self, - environment: "Environment", - name: str, - filename: t.Optional[str], - source: str, - ) -> Bucket: - """Return a cache bucket for the given template. All arguments are - mandatory but filename may be `None`. - """ - key = self.get_cache_key(name, filename) - checksum = self.get_source_checksum(source) - bucket = Bucket(environment, key, checksum) - self.load_bytecode(bucket) - return bucket - - def set_bucket(self, bucket: Bucket) -> None: - """Put the bucket into the cache.""" - self.dump_bytecode(bucket) - - -class FileSystemBytecodeCache(BytecodeCache): - """A bytecode cache that stores bytecode on the filesystem. It accepts - two arguments: The directory where the cache items are stored and a - pattern string that is used to build the filename. - - If no directory is specified a default cache directory is selected. On - Windows the user's temp directory is used, on UNIX systems a directory - is created for the user in the system temp directory. - - The pattern can be used to have multiple separate caches operate on the - same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` - is replaced with the cache key. - - >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') - - This bytecode cache supports clearing of the cache using the clear method. - """ - - def __init__( - self, directory: t.Optional[str] = None, pattern: str = "__jinja2_%s.cache" - ) -> None: - if directory is None: - directory = self._get_default_cache_dir() - self.directory = directory - self.pattern = pattern - - def _get_default_cache_dir(self) -> str: - def _unsafe_dir() -> "te.NoReturn": - raise RuntimeError( - "Cannot determine safe temp directory. You " - "need to explicitly provide one." - ) - - tmpdir = tempfile.gettempdir() - - # On windows the temporary directory is used specific unless - # explicitly forced otherwise. We can just use that. - if os.name == "nt": - return tmpdir - if not hasattr(os, "getuid"): - _unsafe_dir() - - dirname = f"_jinja2-cache-{os.getuid()}" - actual_dir = os.path.join(tmpdir, dirname) - - try: - os.mkdir(actual_dir, stat.S_IRWXU) - except OSError as e: - if e.errno != errno.EEXIST: - raise - try: - os.chmod(actual_dir, stat.S_IRWXU) - actual_dir_stat = os.lstat(actual_dir) - if ( - actual_dir_stat.st_uid != os.getuid() - or not stat.S_ISDIR(actual_dir_stat.st_mode) - or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU - ): - _unsafe_dir() - except OSError as e: - if e.errno != errno.EEXIST: - raise - - actual_dir_stat = os.lstat(actual_dir) - if ( - actual_dir_stat.st_uid != os.getuid() - or not stat.S_ISDIR(actual_dir_stat.st_mode) - or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU - ): - _unsafe_dir() - - return actual_dir - - def _get_cache_filename(self, bucket: Bucket) -> str: - return os.path.join(self.directory, self.pattern % (bucket.key,)) - - def load_bytecode(self, bucket: Bucket) -> None: - filename = self._get_cache_filename(bucket) - - # Don't test for existence before opening the file, since the - # file could disappear after the test before the open. - try: - f = open(filename, "rb") - except (FileNotFoundError, IsADirectoryError, PermissionError): - # PermissionError can occur on Windows when an operation is - # in progress, such as calling clear(). - return - - with f: - bucket.load_bytecode(f) - - def dump_bytecode(self, bucket: Bucket) -> None: - # Write to a temporary file, then rename to the real name after - # writing. This avoids another process reading the file before - # it is fully written. - name = self._get_cache_filename(bucket) - f = tempfile.NamedTemporaryFile( - mode="wb", - dir=os.path.dirname(name), - prefix=os.path.basename(name), - suffix=".tmp", - delete=False, - ) - - def remove_silent() -> None: - try: - os.remove(f.name) - except OSError: - # Another process may have called clear(). On Windows, - # another program may be holding the file open. - pass - - try: - with f: - bucket.write_bytecode(f) - except BaseException: - remove_silent() - raise - - try: - os.replace(f.name, name) - except OSError: - # Another process may have called clear(). On Windows, - # another program may be holding the file open. - remove_silent() - except BaseException: - remove_silent() - raise - - def clear(self) -> None: - # imported lazily here because google app-engine doesn't support - # write access on the file system and the function does not exist - # normally. - from os import remove - - files = fnmatch.filter(os.listdir(self.directory), self.pattern % ("*",)) - for filename in files: - try: - remove(os.path.join(self.directory, filename)) - except OSError: - pass - - -class MemcachedBytecodeCache(BytecodeCache): - """This class implements a bytecode cache that uses a memcache cache for - storing the information. It does not enforce a specific memcache library - (tummy's memcache or cmemcache) but will accept any class that provides - the minimal interface required. - - Libraries compatible with this class: - - - `cachelib `_ - - `python-memcached `_ - - (Unfortunately the django cache interface is not compatible because it - does not support storing binary data, only text. You can however pass - the underlying cache client to the bytecode cache which is available - as `django.core.cache.cache._client`.) - - The minimal interface for the client passed to the constructor is this: - - .. class:: MinimalClientInterface - - .. method:: set(key, value[, timeout]) - - Stores the bytecode in the cache. `value` is a string and - `timeout` the timeout of the key. If timeout is not provided - a default timeout or no timeout should be assumed, if it's - provided it's an integer with the number of seconds the cache - item should exist. - - .. method:: get(key) - - Returns the value for the cache key. If the item does not - exist in the cache the return value must be `None`. - - The other arguments to the constructor are the prefix for all keys that - is added before the actual cache key and the timeout for the bytecode in - the cache system. We recommend a high (or no) timeout. - - This bytecode cache does not support clearing of used items in the cache. - The clear method is a no-operation function. - - .. versionadded:: 2.7 - Added support for ignoring memcache errors through the - `ignore_memcache_errors` parameter. - """ - - def __init__( - self, - client: "_MemcachedClient", - prefix: str = "jinja2/bytecode/", - timeout: t.Optional[int] = None, - ignore_memcache_errors: bool = True, - ): - self.client = client - self.prefix = prefix - self.timeout = timeout - self.ignore_memcache_errors = ignore_memcache_errors - - def load_bytecode(self, bucket: Bucket) -> None: - try: - code = self.client.get(self.prefix + bucket.key) - except Exception: - if not self.ignore_memcache_errors: - raise - else: - bucket.bytecode_from_string(code) - - def dump_bytecode(self, bucket: Bucket) -> None: - key = self.prefix + bucket.key - value = bucket.bytecode_to_string() - - try: - if self.timeout is not None: - self.client.set(key, value, self.timeout) - else: - self.client.set(key, value) - except Exception: - if not self.ignore_memcache_errors: - raise diff --git a/bundle/python-cpu/Lib/site-packages/jinja2/compiler.py b/bundle/python-cpu/Lib/site-packages/jinja2/compiler.py deleted file mode 100644 index a4ff6a1b11af3e1a868d1a74c48d842390259b43..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2/compiler.py +++ /dev/null @@ -1,1998 +0,0 @@ -"""Compiles nodes from the parser into Python code.""" - -import typing as t -from contextlib import contextmanager -from functools import update_wrapper -from io import StringIO -from itertools import chain -from keyword import iskeyword as is_python_keyword - -from markupsafe import escape -from markupsafe import Markup - -from . import nodes -from .exceptions import TemplateAssertionError -from .idtracking import Symbols -from .idtracking import VAR_LOAD_ALIAS -from .idtracking import VAR_LOAD_PARAMETER -from .idtracking import VAR_LOAD_RESOLVE -from .idtracking import VAR_LOAD_UNDEFINED -from .nodes import EvalContext -from .optimizer import Optimizer -from .utils import _PassArg -from .utils import concat -from .visitor import NodeVisitor - -if t.TYPE_CHECKING: - import typing_extensions as te - - from .environment import Environment - -F = t.TypeVar("F", bound=t.Callable[..., t.Any]) - -operators = { - "eq": "==", - "ne": "!=", - "gt": ">", - "gteq": ">=", - "lt": "<", - "lteq": "<=", - "in": "in", - "notin": "not in", -} - - -def optimizeconst(f: F) -> F: - def new_func( - self: "CodeGenerator", node: nodes.Expr, frame: "Frame", **kwargs: t.Any - ) -> t.Any: - # Only optimize if the frame is not volatile - if self.optimizer is not None and not frame.eval_ctx.volatile: - new_node = self.optimizer.visit(node, frame.eval_ctx) - - if new_node != node: - return self.visit(new_node, frame) - - return f(self, node, frame, **kwargs) - - return update_wrapper(new_func, f) # type: ignore[return-value] - - -def _make_binop(op: str) -> t.Callable[["CodeGenerator", nodes.BinExpr, "Frame"], None]: - @optimizeconst - def visitor(self: "CodeGenerator", node: nodes.BinExpr, frame: Frame) -> None: - if ( - self.environment.sandboxed and op in self.environment.intercepted_binops # type: ignore - ): - self.write(f"environment.call_binop(context, {op!r}, ") - self.visit(node.left, frame) - self.write(", ") - self.visit(node.right, frame) - else: - self.write("(") - self.visit(node.left, frame) - self.write(f" {op} ") - self.visit(node.right, frame) - - self.write(")") - - return visitor - - -def _make_unop( - op: str, -) -> t.Callable[["CodeGenerator", nodes.UnaryExpr, "Frame"], None]: - @optimizeconst - def visitor(self: "CodeGenerator", node: nodes.UnaryExpr, frame: Frame) -> None: - if ( - self.environment.sandboxed and op in self.environment.intercepted_unops # type: ignore - ): - self.write(f"environment.call_unop(context, {op!r}, ") - self.visit(node.node, frame) - else: - self.write("(" + op) - self.visit(node.node, frame) - - self.write(")") - - return visitor - - -def generate( - node: nodes.Template, - environment: "Environment", - name: t.Optional[str], - filename: t.Optional[str], - stream: t.Optional[t.TextIO] = None, - defer_init: bool = False, - optimized: bool = True, -) -> t.Optional[str]: - """Generate the python source for a node tree.""" - if not isinstance(node, nodes.Template): - raise TypeError("Can't compile non template nodes") - - generator = environment.code_generator_class( - environment, name, filename, stream, defer_init, optimized - ) - generator.visit(node) - - if stream is None: - return generator.stream.getvalue() # type: ignore - - return None - - -def has_safe_repr(value: t.Any) -> bool: - """Does the node have a safe representation?""" - if value is None or value is NotImplemented or value is Ellipsis: - return True - - if type(value) in {bool, int, float, complex, range, str, Markup}: - return True - - if type(value) in {tuple, list, set, frozenset}: - return all(has_safe_repr(v) for v in value) - - if type(value) is dict: # noqa E721 - return all(has_safe_repr(k) and has_safe_repr(v) for k, v in value.items()) - - return False - - -def find_undeclared( - nodes: t.Iterable[nodes.Node], names: t.Iterable[str] -) -> t.Set[str]: - """Check if the names passed are accessed undeclared. The return value - is a set of all the undeclared names from the sequence of names found. - """ - visitor = UndeclaredNameVisitor(names) - try: - for node in nodes: - visitor.visit(node) - except VisitorExit: - pass - return visitor.undeclared - - -class MacroRef: - def __init__(self, node: t.Union[nodes.Macro, nodes.CallBlock]) -> None: - self.node = node - self.accesses_caller = False - self.accesses_kwargs = False - self.accesses_varargs = False - - -class Frame: - """Holds compile time information for us.""" - - def __init__( - self, - eval_ctx: EvalContext, - parent: t.Optional["Frame"] = None, - level: t.Optional[int] = None, - ) -> None: - self.eval_ctx = eval_ctx - - # the parent of this frame - self.parent = parent - - if parent is None: - self.symbols = Symbols(level=level) - - # in some dynamic inheritance situations the compiler needs to add - # write tests around output statements. - self.require_output_check = False - - # inside some tags we are using a buffer rather than yield statements. - # this for example affects {% filter %} or {% macro %}. If a frame - # is buffered this variable points to the name of the list used as - # buffer. - self.buffer: t.Optional[str] = None - - # the name of the block we're in, otherwise None. - self.block: t.Optional[str] = None - - else: - self.symbols = Symbols(parent.symbols, level=level) - self.require_output_check = parent.require_output_check - self.buffer = parent.buffer - self.block = parent.block - - # a toplevel frame is the root + soft frames such as if conditions. - self.toplevel = False - - # the root frame is basically just the outermost frame, so no if - # conditions. This information is used to optimize inheritance - # situations. - self.rootlevel = False - - # variables set inside of loops and blocks should not affect outer frames, - # but they still needs to be kept track of as part of the active context. - self.loop_frame = False - self.block_frame = False - - # track whether the frame is being used in an if-statement or conditional - # expression as it determines which errors should be raised during runtime - # or compile time. - self.soft_frame = False - - def copy(self) -> "te.Self": - """Create a copy of the current one.""" - rv = object.__new__(self.__class__) - rv.__dict__.update(self.__dict__) - rv.symbols = self.symbols.copy() - return rv - - def inner(self, isolated: bool = False) -> "Frame": - """Return an inner frame.""" - if isolated: - return Frame(self.eval_ctx, level=self.symbols.level + 1) - return Frame(self.eval_ctx, self) - - def soft(self) -> "te.Self": - """Return a soft frame. A soft frame may not be modified as - standalone thing as it shares the resources with the frame it - was created of, but it's not a rootlevel frame any longer. - - This is only used to implement if-statements and conditional - expressions. - """ - rv = self.copy() - rv.rootlevel = False - rv.soft_frame = True - return rv - - __copy__ = copy - - -class VisitorExit(RuntimeError): - """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" - - -class DependencyFinderVisitor(NodeVisitor): - """A visitor that collects filter and test calls.""" - - def __init__(self) -> None: - self.filters: t.Set[str] = set() - self.tests: t.Set[str] = set() - - def visit_Filter(self, node: nodes.Filter) -> None: - self.generic_visit(node) - self.filters.add(node.name) - - def visit_Test(self, node: nodes.Test) -> None: - self.generic_visit(node) - self.tests.add(node.name) - - def visit_Block(self, node: nodes.Block) -> None: - """Stop visiting at blocks.""" - - -class UndeclaredNameVisitor(NodeVisitor): - """A visitor that checks if a name is accessed without being - declared. This is different from the frame visitor as it will - not stop at closure frames. - """ - - def __init__(self, names: t.Iterable[str]) -> None: - self.names = set(names) - self.undeclared: t.Set[str] = set() - - def visit_Name(self, node: nodes.Name) -> None: - if node.ctx == "load" and node.name in self.names: - self.undeclared.add(node.name) - if self.undeclared == self.names: - raise VisitorExit() - else: - self.names.discard(node.name) - - def visit_Block(self, node: nodes.Block) -> None: - """Stop visiting a blocks.""" - - -class CompilerExit(Exception): - """Raised if the compiler encountered a situation where it just - doesn't make sense to further process the code. Any block that - raises such an exception is not further processed. - """ - - -class CodeGenerator(NodeVisitor): - def __init__( - self, - environment: "Environment", - name: t.Optional[str], - filename: t.Optional[str], - stream: t.Optional[t.TextIO] = None, - defer_init: bool = False, - optimized: bool = True, - ) -> None: - if stream is None: - stream = StringIO() - self.environment = environment - self.name = name - self.filename = filename - self.stream = stream - self.created_block_context = False - self.defer_init = defer_init - self.optimizer: t.Optional[Optimizer] = None - - if optimized: - self.optimizer = Optimizer(environment) - - # aliases for imports - self.import_aliases: t.Dict[str, str] = {} - - # a registry for all blocks. Because blocks are moved out - # into the global python scope they are registered here - self.blocks: t.Dict[str, nodes.Block] = {} - - # the number of extends statements so far - self.extends_so_far = 0 - - # some templates have a rootlevel extends. In this case we - # can safely assume that we're a child template and do some - # more optimizations. - self.has_known_extends = False - - # the current line number - self.code_lineno = 1 - - # registry of all filters and tests (global, not block local) - self.tests: t.Dict[str, str] = {} - self.filters: t.Dict[str, str] = {} - - # the debug information - self.debug_info: t.List[t.Tuple[int, int]] = [] - self._write_debug_info: t.Optional[int] = None - - # the number of new lines before the next write() - self._new_lines = 0 - - # the line number of the last written statement - self._last_line = 0 - - # true if nothing was written so far. - self._first_write = True - - # used by the `temporary_identifier` method to get new - # unique, temporary identifier - self._last_identifier = 0 - - # the current indentation - self._indentation = 0 - - # Tracks toplevel assignments - self._assign_stack: t.List[t.Set[str]] = [] - - # Tracks parameter definition blocks - self._param_def_block: t.List[t.Set[str]] = [] - - # Tracks the current context. - self._context_reference_stack = ["context"] - - @property - def optimized(self) -> bool: - return self.optimizer is not None - - # -- Various compilation helpers - - def fail(self, msg: str, lineno: int) -> "te.NoReturn": - """Fail with a :exc:`TemplateAssertionError`.""" - raise TemplateAssertionError(msg, lineno, self.name, self.filename) - - def temporary_identifier(self) -> str: - """Get a new unique identifier.""" - self._last_identifier += 1 - return f"t_{self._last_identifier}" - - def buffer(self, frame: Frame) -> None: - """Enable buffering for the frame from that point onwards.""" - frame.buffer = self.temporary_identifier() - self.writeline(f"{frame.buffer} = []") - - def return_buffer_contents( - self, frame: Frame, force_unescaped: bool = False - ) -> None: - """Return the buffer contents of the frame.""" - if not force_unescaped: - if frame.eval_ctx.volatile: - self.writeline("if context.eval_ctx.autoescape:") - self.indent() - self.writeline(f"return Markup(concat({frame.buffer}))") - self.outdent() - self.writeline("else:") - self.indent() - self.writeline(f"return concat({frame.buffer})") - self.outdent() - return - elif frame.eval_ctx.autoescape: - self.writeline(f"return Markup(concat({frame.buffer}))") - return - self.writeline(f"return concat({frame.buffer})") - - def indent(self) -> None: - """Indent by one.""" - self._indentation += 1 - - def outdent(self, step: int = 1) -> None: - """Outdent by step.""" - self._indentation -= step - - def start_write(self, frame: Frame, node: t.Optional[nodes.Node] = None) -> None: - """Yield or write into the frame buffer.""" - if frame.buffer is None: - self.writeline("yield ", node) - else: - self.writeline(f"{frame.buffer}.append(", node) - - def end_write(self, frame: Frame) -> None: - """End the writing process started by `start_write`.""" - if frame.buffer is not None: - self.write(")") - - def simple_write( - self, s: str, frame: Frame, node: t.Optional[nodes.Node] = None - ) -> None: - """Simple shortcut for start_write + write + end_write.""" - self.start_write(frame, node) - self.write(s) - self.end_write(frame) - - def blockvisit(self, nodes: t.Iterable[nodes.Node], frame: Frame) -> None: - """Visit a list of nodes as block in a frame. If the current frame - is no buffer a dummy ``if 0: yield None`` is written automatically. - """ - try: - self.writeline("pass") - for node in nodes: - self.visit(node, frame) - except CompilerExit: - pass - - def write(self, x: str) -> None: - """Write a string into the output stream.""" - if self._new_lines: - if not self._first_write: - self.stream.write("\n" * self._new_lines) - self.code_lineno += self._new_lines - if self._write_debug_info is not None: - self.debug_info.append((self._write_debug_info, self.code_lineno)) - self._write_debug_info = None - self._first_write = False - self.stream.write(" " * self._indentation) - self._new_lines = 0 - self.stream.write(x) - - def writeline( - self, x: str, node: t.Optional[nodes.Node] = None, extra: int = 0 - ) -> None: - """Combination of newline and write.""" - self.newline(node, extra) - self.write(x) - - def newline(self, node: t.Optional[nodes.Node] = None, extra: int = 0) -> None: - """Add one or more newlines before the next write.""" - self._new_lines = max(self._new_lines, 1 + extra) - if node is not None and node.lineno != self._last_line: - self._write_debug_info = node.lineno - self._last_line = node.lineno - - def signature( - self, - node: t.Union[nodes.Call, nodes.Filter, nodes.Test], - frame: Frame, - extra_kwargs: t.Optional[t.Mapping[str, t.Any]] = None, - ) -> None: - """Writes a function call to the stream for the current node. - A leading comma is added automatically. The extra keyword - arguments may not include python keywords otherwise a syntax - error could occur. The extra keyword arguments should be given - as python dict. - """ - # if any of the given keyword arguments is a python keyword - # we have to make sure that no invalid call is created. - kwarg_workaround = any( - is_python_keyword(t.cast(str, k)) - for k in chain((x.key for x in node.kwargs), extra_kwargs or ()) - ) - - for arg in node.args: - self.write(", ") - self.visit(arg, frame) - - if not kwarg_workaround: - for kwarg in node.kwargs: - self.write(", ") - self.visit(kwarg, frame) - if extra_kwargs is not None: - for key, value in extra_kwargs.items(): - self.write(f", {key}={value}") - if node.dyn_args: - self.write(", *") - self.visit(node.dyn_args, frame) - - if kwarg_workaround: - if node.dyn_kwargs is not None: - self.write(", **dict({") - else: - self.write(", **{") - for kwarg in node.kwargs: - self.write(f"{kwarg.key!r}: ") - self.visit(kwarg.value, frame) - self.write(", ") - if extra_kwargs is not None: - for key, value in extra_kwargs.items(): - self.write(f"{key!r}: {value}, ") - if node.dyn_kwargs is not None: - self.write("}, **") - self.visit(node.dyn_kwargs, frame) - self.write(")") - else: - self.write("}") - - elif node.dyn_kwargs is not None: - self.write(", **") - self.visit(node.dyn_kwargs, frame) - - def pull_dependencies(self, nodes: t.Iterable[nodes.Node]) -> None: - """Find all filter and test names used in the template and - assign them to variables in the compiled namespace. Checking - that the names are registered with the environment is done when - compiling the Filter and Test nodes. If the node is in an If or - CondExpr node, the check is done at runtime instead. - - .. versionchanged:: 3.0 - Filters and tests in If and CondExpr nodes are checked at - runtime instead of compile time. - """ - visitor = DependencyFinderVisitor() - - for node in nodes: - visitor.visit(node) - - for id_map, names, dependency in ( - (self.filters, visitor.filters, "filters"), - ( - self.tests, - visitor.tests, - "tests", - ), - ): - for name in sorted(names): - if name not in id_map: - id_map[name] = self.temporary_identifier() - - # add check during runtime that dependencies used inside of executed - # blocks are defined, as this step may be skipped during compile time - self.writeline("try:") - self.indent() - self.writeline(f"{id_map[name]} = environment.{dependency}[{name!r}]") - self.outdent() - self.writeline("except KeyError:") - self.indent() - self.writeline("@internalcode") - self.writeline(f"def {id_map[name]}(*unused):") - self.indent() - self.writeline( - f'raise TemplateRuntimeError("No {dependency[:-1]}' - f' named {name!r} found.")' - ) - self.outdent() - self.outdent() - - def enter_frame(self, frame: Frame) -> None: - undefs = [] - for target, (action, param) in frame.symbols.loads.items(): - if action == VAR_LOAD_PARAMETER: - pass - elif action == VAR_LOAD_RESOLVE: - self.writeline(f"{target} = {self.get_resolve_func()}({param!r})") - elif action == VAR_LOAD_ALIAS: - self.writeline(f"{target} = {param}") - elif action == VAR_LOAD_UNDEFINED: - undefs.append(target) - else: - raise NotImplementedError("unknown load instruction") - if undefs: - self.writeline(f"{' = '.join(undefs)} = missing") - - def leave_frame(self, frame: Frame, with_python_scope: bool = False) -> None: - if not with_python_scope: - undefs = [] - for target in frame.symbols.loads: - undefs.append(target) - if undefs: - self.writeline(f"{' = '.join(undefs)} = missing") - - def choose_async(self, async_value: str = "async ", sync_value: str = "") -> str: - return async_value if self.environment.is_async else sync_value - - def func(self, name: str) -> str: - return f"{self.choose_async()}def {name}" - - def macro_body( - self, node: t.Union[nodes.Macro, nodes.CallBlock], frame: Frame - ) -> t.Tuple[Frame, MacroRef]: - """Dump the function def of a macro or call block.""" - frame = frame.inner() - frame.symbols.analyze_node(node) - macro_ref = MacroRef(node) - - explicit_caller = None - skip_special_params = set() - args = [] - - for idx, arg in enumerate(node.args): - if arg.name == "caller": - explicit_caller = idx - if arg.name in ("kwargs", "varargs"): - skip_special_params.add(arg.name) - args.append(frame.symbols.ref(arg.name)) - - undeclared = find_undeclared(node.body, ("caller", "kwargs", "varargs")) - - if "caller" in undeclared: - # In older Jinja versions there was a bug that allowed caller - # to retain the special behavior even if it was mentioned in - # the argument list. However thankfully this was only really - # working if it was the last argument. So we are explicitly - # checking this now and error out if it is anywhere else in - # the argument list. - if explicit_caller is not None: - try: - node.defaults[explicit_caller - len(node.args)] - except IndexError: - self.fail( - "When defining macros or call blocks the " - 'special "caller" argument must be omitted ' - "or be given a default.", - node.lineno, - ) - else: - args.append(frame.symbols.declare_parameter("caller")) - macro_ref.accesses_caller = True - if "kwargs" in undeclared and "kwargs" not in skip_special_params: - args.append(frame.symbols.declare_parameter("kwargs")) - macro_ref.accesses_kwargs = True - if "varargs" in undeclared and "varargs" not in skip_special_params: - args.append(frame.symbols.declare_parameter("varargs")) - macro_ref.accesses_varargs = True - - # macros are delayed, they never require output checks - frame.require_output_check = False - frame.symbols.analyze_node(node) - self.writeline(f"{self.func('macro')}({', '.join(args)}):", node) - self.indent() - - self.buffer(frame) - self.enter_frame(frame) - - self.push_parameter_definitions(frame) - for idx, arg in enumerate(node.args): - ref = frame.symbols.ref(arg.name) - self.writeline(f"if {ref} is missing:") - self.indent() - try: - default = node.defaults[idx - len(node.args)] - except IndexError: - self.writeline( - f'{ref} = undefined("parameter {arg.name!r} was not provided",' - f" name={arg.name!r})" - ) - else: - self.writeline(f"{ref} = ") - self.visit(default, frame) - self.mark_parameter_stored(ref) - self.outdent() - self.pop_parameter_definitions() - - self.blockvisit(node.body, frame) - self.return_buffer_contents(frame, force_unescaped=True) - self.leave_frame(frame, with_python_scope=True) - self.outdent() - - return frame, macro_ref - - def macro_def(self, macro_ref: MacroRef, frame: Frame) -> None: - """Dump the macro definition for the def created by macro_body.""" - arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args) - name = getattr(macro_ref.node, "name", None) - if len(macro_ref.node.args) == 1: - arg_tuple += "," - self.write( - f"Macro(environment, macro, {name!r}, ({arg_tuple})," - f" {macro_ref.accesses_kwargs!r}, {macro_ref.accesses_varargs!r}," - f" {macro_ref.accesses_caller!r}, context.eval_ctx.autoescape)" - ) - - def position(self, node: nodes.Node) -> str: - """Return a human readable position for the node.""" - rv = f"line {node.lineno}" - if self.name is not None: - rv = f"{rv} in {self.name!r}" - return rv - - def dump_local_context(self, frame: Frame) -> str: - items_kv = ", ".join( - f"{name!r}: {target}" - for name, target in frame.symbols.dump_stores().items() - ) - return f"{{{items_kv}}}" - - def write_commons(self) -> None: - """Writes a common preamble that is used by root and block functions. - Primarily this sets up common local helpers and enforces a generator - through a dead branch. - """ - self.writeline("resolve = context.resolve_or_missing") - self.writeline("undefined = environment.undefined") - self.writeline("concat = environment.concat") - # always use the standard Undefined class for the implicit else of - # conditional expressions - self.writeline("cond_expr_undefined = Undefined") - self.writeline("if 0: yield None") - - def push_parameter_definitions(self, frame: Frame) -> None: - """Pushes all parameter targets from the given frame into a local - stack that permits tracking of yet to be assigned parameters. In - particular this enables the optimization from `visit_Name` to skip - undefined expressions for parameters in macros as macros can reference - otherwise unbound parameters. - """ - self._param_def_block.append(frame.symbols.dump_param_targets()) - - def pop_parameter_definitions(self) -> None: - """Pops the current parameter definitions set.""" - self._param_def_block.pop() - - def mark_parameter_stored(self, target: str) -> None: - """Marks a parameter in the current parameter definitions as stored. - This will skip the enforced undefined checks. - """ - if self._param_def_block: - self._param_def_block[-1].discard(target) - - def push_context_reference(self, target: str) -> None: - self._context_reference_stack.append(target) - - def pop_context_reference(self) -> None: - self._context_reference_stack.pop() - - def get_context_ref(self) -> str: - return self._context_reference_stack[-1] - - def get_resolve_func(self) -> str: - target = self._context_reference_stack[-1] - if target == "context": - return "resolve" - return f"{target}.resolve" - - def derive_context(self, frame: Frame) -> str: - return f"{self.get_context_ref()}.derived({self.dump_local_context(frame)})" - - def parameter_is_undeclared(self, target: str) -> bool: - """Checks if a given target is an undeclared parameter.""" - if not self._param_def_block: - return False - return target in self._param_def_block[-1] - - def push_assign_tracking(self) -> None: - """Pushes a new layer for assignment tracking.""" - self._assign_stack.append(set()) - - def pop_assign_tracking(self, frame: Frame) -> None: - """Pops the topmost level for assignment tracking and updates the - context variables if necessary. - """ - vars = self._assign_stack.pop() - if ( - not frame.block_frame - and not frame.loop_frame - and not frame.toplevel - or not vars - ): - return - public_names = [x for x in vars if x[:1] != "_"] - if len(vars) == 1: - name = next(iter(vars)) - ref = frame.symbols.ref(name) - if frame.loop_frame: - self.writeline(f"_loop_vars[{name!r}] = {ref}") - return - if frame.block_frame: - self.writeline(f"_block_vars[{name!r}] = {ref}") - return - self.writeline(f"context.vars[{name!r}] = {ref}") - else: - if frame.loop_frame: - self.writeline("_loop_vars.update({") - elif frame.block_frame: - self.writeline("_block_vars.update({") - else: - self.writeline("context.vars.update({") - for idx, name in enumerate(sorted(vars)): - if idx: - self.write(", ") - ref = frame.symbols.ref(name) - self.write(f"{name!r}: {ref}") - self.write("})") - if not frame.block_frame and not frame.loop_frame and public_names: - if len(public_names) == 1: - self.writeline(f"context.exported_vars.add({public_names[0]!r})") - else: - names_str = ", ".join(map(repr, sorted(public_names))) - self.writeline(f"context.exported_vars.update(({names_str}))") - - # -- Statement Visitors - - def visit_Template( - self, node: nodes.Template, frame: t.Optional[Frame] = None - ) -> None: - assert frame is None, "no root frame allowed" - eval_ctx = EvalContext(self.environment, self.name) - - from .runtime import async_exported - from .runtime import exported - - if self.environment.is_async: - exported_names = sorted(exported + async_exported) - else: - exported_names = sorted(exported) - - self.writeline("from jinja2.runtime import " + ", ".join(exported_names)) - - # if we want a deferred initialization we cannot move the - # environment into a local name - envenv = "" if self.defer_init else ", environment=environment" - - # do we have an extends tag at all? If not, we can save some - # overhead by just not processing any inheritance code. - have_extends = node.find(nodes.Extends) is not None - - # find all blocks - for block in node.find_all(nodes.Block): - if block.name in self.blocks: - self.fail(f"block {block.name!r} defined twice", block.lineno) - self.blocks[block.name] = block - - # find all imports and import them - for import_ in node.find_all(nodes.ImportedName): - if import_.importname not in self.import_aliases: - imp = import_.importname - self.import_aliases[imp] = alias = self.temporary_identifier() - if "." in imp: - module, obj = imp.rsplit(".", 1) - self.writeline(f"from {module} import {obj} as {alias}") - else: - self.writeline(f"import {imp} as {alias}") - - # add the load name - self.writeline(f"name = {self.name!r}") - - # generate the root render function. - self.writeline( - f"{self.func('root')}(context, missing=missing{envenv}):", extra=1 - ) - self.indent() - self.write_commons() - - # process the root - frame = Frame(eval_ctx) - if "self" in find_undeclared(node.body, ("self",)): - ref = frame.symbols.declare_parameter("self") - self.writeline(f"{ref} = TemplateReference(context)") - frame.symbols.analyze_node(node) - frame.toplevel = frame.rootlevel = True - frame.require_output_check = have_extends and not self.has_known_extends - if have_extends: - self.writeline("parent_template = None") - self.enter_frame(frame) - self.pull_dependencies(node.body) - self.blockvisit(node.body, frame) - self.leave_frame(frame, with_python_scope=True) - self.outdent() - - # make sure that the parent root is called. - if have_extends: - if not self.has_known_extends: - self.indent() - self.writeline("if parent_template is not None:") - self.indent() - if not self.environment.is_async: - self.writeline("yield from parent_template.root_render_func(context)") - else: - self.writeline("agen = parent_template.root_render_func(context)") - self.writeline("try:") - self.indent() - self.writeline("async for event in agen:") - self.indent() - self.writeline("yield event") - self.outdent() - self.outdent() - self.writeline("finally: await agen.aclose()") - self.outdent(1 + (not self.has_known_extends)) - - # at this point we now have the blocks collected and can visit them too. - for name, block in self.blocks.items(): - self.writeline( - f"{self.func('block_' + name)}(context, missing=missing{envenv}):", - block, - 1, - ) - self.indent() - self.write_commons() - # It's important that we do not make this frame a child of the - # toplevel template. This would cause a variety of - # interesting issues with identifier tracking. - block_frame = Frame(eval_ctx) - block_frame.block_frame = True - undeclared = find_undeclared(block.body, ("self", "super")) - if "self" in undeclared: - ref = block_frame.symbols.declare_parameter("self") - self.writeline(f"{ref} = TemplateReference(context)") - if "super" in undeclared: - ref = block_frame.symbols.declare_parameter("super") - self.writeline(f"{ref} = context.super({name!r}, block_{name})") - block_frame.symbols.analyze_node(block) - block_frame.block = name - self.writeline("_block_vars = {}") - self.enter_frame(block_frame) - self.pull_dependencies(block.body) - self.blockvisit(block.body, block_frame) - self.leave_frame(block_frame, with_python_scope=True) - self.outdent() - - blocks_kv_str = ", ".join(f"{x!r}: block_{x}" for x in self.blocks) - self.writeline(f"blocks = {{{blocks_kv_str}}}", extra=1) - debug_kv_str = "&".join(f"{k}={v}" for k, v in self.debug_info) - self.writeline(f"debug_info = {debug_kv_str!r}") - - def visit_Block(self, node: nodes.Block, frame: Frame) -> None: - """Call a block and register it for the template.""" - level = 0 - if frame.toplevel: - # if we know that we are a child template, there is no need to - # check if we are one - if self.has_known_extends: - return - if self.extends_so_far > 0: - self.writeline("if parent_template is None:") - self.indent() - level += 1 - - if node.scoped: - context = self.derive_context(frame) - else: - context = self.get_context_ref() - - if node.required: - self.writeline(f"if len(context.blocks[{node.name!r}]) <= 1:", node) - self.indent() - self.writeline( - f'raise TemplateRuntimeError("Required block {node.name!r} not found")', - node, - ) - self.outdent() - - if not self.environment.is_async and frame.buffer is None: - self.writeline( - f"yield from context.blocks[{node.name!r}][0]({context})", node - ) - else: - self.writeline(f"gen = context.blocks[{node.name!r}][0]({context})") - self.writeline("try:") - self.indent() - self.writeline( - f"{self.choose_async()}for event in gen:", - node, - ) - self.indent() - self.simple_write("event", frame) - self.outdent() - self.outdent() - self.writeline( - f"finally: {self.choose_async('await gen.aclose()', 'gen.close()')}" - ) - - self.outdent(level) - - def visit_Extends(self, node: nodes.Extends, frame: Frame) -> None: - """Calls the extender.""" - if not frame.toplevel: - self.fail("cannot use extend from a non top-level scope", node.lineno) - - # if the number of extends statements in general is zero so - # far, we don't have to add a check if something extended - # the template before this one. - if self.extends_so_far > 0: - # if we have a known extends we just add a template runtime - # error into the generated code. We could catch that at compile - # time too, but i welcome it not to confuse users by throwing the - # same error at different times just "because we can". - if not self.has_known_extends: - self.writeline("if parent_template is not None:") - self.indent() - self.writeline('raise TemplateRuntimeError("extended multiple times")') - - # if we have a known extends already we don't need that code here - # as we know that the template execution will end here. - if self.has_known_extends: - raise CompilerExit() - else: - self.outdent() - - self.writeline("parent_template = environment.get_template(", node) - self.visit(node.template, frame) - self.write(f", {self.name!r})") - self.writeline("for name, parent_block in parent_template.blocks.items():") - self.indent() - self.writeline("context.blocks.setdefault(name, []).append(parent_block)") - self.outdent() - - # if this extends statement was in the root level we can take - # advantage of that information and simplify the generated code - # in the top level from this point onwards - if frame.rootlevel: - self.has_known_extends = True - - # and now we have one more - self.extends_so_far += 1 - - def visit_Include(self, node: nodes.Include, frame: Frame) -> None: - """Handles includes.""" - if node.ignore_missing: - self.writeline("try:") - self.indent() - - func_name = "get_or_select_template" - if isinstance(node.template, nodes.Const): - if isinstance(node.template.value, str): - func_name = "get_template" - elif isinstance(node.template.value, (tuple, list)): - func_name = "select_template" - elif isinstance(node.template, (nodes.Tuple, nodes.List)): - func_name = "select_template" - - self.writeline(f"template = environment.{func_name}(", node) - self.visit(node.template, frame) - self.write(f", {self.name!r})") - if node.ignore_missing: - self.outdent() - self.writeline("except TemplateNotFound:") - self.indent() - self.writeline("pass") - self.outdent() - self.writeline("else:") - self.indent() - - def loop_body() -> None: - self.indent() - self.simple_write("event", frame) - self.outdent() - - if node.with_context: - self.writeline( - f"gen = template.root_render_func(" - "template.new_context(context.get_all(), True," - f" {self.dump_local_context(frame)}))" - ) - self.writeline("try:") - self.indent() - self.writeline(f"{self.choose_async()}for event in gen:") - loop_body() - self.outdent() - self.writeline( - f"finally: {self.choose_async('await gen.aclose()', 'gen.close()')}" - ) - elif self.environment.is_async: - self.writeline( - "for event in (await template._get_default_module_async())" - "._body_stream:" - ) - loop_body() - else: - self.writeline("yield from template._get_default_module()._body_stream") - - if node.ignore_missing: - self.outdent() - - def _import_common( - self, node: t.Union[nodes.Import, nodes.FromImport], frame: Frame - ) -> None: - self.write(f"{self.choose_async('await ')}environment.get_template(") - self.visit(node.template, frame) - self.write(f", {self.name!r}).") - - if node.with_context: - f_name = f"make_module{self.choose_async('_async')}" - self.write( - f"{f_name}(context.get_all(), True, {self.dump_local_context(frame)})" - ) - else: - self.write(f"_get_default_module{self.choose_async('_async')}(context)") - - def visit_Import(self, node: nodes.Import, frame: Frame) -> None: - """Visit regular imports.""" - self.writeline(f"{frame.symbols.ref(node.target)} = ", node) - if frame.toplevel: - self.write(f"context.vars[{node.target!r}] = ") - - self._import_common(node, frame) - - if frame.toplevel and not node.target.startswith("_"): - self.writeline(f"context.exported_vars.discard({node.target!r})") - - def visit_FromImport(self, node: nodes.FromImport, frame: Frame) -> None: - """Visit named imports.""" - self.newline(node) - self.write("included_template = ") - self._import_common(node, frame) - var_names = [] - discarded_names = [] - for name in node.names: - if isinstance(name, tuple): - name, alias = name - else: - alias = name - self.writeline( - f"{frame.symbols.ref(alias)} =" - f" getattr(included_template, {name!r}, missing)" - ) - self.writeline(f"if {frame.symbols.ref(alias)} is missing:") - self.indent() - # The position will contain the template name, and will be formatted - # into a string that will be compiled into an f-string. Curly braces - # in the name must be replaced with escapes so that they will not be - # executed as part of the f-string. - position = self.position(node).replace("{", "{{").replace("}", "}}") - message = ( - "the template {included_template.__name__!r}" - f" (imported on {position})" - f" does not export the requested name {name!r}" - ) - self.writeline( - f"{frame.symbols.ref(alias)} = undefined(f{message!r}, name={name!r})" - ) - self.outdent() - if frame.toplevel: - var_names.append(alias) - if not alias.startswith("_"): - discarded_names.append(alias) - - if var_names: - if len(var_names) == 1: - name = var_names[0] - self.writeline(f"context.vars[{name!r}] = {frame.symbols.ref(name)}") - else: - names_kv = ", ".join( - f"{name!r}: {frame.symbols.ref(name)}" for name in var_names - ) - self.writeline(f"context.vars.update({{{names_kv}}})") - if discarded_names: - if len(discarded_names) == 1: - self.writeline(f"context.exported_vars.discard({discarded_names[0]!r})") - else: - names_str = ", ".join(map(repr, discarded_names)) - self.writeline( - f"context.exported_vars.difference_update(({names_str}))" - ) - - def visit_For(self, node: nodes.For, frame: Frame) -> None: - loop_frame = frame.inner() - loop_frame.loop_frame = True - test_frame = frame.inner() - else_frame = frame.inner() - - # try to figure out if we have an extended loop. An extended loop - # is necessary if the loop is in recursive mode if the special loop - # variable is accessed in the body if the body is a scoped block. - extended_loop = ( - node.recursive - or "loop" - in find_undeclared(node.iter_child_nodes(only=("body",)), ("loop",)) - or any(block.scoped for block in node.find_all(nodes.Block)) - ) - - loop_ref = None - if extended_loop: - loop_ref = loop_frame.symbols.declare_parameter("loop") - - loop_frame.symbols.analyze_node(node, for_branch="body") - if node.else_: - else_frame.symbols.analyze_node(node, for_branch="else") - - if node.test: - loop_filter_func = self.temporary_identifier() - test_frame.symbols.analyze_node(node, for_branch="test") - self.writeline(f"{self.func(loop_filter_func)}(fiter):", node.test) - self.indent() - self.enter_frame(test_frame) - self.writeline(self.choose_async("async for ", "for ")) - self.visit(node.target, loop_frame) - self.write(" in ") - self.write(self.choose_async("auto_aiter(fiter)", "fiter")) - self.write(":") - self.indent() - self.writeline("if ", node.test) - self.visit(node.test, test_frame) - self.write(":") - self.indent() - self.writeline("yield ") - self.visit(node.target, loop_frame) - self.outdent(3) - self.leave_frame(test_frame, with_python_scope=True) - - # if we don't have an recursive loop we have to find the shadowed - # variables at that point. Because loops can be nested but the loop - # variable is a special one we have to enforce aliasing for it. - if node.recursive: - self.writeline( - f"{self.func('loop')}(reciter, loop_render_func, depth=0):", node - ) - self.indent() - self.buffer(loop_frame) - - # Use the same buffer for the else frame - else_frame.buffer = loop_frame.buffer - - # make sure the loop variable is a special one and raise a template - # assertion error if a loop tries to write to loop - if extended_loop: - self.writeline(f"{loop_ref} = missing") - - for name in node.find_all(nodes.Name): - if name.ctx == "store" and name.name == "loop": - self.fail( - "Can't assign to special loop variable in for-loop target", - name.lineno, - ) - - if node.else_: - iteration_indicator = self.temporary_identifier() - self.writeline(f"{iteration_indicator} = 1") - - self.writeline(self.choose_async("async for ", "for "), node) - self.visit(node.target, loop_frame) - if extended_loop: - self.write(f", {loop_ref} in {self.choose_async('Async')}LoopContext(") - else: - self.write(" in ") - - if node.test: - self.write(f"{loop_filter_func}(") - if node.recursive: - self.write("reciter") - else: - if self.environment.is_async and not extended_loop: - self.write("auto_aiter(") - self.visit(node.iter, frame) - if self.environment.is_async and not extended_loop: - self.write(")") - if node.test: - self.write(")") - - if node.recursive: - self.write(", undefined, loop_render_func, depth):") - else: - self.write(", undefined):" if extended_loop else ":") - - self.indent() - self.enter_frame(loop_frame) - - self.writeline("_loop_vars = {}") - self.blockvisit(node.body, loop_frame) - if node.else_: - self.writeline(f"{iteration_indicator} = 0") - self.outdent() - self.leave_frame( - loop_frame, with_python_scope=node.recursive and not node.else_ - ) - - if node.else_: - self.writeline(f"if {iteration_indicator}:") - self.indent() - self.enter_frame(else_frame) - self.blockvisit(node.else_, else_frame) - self.leave_frame(else_frame) - self.outdent() - - # if the node was recursive we have to return the buffer contents - # and start the iteration code - if node.recursive: - self.return_buffer_contents(loop_frame) - self.outdent() - self.start_write(frame, node) - self.write(f"{self.choose_async('await ')}loop(") - if self.environment.is_async: - self.write("auto_aiter(") - self.visit(node.iter, frame) - if self.environment.is_async: - self.write(")") - self.write(", loop)") - self.end_write(frame) - - # at the end of the iteration, clear any assignments made in the - # loop from the top level - if self._assign_stack: - self._assign_stack[-1].difference_update(loop_frame.symbols.stores) - - def visit_If(self, node: nodes.If, frame: Frame) -> None: - if_frame = frame.soft() - self.writeline("if ", node) - self.visit(node.test, if_frame) - self.write(":") - self.indent() - self.blockvisit(node.body, if_frame) - self.outdent() - for elif_ in node.elif_: - self.writeline("elif ", elif_) - self.visit(elif_.test, if_frame) - self.write(":") - self.indent() - self.blockvisit(elif_.body, if_frame) - self.outdent() - if node.else_: - self.writeline("else:") - self.indent() - self.blockvisit(node.else_, if_frame) - self.outdent() - - def visit_Macro(self, node: nodes.Macro, frame: Frame) -> None: - macro_frame, macro_ref = self.macro_body(node, frame) - self.newline() - if frame.toplevel: - if not node.name.startswith("_"): - self.write(f"context.exported_vars.add({node.name!r})") - self.writeline(f"context.vars[{node.name!r}] = ") - self.write(f"{frame.symbols.ref(node.name)} = ") - self.macro_def(macro_ref, macro_frame) - - def visit_CallBlock(self, node: nodes.CallBlock, frame: Frame) -> None: - call_frame, macro_ref = self.macro_body(node, frame) - self.writeline("caller = ") - self.macro_def(macro_ref, call_frame) - self.start_write(frame, node) - self.visit_Call(node.call, frame, forward_caller=True) - self.end_write(frame) - - def visit_FilterBlock(self, node: nodes.FilterBlock, frame: Frame) -> None: - filter_frame = frame.inner() - filter_frame.symbols.analyze_node(node) - self.enter_frame(filter_frame) - self.buffer(filter_frame) - self.blockvisit(node.body, filter_frame) - self.start_write(frame, node) - self.visit_Filter(node.filter, filter_frame) - self.end_write(frame) - self.leave_frame(filter_frame) - - def visit_With(self, node: nodes.With, frame: Frame) -> None: - with_frame = frame.inner() - with_frame.symbols.analyze_node(node) - self.enter_frame(with_frame) - for target, expr in zip(node.targets, node.values): - self.newline() - self.visit(target, with_frame) - self.write(" = ") - self.visit(expr, frame) - self.blockvisit(node.body, with_frame) - self.leave_frame(with_frame) - - def visit_ExprStmt(self, node: nodes.ExprStmt, frame: Frame) -> None: - self.newline(node) - self.visit(node.node, frame) - - class _FinalizeInfo(t.NamedTuple): - const: t.Optional[t.Callable[..., str]] - src: t.Optional[str] - - @staticmethod - def _default_finalize(value: t.Any) -> t.Any: - """The default finalize function if the environment isn't - configured with one. Or, if the environment has one, this is - called on that function's output for constants. - """ - return str(value) - - _finalize: t.Optional[_FinalizeInfo] = None - - def _make_finalize(self) -> _FinalizeInfo: - """Build the finalize function to be used on constants and at - runtime. Cached so it's only created once for all output nodes. - - Returns a ``namedtuple`` with the following attributes: - - ``const`` - A function to finalize constant data at compile time. - - ``src`` - Source code to output around nodes to be evaluated at - runtime. - """ - if self._finalize is not None: - return self._finalize - - finalize: t.Optional[t.Callable[..., t.Any]] - finalize = default = self._default_finalize - src = None - - if self.environment.finalize: - src = "environment.finalize(" - env_finalize = self.environment.finalize - pass_arg = { - _PassArg.context: "context", - _PassArg.eval_context: "context.eval_ctx", - _PassArg.environment: "environment", - }.get( - _PassArg.from_obj(env_finalize) # type: ignore - ) - finalize = None - - if pass_arg is None: - - def finalize(value: t.Any) -> t.Any: # noqa: F811 - return default(env_finalize(value)) - - else: - src = f"{src}{pass_arg}, " - - if pass_arg == "environment": - - def finalize(value: t.Any) -> t.Any: # noqa: F811 - return default(env_finalize(self.environment, value)) - - self._finalize = self._FinalizeInfo(finalize, src) - return self._finalize - - def _output_const_repr(self, group: t.Iterable[t.Any]) -> str: - """Given a group of constant values converted from ``Output`` - child nodes, produce a string to write to the template module - source. - """ - return repr(concat(group)) - - def _output_child_to_const( - self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo - ) -> str: - """Try to optimize a child of an ``Output`` node by trying to - convert it to constant, finalized data at compile time. - - If :exc:`Impossible` is raised, the node is not constant and - will be evaluated at runtime. Any other exception will also be - evaluated at runtime for easier debugging. - """ - const = node.as_const(frame.eval_ctx) - - if frame.eval_ctx.autoescape: - const = escape(const) - - # Template data doesn't go through finalize. - if isinstance(node, nodes.TemplateData): - return str(const) - - return finalize.const(const) # type: ignore - - def _output_child_pre( - self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo - ) -> None: - """Output extra source code before visiting a child of an - ``Output`` node. - """ - if frame.eval_ctx.volatile: - self.write("(escape if context.eval_ctx.autoescape else str)(") - elif frame.eval_ctx.autoescape: - self.write("escape(") - else: - self.write("str(") - - if finalize.src is not None: - self.write(finalize.src) - - def _output_child_post( - self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo - ) -> None: - """Output extra source code after visiting a child of an - ``Output`` node. - """ - self.write(")") - - if finalize.src is not None: - self.write(")") - - def visit_Output(self, node: nodes.Output, frame: Frame) -> None: - # If an extends is active, don't render outside a block. - if frame.require_output_check: - # A top-level extends is known to exist at compile time. - if self.has_known_extends: - return - - self.writeline("if parent_template is None:") - self.indent() - - finalize = self._make_finalize() - body: t.List[t.Union[t.List[t.Any], nodes.Expr]] = [] - - # Evaluate constants at compile time if possible. Each item in - # body will be either a list of static data or a node to be - # evaluated at runtime. - for child in node.nodes: - try: - if not ( - # If the finalize function requires runtime context, - # constants can't be evaluated at compile time. - finalize.const - # Unless it's basic template data that won't be - # finalized anyway. - or isinstance(child, nodes.TemplateData) - ): - raise nodes.Impossible() - - const = self._output_child_to_const(child, frame, finalize) - except (nodes.Impossible, Exception): - # The node was not constant and needs to be evaluated at - # runtime. Or another error was raised, which is easier - # to debug at runtime. - body.append(child) - continue - - if body and isinstance(body[-1], list): - body[-1].append(const) - else: - body.append([const]) - - if frame.buffer is not None: - if len(body) == 1: - self.writeline(f"{frame.buffer}.append(") - else: - self.writeline(f"{frame.buffer}.extend((") - - self.indent() - - for item in body: - if isinstance(item, list): - # A group of constant data to join and output. - val = self._output_const_repr(item) - - if frame.buffer is None: - self.writeline("yield " + val) - else: - self.writeline(val + ",") - else: - if frame.buffer is None: - self.writeline("yield ", item) - else: - self.newline(item) - - # A node to be evaluated at runtime. - self._output_child_pre(item, frame, finalize) - self.visit(item, frame) - self._output_child_post(item, frame, finalize) - - if frame.buffer is not None: - self.write(",") - - if frame.buffer is not None: - self.outdent() - self.writeline(")" if len(body) == 1 else "))") - - if frame.require_output_check: - self.outdent() - - def visit_Assign(self, node: nodes.Assign, frame: Frame) -> None: - self.push_assign_tracking() - - # ``a.b`` is allowed for assignment, and is parsed as an NSRef. However, - # it is only valid if it references a Namespace object. Emit a check for - # that for each ref here, before assignment code is emitted. This can't - # be done in visit_NSRef as the ref could be in the middle of a tuple. - seen_refs: t.Set[str] = set() - - for nsref in node.find_all(nodes.NSRef): - if nsref.name in seen_refs: - # Only emit the check for each reference once, in case the same - # ref is used multiple times in a tuple, `ns.a, ns.b = c, d`. - continue - - seen_refs.add(nsref.name) - ref = frame.symbols.ref(nsref.name) - self.writeline(f"if not isinstance({ref}, Namespace):") - self.indent() - self.writeline( - "raise TemplateRuntimeError" - '("cannot assign attribute on non-namespace object")' - ) - self.outdent() - - self.newline(node) - self.visit(node.target, frame) - self.write(" = ") - self.visit(node.node, frame) - self.pop_assign_tracking(frame) - - def visit_AssignBlock(self, node: nodes.AssignBlock, frame: Frame) -> None: - self.push_assign_tracking() - block_frame = frame.inner() - # This is a special case. Since a set block always captures we - # will disable output checks. This way one can use set blocks - # toplevel even in extended templates. - block_frame.require_output_check = False - block_frame.symbols.analyze_node(node) - self.enter_frame(block_frame) - self.buffer(block_frame) - self.blockvisit(node.body, block_frame) - self.newline(node) - self.visit(node.target, frame) - self.write(" = (Markup if context.eval_ctx.autoescape else identity)(") - if node.filter is not None: - self.visit_Filter(node.filter, block_frame) - else: - self.write(f"concat({block_frame.buffer})") - self.write(")") - self.pop_assign_tracking(frame) - self.leave_frame(block_frame) - - # -- Expression Visitors - - def visit_Name(self, node: nodes.Name, frame: Frame) -> None: - if node.ctx == "store" and ( - frame.toplevel or frame.loop_frame or frame.block_frame - ): - if self._assign_stack: - self._assign_stack[-1].add(node.name) - ref = frame.symbols.ref(node.name) - - # If we are looking up a variable we might have to deal with the - # case where it's undefined. We can skip that case if the load - # instruction indicates a parameter which are always defined. - if node.ctx == "load": - load = frame.symbols.find_load(ref) - if not ( - load is not None - and load[0] == VAR_LOAD_PARAMETER - and not self.parameter_is_undeclared(ref) - ): - self.write( - f"(undefined(name={node.name!r}) if {ref} is missing else {ref})" - ) - return - - self.write(ref) - - def visit_NSRef(self, node: nodes.NSRef, frame: Frame) -> None: - # NSRef is a dotted assignment target a.b=c, but uses a[b]=c internally. - # visit_Assign emits code to validate that each ref is to a Namespace - # object only. That can't be emitted here as the ref could be in the - # middle of a tuple assignment. - ref = frame.symbols.ref(node.name) - self.writeline(f"{ref}[{node.attr!r}]") - - def visit_Const(self, node: nodes.Const, frame: Frame) -> None: - val = node.as_const(frame.eval_ctx) - if isinstance(val, float): - self.write(str(val)) - else: - self.write(repr(val)) - - def visit_TemplateData(self, node: nodes.TemplateData, frame: Frame) -> None: - try: - self.write(repr(node.as_const(frame.eval_ctx))) - except nodes.Impossible: - self.write( - f"(Markup if context.eval_ctx.autoescape else identity)({node.data!r})" - ) - - def visit_Tuple(self, node: nodes.Tuple, frame: Frame) -> None: - self.write("(") - idx = -1 - for idx, item in enumerate(node.items): - if idx: - self.write(", ") - self.visit(item, frame) - self.write(",)" if idx == 0 else ")") - - def visit_List(self, node: nodes.List, frame: Frame) -> None: - self.write("[") - for idx, item in enumerate(node.items): - if idx: - self.write(", ") - self.visit(item, frame) - self.write("]") - - def visit_Dict(self, node: nodes.Dict, frame: Frame) -> None: - self.write("{") - for idx, item in enumerate(node.items): - if idx: - self.write(", ") - self.visit(item.key, frame) - self.write(": ") - self.visit(item.value, frame) - self.write("}") - - visit_Add = _make_binop("+") - visit_Sub = _make_binop("-") - visit_Mul = _make_binop("*") - visit_Div = _make_binop("/") - visit_FloorDiv = _make_binop("//") - visit_Pow = _make_binop("**") - visit_Mod = _make_binop("%") - visit_And = _make_binop("and") - visit_Or = _make_binop("or") - visit_Pos = _make_unop("+") - visit_Neg = _make_unop("-") - visit_Not = _make_unop("not ") - - @optimizeconst - def visit_Concat(self, node: nodes.Concat, frame: Frame) -> None: - if frame.eval_ctx.volatile: - func_name = "(markup_join if context.eval_ctx.volatile else str_join)" - elif frame.eval_ctx.autoescape: - func_name = "markup_join" - else: - func_name = "str_join" - self.write(f"{func_name}((") - for arg in node.nodes: - self.visit(arg, frame) - self.write(", ") - self.write("))") - - @optimizeconst - def visit_Compare(self, node: nodes.Compare, frame: Frame) -> None: - self.write("(") - self.visit(node.expr, frame) - for op in node.ops: - self.visit(op, frame) - self.write(")") - - def visit_Operand(self, node: nodes.Operand, frame: Frame) -> None: - self.write(f" {operators[node.op]} ") - self.visit(node.expr, frame) - - @optimizeconst - def visit_Getattr(self, node: nodes.Getattr, frame: Frame) -> None: - if self.environment.is_async: - self.write("(await auto_await(") - - self.write("environment.getattr(") - self.visit(node.node, frame) - self.write(f", {node.attr!r})") - - if self.environment.is_async: - self.write("))") - - @optimizeconst - def visit_Getitem(self, node: nodes.Getitem, frame: Frame) -> None: - # slices bypass the environment getitem method. - if isinstance(node.arg, nodes.Slice): - self.visit(node.node, frame) - self.write("[") - self.visit(node.arg, frame) - self.write("]") - else: - if self.environment.is_async: - self.write("(await auto_await(") - - self.write("environment.getitem(") - self.visit(node.node, frame) - self.write(", ") - self.visit(node.arg, frame) - self.write(")") - - if self.environment.is_async: - self.write("))") - - def visit_Slice(self, node: nodes.Slice, frame: Frame) -> None: - if node.start is not None: - self.visit(node.start, frame) - self.write(":") - if node.stop is not None: - self.visit(node.stop, frame) - if node.step is not None: - self.write(":") - self.visit(node.step, frame) - - @contextmanager - def _filter_test_common( - self, node: t.Union[nodes.Filter, nodes.Test], frame: Frame, is_filter: bool - ) -> t.Iterator[None]: - if self.environment.is_async: - self.write("(await auto_await(") - - if is_filter: - self.write(f"{self.filters[node.name]}(") - func = self.environment.filters.get(node.name) - else: - self.write(f"{self.tests[node.name]}(") - func = self.environment.tests.get(node.name) - - # When inside an If or CondExpr frame, allow the filter to be - # undefined at compile time and only raise an error if it's - # actually called at runtime. See pull_dependencies. - if func is None and not frame.soft_frame: - type_name = "filter" if is_filter else "test" - self.fail(f"No {type_name} named {node.name!r}.", node.lineno) - - pass_arg = { - _PassArg.context: "context", - _PassArg.eval_context: "context.eval_ctx", - _PassArg.environment: "environment", - }.get( - _PassArg.from_obj(func) # type: ignore - ) - - if pass_arg is not None: - self.write(f"{pass_arg}, ") - - # Back to the visitor function to handle visiting the target of - # the filter or test. - yield - - self.signature(node, frame) - self.write(")") - - if self.environment.is_async: - self.write("))") - - @optimizeconst - def visit_Filter(self, node: nodes.Filter, frame: Frame) -> None: - with self._filter_test_common(node, frame, True): - # if the filter node is None we are inside a filter block - # and want to write to the current buffer - if node.node is not None: - self.visit(node.node, frame) - elif frame.eval_ctx.volatile: - self.write( - f"(Markup(concat({frame.buffer}))" - f" if context.eval_ctx.autoescape else concat({frame.buffer}))" - ) - elif frame.eval_ctx.autoescape: - self.write(f"Markup(concat({frame.buffer}))") - else: - self.write(f"concat({frame.buffer})") - - @optimizeconst - def visit_Test(self, node: nodes.Test, frame: Frame) -> None: - with self._filter_test_common(node, frame, False): - self.visit(node.node, frame) - - @optimizeconst - def visit_CondExpr(self, node: nodes.CondExpr, frame: Frame) -> None: - frame = frame.soft() - - def write_expr2() -> None: - if node.expr2 is not None: - self.visit(node.expr2, frame) - return - - self.write( - f'cond_expr_undefined("the inline if-expression on' - f" {self.position(node)} evaluated to false and no else" - f' section was defined.")' - ) - - self.write("(") - self.visit(node.expr1, frame) - self.write(" if ") - self.visit(node.test, frame) - self.write(" else ") - write_expr2() - self.write(")") - - @optimizeconst - def visit_Call( - self, node: nodes.Call, frame: Frame, forward_caller: bool = False - ) -> None: - if self.environment.is_async: - self.write("(await auto_await(") - if self.environment.sandboxed: - self.write("environment.call(context, ") - else: - self.write("context.call(") - self.visit(node.node, frame) - extra_kwargs = {"caller": "caller"} if forward_caller else None - loop_kwargs = {"_loop_vars": "_loop_vars"} if frame.loop_frame else {} - block_kwargs = {"_block_vars": "_block_vars"} if frame.block_frame else {} - if extra_kwargs: - extra_kwargs.update(loop_kwargs, **block_kwargs) - elif loop_kwargs or block_kwargs: - extra_kwargs = dict(loop_kwargs, **block_kwargs) - self.signature(node, frame, extra_kwargs) - self.write(")") - if self.environment.is_async: - self.write("))") - - def visit_Keyword(self, node: nodes.Keyword, frame: Frame) -> None: - self.write(node.key + "=") - self.visit(node.value, frame) - - # -- Unused nodes for extensions - - def visit_MarkSafe(self, node: nodes.MarkSafe, frame: Frame) -> None: - self.write("Markup(") - self.visit(node.expr, frame) - self.write(")") - - def visit_MarkSafeIfAutoescape( - self, node: nodes.MarkSafeIfAutoescape, frame: Frame - ) -> None: - self.write("(Markup if context.eval_ctx.autoescape else identity)(") - self.visit(node.expr, frame) - self.write(")") - - def visit_EnvironmentAttribute( - self, node: nodes.EnvironmentAttribute, frame: Frame - ) -> None: - self.write("environment." + node.name) - - def visit_ExtensionAttribute( - self, node: nodes.ExtensionAttribute, frame: Frame - ) -> None: - self.write(f"environment.extensions[{node.identifier!r}].{node.name}") - - def visit_ImportedName(self, node: nodes.ImportedName, frame: Frame) -> None: - self.write(self.import_aliases[node.importname]) - - def visit_InternalName(self, node: nodes.InternalName, frame: Frame) -> None: - self.write(node.name) - - def visit_ContextReference( - self, node: nodes.ContextReference, frame: Frame - ) -> None: - self.write("context") - - def visit_DerivedContextReference( - self, node: nodes.DerivedContextReference, frame: Frame - ) -> None: - self.write(self.derive_context(frame)) - - def visit_Continue(self, node: nodes.Continue, frame: Frame) -> None: - self.writeline("continue", node) - - def visit_Break(self, node: nodes.Break, frame: Frame) -> None: - self.writeline("break", node) - - def visit_Scope(self, node: nodes.Scope, frame: Frame) -> None: - scope_frame = frame.inner() - scope_frame.symbols.analyze_node(node) - self.enter_frame(scope_frame) - self.blockvisit(node.body, scope_frame) - self.leave_frame(scope_frame) - - def visit_OverlayScope(self, node: nodes.OverlayScope, frame: Frame) -> None: - ctx = self.temporary_identifier() - self.writeline(f"{ctx} = {self.derive_context(frame)}") - self.writeline(f"{ctx}.vars = ") - self.visit(node.context, frame) - self.push_context_reference(ctx) - - scope_frame = frame.inner(isolated=True) - scope_frame.symbols.analyze_node(node) - self.enter_frame(scope_frame) - self.blockvisit(node.body, scope_frame) - self.leave_frame(scope_frame) - self.pop_context_reference() - - def visit_EvalContextModifier( - self, node: nodes.EvalContextModifier, frame: Frame - ) -> None: - for keyword in node.options: - self.writeline(f"context.eval_ctx.{keyword.key} = ") - self.visit(keyword.value, frame) - try: - val = keyword.value.as_const(frame.eval_ctx) - except nodes.Impossible: - frame.eval_ctx.volatile = True - else: - setattr(frame.eval_ctx, keyword.key, val) - - def visit_ScopedEvalContextModifier( - self, node: nodes.ScopedEvalContextModifier, frame: Frame - ) -> None: - old_ctx_name = self.temporary_identifier() - saved_ctx = frame.eval_ctx.save() - self.writeline(f"{old_ctx_name} = context.eval_ctx.save()") - self.visit_EvalContextModifier(node, frame) - for child in node.body: - self.visit(child, frame) - frame.eval_ctx.revert(saved_ctx) - self.writeline(f"context.eval_ctx.revert({old_ctx_name})") diff --git a/bundle/python-cpu/Lib/site-packages/jinja2/constants.py b/bundle/python-cpu/Lib/site-packages/jinja2/constants.py deleted file mode 100644 index 41a1c23b0a7fe134b1f662545876eb65b31b071e..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2/constants.py +++ /dev/null @@ -1,20 +0,0 @@ -#: list of lorem ipsum words used by the lipsum() helper function -LOREM_IPSUM_WORDS = """\ -a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at -auctor augue bibendum blandit class commodo condimentum congue consectetuer -consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus -diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend -elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames -faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac -hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum -justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem -luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie -mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non -nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque -penatibus per pharetra phasellus placerat platea porta porttitor posuere -potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus -ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit -sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor -tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices -ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus -viverra volutpat vulputate""" diff --git a/bundle/python-cpu/Lib/site-packages/jinja2/debug.py b/bundle/python-cpu/Lib/site-packages/jinja2/debug.py deleted file mode 100644 index eeeeee78b620f5d0745133b4629647973cd7af87..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2/debug.py +++ /dev/null @@ -1,191 +0,0 @@ -import sys -import typing as t -from types import CodeType -from types import TracebackType - -from .exceptions import TemplateSyntaxError -from .utils import internal_code -from .utils import missing - -if t.TYPE_CHECKING: - from .runtime import Context - - -def rewrite_traceback_stack(source: t.Optional[str] = None) -> BaseException: - """Rewrite the current exception to replace any tracebacks from - within compiled template code with tracebacks that look like they - came from the template source. - - This must be called within an ``except`` block. - - :param source: For ``TemplateSyntaxError``, the original source if - known. - :return: The original exception with the rewritten traceback. - """ - _, exc_value, tb = sys.exc_info() - exc_value = t.cast(BaseException, exc_value) - tb = t.cast(TracebackType, tb) - - if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated: - exc_value.translated = True - exc_value.source = source - # Remove the old traceback, otherwise the frames from the - # compiler still show up. - exc_value.with_traceback(None) - # Outside of runtime, so the frame isn't executing template - # code, but it still needs to point at the template. - tb = fake_traceback( - exc_value, None, exc_value.filename or "", exc_value.lineno - ) - else: - # Skip the frame for the render function. - tb = tb.tb_next - - stack = [] - - # Build the stack of traceback object, replacing any in template - # code with the source file and line information. - while tb is not None: - # Skip frames decorated with @internalcode. These are internal - # calls that aren't useful in template debugging output. - if tb.tb_frame.f_code in internal_code: - tb = tb.tb_next - continue - - template = tb.tb_frame.f_globals.get("__jinja_template__") - - if template is not None: - lineno = template.get_corresponding_lineno(tb.tb_lineno) - fake_tb = fake_traceback(exc_value, tb, template.filename, lineno) - stack.append(fake_tb) - else: - stack.append(tb) - - tb = tb.tb_next - - tb_next = None - - # Assign tb_next in reverse to avoid circular references. - for tb in reversed(stack): - tb.tb_next = tb_next - tb_next = tb - - return exc_value.with_traceback(tb_next) - - -def fake_traceback( # type: ignore - exc_value: BaseException, tb: t.Optional[TracebackType], filename: str, lineno: int -) -> TracebackType: - """Produce a new traceback object that looks like it came from the - template source instead of the compiled code. The filename, line - number, and location name will point to the template, and the local - variables will be the current template context. - - :param exc_value: The original exception to be re-raised to create - the new traceback. - :param tb: The original traceback to get the local variables and - code info from. - :param filename: The template filename. - :param lineno: The line number in the template source. - """ - if tb is not None: - # Replace the real locals with the context that would be - # available at that point in the template. - locals = get_template_locals(tb.tb_frame.f_locals) - locals.pop("__jinja_exception__", None) - else: - locals = {} - - globals = { - "__name__": filename, - "__file__": filename, - "__jinja_exception__": exc_value, - } - # Raise an exception at the correct line number. - code: CodeType = compile( - "\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec" - ) - - # Build a new code object that points to the template file and - # replaces the location with a block name. - location = "template" - - if tb is not None: - function = tb.tb_frame.f_code.co_name - - if function == "root": - location = "top-level template code" - elif function.startswith("block_"): - location = f"block {function[6:]!r}" - - if sys.version_info >= (3, 8): - code = code.replace(co_name=location) - else: - code = CodeType( - code.co_argcount, - code.co_kwonlyargcount, - code.co_nlocals, - code.co_stacksize, - code.co_flags, - code.co_code, - code.co_consts, - code.co_names, - code.co_varnames, - code.co_filename, - location, - code.co_firstlineno, - code.co_lnotab, - code.co_freevars, - code.co_cellvars, - ) - - # Execute the new code, which is guaranteed to raise, and return - # the new traceback without this frame. - try: - exec(code, globals, locals) - except BaseException: - return sys.exc_info()[2].tb_next # type: ignore - - -def get_template_locals(real_locals: t.Mapping[str, t.Any]) -> t.Dict[str, t.Any]: - """Based on the runtime locals, get the context that would be - available at that point in the template. - """ - # Start with the current template context. - ctx: t.Optional[Context] = real_locals.get("context") - - if ctx is not None: - data: t.Dict[str, t.Any] = ctx.get_all().copy() - else: - data = {} - - # Might be in a derived context that only sets local variables - # rather than pushing a context. Local variables follow the scheme - # l_depth_name. Find the highest-depth local that has a value for - # each name. - local_overrides: t.Dict[str, t.Tuple[int, t.Any]] = {} - - for name, value in real_locals.items(): - if not name.startswith("l_") or value is missing: - # Not a template variable, or no longer relevant. - continue - - try: - _, depth_str, name = name.split("_", 2) - depth = int(depth_str) - except ValueError: - continue - - cur_depth = local_overrides.get(name, (-1,))[0] - - if cur_depth < depth: - local_overrides[name] = (depth, value) - - # Modify the context with any derived context. - for name, (_, value) in local_overrides.items(): - if value is missing: - data.pop(name, None) - else: - data[name] = value - - return data diff --git a/bundle/python-cpu/Lib/site-packages/jinja2/defaults.py b/bundle/python-cpu/Lib/site-packages/jinja2/defaults.py deleted file mode 100644 index 638cad3d2d8907330bde56e2b76c9b185c523b45..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2/defaults.py +++ /dev/null @@ -1,48 +0,0 @@ -import typing as t - -from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 -from .tests import TESTS as DEFAULT_TESTS # noqa: F401 -from .utils import Cycler -from .utils import generate_lorem_ipsum -from .utils import Joiner -from .utils import Namespace - -if t.TYPE_CHECKING: - import typing_extensions as te - -# defaults for the parser / lexer -BLOCK_START_STRING = "{%" -BLOCK_END_STRING = "%}" -VARIABLE_START_STRING = "{{" -VARIABLE_END_STRING = "}}" -COMMENT_START_STRING = "{#" -COMMENT_END_STRING = "#}" -LINE_STATEMENT_PREFIX: t.Optional[str] = None -LINE_COMMENT_PREFIX: t.Optional[str] = None -TRIM_BLOCKS = False -LSTRIP_BLOCKS = False -NEWLINE_SEQUENCE: "te.Literal['\\n', '\\r\\n', '\\r']" = "\n" -KEEP_TRAILING_NEWLINE = False - -# default filters, tests and namespace - -DEFAULT_NAMESPACE = { - "range": range, - "dict": dict, - "lipsum": generate_lorem_ipsum, - "cycler": Cycler, - "joiner": Joiner, - "namespace": Namespace, -} - -# default policies -DEFAULT_POLICIES: t.Dict[str, t.Any] = { - "compiler.ascii_str": True, - "urlize.rel": "noopener", - "urlize.target": None, - "urlize.extra_schemes": None, - "truncate.leeway": 5, - "json.dumps_function": None, - "json.dumps_kwargs": {"sort_keys": True}, - "ext.i18n.trimmed": False, -} diff --git a/bundle/python-cpu/Lib/site-packages/jinja2/environment.py b/bundle/python-cpu/Lib/site-packages/jinja2/environment.py deleted file mode 100644 index 0fc6e5be87ab8273f6056ddfede07e1be28f1495..0000000000000000000000000000000000000000 --- a/bundle/python-cpu/Lib/site-packages/jinja2/environment.py +++ /dev/null @@ -1,1672 +0,0 @@ -"""Classes for managing templates and their runtime and compile time -options. -""" - -import os -import typing -import typing as t -import weakref -from collections import ChainMap -from functools import lru_cache -from functools import partial -from functools import reduce -from types import CodeType - -from markupsafe import Markup - -from . import nodes -from .compiler import CodeGenerator -from .compiler import generate -from .defaults import BLOCK_END_STRING -from .defaults import BLOCK_START_STRING -from .defaults import COMMENT_END_STRING -from .defaults import COMMENT_START_STRING -from .defaults import DEFAULT_FILTERS # type: ignore[attr-defined] -from .defaults import DEFAULT_NAMESPACE -from .defaults import DEFAULT_POLICIES -from .defaults import DEFAULT_TESTS # type: ignore[attr-defined] -from .defaults import KEEP_TRAILING_NEWLINE -from .defaults import LINE_COMMENT_PREFIX -from .defaults import LINE_STATEMENT_PREFIX -from .defaults import LSTRIP_BLOCKS -from .defaults import NEWLINE_SEQUENCE -from .defaults import TRIM_BLOCKS -from .defaults import VARIABLE_END_STRING -from .defaults import VARIABLE_START_STRING -from .exceptions import TemplateNotFound -from .exceptions import TemplateRuntimeError -from .exceptions import TemplatesNotFound -from .exceptions import TemplateSyntaxError -from .exceptions import UndefinedError -from .lexer import get_lexer -from .lexer import Lexer -from .lexer import TokenStream -from .nodes import EvalContext -from .parser import Parser -from .runtime import Context -from .runtime import new_context -from .runtime import Undefined -from .utils import _PassArg -from .utils import concat -from .utils import consume -from .utils import import_string -from .utils import internalcode -from .utils import LRUCache -from .utils import missing - -if t.TYPE_CHECKING: - import typing_extensions as te - - from .bccache import BytecodeCache - from .ext import Extension - from .loaders import BaseLoader - -_env_bound = t.TypeVar("_env_bound", bound="Environment") - - -# for direct template usage we have up to ten living environments -@lru_cache(maxsize=10) -def get_spontaneous_environment(cls: t.Type[_env_bound], *args: t.Any) -> _env_bound: - """Return a new spontaneous environment. A spontaneous environment - is used for templates created directly rather than through an - existing environment. - - :param cls: Environment class to create. - :param args: Positional arguments passed to environment. - """ - env = cls(*args) - env.shared = True - return env - - -def create_cache( - size: int, -) -> t.Optional[t.MutableMapping[t.Tuple["weakref.ref[t.Any]", str], "Template"]]: - """Return the cache class for the given size.""" - if size == 0: - return None - - if size < 0: - return {} - - return LRUCache(size) # type: ignore - - -def copy_cache( - cache: t.Optional[t.MutableMapping[t.Any, t.Any]], -) -> t.Optional[t.MutableMapping[t.Tuple["weakref.ref[t.Any]", str], "Template"]]: - """Create an empty copy of the given cache.""" - if cache is None: - return None - - if type(cache) is dict: # noqa E721 - return {} - - return LRUCache(cache.capacity) # type: ignore - - -def load_extensions( - environment: "Environment", - extensions: t.Sequence[t.Union[str, t.Type["Extension"]]], -) -> t.Dict[str, "Extension"]: - """Load the extensions from the list and bind it to the environment. - Returns a dict of instantiated extensions. - """ - result = {} - - for extension in extensions: - if isinstance(extension, str): - extension = t.cast(t.Type["Extension"], import_string(extension)) - - result[extension.identifier] = extension(environment) - - return result - - -def _environment_config_check(environment: _env_bound) -> _env_bound: - """Perform a sanity check on the environment.""" - assert issubclass( - environment.undefined, Undefined - ), "'undefined' must be a subclass of 'jinja2.Undefined'." - assert ( - environment.block_start_string - != environment.variable_start_string - != environment.comment_start_string - ), "block, variable and comment start strings must be different." - assert environment.newline_sequence in { - "\r", - "\r\n", - "\n", - }, "'newline_sequence' must be one of '\\n', '\\r\\n', or '\\r'." - return environment - - -class Environment: - r"""The core component of Jinja is the `Environment`. It contains - important shared variables like configuration, filters, tests, - globals and others. Instances of this class may be modified if - they are not shared and if no template was loaded so far. - Modifications on environments after the first template was loaded - will lead to surprising effects and undefined behavior. - - Here are the possible initialization parameters: - - `block_start_string` - The string marking the beginning of a block. Defaults to ``'{%'``. - - `block_end_string` - The string marking the end of a block. Defaults to ``'%}'``. - - `variable_start_string` - The string marking the beginning of a print statement. - Defaults to ``'{{'``. - - `variable_end_string` - The string marking the end of a print statement. Defaults to - ``'}}'``. - - `comment_start_string` - The string marking the beginning of a comment. Defaults to ``'{#'``. - - `comment_end_string` - The string marking the end of a comment. Defaults to ``'#}'``. - - `line_statement_prefix` - If given and a string, this will be used as prefix for line based - statements. See also :ref:`line-statements`. - - `line_comment_prefix` - If given and a string, this will be used as prefix for line based - comments. See also :ref:`line-statements`. - - .. versionadded:: 2.2 - - `trim_blocks` - If this is set to ``True`` the first newline after a block is - removed (block, not variable tag!). Defaults to `False`. - - `lstrip_blocks` - If this is set to ``True`` leading spaces and tabs are stripped - from the start of a line to a block. Defaults to `False`. - - `newline_sequence` - The sequence that starts a newline. Must be one of ``'\r'``, - ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a - useful default for Linux and OS X systems as well as web - applications. - - `keep_trailing_newline` - Preserve the trailing newline when rendering templates. - The default is ``False``, which causes a single newline, - if present, to be stripped from the end of the template. - - .. versionadded:: 2.7 - - `extensions` - List of Jinja extensions to use. This can either be import paths - as strings or extension classes. For more information have a - look at :ref:`the extensions documentation `. - - `optimized` - should the optimizer be enabled? Default is ``True``. - - `undefined` - :class:`Undefined` or a subclass of it that is used to represent - undefined values in the template. - - `finalize` - A callable that can be used to process the result of a variable - expression before it is output. For example one can convert - ``None`` implicitly into an empty string here. - - `autoescape` - If set to ``True`` the XML/HTML autoescaping feature is enabled by - default. For more details about autoescaping see - :class:`~markupsafe.Markup`. As of Jinja 2.4 this can also - be a callable that is passed the template name and has to - return ``True`` or ``False`` depending on autoescape should be - enabled by default. - - .. versionchanged:: 2.4 - `autoescape` can now be a function - - `loader` - The template loader for this environment. - - `cache_size` - The size of the cache. Per default this is ``400`` which means - that if more than 400 templates are loaded the loader will clean - out the least recently used template. If the cache size is set to - ``0`` templates are recompiled all the time, if the cache size is - ``-1`` the cache will not be cleaned. - - .. versionchanged:: 2.8 - The cache size was increased to 400 from a low 50. - - `auto_reload` - Some loaders load templates from locations where the template - sources may change (ie: file system or database). If - ``auto_reload`` is set to ``True`` (default) every time a template is - requested the loader checks if the source changed and if yes, it - will reload the template. For higher performance it's possible to - disable that. - - `bytecode_cache` - If set to a bytecode cache object, this object will provide a - cache for the internal Jinja bytecode so that templates don't - have to be parsed if they were not changed. - - See :ref:`bytecode-cache` for more information. - - `enable_async` - If set to true this enables async template execution which - allows using async functions and generators. - """ - - #: if this environment is sandboxed. Modifying this variable won't make - #: the environment sandboxed though. For a real sandboxed environment - #: have a look at jinja2.sandbox. This flag alone controls the code - #: generation by the compiler. - sandboxed = False - - #: True if the environment is just an overlay - overlayed = False - - #: the environment this environment is linked to if it is an overlay - linked_to: t.Optional["Environment"] = None - - #: shared environments have this set to `True`. A shared environment - #: must not be modified - shared = False - - #: the class that is used for code generation. See - #: :class:`~jinja2.compiler.CodeGenerator` for more information. - code_generator_class: t.Type["CodeGenerator"] = CodeGenerator - - concat = "".join - - #: the context class that is used for templates. See - #: :class:`~jinja2.runtime.Context` for more information. - context_class: t.Type[Context] = Context - - template_class: t.Type["Template"] - - def __init__( - self, - block_start_string: str = BLOCK_START_STRING, - block_end_string: str = BLOCK_END_STRING, - variable_start_string: str = VARIABLE_START_STRING, - variable_end_string: str = VARIABLE_END_STRING, - comment_start_string: str = COMMENT_START_STRING, - comment_end_string: str = COMMENT_END_STRING, - line_statement_prefix: t.Optional[str] = LINE_STATEMENT_PREFIX, - line_comment_prefix: t.Optional[str] = LINE_COMMENT_PREFIX, - trim_blocks: bool = TRIM_BLOCKS, - lstrip_blocks: bool = LSTRIP_BLOCKS, - newline_sequence: "te.Literal['\\n', '\\r\\n', '\\r']" = NEWLINE_SEQUENCE, - keep_trailing_newline: bool = KEEP_TRAILING_NEWLINE, - extensions: t.Sequence[t.Union[str, t.Type["Extension"]]] = (), - optimized: bool = True, - undefined: t.Type[Undefined] = Undefined, - finalize: t.Optional[t.Callable[..., t.Any]] = None, - autoescape: t.Union[bool, t.Callable[[t.Optional[str]], bool]] = False, - loader: t.Optional["BaseLoader"] = None, - cache_size: int = 400, - auto_reload: bool = True, - bytecode_cache: t.Optional["BytecodeCache"] = None, - enable_async: bool = False, - ): - # !!Important notice!! - # The constructor accepts quite a few arguments that should be - # passed by keyword rather than position. However it's important to - # not change the order of arguments because it's used at least - # internally in those cases: - # - spontaneous environments (i18n extension and Template) - # - unittests - # If parameter changes are required only add parameters at the end - # and don't change the arguments (or the defaults!) of the arguments - # existing already. - - # lexer / parser information - self.block_start_string = block_start_string - self.block_end_string = block_end_string - self.variable_start_string = variable_start_string - self.variable_end_string = variable_end_string - self.comment_start_string = comment_start_string - self.comment_end_string = comment_end_string - self.line_statement_prefix = line_statement_prefix - self.line_comment_prefix = line_comment_prefix - self.trim_blocks = trim_blocks - self.lstrip_blocks = lstrip_blocks - self.newline_sequence = newline_sequence - self.keep_trailing_newline = keep_trailing_newline - - # runtime information - self.undefined: t.Type[Undefined] = undefined - self.optimized = optimized - self.finalize = finalize - self.autoescape = autoescape - - # defaults - self.filters = DEFAULT_FILTERS.copy() - self.tests = DEFAULT_TESTS.copy() - self.globals = DEFAULT_NAMESPACE.copy() - - # set the loader provided - self.loader = loader - self.cache = create_cache(cache_size) - self.bytecode_cache = bytecode_cache - self.auto_reload = auto_reload - - # configurable policies - self.policies = DEFAULT_POLICIES.copy() - - # load extensions - self.extensions = load_extensions(self, extensions) - - self.is_async = enable_async - _environment_config_check(self) - - def add_extension(self, extension: t.Union[str, t.Type["Extension"]]) -> None: - """Adds an extension after the environment was created. - - .. versionadded:: 2.5 - """ - self.extensions.update(load_extensions(self, [extension])) - - def extend(self, **attributes: t.Any) -> None: - """Add the items to the instance of the environment if they do not exist - yet. This is used by :ref:`extensions ` to register - callbacks and configuration values without breaking inheritance. - """ - for key, value in attributes.items(): - if not hasattr(self, key): - setattr(self, key, value) - - def overlay( - self, - block_start_string: str = missing, - block_end_string: str = missing, - variable_start_string: str = missing, - variable_end_string: str = missing, - comment_start_string: str = missing, - comment_end_string: str = missing, - line_statement_prefix: t.Optional[str] = missing, - line_comment_prefix: t.Optional[str] = missing, - trim_blocks: bool = missing, - lstrip_blocks: bool = missing, - newline_sequence: "te.Literal['\\n', '\\r\\n', '\\r']" = missing, - keep_trailing_newline: bool = missing, - extensions: t.Sequence[t.Union[str, t.Type["Extension"]]] = missing, - optimized: bool = missing, - undefined: t.Type[Undefined] = missing, - finalize: t.Optional[t.Callable[..., t.Any]] = missing, - autoescape: t.Union[bool, t.Callable[[t.Optional[str]], bool]] = missing, - loader: t.Optional["BaseLoader"] = missing, - cache_size: int = missing, - auto_reload: bool = missing, - bytecode_cache: t.Optional["BytecodeCache"] = missing, - enable_async: bool = missing, - ) -> "te.Self": - """Create a new overlay environment that shares all the data with the - current environment except for cache and the overridden attributes. - Extensions cannot be removed for an overlayed environment. An overlayed - environment automatically gets all the extensions of the environment it - is linked to plus optional extra extensions. - - Creating overlays should happen after the initial environment was set - up completely. Not all attributes are truly linked, some are just - copied over so modifications on the original environment may not shine - through. - - .. versionchanged:: 3.1.5 - ``enable_async`` is applied correctly. - - .. versionchanged:: 3.1.2 - Added the ``newline_sequence``, ``keep_trailing_newline``, - and ``enable_async`` parameters to match ``__init__``. - """ - args = dict(locals()) - del args["self"], args["cache_size"], args["extensions"], args["enable_async"] - - rv = object.__new__(self.__class__) - rv.__dict__.update(self.__dict__) - rv.overlayed = True - rv.linked_to = self - - for key, value in args.items(): - if value is not missing: - setattr(rv, key, value) - - if cache_size is not missing: - rv.cache = create_cache(cache_size) - else: - rv.cache = copy_cache(self.cache) - - rv.extensions = {} - for key, value in self.extensions.items(): - rv.extensions[key] = value.bind(rv) - if extensions is not missing: - rv.extensions.update(load_extensions(rv, extensions)) - - if enable_async is not missing: - rv.is_async = enable_async - - return _environment_config_check(rv) - - @property - def lexer(self) -> Lexer: - """The lexer for this environment.""" - return get_lexer(self) - - def iter_extensions(self) -> t.Iterator["Extension"]: - """Iterates over the extensions by priority.""" - return iter(sorted(self.extensions.values(), key=lambda x: x.priority)) - - def getitem( - self, obj: t.Any, argument: t.Union[str, t.Any] - ) -> t.Union[t.Any, Undefined]: - """Get an item or attribute of an object but prefer the item.""" - try: - return obj[argument] - except (AttributeError, TypeError, LookupError): - if isinstance(argument, str): - try: - attr = str(argument) - except Exception: - pass - else: - try: - return getattr(obj, attr) - except AttributeError: - pass - return self.undefined(obj=obj, name=argument) - - def getattr(self, obj: t.Any, attribute: str) -> t.Any: - """Get an item or attribute of an object but prefer the attribute. - Unlike :meth:`getitem` the attribute *must* be a string. - """ - try: - return getattr(obj, attribute) - except AttributeError: - pass - try: - return obj[attribute] - except (TypeError, LookupError, AttributeError): - return self.undefined(obj=obj, name=attribute) - - def _filter_test_common( - self, - name: t.Union[str, Undefined], - value: t.Any, - args: t.Optional[t.Sequence[t.Any]], - kwargs: t.Optional[t.Mapping[str, t.Any]], - context: t.Optional[Context], - eval_ctx: t.Optional[EvalContext], - is_filter: bool, - ) -> t.Any: - if is_filter: - env_map = self.filters - type_name = "filter" - else: - env_map = self.tests - type_name = "test" - - func = env_map.get(name) # type: ignore - - if func is None: - msg = f"No {type_name} named {name!r}." - - if isinstance(name, Undefined): - try: - name._fail_with_undefined_error() - except Exception as e: - msg = f"{msg} ({e}; did you forget to quote the callable name?)" - - raise TemplateRuntimeError(msg) - - args = [value, *(args if args is not None else ())] - kwargs = kwargs if kwargs is not None else {} - pass_arg = _PassArg.from_obj(func) - - if pass_arg is _PassArg.context: - if context is None: - raise TemplateRuntimeError( - f"Attempted to invoke a context {type_name} without context." - ) - - args.insert(0, context) - elif pass_arg is _PassArg.eval_context: - if eval_ctx is None: - if context is not None: - eval_ctx = context.eval_ctx - else: - eval_ctx = EvalContext(self) - - args.insert(0, eval_ctx) - elif pass_arg is _PassArg.environment: - args.insert(0, self) - - return func(*args, **kwargs) - - def call_filter( - self, - name: str, - value: t.Any, - args: t.Optional[t.Sequence[t.Any]] = None, - kwargs: t.Optional[t.Mapping[str, t.Any]] = None, - context: t.Optional[Context] = None, - eval_ctx: t.Optional[EvalContext] = None, - ) -> t.Any: - """Invoke a filter on a value the same way the compiler does. - - This might return a coroutine if the filter is running from an - environment in async mode and the filter supports async - execution. It's your responsibility to await this if needed. - - .. versionadded:: 2.7 - """ - return self._filter_test_common( - name, value, args, kwargs, context, eval_ctx, True - ) - - def call_test( - self, - name: str, - value: t.Any, - args: t.Optional[t.Sequence[t.Any]] = None, - kwargs: t.Optional[t.Mapping[str, t.Any]] = None, - context: t.Optional[Context] = None, - eval_ctx: t.Optional[EvalContext] = None, - ) -> t.Any: - """Invoke a test on a value the same way the compiler does. - - This might return a coroutine if the test is running from an - environment in async mode and the test supports async execution. - It's your responsibility to await this if needed. - - .. versionchanged:: 3.0 - Tests support ``@pass_context``, etc. decorators. Added - the ``context`` and ``eval_ctx`` parameters. - - .. versionadded:: 2.7 - """ - return self._filter_test_common( - name, value, args, kwargs, context, eval_ctx, False - ) - - @internalcode - def parse( - self, - source: str, - name: t.Optional[str] = None, - filename: t.Optional[str] = None, - ) -> nodes.Template: - """Parse the sourcecode and return the abstract syntax tree. This - tree of nodes is used by the compiler to convert the template into - executable source- or bytecode. This is useful for debugging or to - extract information from templates. - - If you are :ref:`developing Jinja extensions ` - this gives you a good overview of the node tree generated. - """ - try: - return self._parse(source, name, filename) - except TemplateSyntaxError: - self.handle_exception(source=source) - - def _parse( - self, source: str, name: t.Optional[str], filename: t.Optional[str] - ) -> nodes.Template: - """Internal parsing function used by `parse` and `compile`.""" - return Parser(self, source, name, filename).parse() - - def lex( - self, - source: str, - name: t.Optional[str] = None, - filename: t.Optional[str] = None, - ) -> t.Iterator[t.Tuple[int, str, str]]: - """Lex the given sourcecode and return a generator that yields - tokens as tuples in the form ``(lineno, token_type, value)``. - This can be useful for :ref:`extension development ` - and debugging templates. - - This does not perform preprocessing. If you want the preprocessing - of the extensions to be applied you have to filter source through - the :meth:`preprocess` method. - """ - source = str(source) - try: - return self.lexer.tokeniter(source, name, filename) - except TemplateSyntaxError: - self.handle_exception(source=source) - - def preprocess( - self, - source: str, - name: t.Optional[str] = None, - filename: t.Optional[str] = None, - ) -> str: - """Preprocesses the source with all extensions. This is automatically - called for all parsing and compiling methods but *not* for :meth:`lex` - because there you usually only want the actual source tokenized. - """ - return reduce( - lambda s, e: e.preprocess(s, name, filename), - self.iter_extensions(), - str(source), - ) - - def _tokenize( - self, - source: str, - name: t.Optional[str], - filename: t.Optional[str] = None, - state: t.Optional[str] = None, - ) -> TokenStream: - """Called by the parser to do the preprocessing and filtering - for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`. - """ - source = self.preprocess(source, name, filename) - stream = self.lexer.tokenize(source, name, filename, state) - - for ext in self.iter_extensions(): - stream = ext.filter_stream(stream) # type: ignore - - if not isinstance(stream, TokenStream): - stream = TokenStream(stream, name, filename) - - return stream - - def _generate( - self, - source: nodes.Template, - name: t.Optional[str], - filename: t.Optional[str], - defer_init: bool = False, - ) -> str: - """Internal hook that can be overridden to hook a different generate - method in. - - .. versionadded:: 2.5 - """ - return generate( # type: ignore - source, - self, - name, - filename, - defer_init=defer_init, - optimized=self.optimized, - ) - - def _compile(self, source: str, filename: str) -> CodeType: - """Internal hook that can be overridden to hook a different compile - method in. - - .. versionadded:: 2.5 - """ - return compile(source, filename, "exec") - - @typing.overload - def compile( - self, - source: t.Union[str, nodes.Template], - name: t.Optional[str] = None, - filename: t.Optional[str] = None, - raw: "te.Literal[False]" = False, - defer_init: bool = False, - ) -> CodeType: ... - - @typing.overload - def compile( - self, - source: t.Union[str, nodes.Template], - name: t.Optional[str] = None, - filename: t.Optional[str] = None, - raw: "te.Literal[True]" = ..., - defer_init: bool = False, - ) -> str: ... - - @internalcode - def compile( - self, - source: t.Union[str, nodes.Template], - name: t.Optional[str] = None, - filename: t.Optional[str] = None, - raw: bool = False, - defer_init: bool = False, - ) -> t.Union[str, CodeType]: - """Compile a node or template source code. The `name` parameter is - the load name of the template after it was joined using - :meth:`join_path` if necessary, not the filename on the file system. - the `filename` parameter is the estimated filename of the template on - the file system. If the template came from a database or memory this - can be omitted. - - The return value of this method is a python code object. If the `raw` - parameter is `True` the return value will be a string with python - code equivalent to the bytecode returned otherwise. This method is - mainly used internally. - - `defer_init` is use internally to aid the module code generator. This - causes the generated code to be able to import without the global - environment variable to be set. - - .. versionadded:: 2.4 - `defer_init` parameter added. - """ - source_hint = None - try: - if isinstance(source, str): - source_hint = source - source = self._parse(source, name, filename) - source = self._generate(source, name, filename, defer_init=defer_init) - if raw: - return source - if filename is None: - filename = "