Spaces:
Configuration error
Configuration error
Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +2 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/_output.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/_skills.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/auth.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/buckets.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/cache.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/collections.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/datasets.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/deprecated_cli.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/discussions.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/download.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/extensions.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/hf.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/inference_endpoints.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/jobs.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/lfs.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/models.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/papers.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/repo_files.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/repos.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/skills.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/spaces.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/system.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/upload.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/upload_large_folder.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/__pycache__/webhooks.cpython-311.pyc +0 -0
- venv/Lib/site-packages/huggingface_hub/cli/_city_game.py +661 -0
- venv/Lib/site-packages/huggingface_hub/cli/_cli_utils.py +1068 -0
- venv/Lib/site-packages/huggingface_hub/cli/_cp.py +252 -0
- venv/Lib/site-packages/huggingface_hub/cli/_errors.py +130 -0
- venv/Lib/site-packages/huggingface_hub/cli/_file_listing.py +225 -0
- venv/Lib/site-packages/huggingface_hub/cli/_help_formatter.py +34 -0
- venv/Lib/site-packages/huggingface_hub/cli/_output.py +345 -0
- venv/Lib/site-packages/huggingface_hub/cli/_skills.py +260 -0
- venv/Lib/site-packages/huggingface_hub/cli/auth.py +174 -0
- venv/Lib/site-packages/huggingface_hub/cli/buckets.py +678 -0
- venv/Lib/site-packages/huggingface_hub/cli/cache.py +775 -0
- venv/Lib/site-packages/huggingface_hub/cli/collections.py +316 -0
- venv/Lib/site-packages/huggingface_hub/cli/datasets.py +284 -0
- venv/Lib/site-packages/huggingface_hub/cli/deprecated_cli.py +35 -0
- venv/Lib/site-packages/huggingface_hub/cli/discussions.py +448 -0
- venv/Lib/site-packages/huggingface_hub/cli/download.py +254 -0
- venv/Lib/site-packages/huggingface_hub/cli/extensions.py +627 -0
- venv/Lib/site-packages/huggingface_hub/cli/hf.py +130 -0
- venv/Lib/site-packages/huggingface_hub/cli/inference_endpoints.py +438 -0
- venv/Lib/site-packages/huggingface_hub/cli/jobs.py +1172 -0
- venv/Lib/site-packages/huggingface_hub/cli/lfs.py +176 -0
- venv/Lib/site-packages/huggingface_hub/cli/models.py +217 -0
- venv/Lib/site-packages/huggingface_hub/cli/papers.py +197 -0
- venv/Lib/site-packages/huggingface_hub/cli/repo_files.py +84 -0
.gitattributes
CHANGED
|
@@ -92,3 +92,5 @@ venv/Lib/site-packages/h5py/hdf5_hl.dll filter=lfs diff=lfs merge=lfs -text
|
|
| 92 |
venv/Lib/site-packages/h5py/tests/__pycache__/test_dataset.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
| 93 |
venv/Lib/site-packages/hf_xet/hf_xet.pyd filter=lfs diff=lfs merge=lfs -text
|
| 94 |
venv/Lib/site-packages/huggingface_hub/__pycache__/hf_api.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 92 |
venv/Lib/site-packages/h5py/tests/__pycache__/test_dataset.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
| 93 |
venv/Lib/site-packages/hf_xet/hf_xet.pyd filter=lfs diff=lfs merge=lfs -text
|
| 94 |
venv/Lib/site-packages/huggingface_hub/__pycache__/hf_api.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
| 95 |
+
venv/Lib/site-packages/huggingface_hub/inference/__pycache__/_client.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
| 96 |
+
venv/Lib/site-packages/huggingface_hub/inference/_generated/__pycache__/_async_client.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/_output.cpython-311.pyc
ADDED
|
Binary file (23.5 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/_skills.cpython-311.pyc
ADDED
|
Binary file (16 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/auth.cpython-311.pyc
ADDED
|
Binary file (7.64 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/buckets.cpython-311.pyc
ADDED
|
Binary file (24.8 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/cache.cpython-311.pyc
ADDED
|
Binary file (37.5 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/collections.cpython-311.pyc
ADDED
|
Binary file (13.5 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/datasets.cpython-311.pyc
ADDED
|
Binary file (14.4 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/deprecated_cli.cpython-311.pyc
ADDED
|
Binary file (1.63 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/discussions.cpython-311.pyc
ADDED
|
Binary file (15.1 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/download.cpython-311.pyc
ADDED
|
Binary file (11.2 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/extensions.cpython-311.pyc
ADDED
|
Binary file (31.2 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/hf.cpython-311.pyc
ADDED
|
Binary file (7.17 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/inference_endpoints.cpython-311.pyc
ADDED
|
Binary file (18.3 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/jobs.cpython-311.pyc
ADDED
|
Binary file (54.1 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/lfs.cpython-311.pyc
ADDED
|
Binary file (7.19 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/models.cpython-311.pyc
ADDED
|
Binary file (9.62 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/papers.cpython-311.pyc
ADDED
|
Binary file (7.81 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/repo_files.cpython-311.pyc
ADDED
|
Binary file (2.68 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/repos.cpython-311.pyc
ADDED
|
Binary file (25.7 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/skills.cpython-311.pyc
ADDED
|
Binary file (26.9 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/spaces.cpython-311.pyc
ADDED
|
Binary file (54.9 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/system.cpython-311.pyc
ADDED
|
Binary file (2.09 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/upload.cpython-311.pyc
ADDED
|
Binary file (12.5 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/upload_large_folder.cpython-311.pyc
ADDED
|
Binary file (4.77 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/__pycache__/webhooks.cpython-311.pyc
ADDED
|
Binary file (12.8 kB). View file
|
|
|
venv/Lib/site-packages/huggingface_hub/cli/_city_game.py
ADDED
|
@@ -0,0 +1,661 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Interactive isometric city explorer — easter egg for `hf repos ls --explore`."""
|
| 2 |
+
|
| 3 |
+
import dataclasses
|
| 4 |
+
import math
|
| 5 |
+
import os
|
| 6 |
+
import random
|
| 7 |
+
import re
|
| 8 |
+
import select
|
| 9 |
+
import shutil
|
| 10 |
+
import sys
|
| 11 |
+
import time
|
| 12 |
+
|
| 13 |
+
from huggingface_hub.hf_api import RepoStorageInfo
|
| 14 |
+
|
| 15 |
+
from ._file_listing import format_size
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
Color = tuple[int, int, int]
|
| 19 |
+
|
| 20 |
+
# (top_face, left_face, right_face) — lighter to darker for 3D effect
|
| 21 |
+
_TYPE_COLORS: dict[str, tuple[Color, Color, Color]] = {
|
| 22 |
+
"model": ((175, 148, 240), (138, 112, 208), (105, 80, 180)),
|
| 23 |
+
"dataset": ((245, 128, 128), (222, 92, 92), (190, 60, 60)),
|
| 24 |
+
"space": ((245, 175, 85), (218, 140, 55), (185, 110, 30)),
|
| 25 |
+
"bucket": ((112, 185, 242), (70, 150, 220), (40, 118, 192)),
|
| 26 |
+
}
|
| 27 |
+
_EXTRA_COLORS: tuple[Color, Color, Color] = ((168, 176, 188), (128, 136, 148), (90, 98, 110))
|
| 28 |
+
_GRID_COLOR: Color = (178, 182, 190)
|
| 29 |
+
|
| 30 |
+
_DX = 4 # isometric half-width (pixels)
|
| 31 |
+
_DY = 2 # isometric half-height (pixels)
|
| 32 |
+
_MAX_H = 16 # tallest tile (pixels)
|
| 33 |
+
_MIN_H = 1
|
| 34 |
+
_COLS = 6
|
| 35 |
+
_EXT = 1 # grid extension beyond tiles
|
| 36 |
+
_MAX_TILES = 30
|
| 37 |
+
|
| 38 |
+
# Cursor sprite — pixel art arrow pointer
|
| 39 |
+
_OUTLINE: Color = (30, 30, 30)
|
| 40 |
+
_FILL: Color = (255, 255, 255)
|
| 41 |
+
|
| 42 |
+
_CURSOR_GRID = [
|
| 43 |
+
" X ",
|
| 44 |
+
" XWX ",
|
| 45 |
+
"XWWWX",
|
| 46 |
+
" XWX ",
|
| 47 |
+
" X ",
|
| 48 |
+
]
|
| 49 |
+
_CURSOR_PALETTE: dict[str, Color] = {
|
| 50 |
+
"X": _OUTLINE,
|
| 51 |
+
"W": _FILL,
|
| 52 |
+
}
|
| 53 |
+
_CURSOR_H = len(_CURSOR_GRID)
|
| 54 |
+
|
| 55 |
+
_MOVE_FRAMES = 8
|
| 56 |
+
_MOVE_DELAY = 0.03
|
| 57 |
+
_CURSOR_PAD = _CURSOR_H + 16
|
| 58 |
+
_GAP = 3
|
| 59 |
+
_MIN_TERM_W = 100
|
| 60 |
+
_MIN_TERM_H = 24
|
| 61 |
+
_SUMMARY_W = 24
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
# ---------------------------------------------------------------------------
|
| 65 |
+
# Data structures
|
| 66 |
+
# ---------------------------------------------------------------------------
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
@dataclasses.dataclass
|
| 70 |
+
class TileInfo:
|
| 71 |
+
grid_row: int
|
| 72 |
+
grid_col: int
|
| 73 |
+
height: int
|
| 74 |
+
top: Color
|
| 75 |
+
left: Color
|
| 76 |
+
right: Color
|
| 77 |
+
repo: RepoStorageInfo | None
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
@dataclasses.dataclass
|
| 81 |
+
class CityData:
|
| 82 |
+
tiles: list[TileInfo]
|
| 83 |
+
rows: int
|
| 84 |
+
cols: int
|
| 85 |
+
x_off: int
|
| 86 |
+
y_off: int
|
| 87 |
+
buf_w: int
|
| 88 |
+
buf_h: int
|
| 89 |
+
total_storage: int
|
| 90 |
+
extra_count: int
|
| 91 |
+
extra_storage: int
|
| 92 |
+
all_repos: list[RepoStorageInfo]
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
# ---------------------------------------------------------------------------
|
| 96 |
+
# City layout
|
| 97 |
+
# ---------------------------------------------------------------------------
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _prepare_city_data(repos: list[RepoStorageInfo]) -> CityData:
|
| 101 |
+
sorted_repos = sorted(repos, key=lambda r: r.storage, reverse=True)
|
| 102 |
+
display = sorted_repos[:_MAX_TILES]
|
| 103 |
+
extra_count = max(0, len(sorted_repos) - _MAX_TILES)
|
| 104 |
+
extra_storage = sum(r.storage for r in sorted_repos[_MAX_TILES:])
|
| 105 |
+
total_storage = sum(r.storage for r in repos)
|
| 106 |
+
max_storage = max(1, display[0].storage)
|
| 107 |
+
|
| 108 |
+
n = len(display) + (1 if extra_count > 0 else 0)
|
| 109 |
+
cols = min(n, _COLS)
|
| 110 |
+
rows = math.ceil(n / cols) if cols > 0 else 1
|
| 111 |
+
|
| 112 |
+
tiles: list[TileInfo] = []
|
| 113 |
+
for i, repo in enumerate(display):
|
| 114 |
+
r, c = divmod(i, cols)
|
| 115 |
+
h = max(_MIN_H, round(math.sqrt(repo.storage / max_storage) * _MAX_H))
|
| 116 |
+
top, left, right = _TYPE_COLORS.get(repo.type, _EXTRA_COLORS)
|
| 117 |
+
tiles.append(TileInfo(r, c, h, top, left, right, repo))
|
| 118 |
+
if extra_count > 0:
|
| 119 |
+
r, c = divmod(len(display), cols)
|
| 120 |
+
h = max(_MIN_H, round(math.sqrt(extra_storage / max_storage) * _MAX_H))
|
| 121 |
+
tiles.append(TileInfo(r, c, h, *_EXTRA_COLORS, None))
|
| 122 |
+
|
| 123 |
+
r_lo, r_hi = -_EXT, rows - 1 + _EXT
|
| 124 |
+
c_lo, c_hi = -_EXT, cols - 1 + _EXT
|
| 125 |
+
|
| 126 |
+
xs: list[int] = []
|
| 127 |
+
ys: list[int] = []
|
| 128 |
+
for rr in range(r_lo, r_hi + 1):
|
| 129 |
+
for cc in range(c_lo, c_hi + 1):
|
| 130 |
+
cx, cy = (cc - rr) * _DX, (cc + rr) * _DY
|
| 131 |
+
xs.extend([cx - _DX, cx + _DX])
|
| 132 |
+
ys.extend([cy, cy + 2 * _DY])
|
| 133 |
+
for tile in tiles:
|
| 134 |
+
ys.append((tile.grid_col + tile.grid_row) * _DY - tile.height)
|
| 135 |
+
|
| 136 |
+
x_off = -min(xs)
|
| 137 |
+
y_off = -min(ys)
|
| 138 |
+
buf_w = max(xs) - min(xs) + 1
|
| 139 |
+
buf_h = max(ys) - min(ys) + 1
|
| 140 |
+
if buf_h % 2:
|
| 141 |
+
buf_h += 1
|
| 142 |
+
|
| 143 |
+
return CityData(
|
| 144 |
+
tiles=tiles,
|
| 145 |
+
rows=rows,
|
| 146 |
+
cols=cols,
|
| 147 |
+
x_off=x_off,
|
| 148 |
+
y_off=y_off,
|
| 149 |
+
buf_w=buf_w,
|
| 150 |
+
buf_h=buf_h,
|
| 151 |
+
total_storage=total_storage,
|
| 152 |
+
extra_count=extra_count,
|
| 153 |
+
extra_storage=extra_storage,
|
| 154 |
+
all_repos=repos,
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
# ---------------------------------------------------------------------------
|
| 159 |
+
# Drawing primitives
|
| 160 |
+
# ---------------------------------------------------------------------------
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def _draw_diamond_outline(buf: list[list[Color | None]], cx: int, cy: int) -> None:
|
| 164 |
+
t = (cx, cy)
|
| 165 |
+
r = (cx + _DX, cy + _DY)
|
| 166 |
+
b = (cx, cy + 2 * _DY)
|
| 167 |
+
ll = (cx - _DX, cy + _DY)
|
| 168 |
+
_draw_line(buf, *t, *r, _GRID_COLOR)
|
| 169 |
+
_draw_line(buf, *r, *b, _GRID_COLOR)
|
| 170 |
+
_draw_line(buf, *b, *ll, _GRID_COLOR)
|
| 171 |
+
_draw_line(buf, *ll, *t, _GRID_COLOR)
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def _draw_block(
|
| 175 |
+
buf: list[list[Color | None]],
|
| 176 |
+
cx: int,
|
| 177 |
+
cy: int,
|
| 178 |
+
h: int,
|
| 179 |
+
top: Color,
|
| 180 |
+
left: Color,
|
| 181 |
+
right: Color,
|
| 182 |
+
) -> None:
|
| 183 |
+
_fill_poly(
|
| 184 |
+
buf,
|
| 185 |
+
[(cx - _DX, cy + _DY - h), (cx, cy + 2 * _DY - h), (cx, cy + 2 * _DY), (cx - _DX, cy + _DY)],
|
| 186 |
+
left,
|
| 187 |
+
)
|
| 188 |
+
_fill_poly(
|
| 189 |
+
buf,
|
| 190 |
+
[(cx, cy + 2 * _DY - h), (cx + _DX, cy + _DY - h), (cx + _DX, cy + _DY), (cx, cy + 2 * _DY)],
|
| 191 |
+
right,
|
| 192 |
+
)
|
| 193 |
+
_fill_poly(
|
| 194 |
+
buf,
|
| 195 |
+
[(cx, cy - h), (cx + _DX, cy + _DY - h), (cx, cy + 2 * _DY - h), (cx - _DX, cy + _DY - h)],
|
| 196 |
+
top,
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _fill_poly(buf: list[list[Color | None]], verts: list[tuple[int, int]], color: Color) -> None:
|
| 201 |
+
bh = len(buf)
|
| 202 |
+
bw = len(buf[0]) if buf else 0
|
| 203 |
+
all_y = [v[1] for v in verts]
|
| 204 |
+
y0 = max(0, min(all_y))
|
| 205 |
+
y1 = min(bh - 1, max(all_y))
|
| 206 |
+
n = len(verts)
|
| 207 |
+
for y in range(y0, y1 + 1):
|
| 208 |
+
xl: float = float("inf")
|
| 209 |
+
xr: float = float("-inf")
|
| 210 |
+
for i in range(n):
|
| 211 |
+
ax, ay = verts[i]
|
| 212 |
+
bx, by = verts[(i + 1) % n]
|
| 213 |
+
if ay == by:
|
| 214 |
+
if y == ay:
|
| 215 |
+
xl = min(xl, float(min(ax, bx)))
|
| 216 |
+
xr = max(xr, float(max(ax, bx)))
|
| 217 |
+
continue
|
| 218 |
+
if not (min(ay, by) <= y <= max(ay, by)):
|
| 219 |
+
continue
|
| 220 |
+
t = (y - ay) / (by - ay)
|
| 221 |
+
ix = ax + t * (bx - ax)
|
| 222 |
+
xl = min(xl, ix)
|
| 223 |
+
xr = max(xr, ix)
|
| 224 |
+
if xl <= xr:
|
| 225 |
+
for x in range(max(0, round(xl)), min(bw, round(xr) + 1)):
|
| 226 |
+
buf[y][x] = color
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def _draw_line(buf: list[list[Color | None]], x0: int, y0: int, x1: int, y1: int, color: Color) -> None:
|
| 230 |
+
bh = len(buf)
|
| 231 |
+
bw = len(buf[0]) if buf else 0
|
| 232 |
+
dx = abs(x1 - x0)
|
| 233 |
+
dy = abs(y1 - y0)
|
| 234 |
+
steps = max(dx, dy)
|
| 235 |
+
if steps == 0:
|
| 236 |
+
if 0 <= y0 < bh and 0 <= x0 < bw:
|
| 237 |
+
buf[y0][x0] = color
|
| 238 |
+
return
|
| 239 |
+
xi = (x1 - x0) / steps
|
| 240 |
+
yi = (y1 - y0) / steps
|
| 241 |
+
fx, fy = float(x0), float(y0)
|
| 242 |
+
for _ in range(steps + 1):
|
| 243 |
+
px, py = round(fx), round(fy)
|
| 244 |
+
if 0 <= py < bh and 0 <= px < bw:
|
| 245 |
+
buf[py][px] = color
|
| 246 |
+
fx += xi
|
| 247 |
+
fy += yi
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
# ---------------------------------------------------------------------------
|
| 251 |
+
# Pixel buffer → terminal
|
| 252 |
+
# ---------------------------------------------------------------------------
|
| 253 |
+
|
| 254 |
+
_ANSI_RE = re.compile(r"\033\[[0-9;]*m")
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def _strip_ansi(s: str) -> str:
|
| 258 |
+
return _ANSI_RE.sub("", s)
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def _visible_len(s: str) -> int:
|
| 262 |
+
return len(_strip_ansi(s))
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def _pixels_to_lines(buf: list[list[Color | None]]) -> list[str]:
|
| 266 |
+
height = len(buf)
|
| 267 |
+
width = len(buf[0]) if buf else 0
|
| 268 |
+
lines: list[str] = []
|
| 269 |
+
for row in range(0, height, 2):
|
| 270 |
+
last = -1
|
| 271 |
+
for col in range(width - 1, -1, -1):
|
| 272 |
+
top = buf[row][col]
|
| 273 |
+
bot = buf[row + 1][col] if row + 1 < height else None
|
| 274 |
+
if top or bot:
|
| 275 |
+
last = col
|
| 276 |
+
break
|
| 277 |
+
if last < 0:
|
| 278 |
+
lines.append("")
|
| 279 |
+
continue
|
| 280 |
+
|
| 281 |
+
parts: list[str] = []
|
| 282 |
+
cfg: Color | None = None
|
| 283 |
+
cbg: Color | None = None
|
| 284 |
+
|
| 285 |
+
for col in range(last + 1):
|
| 286 |
+
top = buf[row][col]
|
| 287 |
+
bot = buf[row + 1][col] if row + 1 < height else None
|
| 288 |
+
|
| 289 |
+
if not top and not bot:
|
| 290 |
+
if cfg is not None or cbg is not None:
|
| 291 |
+
parts.append("\033[0m")
|
| 292 |
+
cfg = cbg = None
|
| 293 |
+
parts.append(" ")
|
| 294 |
+
continue
|
| 295 |
+
|
| 296 |
+
if top and bot and top == bot:
|
| 297 |
+
nfg, nbg, ch = top, None, "█"
|
| 298 |
+
elif top and bot:
|
| 299 |
+
nfg, nbg, ch = bot, top, "▄"
|
| 300 |
+
elif top:
|
| 301 |
+
nfg, nbg, ch = top, None, "▀"
|
| 302 |
+
else:
|
| 303 |
+
nfg, nbg, ch = bot, None, "▄" # type: ignore[assignment]
|
| 304 |
+
|
| 305 |
+
esc = ""
|
| 306 |
+
if nfg != cfg:
|
| 307 |
+
esc += f"\033[38;2;{nfg[0]};{nfg[1]};{nfg[2]}m"
|
| 308 |
+
cfg = nfg
|
| 309 |
+
if nbg != cbg:
|
| 310 |
+
esc += "\033[49m" if nbg is None else f"\033[48;2;{nbg[0]};{nbg[1]};{nbg[2]}m"
|
| 311 |
+
cbg = nbg
|
| 312 |
+
parts.append(esc + ch)
|
| 313 |
+
|
| 314 |
+
if cfg is not None or cbg is not None:
|
| 315 |
+
parts.append("\033[0m")
|
| 316 |
+
lines.append("".join(parts))
|
| 317 |
+
return lines
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
# ---------------------------------------------------------------------------
|
| 321 |
+
# Rendering
|
| 322 |
+
# ---------------------------------------------------------------------------
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
def _render_base_buffer(city: CityData) -> list[list[Color | None]]:
|
| 326 |
+
buf: list[list[Color | None]] = [[None] * city.buf_w for _ in range(city.buf_h)]
|
| 327 |
+
|
| 328 |
+
for tile in city.tiles:
|
| 329 |
+
cx = city.x_off + (tile.grid_col - tile.grid_row) * _DX
|
| 330 |
+
cy = city.y_off + (tile.grid_col + tile.grid_row) * _DY
|
| 331 |
+
_draw_diamond_outline(buf, cx, cy)
|
| 332 |
+
|
| 333 |
+
sorted_tiles = sorted(city.tiles, key=lambda t: (t.grid_row + t.grid_col, t.grid_col))
|
| 334 |
+
for tile in sorted_tiles:
|
| 335 |
+
cx = city.x_off + (tile.grid_col - tile.grid_row) * _DX
|
| 336 |
+
cy = city.y_off + (tile.grid_col + tile.grid_row) * _DY
|
| 337 |
+
_draw_block(buf, cx, cy, tile.height, tile.top, tile.left, tile.right)
|
| 338 |
+
|
| 339 |
+
return buf
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
# ---------------------------------------------------------------------------
|
| 343 |
+
# Summary panel
|
| 344 |
+
# ---------------------------------------------------------------------------
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
def _colored_square(color: Color) -> str:
|
| 348 |
+
return f"\033[38;2;{color[0]};{color[1]};{color[2]}m■\033[0m"
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def _build_summary(
|
| 352 |
+
repos: list[RepoStorageInfo],
|
| 353 |
+
total_storage: int,
|
| 354 |
+
extra_count: int,
|
| 355 |
+
) -> list[str]:
|
| 356 |
+
lines: list[str] = [""]
|
| 357 |
+
lines.append(" Storage Overview")
|
| 358 |
+
lines.append(" " + "─" * 16)
|
| 359 |
+
lines.append(f" {format_size(total_storage, human_readable=True)} total")
|
| 360 |
+
lines.append("")
|
| 361 |
+
|
| 362 |
+
order = ["model", "dataset", "space", "bucket"]
|
| 363 |
+
labels = {"model": "Models", "dataset": "Datasets", "space": "Spaces", "bucket": "Buckets"}
|
| 364 |
+
for rtype in order:
|
| 365 |
+
group = [r for r in repos if r.type == rtype]
|
| 366 |
+
if not group:
|
| 367 |
+
continue
|
| 368 |
+
storage = sum(r.storage for r in group)
|
| 369 |
+
sq = _colored_square(_TYPE_COLORS[rtype][0])
|
| 370 |
+
lines.append(f" {sq} {labels[rtype]}")
|
| 371 |
+
lines.append(f" {len(group)} repos · {format_size(storage, human_readable=True)}")
|
| 372 |
+
lines.append("")
|
| 373 |
+
|
| 374 |
+
if extra_count > 0:
|
| 375 |
+
sq = _colored_square(_EXTRA_COLORS[0])
|
| 376 |
+
lines.append(f" {sq} +{extra_count} more repos")
|
| 377 |
+
|
| 378 |
+
return lines
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
# ---------------------------------------------------------------------------
|
| 382 |
+
# Cursor
|
| 383 |
+
# ---------------------------------------------------------------------------
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
def _build_cursor() -> list[tuple[int, int, Color]]:
|
| 387 |
+
pixels: list[tuple[int, int, Color]] = []
|
| 388 |
+
for ri, row in enumerate(_CURSOR_GRID):
|
| 389 |
+
for ci, ch in enumerate(row):
|
| 390 |
+
if ch in _CURSOR_PALETTE:
|
| 391 |
+
pixels.append((ci - len(row) // 2, ri - _CURSOR_H + 1, _CURSOR_PALETTE[ch]))
|
| 392 |
+
return pixels
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
_CURSOR_PIXELS = _build_cursor()
|
| 396 |
+
|
| 397 |
+
|
| 398 |
+
# ---------------------------------------------------------------------------
|
| 399 |
+
# Interactive game
|
| 400 |
+
# ---------------------------------------------------------------------------
|
| 401 |
+
|
| 402 |
+
|
| 403 |
+
def run_city_game(repos: list[RepoStorageInfo]) -> None:
|
| 404 |
+
"""Launch the interactive city explorer."""
|
| 405 |
+
if not repos:
|
| 406 |
+
print("No repositories found.")
|
| 407 |
+
return
|
| 408 |
+
|
| 409 |
+
try:
|
| 410 |
+
import termios
|
| 411 |
+
import tty
|
| 412 |
+
except ImportError:
|
| 413 |
+
print("Interactive mode requires a Unix-like terminal (Linux/macOS).")
|
| 414 |
+
return
|
| 415 |
+
|
| 416 |
+
if not sys.stdin.isatty() or not sys.stdout.isatty():
|
| 417 |
+
print("Interactive mode requires a terminal.")
|
| 418 |
+
return
|
| 419 |
+
|
| 420 |
+
term = shutil.get_terminal_size()
|
| 421 |
+
if term.columns < _MIN_TERM_W or term.lines < _MIN_TERM_H:
|
| 422 |
+
print(f"Your terminal is {term.columns}×{term.lines} characters.")
|
| 423 |
+
print(f"Please resize to at least {_MIN_TERM_W}×{_MIN_TERM_H} to explore the city!")
|
| 424 |
+
return
|
| 425 |
+
|
| 426 |
+
city = _prepare_city_data(repos)
|
| 427 |
+
|
| 428 |
+
tiles_with_repos = [t for t in city.tiles if t.repo is not None]
|
| 429 |
+
start_tile = random.choice(tiles_with_repos) if tiles_with_repos else city.tiles[0]
|
| 430 |
+
|
| 431 |
+
fd = sys.stdin.fileno()
|
| 432 |
+
old_settings = termios.tcgetattr(fd)
|
| 433 |
+
try:
|
| 434 |
+
tty.setraw(fd)
|
| 435 |
+
sys.stdout.write("\033[?1049h\033[?25l\033[2J")
|
| 436 |
+
sys.stdout.flush()
|
| 437 |
+
_game_loop(city, start_tile.grid_row, start_tile.grid_col)
|
| 438 |
+
finally:
|
| 439 |
+
sys.stdout.write("\033[?25h\033[?1049l")
|
| 440 |
+
sys.stdout.flush()
|
| 441 |
+
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
|
| 442 |
+
|
| 443 |
+
|
| 444 |
+
def _game_loop(city: CityData, cur_row: int, cur_col: int) -> None:
|
| 445 |
+
tile_map: dict[tuple[int, int], TileInfo] = {(t.grid_row, t.grid_col): t for t in city.tiles}
|
| 446 |
+
city = dataclasses.replace(city, buf_h=city.buf_h + _CURSOR_PAD, y_off=city.y_off + _CURSOR_PAD)
|
| 447 |
+
base_buf = _render_base_buffer(city)
|
| 448 |
+
|
| 449 |
+
summary = _build_summary(city.all_repos, city.total_storage, city.extra_count)
|
| 450 |
+
|
| 451 |
+
# Intro: cursor drops onto starting tile
|
| 452 |
+
tx, ty = _tile_top_center(city, cur_row, cur_col, tile_map)
|
| 453 |
+
for i in range(1, _MOVE_FRAMES + 1):
|
| 454 |
+
t = i / _MOVE_FRAMES
|
| 455 |
+
t = t * t * (3 - 2 * t)
|
| 456 |
+
drop_y = ty - 16 * (1 - t)
|
| 457 |
+
frame = _copy_buf(base_buf)
|
| 458 |
+
_highlight_tile(frame, city, tile_map[(cur_row, cur_col)])
|
| 459 |
+
_draw_cursor(frame, tx, round(drop_y))
|
| 460 |
+
_present(city, frame, tile_map.get((cur_row, cur_col)), summary)
|
| 461 |
+
time.sleep(_MOVE_DELAY)
|
| 462 |
+
|
| 463 |
+
while True:
|
| 464 |
+
cx, cy = _tile_top_center(city, cur_row, cur_col, tile_map)
|
| 465 |
+
frame = _copy_buf(base_buf)
|
| 466 |
+
_highlight_tile(frame, city, tile_map[(cur_row, cur_col)])
|
| 467 |
+
_draw_cursor(frame, cx, cy)
|
| 468 |
+
_present(city, frame, tile_map.get((cur_row, cur_col)), summary)
|
| 469 |
+
|
| 470 |
+
key = _read_key()
|
| 471 |
+
if key in ("q", "Q", "esc", "\x03"):
|
| 472 |
+
return
|
| 473 |
+
|
| 474 |
+
dr, dc = _key_to_direction(key)
|
| 475 |
+
if dr == 0 and dc == 0:
|
| 476 |
+
continue
|
| 477 |
+
|
| 478 |
+
nr, nc = cur_row + dr, cur_col + dc
|
| 479 |
+
if (nr, nc) not in tile_map:
|
| 480 |
+
continue
|
| 481 |
+
|
| 482 |
+
ex, ey = _tile_top_center(city, nr, nc, tile_map)
|
| 483 |
+
for i in range(1, _MOVE_FRAMES + 1):
|
| 484 |
+
t = i / _MOVE_FRAMES
|
| 485 |
+
t = t * t * (3 - 2 * t)
|
| 486 |
+
bx = cx + (ex - cx) * t
|
| 487 |
+
by = cy + (ey - cy) * t
|
| 488 |
+
frame = _copy_buf(base_buf)
|
| 489 |
+
_highlight_tile(frame, city, tile_map[(nr, nc)])
|
| 490 |
+
_draw_cursor(frame, round(bx), round(by))
|
| 491 |
+
_present(city, frame, tile_map.get((nr, nc)), summary)
|
| 492 |
+
time.sleep(_MOVE_DELAY)
|
| 493 |
+
|
| 494 |
+
cur_row, cur_col = nr, nc
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def _tile_top_center(city: CityData, row: int, col: int, tile_map: dict[tuple[int, int], TileInfo]) -> tuple[int, int]:
|
| 498 |
+
tile = tile_map.get((row, col))
|
| 499 |
+
h = tile.height if tile else 1
|
| 500 |
+
cx = city.x_off + (col - row) * _DX
|
| 501 |
+
cy = city.y_off + (col + row) * _DY
|
| 502 |
+
return cx, cy + _DY - h
|
| 503 |
+
|
| 504 |
+
|
| 505 |
+
def _key_to_direction(key: str) -> tuple[int, int]:
|
| 506 |
+
match key:
|
| 507 |
+
case "w" | "W" | "\x1b[A":
|
| 508 |
+
return -1, 0
|
| 509 |
+
case "s" | "S" | "\x1b[B":
|
| 510 |
+
return 1, 0
|
| 511 |
+
case "a" | "A" | "\x1b[D":
|
| 512 |
+
return 0, -1
|
| 513 |
+
case "d" | "D" | "\x1b[C":
|
| 514 |
+
return 0, 1
|
| 515 |
+
case _:
|
| 516 |
+
return 0, 0
|
| 517 |
+
|
| 518 |
+
|
| 519 |
+
def _draw_cursor(buf: list[list[Color | None]], cx: int, cy: int) -> None:
|
| 520 |
+
bh = len(buf)
|
| 521 |
+
bw = len(buf[0]) if buf else 0
|
| 522 |
+
for dx, dy, color in _CURSOR_PIXELS:
|
| 523 |
+
px, py = cx + dx, cy + dy
|
| 524 |
+
if 0 <= py < bh and 0 <= px < bw:
|
| 525 |
+
buf[py][px] = color
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def _highlight_tile(buf: list[list[Color | None]], city: CityData, tile: TileInfo) -> None:
|
| 529 |
+
cx = city.x_off + (tile.grid_col - tile.grid_row) * _DX
|
| 530 |
+
cy = city.y_off + (tile.grid_col + tile.grid_row) * _DY
|
| 531 |
+
h = tile.height
|
| 532 |
+
_fill_poly(
|
| 533 |
+
buf,
|
| 534 |
+
[(cx, cy - h), (cx + _DX, cy + _DY - h), (cx, cy + 2 * _DY - h), (cx - _DX, cy + _DY - h)],
|
| 535 |
+
_brighten(tile.top, 35),
|
| 536 |
+
)
|
| 537 |
+
|
| 538 |
+
|
| 539 |
+
def _brighten(color: Color, amount: int) -> Color:
|
| 540 |
+
return (min(255, color[0] + amount), min(255, color[1] + amount), min(255, color[2] + amount))
|
| 541 |
+
|
| 542 |
+
|
| 543 |
+
def _present(
|
| 544 |
+
city: CityData,
|
| 545 |
+
buf: list[list[Color | None]],
|
| 546 |
+
tile: TileInfo | None,
|
| 547 |
+
summary: list[str],
|
| 548 |
+
) -> None:
|
| 549 |
+
city_lines = _pixels_to_lines(buf)
|
| 550 |
+
while city_lines and not _strip_ansi(city_lines[0]).strip():
|
| 551 |
+
city_lines.pop(0)
|
| 552 |
+
while city_lines and not _strip_ansi(city_lines[-1]).strip():
|
| 553 |
+
city_lines.pop()
|
| 554 |
+
|
| 555 |
+
city_w = max((_visible_len(line) for line in city_lines), default=0)
|
| 556 |
+
term = shutil.get_terminal_size()
|
| 557 |
+
panel_max_w = max(20, term.columns - city_w - _SUMMARY_W - 2 * _GAP)
|
| 558 |
+
|
| 559 |
+
info = _build_info_panel(tile, city, panel_max_w)
|
| 560 |
+
|
| 561 |
+
n = max(len(summary), len(city_lines), len(info))
|
| 562 |
+
summary_lo = max(0, (n - len(summary)) // 2)
|
| 563 |
+
info_lo = max(0, (n - len(info)) // 2)
|
| 564 |
+
|
| 565 |
+
lines: list[str] = []
|
| 566 |
+
for i in range(n):
|
| 567 |
+
si = i - summary_lo
|
| 568 |
+
lt = summary[si] if 0 <= si < len(summary) else ""
|
| 569 |
+
lpad = max(0, _SUMMARY_W - _visible_len(lt))
|
| 570 |
+
|
| 571 |
+
ct = city_lines[i] if i < len(city_lines) else ""
|
| 572 |
+
cpad = max(0, city_w - _visible_len(ct))
|
| 573 |
+
|
| 574 |
+
ri = i - info_lo
|
| 575 |
+
rt = info[ri] if 0 <= ri < len(info) else ""
|
| 576 |
+
|
| 577 |
+
lines.append(lt + " " * lpad + " " * _GAP + ct + " " * cpad + " " * _GAP + rt)
|
| 578 |
+
|
| 579 |
+
lines.append("")
|
| 580 |
+
lines.append(" \033[90mWASD/Arrows: move · Q/ESC: quit\033[0m")
|
| 581 |
+
|
| 582 |
+
while len(lines) < term.lines - 1:
|
| 583 |
+
lines.append("")
|
| 584 |
+
|
| 585 |
+
output = "\033[H"
|
| 586 |
+
for line in lines[: term.lines - 1]:
|
| 587 |
+
output += line + "\033[K\r\n"
|
| 588 |
+
sys.stdout.write(output)
|
| 589 |
+
sys.stdout.flush()
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
def _build_info_panel(tile: TileInfo | None, city: CityData, max_w: int) -> list[str]:
|
| 593 |
+
reset = "\033[0m"
|
| 594 |
+
gray = "\033[90m"
|
| 595 |
+
bold = "\033[1m"
|
| 596 |
+
indent = " "
|
| 597 |
+
content_w = max_w - len(indent)
|
| 598 |
+
|
| 599 |
+
lines: list[str] = [""]
|
| 600 |
+
lines.append(f"{indent}{bold}City Explorer{reset}")
|
| 601 |
+
lines.append(indent + "─" * min(22, content_w))
|
| 602 |
+
lines.append("")
|
| 603 |
+
|
| 604 |
+
if tile is None:
|
| 605 |
+
lines.append(f"{indent}{gray}Move to a tile")
|
| 606 |
+
lines.append(f"{indent}to see details.{reset}")
|
| 607 |
+
return lines
|
| 608 |
+
|
| 609 |
+
if tile.repo is None:
|
| 610 |
+
lines.append(f"{indent}{gray}+{city.extra_count} more repos{reset}")
|
| 611 |
+
lines.append(f"{indent}{gray}{format_size(city.extra_storage, human_readable=True)} combined{reset}")
|
| 612 |
+
return lines
|
| 613 |
+
|
| 614 |
+
repo = tile.repo
|
| 615 |
+
name = repo.id
|
| 616 |
+
if len(name) > content_w:
|
| 617 |
+
name = name[: content_w - 3] + "..."
|
| 618 |
+
lines.append(f"{indent}{bold}{name}{reset}")
|
| 619 |
+
lines.append("")
|
| 620 |
+
|
| 621 |
+
type_ansi = {
|
| 622 |
+
"model": "\033[38;2;175;148;240m",
|
| 623 |
+
"dataset": "\033[38;2;245;128;128m",
|
| 624 |
+
"space": "\033[38;2;245;175;85m",
|
| 625 |
+
"bucket": "\033[38;2;112;185;242m",
|
| 626 |
+
}
|
| 627 |
+
tc = type_ansi.get(repo.type, "")
|
| 628 |
+
|
| 629 |
+
lines.append(f"{indent}Type {tc}{repo.type}{reset}")
|
| 630 |
+
lines.append(f"{indent}Visibility {repo.visibility}")
|
| 631 |
+
lines.append(f"{indent}Storage {format_size(repo.storage, human_readable=True)}")
|
| 632 |
+
lines.append(f"{indent}Usage {repo.storage_percent:.1f}%")
|
| 633 |
+
lines.append("")
|
| 634 |
+
|
| 635 |
+
bar_w = min(18, content_w)
|
| 636 |
+
filled = max(0, min(bar_w, round(repo.storage_percent / 100 * bar_w)))
|
| 637 |
+
lines.append(f"{indent}{tc}{'█' * filled}{gray}{'░' * (bar_w - filled)}{reset}")
|
| 638 |
+
|
| 639 |
+
return lines
|
| 640 |
+
|
| 641 |
+
|
| 642 |
+
def _copy_buf(buf: list[list[Color | None]]) -> list[list[Color | None]]:
|
| 643 |
+
return [row[:] for row in buf]
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
def _read_key() -> str:
|
| 647 |
+
fd = sys.stdin.fileno()
|
| 648 |
+
ch = os.read(fd, 1)
|
| 649 |
+
if ch == b"\x1b":
|
| 650 |
+
if _has_input(fd, 0.05):
|
| 651 |
+
ch2 = os.read(fd, 1)
|
| 652 |
+
if ch2 == b"[" and _has_input(fd, 0.05):
|
| 653 |
+
ch3 = os.read(fd, 1)
|
| 654 |
+
return f"\x1b[{ch3.decode()}"
|
| 655 |
+
return "esc"
|
| 656 |
+
return ch.decode("utf-8", errors="replace")
|
| 657 |
+
|
| 658 |
+
|
| 659 |
+
def _has_input(fd: int, timeout: float) -> bool:
|
| 660 |
+
r, _, _ = select.select([fd], [], [], timeout)
|
| 661 |
+
return bool(r)
|
venv/Lib/site-packages/huggingface_hub/cli/_cli_utils.py
ADDED
|
@@ -0,0 +1,1068 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains CLI utilities (styling, helpers)."""
|
| 15 |
+
|
| 16 |
+
import difflib
|
| 17 |
+
import importlib.metadata
|
| 18 |
+
import os
|
| 19 |
+
import re
|
| 20 |
+
import subprocess
|
| 21 |
+
import sys
|
| 22 |
+
import time
|
| 23 |
+
from collections.abc import Callable, Sequence
|
| 24 |
+
from enum import Enum
|
| 25 |
+
from pathlib import Path
|
| 26 |
+
from typing import TYPE_CHECKING, Annotated, Any, Literal, TypeVar, cast
|
| 27 |
+
|
| 28 |
+
import click
|
| 29 |
+
import typer
|
| 30 |
+
from typer.core import TyperCommand, TyperGroup
|
| 31 |
+
|
| 32 |
+
from huggingface_hub import Volume, __version__, constants
|
| 33 |
+
from huggingface_hub.errors import CLIError
|
| 34 |
+
from huggingface_hub.utils import (
|
| 35 |
+
get_session,
|
| 36 |
+
hf_raise_for_status,
|
| 37 |
+
installation_method,
|
| 38 |
+
logging,
|
| 39 |
+
parse_hf_mount,
|
| 40 |
+
)
|
| 41 |
+
from huggingface_hub.utils._dotenv import load_dotenv
|
| 42 |
+
|
| 43 |
+
from ._help_formatter import StyledContext
|
| 44 |
+
from ._output import OutputFormat, out
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
logger = logging.get_logger()
|
| 48 |
+
|
| 49 |
+
# Arbitrary default limit for models/datasets/spaces list commands.
|
| 50 |
+
REPO_LIST_DEFAULT_LIMIT = 30
|
| 51 |
+
|
| 52 |
+
if TYPE_CHECKING:
|
| 53 |
+
from huggingface_hub.hf_api import HfApi
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def get_hf_api(token: str | None = None) -> "HfApi":
|
| 57 |
+
# Import here to avoid circular import
|
| 58 |
+
from huggingface_hub.hf_api import HfApi
|
| 59 |
+
|
| 60 |
+
return HfApi(token=token, library_name="huggingface-cli", library_version=__version__)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
#### TYPER UTILS
|
| 64 |
+
|
| 65 |
+
CLI_REFERENCE_URL = "https://huggingface.co/docs/huggingface_hub/en/guides/cli"
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def generate_epilog(examples: list[str], docs_anchor: str | None = None) -> str:
|
| 69 |
+
"""Generate an epilog with examples and a Learn More section.
|
| 70 |
+
|
| 71 |
+
Args:
|
| 72 |
+
examples: List of example commands (without the `$ ` prefix).
|
| 73 |
+
docs_anchor: Optional anchor for the docs URL (e.g., "#hf-download").
|
| 74 |
+
|
| 75 |
+
Returns:
|
| 76 |
+
Formatted epilog string.
|
| 77 |
+
"""
|
| 78 |
+
docs_url = f"{CLI_REFERENCE_URL}{docs_anchor}" if docs_anchor else CLI_REFERENCE_URL
|
| 79 |
+
examples_str = "\n".join(f" $ {ex}" for ex in examples)
|
| 80 |
+
return f"""\
|
| 81 |
+
Examples
|
| 82 |
+
{examples_str}
|
| 83 |
+
|
| 84 |
+
Learn more
|
| 85 |
+
Use `hf <command> --help` for more information about a command.
|
| 86 |
+
Read the documentation at {docs_url}
|
| 87 |
+
"""
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
TOPIC_T = Literal["main", "help"] | str
|
| 91 |
+
FallbackHandlerT = Callable[[list[str], set[str]], int | None]
|
| 92 |
+
ExpandPropertyT = TypeVar("ExpandPropertyT", bound=str)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _format_epilog_no_indent(epilog: str | None, ctx: click.Context, formatter: click.HelpFormatter) -> None:
|
| 96 |
+
"""Write the epilog without indentation."""
|
| 97 |
+
if epilog:
|
| 98 |
+
formatter.write_paragraph()
|
| 99 |
+
for line in epilog.split("\n"):
|
| 100 |
+
formatter.write_text(line)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
_ALIAS_SPLIT = re.compile(r"\s*\|\s*")
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
class HFCliTyperGroup(TyperGroup):
|
| 107 |
+
"""
|
| 108 |
+
Typer Group that:
|
| 109 |
+
- lists commands alphabetically within sections.
|
| 110 |
+
- separates commands by topic (main, help, etc.).
|
| 111 |
+
- formats epilog without extra indentation.
|
| 112 |
+
- supports aliases via pipe-separated names (e.g. ``name="list | ls"``).
|
| 113 |
+
- consumes the global formatting flags (``--format``, ``--json``, ``-q`` / ``--quiet``, ``--no-truncate``)
|
| 114 |
+
anywhere in the args of a leaf command and applies them to ``out``, so leaf
|
| 115 |
+
commands don't need to declare these options themselves.
|
| 116 |
+
- rewrites ``spaces/user/repo`` to ``user/repo --type space`` for commands that accept ``--type``.
|
| 117 |
+
- enriches "No such option" / "No such command" errors with available options or commands.
|
| 118 |
+
"""
|
| 119 |
+
|
| 120 |
+
context_class = StyledContext
|
| 121 |
+
|
| 122 |
+
def invoke(self, ctx: click.Context) -> None:
|
| 123 |
+
"""Enrich unknown-option errors with available options or subcommands.
|
| 124 |
+
|
| 125 |
+
Catches `NoSuchOption` raised during subcommand `make_context()`
|
| 126 |
+
(option parsing). For leaf commands (e.g. `hf repos create --test`)
|
| 127 |
+
we list the command's options; for groups (e.g. `hf cache --test`)
|
| 128 |
+
we list subcommands since groups have no user-facing options.
|
| 129 |
+
"""
|
| 130 |
+
try:
|
| 131 |
+
return super().invoke(ctx)
|
| 132 |
+
except click.NoSuchOption as e:
|
| 133 |
+
if e.ctx is not None and e.ctx.command is not None:
|
| 134 |
+
cmd = e.ctx.command
|
| 135 |
+
if isinstance(cmd, click.Group):
|
| 136 |
+
# Group has no user-facing options -> show subcommands instead
|
| 137 |
+
items = [
|
| 138 |
+
(name, sub.get_short_help_str(limit=80))
|
| 139 |
+
for name in cmd.list_commands(e.ctx)
|
| 140 |
+
if (sub := cmd.get_command(e.ctx, name)) is not None and not sub.hidden
|
| 141 |
+
]
|
| 142 |
+
_enrich_usage_error(e, "commands", items)
|
| 143 |
+
else:
|
| 144 |
+
# Leaf command -> show its options using Click's rich formatting
|
| 145 |
+
items = [
|
| 146 |
+
record
|
| 147 |
+
for p in cmd.get_params(e.ctx)
|
| 148 |
+
if isinstance(p, click.Option) and not p.hidden and (record := p.get_help_record(e.ctx))
|
| 149 |
+
]
|
| 150 |
+
_enrich_usage_error(e, "options", items)
|
| 151 |
+
raise
|
| 152 |
+
|
| 153 |
+
def resolve_command(self, ctx: click.Context, args: list[str]) -> tuple:
|
| 154 |
+
cmd_name = args[0] if args and not args[0].startswith("-") else None
|
| 155 |
+
cmd = self.get_command(ctx, cmd_name) if cmd_name else None
|
| 156 |
+
|
| 157 |
+
if cmd is not None:
|
| 158 |
+
self._rewrite_repo_type_prefix(cmd, args)
|
| 159 |
+
|
| 160 |
+
try:
|
| 161 |
+
name, resolved_cmd, sub_args = super().resolve_command(ctx, args)
|
| 162 |
+
except click.UsageError as e:
|
| 163 |
+
# Unknown subcommand -> add fuzzy suggestions and list available commands.
|
| 164 |
+
if cmd is None and cmd_name is not None:
|
| 165 |
+
# Expand aliases ("list | ls" → ["list", "ls"]) for accurate fuzzy matching.
|
| 166 |
+
visible_names = [
|
| 167 |
+
alias
|
| 168 |
+
for key, registered in self.commands.items()
|
| 169 |
+
if not registered.hidden
|
| 170 |
+
for alias in _ALIAS_SPLIT.split(key)
|
| 171 |
+
]
|
| 172 |
+
matches = difflib.get_close_matches(cmd_name, visible_names)
|
| 173 |
+
if matches:
|
| 174 |
+
suggestions = ", ".join(f"'{m}'" for m in matches)
|
| 175 |
+
e.message = f"{e.message.rstrip('.')}. Did you mean {suggestions}?"
|
| 176 |
+
items = [
|
| 177 |
+
(name, sub.get_short_help_str(limit=80))
|
| 178 |
+
for name in self.list_commands(ctx)
|
| 179 |
+
if (sub := self.get_command(ctx, name)) is not None and not sub.hidden
|
| 180 |
+
]
|
| 181 |
+
_enrich_usage_error(e, "commands", items)
|
| 182 |
+
raise
|
| 183 |
+
|
| 184 |
+
# If we just resolved a leaf command, eagerly consume any global formatting
|
| 185 |
+
# flags (--format / --json / -q / --quiet / --no-truncate) from its args before click parses
|
| 186 |
+
# them. Group resolution is recursive — leaves (and only leaves) need this.
|
| 187 |
+
if resolved_cmd is not None and not isinstance(resolved_cmd, click.Group):
|
| 188 |
+
_consume_format_flags_for_leaf(resolved_cmd, sub_args)
|
| 189 |
+
|
| 190 |
+
return name, resolved_cmd, sub_args
|
| 191 |
+
|
| 192 |
+
@staticmethod
|
| 193 |
+
def _rewrite_repo_type_prefix(cmd: click.Command, args: list[str]) -> None:
|
| 194 |
+
"""Rewrite prefixed repo IDs (e.g. ``spaces/user/repo``) to ``user/repo --type space``.
|
| 195 |
+
|
| 196 |
+
Only applies to commands that have a ``--type`` / ``--repo-type`` option and
|
| 197 |
+
at least one repo-ID positional argument (any ``click.Argument`` whose name
|
| 198 |
+
ends with ``_id``, e.g. ``repo_id``, ``from_id``, ``to_id``). When the
|
| 199 |
+
token that maps to such an argument matches ``{prefix}/org/repo`` (where
|
| 200 |
+
*prefix* is one of ``spaces``, ``datasets``, or ``models``), the prefix is
|
| 201 |
+
stripped and an implicit ``--type {type}`` is appended. An error is raised
|
| 202 |
+
if ``--type`` is also provided explicitly or if multiple prefixed arguments
|
| 203 |
+
disagree on the repo type.
|
| 204 |
+
|
| 205 |
+
Only repo-ID positional slots are inspected so that other positional
|
| 206 |
+
arguments (filenames, local paths, patterns …) are never misinterpreted as
|
| 207 |
+
prefixed repo IDs.
|
| 208 |
+
"""
|
| 209 |
+
has_type_option = any(isinstance(param, click.Option) and "--type" in param.opts for param in cmd.params)
|
| 210 |
+
if not has_type_option:
|
| 211 |
+
return
|
| 212 |
+
|
| 213 |
+
# Locate all repo-ID positional arguments and their indices among Arguments.
|
| 214 |
+
repo_id_positions: set[int] = set()
|
| 215 |
+
arg_idx = 0
|
| 216 |
+
for param in cmd.params:
|
| 217 |
+
if isinstance(param, click.Argument):
|
| 218 |
+
if param.name in ("repo_id", "from_id", "to_id"):
|
| 219 |
+
repo_id_positions.add(arg_idx)
|
| 220 |
+
arg_idx += 1
|
| 221 |
+
|
| 222 |
+
if not repo_id_positions:
|
| 223 |
+
return
|
| 224 |
+
|
| 225 |
+
# Build a set of option names that consume a following value token.
|
| 226 |
+
value_options: set[str] = set()
|
| 227 |
+
for param in cmd.params:
|
| 228 |
+
if isinstance(param, click.Option) and not param.is_flag:
|
| 229 |
+
for opt in (*param.opts, *param.secondary_opts):
|
| 230 |
+
value_options.add(opt)
|
| 231 |
+
|
| 232 |
+
# Walk through args (skipping args[0] = command name) to map positional
|
| 233 |
+
# slots to their indices in `args`.
|
| 234 |
+
positional_count = 0
|
| 235 |
+
repo_id_arg_indices: list[int] = []
|
| 236 |
+
i = 1
|
| 237 |
+
while i < len(args):
|
| 238 |
+
arg = args[i]
|
| 239 |
+
if arg == "--":
|
| 240 |
+
break # everything after -- is positional literal; stop rewriting
|
| 241 |
+
if arg.startswith("-"):
|
| 242 |
+
if "=" in arg or arg not in value_options:
|
| 243 |
+
i += 1 # flag or --opt=val — single token
|
| 244 |
+
else:
|
| 245 |
+
i += 2 # value-taking option — skip the value too
|
| 246 |
+
else:
|
| 247 |
+
if positional_count in repo_id_positions:
|
| 248 |
+
repo_id_arg_indices.append(i)
|
| 249 |
+
positional_count += 1
|
| 250 |
+
i += 1
|
| 251 |
+
|
| 252 |
+
if not repo_id_arg_indices:
|
| 253 |
+
return
|
| 254 |
+
|
| 255 |
+
# Check each repo-ID arg for a type prefix and collect rewrites.
|
| 256 |
+
inferred_type: str | None = None
|
| 257 |
+
first_prefix: str | None = None
|
| 258 |
+
rewrites: list[tuple[int, str]] = [] # (args index, new value without prefix)
|
| 259 |
+
|
| 260 |
+
for arg_index in repo_id_arg_indices:
|
| 261 |
+
parts = args[arg_index].split("/", 2)
|
| 262 |
+
if len(parts) != 3 or parts[0] not in constants.REPO_TYPES_MAPPING:
|
| 263 |
+
continue
|
| 264 |
+
prefix = parts[0]
|
| 265 |
+
mapped_type = constants.REPO_TYPES_MAPPING[prefix]
|
| 266 |
+
if inferred_type is not None and mapped_type != inferred_type:
|
| 267 |
+
raise click.UsageError(f"Conflicting repo type prefixes: '{first_prefix}/' and '{prefix}/'.")
|
| 268 |
+
inferred_type = mapped_type
|
| 269 |
+
first_prefix = prefix
|
| 270 |
+
rewrites.append((arg_index, f"{parts[1]}/{parts[2]}"))
|
| 271 |
+
|
| 272 |
+
if not rewrites:
|
| 273 |
+
return
|
| 274 |
+
|
| 275 |
+
# Error if --type / --repo-type was also provided explicitly.
|
| 276 |
+
if any(
|
| 277 |
+
arg == "--type" or arg.startswith("--type=") or arg == "--repo-type" or arg.startswith("--repo-type=")
|
| 278 |
+
for arg in args
|
| 279 |
+
):
|
| 280 |
+
raise click.UsageError(
|
| 281 |
+
f"Ambiguous repo type: got prefix '{first_prefix}/' in repo ID and explicit --type. Use one or the other."
|
| 282 |
+
)
|
| 283 |
+
|
| 284 |
+
# Apply all rewrites and append --type once.
|
| 285 |
+
for arg_index, new_value in rewrites:
|
| 286 |
+
args[arg_index] = new_value
|
| 287 |
+
args.extend(["--type", inferred_type]) # type: ignore
|
| 288 |
+
|
| 289 |
+
def get_command(self, ctx: click.Context, cmd_name: str) -> click.Command | None:
|
| 290 |
+
# Try exact match first
|
| 291 |
+
cmd = super().get_command(ctx, cmd_name)
|
| 292 |
+
if cmd is not None:
|
| 293 |
+
return cmd
|
| 294 |
+
# Fall back to alias lookup: check if cmd_name matches any alias
|
| 295 |
+
# taken from https://github.com/fastapi/typer/issues/132#issuecomment-2417492805
|
| 296 |
+
for registered_name, registered_cmd in self.commands.items():
|
| 297 |
+
aliases = _ALIAS_SPLIT.split(registered_name)
|
| 298 |
+
if cmd_name in aliases:
|
| 299 |
+
return registered_cmd
|
| 300 |
+
return None
|
| 301 |
+
|
| 302 |
+
def _alias_map(self) -> dict[str, list[str]]:
|
| 303 |
+
"""Build a mapping from primary command name to its aliases (if any)."""
|
| 304 |
+
result: dict[str, list[str]] = {}
|
| 305 |
+
for registered_name in self.commands:
|
| 306 |
+
parts = _ALIAS_SPLIT.split(registered_name)
|
| 307 |
+
primary = parts[0]
|
| 308 |
+
result[primary] = parts[1:]
|
| 309 |
+
return result
|
| 310 |
+
|
| 311 |
+
def format_commands(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
|
| 312 |
+
topics: dict[str, list] = {}
|
| 313 |
+
alias_map = self._alias_map()
|
| 314 |
+
|
| 315 |
+
for name in self.list_commands(ctx):
|
| 316 |
+
cmd = self.get_command(ctx, name)
|
| 317 |
+
if cmd is None or cmd.hidden:
|
| 318 |
+
continue
|
| 319 |
+
help_text = cmd.get_short_help_str(limit=formatter.width)
|
| 320 |
+
aliases = alias_map.get(name, [])
|
| 321 |
+
if aliases:
|
| 322 |
+
help_text = f"{help_text} [alias: {', '.join(aliases)}]"
|
| 323 |
+
topic = getattr(cmd, "topic", "main")
|
| 324 |
+
topics.setdefault(topic, []).append((name, help_text))
|
| 325 |
+
|
| 326 |
+
with formatter.section("Main commands"):
|
| 327 |
+
formatter.write_dl(topics["main"])
|
| 328 |
+
for topic in sorted(topics.keys()):
|
| 329 |
+
if topic == "main":
|
| 330 |
+
continue
|
| 331 |
+
with formatter.section(f"{topic.capitalize()} commands"):
|
| 332 |
+
formatter.write_dl(topics[topic])
|
| 333 |
+
|
| 334 |
+
def format_epilog(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
|
| 335 |
+
# Collect only the first example from each command (to keep group help concise)
|
| 336 |
+
# Full examples are shown in individual subcommand help (e.g. `hf buckets sync --help`)
|
| 337 |
+
all_examples: list[str] = []
|
| 338 |
+
for name in self.list_commands(ctx):
|
| 339 |
+
cmd = self.get_command(ctx, name)
|
| 340 |
+
if cmd is None or cmd.hidden:
|
| 341 |
+
continue
|
| 342 |
+
cmd_examples = getattr(cmd, "examples", [])
|
| 343 |
+
if cmd_examples:
|
| 344 |
+
all_examples.append(cmd_examples[0])
|
| 345 |
+
|
| 346 |
+
if all_examples:
|
| 347 |
+
epilog = generate_epilog(all_examples)
|
| 348 |
+
_format_epilog_no_indent(epilog, ctx, formatter)
|
| 349 |
+
elif self.epilog:
|
| 350 |
+
_format_epilog_no_indent(self.epilog, ctx, formatter)
|
| 351 |
+
|
| 352 |
+
def list_commands(self, ctx: click.Context) -> list[str]: # type: ignore[name-defined]
|
| 353 |
+
# For aliased commands ("list | ls"), use the primary name (first entry).
|
| 354 |
+
primary_names: list[str] = []
|
| 355 |
+
for name in self.commands:
|
| 356 |
+
primary = _ALIAS_SPLIT.split(name)[0]
|
| 357 |
+
primary_names.append(primary)
|
| 358 |
+
return sorted(primary_names)
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
_FORMATTING_OPTIONS_HELP_RECORDS: list[tuple[str, str]] = [
|
| 362 |
+
(
|
| 363 |
+
"--format [auto|human|agent|json|quiet]",
|
| 364 |
+
"Output format. Defaults to 'auto' which picks 'agent' or 'human' based on the terminal.",
|
| 365 |
+
),
|
| 366 |
+
("--json", "JSON output. Equivalent to '--format json'."),
|
| 367 |
+
("-q, --quiet", "Quiet output (one ID per line). Equivalent to '--format quiet'."),
|
| 368 |
+
("--no-truncate", "Do not truncate scalar values in human tables (list/dict columns stay shortened)."),
|
| 369 |
+
]
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
def _format_formatting_options_section(formatter: click.HelpFormatter) -> None:
|
| 373 |
+
with formatter.section("Formatting options"):
|
| 374 |
+
formatter.write_dl(_FORMATTING_OPTIONS_HELP_RECORDS)
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def _has_local_formatting_option(cmd: click.Command) -> bool:
|
| 378 |
+
"""Return True if the command defines its own --format, --json or --quiet / -q.
|
| 379 |
+
|
| 380 |
+
Used to skip the global formatting flag pre-processor and the duplicated "Formatting options" help section for
|
| 381 |
+
legacy commands like 'hf jobs ps' that have their own format/quiet options.
|
| 382 |
+
"""
|
| 383 |
+
for param in cmd.params:
|
| 384 |
+
if not isinstance(param, click.Option):
|
| 385 |
+
continue
|
| 386 |
+
opts = (*param.opts, *param.secondary_opts)
|
| 387 |
+
if "--format" in opts or "--json" in opts or "--quiet" in opts or "-q" in opts:
|
| 388 |
+
return True
|
| 389 |
+
return False
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
def _consume_format_flags_for_leaf(cmd: click.Command, args: list[str]) -> None:
|
| 393 |
+
"""Apply global formatting flags from 'args' to a leaf command.
|
| 394 |
+
|
| 395 |
+
Two modes, depending on the command:
|
| 396 |
+
|
| 397 |
+
* **Pass-through commands** (ignore_unknown_options=True, e.g. 'hf extensions exec'):
|
| 398 |
+
args are forwarded verbatim to an external binary; we don't touch them.
|
| 399 |
+
|
| 400 |
+
* **Legacy commands with a local --format option** (e.g. 'hf jobs ps' whose '--format' accepts Go templates):
|
| 401 |
+
the global flags are rewritten in-place to the legacy form ('--json' → '--format json', '--quiet'/'-q' → '--format quiet'
|
| 402 |
+
when the cmd has no own '--quiet') so click can parse them locally. This preserves backwards compatibility with the previous shorthand behavior.
|
| 403 |
+
|
| 404 |
+
* **Modern commands** (no local format/quiet/json options): the flags '--format <value>' / '--json' / '--quiet' / '-q' are stripped from 'args' and applied to the singleton 'out'.
|
| 405 |
+
|
| 406 |
+
'--no-truncate' is stripped for all non-pass-through commands; when present, human table cells are not truncated.
|
| 407 |
+
|
| 408 |
+
Raises click.UsageError if multiple conflicting flags are supplied (e.g. '--json' together with '--format table').
|
| 409 |
+
"""
|
| 410 |
+
if cmd.context_settings.get("ignore_unknown_options"):
|
| 411 |
+
return
|
| 412 |
+
|
| 413 |
+
no_truncate = _consume_no_truncate_flags(args)
|
| 414 |
+
out.set_no_truncate(no_truncate)
|
| 415 |
+
|
| 416 |
+
has_local_format = False
|
| 417 |
+
has_local_quiet = False
|
| 418 |
+
has_local_json = False
|
| 419 |
+
for param in cmd.params:
|
| 420 |
+
if not isinstance(param, click.Option):
|
| 421 |
+
continue
|
| 422 |
+
opts = (*param.opts, *param.secondary_opts)
|
| 423 |
+
if "--format" in opts:
|
| 424 |
+
has_local_format = True
|
| 425 |
+
if "--quiet" in opts or "-q" in opts:
|
| 426 |
+
has_local_quiet = True
|
| 427 |
+
if "--json" in opts:
|
| 428 |
+
has_local_json = True
|
| 429 |
+
|
| 430 |
+
if has_local_format:
|
| 431 |
+
_rewrite_legacy_shorthands(args, rewrite_json=not has_local_json, rewrite_quiet=not has_local_quiet)
|
| 432 |
+
return
|
| 433 |
+
|
| 434 |
+
# Strip --format/--json/-q/--quiet from 'args' and apply to 'out'
|
| 435 |
+
chosen_mode: OutputFormat = OutputFormat.auto
|
| 436 |
+
chosen_flag: str | None = None
|
| 437 |
+
|
| 438 |
+
def _check_conflict(new_flag: str) -> None:
|
| 439 |
+
# Reject any second formatting flag before parsing values, so the user gets
|
| 440 |
+
# a "mutually exclusive" error rather than e.g. an "invalid value" error
|
| 441 |
+
# from the second flag's argument.
|
| 442 |
+
if chosen_flag is not None:
|
| 443 |
+
raise click.UsageError(f"'{chosen_flag}' and '{new_flag}' are mutually exclusive.")
|
| 444 |
+
|
| 445 |
+
i = 0
|
| 446 |
+
while i < len(args):
|
| 447 |
+
arg = args[i]
|
| 448 |
+
if arg == "--":
|
| 449 |
+
break # everything after '--' is a positional literal
|
| 450 |
+
if arg == "--format":
|
| 451 |
+
_check_conflict("--format")
|
| 452 |
+
if i + 1 >= len(args):
|
| 453 |
+
raise click.UsageError("Option '--format' requires a value.")
|
| 454 |
+
chosen_mode = _parse_format_value(args[i + 1])
|
| 455 |
+
chosen_flag = "--format"
|
| 456 |
+
del args[i : i + 2] # --format value => 2 args removed
|
| 457 |
+
continue
|
| 458 |
+
if arg.startswith("--format="):
|
| 459 |
+
_check_conflict("--format")
|
| 460 |
+
chosen_mode = _parse_format_value(arg[len("--format=") :])
|
| 461 |
+
chosen_flag = "--format"
|
| 462 |
+
del args[i : i + 1]
|
| 463 |
+
continue
|
| 464 |
+
if arg == "--json":
|
| 465 |
+
_check_conflict("--json")
|
| 466 |
+
chosen_mode = OutputFormat.json
|
| 467 |
+
chosen_flag = "--json"
|
| 468 |
+
del args[i : i + 1]
|
| 469 |
+
continue
|
| 470 |
+
if arg in ("-q", "--quiet"):
|
| 471 |
+
_check_conflict(arg)
|
| 472 |
+
chosen_mode = OutputFormat.quiet
|
| 473 |
+
chosen_flag = arg
|
| 474 |
+
del args[i : i + 1]
|
| 475 |
+
continue
|
| 476 |
+
i += 1
|
| 477 |
+
|
| 478 |
+
out.set_mode(chosen_mode)
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
def _consume_no_truncate_flags(args: list[str]) -> bool:
|
| 482 |
+
"""Strip all global --no-truncate flags from args and return whether any was provided."""
|
| 483 |
+
no_truncate = False
|
| 484 |
+
i = 0
|
| 485 |
+
while i < len(args):
|
| 486 |
+
arg = args[i]
|
| 487 |
+
if arg == "--":
|
| 488 |
+
break # everything after '--' is a positional literal
|
| 489 |
+
if arg == "--no-truncate":
|
| 490 |
+
no_truncate = True
|
| 491 |
+
del args[i : i + 1]
|
| 492 |
+
continue
|
| 493 |
+
if arg.startswith("--no-truncate="):
|
| 494 |
+
raise click.UsageError("Option '--no-truncate' does not take a value.")
|
| 495 |
+
i += 1
|
| 496 |
+
return no_truncate
|
| 497 |
+
|
| 498 |
+
|
| 499 |
+
def _rewrite_legacy_shorthands(args: list[str], *, rewrite_json: bool, rewrite_quiet: bool) -> None:
|
| 500 |
+
"""Rewrite --json / -q / --quiet to --format ... for legacy commands.
|
| 501 |
+
|
| 502 |
+
Used for commands like 'hf jobs ps' that still own their '--format' option.
|
| 503 |
+
The rewrite lets users keep using the global shorthand while click parses
|
| 504 |
+
'--format <value>' locally.
|
| 505 |
+
"""
|
| 506 |
+
has_format_in_args = any(arg == "--format" or arg.startswith("--format=") for arg in args)
|
| 507 |
+
|
| 508 |
+
if rewrite_json and "--json" in args:
|
| 509 |
+
if has_format_in_args:
|
| 510 |
+
raise click.UsageError("'--json' and '--format' are mutually exclusive.")
|
| 511 |
+
idx = args.index("--json")
|
| 512 |
+
args[idx : idx + 1] = ["--format", "json"]
|
| 513 |
+
has_format_in_args = True
|
| 514 |
+
|
| 515 |
+
if rewrite_quiet:
|
| 516 |
+
flag = "-q" if "-q" in args else ("--quiet" if "--quiet" in args else None)
|
| 517 |
+
if flag is not None:
|
| 518 |
+
if has_format_in_args:
|
| 519 |
+
raise click.UsageError(f"'{flag}' and '--format' are mutually exclusive.")
|
| 520 |
+
idx = args.index(flag)
|
| 521 |
+
args[idx : idx + 1] = ["--format", "quiet"]
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def _parse_format_value(value: str) -> "OutputFormat":
|
| 525 |
+
try:
|
| 526 |
+
return OutputFormat(value)
|
| 527 |
+
except ValueError:
|
| 528 |
+
valid = ", ".join(m.value for m in OutputFormat)
|
| 529 |
+
raise click.UsageError(f"Invalid value for '--format': '{value}'. Valid values: {valid}.") from None
|
| 530 |
+
|
| 531 |
+
|
| 532 |
+
def _enrich_usage_error(error: click.UsageError, label: str, items: list[tuple[str, str]]) -> None:
|
| 533 |
+
"""Append a list of available options or commands to a usage error message."""
|
| 534 |
+
if not items or error.ctx is None or f"Available {label} for" in error.message:
|
| 535 |
+
return
|
| 536 |
+
cmd_path = error.ctx.command_path
|
| 537 |
+
lines = [f"\n\nAvailable {label} for '{cmd_path}':"]
|
| 538 |
+
for name, help_text in items:
|
| 539 |
+
lines.append(f" {name:30s} {help_text}")
|
| 540 |
+
lines.append(f"\nRun '{cmd_path} --help' for full details.")
|
| 541 |
+
if isinstance(error, click.NoSuchOption) and error.possibilities:
|
| 542 |
+
lines.append(f"\nDid you mean: {', '.join(sorted(error.possibilities))}?")
|
| 543 |
+
error.possibilities = []
|
| 544 |
+
error.message += "\n".join(lines)
|
| 545 |
+
|
| 546 |
+
|
| 547 |
+
def fallback_typer_group_factory(
|
| 548 |
+
fallback_handler: FallbackHandlerT,
|
| 549 |
+
extra_commands_provider: Callable[[], list[tuple[str, str]]] | None = None,
|
| 550 |
+
) -> type[HFCliTyperGroup]:
|
| 551 |
+
"""Return a Typer group class that runs a fallback handler before command resolution."""
|
| 552 |
+
|
| 553 |
+
class FallbackTyperGroup(HFCliTyperGroup):
|
| 554 |
+
def resolve_command(self, ctx: click.Context, args: list[str]) -> tuple:
|
| 555 |
+
fallback_exit_code = fallback_handler(args, set(self.commands.keys()))
|
| 556 |
+
if fallback_exit_code is not None:
|
| 557 |
+
raise SystemExit(fallback_exit_code)
|
| 558 |
+
return super().resolve_command(ctx, args)
|
| 559 |
+
|
| 560 |
+
def format_commands(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
|
| 561 |
+
super().format_commands(ctx, formatter)
|
| 562 |
+
if extra_commands_provider is not None:
|
| 563 |
+
entries = extra_commands_provider()
|
| 564 |
+
if entries:
|
| 565 |
+
with formatter.section("Extension commands"):
|
| 566 |
+
formatter.write_dl(entries)
|
| 567 |
+
|
| 568 |
+
return FallbackTyperGroup
|
| 569 |
+
|
| 570 |
+
|
| 571 |
+
def HFCliCommand(topic: TOPIC_T, examples: list[str] | None = None) -> type[TyperCommand]:
|
| 572 |
+
def format_epilog(self: click.Command, ctx: click.Context, formatter: click.HelpFormatter) -> None:
|
| 573 |
+
_format_epilog_no_indent(self.epilog, ctx, formatter)
|
| 574 |
+
|
| 575 |
+
def format_options(self: TyperCommand, ctx: click.Context, formatter: click.HelpFormatter) -> None:
|
| 576 |
+
TyperCommand.format_options(self, ctx, formatter)
|
| 577 |
+
# Skip the section for commands that define their own --format / --quiet / --json,
|
| 578 |
+
# or for pass-through commands that forward args to an external binary.
|
| 579 |
+
if _has_local_formatting_option(self):
|
| 580 |
+
return
|
| 581 |
+
if self.context_settings.get("ignore_unknown_options"):
|
| 582 |
+
return
|
| 583 |
+
_format_formatting_options_section(formatter)
|
| 584 |
+
|
| 585 |
+
def parse_args(self: click.Command, ctx: click.Context, args: list[str]) -> list[str]:
|
| 586 |
+
# Show help when a command with required arguments is invoked without any args
|
| 587 |
+
# (mirrors group behavior: `hf jobs` prints help, so `hf download` should too).
|
| 588 |
+
if not args and not ctx.resilient_parsing:
|
| 589 |
+
if any(isinstance(p, click.Argument) and p.required for p in self.params):
|
| 590 |
+
click.echo(ctx.get_help(), color=ctx.color)
|
| 591 |
+
ctx.exit()
|
| 592 |
+
return TyperCommand.parse_args(self, ctx, args)
|
| 593 |
+
|
| 594 |
+
return type(
|
| 595 |
+
f"TyperCommand{topic.capitalize()}",
|
| 596 |
+
(TyperCommand,),
|
| 597 |
+
{
|
| 598 |
+
"context_class": StyledContext,
|
| 599 |
+
"topic": topic,
|
| 600 |
+
"examples": examples or [],
|
| 601 |
+
"format_epilog": format_epilog,
|
| 602 |
+
"format_options": format_options,
|
| 603 |
+
"parse_args": parse_args,
|
| 604 |
+
},
|
| 605 |
+
)
|
| 606 |
+
|
| 607 |
+
|
| 608 |
+
class HFCliApp(typer.Typer):
|
| 609 |
+
"""Custom Typer app for Hugging Face CLI."""
|
| 610 |
+
|
| 611 |
+
def command( # type: ignore
|
| 612 |
+
self,
|
| 613 |
+
name: str | None = None,
|
| 614 |
+
*,
|
| 615 |
+
topic: TOPIC_T = "main",
|
| 616 |
+
examples: list[str] | None = None,
|
| 617 |
+
context_settings: dict[str, Any] | None = None,
|
| 618 |
+
help: str | None = None,
|
| 619 |
+
epilog: str | None = None,
|
| 620 |
+
short_help: str | None = None,
|
| 621 |
+
options_metavar: str = "[OPTIONS]",
|
| 622 |
+
add_help_option: bool = True,
|
| 623 |
+
no_args_is_help: bool = False,
|
| 624 |
+
hidden: bool = False,
|
| 625 |
+
deprecated: bool = False,
|
| 626 |
+
rich_help_panel: str | None = None,
|
| 627 |
+
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
|
| 628 |
+
# Generate epilog from examples if not explicitly provided
|
| 629 |
+
if epilog is None and examples:
|
| 630 |
+
epilog = generate_epilog(examples)
|
| 631 |
+
|
| 632 |
+
def _inner(func: Callable[..., Any]) -> Callable[..., Any]:
|
| 633 |
+
return super(HFCliApp, self).command(
|
| 634 |
+
name,
|
| 635 |
+
cls=HFCliCommand(topic, examples),
|
| 636 |
+
context_settings=context_settings,
|
| 637 |
+
help=help,
|
| 638 |
+
epilog=epilog,
|
| 639 |
+
short_help=short_help,
|
| 640 |
+
options_metavar=options_metavar,
|
| 641 |
+
add_help_option=add_help_option,
|
| 642 |
+
no_args_is_help=no_args_is_help,
|
| 643 |
+
hidden=hidden,
|
| 644 |
+
deprecated=deprecated,
|
| 645 |
+
rich_help_panel=rich_help_panel,
|
| 646 |
+
)(func)
|
| 647 |
+
|
| 648 |
+
return _inner
|
| 649 |
+
|
| 650 |
+
|
| 651 |
+
def typer_factory(help: str, epilog: str | None = None, cls: type[TyperGroup] | None = None) -> "HFCliApp":
|
| 652 |
+
"""Create a Typer app with consistent settings.
|
| 653 |
+
|
| 654 |
+
Args:
|
| 655 |
+
help: Help text for the app.
|
| 656 |
+
epilog: Optional epilog text (use `generate_epilog` to create one).
|
| 657 |
+
cls: Optional Click group class to use (defaults to `HFCliTyperGroup`).
|
| 658 |
+
|
| 659 |
+
Returns:
|
| 660 |
+
A configured Typer app.
|
| 661 |
+
"""
|
| 662 |
+
if cls is None:
|
| 663 |
+
cls = HFCliTyperGroup
|
| 664 |
+
return HFCliApp(
|
| 665 |
+
help=help,
|
| 666 |
+
epilog=epilog,
|
| 667 |
+
add_completion=True,
|
| 668 |
+
no_args_is_help=True,
|
| 669 |
+
cls=cls,
|
| 670 |
+
# Disable rich completely for consistent experience
|
| 671 |
+
rich_markup_mode=None,
|
| 672 |
+
rich_help_panel=None,
|
| 673 |
+
pretty_exceptions_enable=False,
|
| 674 |
+
# Disable TyperGroup's suggest_commands, it matches against raw aliased
|
| 675 |
+
# keys ("list | ls") leaking pipe syntax into user-facing messages.
|
| 676 |
+
# HFCliTyperGroup.resolve_command() handles suggestions with expanded names.
|
| 677 |
+
suggest_commands=False,
|
| 678 |
+
# Increase max content width for better readability
|
| 679 |
+
context_settings={
|
| 680 |
+
"max_content_width": 120,
|
| 681 |
+
"help_option_names": ["-h", "--help"],
|
| 682 |
+
},
|
| 683 |
+
)
|
| 684 |
+
|
| 685 |
+
|
| 686 |
+
class SoftChoice(click.Choice):
|
| 687 |
+
"""A click Choice that suggests choices for autocompletion/docs but accepts any string.
|
| 688 |
+
|
| 689 |
+
Unlike `click.Choice`, unknown values are passed through as-is instead of raising an error.
|
| 690 |
+
This makes CLI options future-compatible when new server-side values are added.
|
| 691 |
+
|
| 692 |
+
Accepts either a sequence of strings or an Enum class:
|
| 693 |
+
```python
|
| 694 |
+
SoftChoice(SpaceHardware) # from an enum
|
| 695 |
+
SoftChoice(["a", "b", "c"]) # from a list
|
| 696 |
+
```
|
| 697 |
+
"""
|
| 698 |
+
|
| 699 |
+
def __init__(self, choices: Sequence[str] | type[Enum]) -> None:
|
| 700 |
+
values = (
|
| 701 |
+
[m.value for m in choices] if isinstance(choices, type) and issubclass(choices, Enum) else list(choices)
|
| 702 |
+
)
|
| 703 |
+
super().__init__(values, case_sensitive=True)
|
| 704 |
+
|
| 705 |
+
def convert(self, value: Any, param: click.Parameter | None, ctx: click.Context | None) -> str:
|
| 706 |
+
try:
|
| 707 |
+
return super().convert(value, param, ctx)
|
| 708 |
+
except click.exceptions.BadParameter:
|
| 709 |
+
return str(value)
|
| 710 |
+
|
| 711 |
+
|
| 712 |
+
class RepoType(str, Enum):
|
| 713 |
+
model = "model"
|
| 714 |
+
dataset = "dataset"
|
| 715 |
+
space = "space"
|
| 716 |
+
|
| 717 |
+
|
| 718 |
+
RepoIdArg = Annotated[
|
| 719 |
+
str,
|
| 720 |
+
typer.Argument(
|
| 721 |
+
help="The ID of the repo (e.g. `username/repo-name` or `spaces/username/repo-name`).",
|
| 722 |
+
),
|
| 723 |
+
]
|
| 724 |
+
|
| 725 |
+
|
| 726 |
+
RepoTypeOpt = Annotated[
|
| 727 |
+
RepoType,
|
| 728 |
+
typer.Option(
|
| 729 |
+
"--type",
|
| 730 |
+
"--repo-type",
|
| 731 |
+
help="The type of repository (model, dataset, or space).",
|
| 732 |
+
),
|
| 733 |
+
]
|
| 734 |
+
|
| 735 |
+
# Same as `RepoTypeOpt` but optional (defaults to `None` rather than `model`). Used by commands that
|
| 736 |
+
# accept an `hf://` URI as repo id: a `None` default lets us tell apart "user did not pass --repo-type"
|
| 737 |
+
# from "user explicitly passed --repo-type model", which is required to detect conflicts with the URI.
|
| 738 |
+
RepoTypeOptionalOpt = Annotated[
|
| 739 |
+
RepoType | None,
|
| 740 |
+
typer.Option(
|
| 741 |
+
"--type",
|
| 742 |
+
"--repo-type",
|
| 743 |
+
help="The type of repository (model, dataset, or space).",
|
| 744 |
+
show_default="model",
|
| 745 |
+
),
|
| 746 |
+
]
|
| 747 |
+
|
| 748 |
+
TokenOpt = Annotated[
|
| 749 |
+
str | None,
|
| 750 |
+
typer.Option(
|
| 751 |
+
help="A User Access Token generated from https://huggingface.co/settings/tokens.",
|
| 752 |
+
),
|
| 753 |
+
]
|
| 754 |
+
|
| 755 |
+
PrivateOpt = Annotated[
|
| 756 |
+
bool | None,
|
| 757 |
+
typer.Option(
|
| 758 |
+
help="Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists.",
|
| 759 |
+
),
|
| 760 |
+
]
|
| 761 |
+
|
| 762 |
+
RevisionOpt = Annotated[
|
| 763 |
+
str | None,
|
| 764 |
+
typer.Option(
|
| 765 |
+
help="Git revision id which can be a branch name, a tag, or a commit hash.",
|
| 766 |
+
),
|
| 767 |
+
]
|
| 768 |
+
|
| 769 |
+
|
| 770 |
+
LimitOpt = Annotated[
|
| 771 |
+
int,
|
| 772 |
+
typer.Option(help="Limit the number of results."),
|
| 773 |
+
]
|
| 774 |
+
|
| 775 |
+
AuthorOpt = Annotated[
|
| 776 |
+
str | None,
|
| 777 |
+
typer.Option(help="Filter by author or organization."),
|
| 778 |
+
]
|
| 779 |
+
|
| 780 |
+
FilterOpt = Annotated[
|
| 781 |
+
list[str] | None,
|
| 782 |
+
typer.Option(help="Filter by tags (e.g. 'text-classification'). Can be used multiple times."),
|
| 783 |
+
]
|
| 784 |
+
|
| 785 |
+
SearchOpt = Annotated[
|
| 786 |
+
str | None,
|
| 787 |
+
typer.Option(help="Search query."),
|
| 788 |
+
]
|
| 789 |
+
|
| 790 |
+
|
| 791 |
+
# --- Env / Secrets shared options and parsing helpers (used by jobs, repos, etc.) ---
|
| 792 |
+
|
| 793 |
+
EnvOpt = Annotated[
|
| 794 |
+
list[str] | None,
|
| 795 |
+
typer.Option(
|
| 796 |
+
"-e",
|
| 797 |
+
"--env",
|
| 798 |
+
help="Set environment variables. E.g. --env ENV=value",
|
| 799 |
+
),
|
| 800 |
+
]
|
| 801 |
+
|
| 802 |
+
SecretsOpt = Annotated[
|
| 803 |
+
list[str] | None,
|
| 804 |
+
typer.Option(
|
| 805 |
+
"-s",
|
| 806 |
+
"--secrets",
|
| 807 |
+
help=(
|
| 808 |
+
"Set secret environment variables. E.g. --secrets SECRET=value"
|
| 809 |
+
" or `--secrets HF_TOKEN` to pass your Hugging Face token."
|
| 810 |
+
),
|
| 811 |
+
),
|
| 812 |
+
]
|
| 813 |
+
|
| 814 |
+
EnvFileOpt = Annotated[
|
| 815 |
+
str | None,
|
| 816 |
+
typer.Option(
|
| 817 |
+
"--env-file",
|
| 818 |
+
help="Read in a file of environment variables.",
|
| 819 |
+
),
|
| 820 |
+
]
|
| 821 |
+
|
| 822 |
+
SecretsFileOpt = Annotated[
|
| 823 |
+
str | None,
|
| 824 |
+
typer.Option(
|
| 825 |
+
help="Read in a file of secret environment variables.",
|
| 826 |
+
),
|
| 827 |
+
]
|
| 828 |
+
|
| 829 |
+
|
| 830 |
+
def _get_extended_environ() -> dict[str, str]:
|
| 831 |
+
"""Return a copy of ``os.environ`` with the user's HF token injected (if available)."""
|
| 832 |
+
from huggingface_hub import get_token
|
| 833 |
+
|
| 834 |
+
extended_environ = os.environ.copy()
|
| 835 |
+
if (token := get_token()) is not None:
|
| 836 |
+
extended_environ["HF_TOKEN"] = token
|
| 837 |
+
return extended_environ
|
| 838 |
+
|
| 839 |
+
|
| 840 |
+
def parse_env_map(
|
| 841 |
+
env: list[str] | None = None,
|
| 842 |
+
env_file: str | None = None,
|
| 843 |
+
) -> dict[str, str | None]:
|
| 844 |
+
"""Parse ``-e``/``--env``/``-s``/``--secrets`` and ``--env-file``/``--secrets-file`` CLI args into a dict.
|
| 845 |
+
|
| 846 |
+
Uses an extended environment that includes the user's HF token so that
|
| 847 |
+
bare ``--secrets HF_TOKEN`` resolves correctly.
|
| 848 |
+
"""
|
| 849 |
+
extended_environ = _get_extended_environ()
|
| 850 |
+
env_map: dict[str, str | None] = {}
|
| 851 |
+
if env_file:
|
| 852 |
+
env_map.update(load_dotenv(Path(env_file).read_text(), environ=extended_environ))
|
| 853 |
+
for env_value in env or []:
|
| 854 |
+
env_map.update(load_dotenv(env_value, environ=extended_environ))
|
| 855 |
+
return env_map
|
| 856 |
+
|
| 857 |
+
|
| 858 |
+
def env_map_to_key_value_list(env_map: dict[str, str | None]) -> list[dict[str, str]] | None:
|
| 859 |
+
"""Convert an env/secrets dict to the ``[{"key": ..., "value": ...}]`` format used by the Hub API."""
|
| 860 |
+
if not env_map:
|
| 861 |
+
return None
|
| 862 |
+
return [{"key": k, "value": v or ""} for k, v in env_map.items()]
|
| 863 |
+
|
| 864 |
+
|
| 865 |
+
VolumesOpt = Annotated[
|
| 866 |
+
list[str] | None,
|
| 867 |
+
typer.Option(
|
| 868 |
+
"-v",
|
| 869 |
+
"--volume",
|
| 870 |
+
help="Mount one or more volumes. Format: hf://[TYPE/]SOURCE:/MOUNT_PATH[:ro]. "
|
| 871 |
+
"TYPE is one of: models, datasets, spaces, buckets. "
|
| 872 |
+
"TYPE defaults to models if omitted. "
|
| 873 |
+
"models, datasets and spaces are always mounted read-only. buckets are read+write by default. "
|
| 874 |
+
"E.g. -v hf://org/m:/data or -v hf://datasets/org/ds:/data or -v hf://buckets/org/b:/mnt:ro",
|
| 875 |
+
),
|
| 876 |
+
]
|
| 877 |
+
|
| 878 |
+
|
| 879 |
+
def parse_volumes(volumes: list[str] | None) -> "list[Volume] | None":
|
| 880 |
+
"""Parse volume specs from CLI arguments.
|
| 881 |
+
|
| 882 |
+
Format: hf://[TYPE/]SOURCE[/PATH]:/MOUNT_PATH[:ro|:rw]
|
| 883 |
+
Where TYPE is one of: models, datasets, spaces, buckets (defaults to models if omitted).
|
| 884 |
+
SOURCE is the repo/bucket identifier (e.g. 'username/my-model').
|
| 885 |
+
PATH is an optional subfolder inside the repo/bucket.
|
| 886 |
+
MOUNT_PATH starts with '/'.
|
| 887 |
+
Optional ':ro' or ':rw' suffix for read-only or read-write.
|
| 888 |
+
|
| 889 |
+
Examples:
|
| 890 |
+
hf://my-org/my-model:/data (model, implicit type)
|
| 891 |
+
hf://models/my-org/my-model:/data (model, explicit type)
|
| 892 |
+
hf://datasets/my-org/my-dataset:/data:ro
|
| 893 |
+
hf://buckets/my-org/my-bucket:/mnt
|
| 894 |
+
hf://spaces/my-org/my-space:/app
|
| 895 |
+
hf://datasets/org/ds/train:/data (with path inside repo)
|
| 896 |
+
hf://buckets/org/b/sub/dir:/mnt (with path inside bucket)
|
| 897 |
+
"""
|
| 898 |
+
if not volumes:
|
| 899 |
+
return None
|
| 900 |
+
|
| 901 |
+
result: list[Volume] = []
|
| 902 |
+
for raw_spec in volumes:
|
| 903 |
+
mount = parse_hf_mount(raw_spec)
|
| 904 |
+
result.append(
|
| 905 |
+
Volume(
|
| 906 |
+
type=mount.source.type,
|
| 907 |
+
source=mount.source.id,
|
| 908 |
+
mount_path=mount.mount_path,
|
| 909 |
+
read_only=mount.read_only,
|
| 910 |
+
path=mount.source.path_in_repo or None,
|
| 911 |
+
revision=mount.source.revision or None,
|
| 912 |
+
)
|
| 913 |
+
)
|
| 914 |
+
return result
|
| 915 |
+
|
| 916 |
+
|
| 917 |
+
def make_expand_properties_parser(valid_properties: Sequence[ExpandPropertyT]):
|
| 918 |
+
"""Create a callback to parse and validate comma-separated expand properties."""
|
| 919 |
+
|
| 920 |
+
def _parse_expand_properties(value: str | None) -> list[ExpandPropertyT] | None:
|
| 921 |
+
if value is None:
|
| 922 |
+
return None
|
| 923 |
+
properties = [p.strip() for p in value.split(",")]
|
| 924 |
+
for prop in properties:
|
| 925 |
+
if prop not in valid_properties:
|
| 926 |
+
raise typer.BadParameter(
|
| 927 |
+
f"Invalid expand property: '{prop}'. Valid values are: {', '.join(valid_properties)}"
|
| 928 |
+
)
|
| 929 |
+
return [cast(ExpandPropertyT, prop) for prop in properties]
|
| 930 |
+
|
| 931 |
+
return _parse_expand_properties
|
| 932 |
+
|
| 933 |
+
|
| 934 |
+
### PyPI VERSION CHECKER
|
| 935 |
+
|
| 936 |
+
|
| 937 |
+
def check_cli_update(library: Literal["huggingface_hub", "transformers"]) -> None:
|
| 938 |
+
"""
|
| 939 |
+
Check whether a newer version of a library is available on PyPI.
|
| 940 |
+
|
| 941 |
+
If a newer version is found, print a hint pointing at `hf update`.
|
| 942 |
+
|
| 943 |
+
If current version is a pre-release (e.g. `1.0.0.rc1`), or a dev version (e.g. `1.0.0.dev1`), no check is performed.
|
| 944 |
+
If `HF_HUB_DISABLE_UPDATE_CHECK` is set, the check is skipped entirely.
|
| 945 |
+
|
| 946 |
+
This function is called at the entry point of the CLI. It only performs the check once every 24 hours, and any error
|
| 947 |
+
during the check is caught and logged, to avoid breaking the CLI.
|
| 948 |
+
|
| 949 |
+
Args:
|
| 950 |
+
library: The library to check for updates. Currently supports "huggingface_hub" and "transformers".
|
| 951 |
+
"""
|
| 952 |
+
try:
|
| 953 |
+
_check_cli_update(library)
|
| 954 |
+
except Exception:
|
| 955 |
+
# We don't want the CLI to fail on version checks, no matter the reason.
|
| 956 |
+
logger.debug("Error while checking for CLI update.", exc_info=True)
|
| 957 |
+
|
| 958 |
+
|
| 959 |
+
def _check_cli_update(library: Literal["huggingface_hub", "transformers"]) -> None:
|
| 960 |
+
if constants.HF_HUB_DISABLE_UPDATE_CHECK:
|
| 961 |
+
return
|
| 962 |
+
|
| 963 |
+
current_version = importlib.metadata.version(library)
|
| 964 |
+
|
| 965 |
+
# Skip if current version is a pre-release or dev version
|
| 966 |
+
if any(tag in current_version for tag in ["rc", "dev"]):
|
| 967 |
+
return
|
| 968 |
+
|
| 969 |
+
# Skip if already checked in the last 24 hours
|
| 970 |
+
if os.path.exists(constants.CHECK_FOR_UPDATE_DONE_PATH):
|
| 971 |
+
mtime = os.path.getmtime(constants.CHECK_FOR_UPDATE_DONE_PATH)
|
| 972 |
+
if (time.time() - mtime) < 24 * 3600:
|
| 973 |
+
return
|
| 974 |
+
|
| 975 |
+
# Touch the file to mark that we did the check now
|
| 976 |
+
Path(constants.CHECK_FOR_UPDATE_DONE_PATH).parent.mkdir(parents=True, exist_ok=True)
|
| 977 |
+
Path(constants.CHECK_FOR_UPDATE_DONE_PATH).touch()
|
| 978 |
+
|
| 979 |
+
# Check latest version from the appropriate registry
|
| 980 |
+
if library == "huggingface_hub" and installation_method() == "brew":
|
| 981 |
+
latest_version = _fetch_latest_brew_version()
|
| 982 |
+
else:
|
| 983 |
+
latest_version = _fetch_latest_pypi_version(library)
|
| 984 |
+
if latest_version is None or current_version == latest_version:
|
| 985 |
+
return
|
| 986 |
+
|
| 987 |
+
if library == "huggingface_hub":
|
| 988 |
+
update_command = _get_huggingface_hub_update_command()
|
| 989 |
+
else:
|
| 990 |
+
update_command = _get_transformers_update_command()
|
| 991 |
+
|
| 992 |
+
message = f"A new version of {library} ({latest_version}) is available! You are using version {current_version}."
|
| 993 |
+
if update_command is not None:
|
| 994 |
+
match library:
|
| 995 |
+
case "huggingface_hub":
|
| 996 |
+
message += "\nTo update, run: hf update"
|
| 997 |
+
case _:
|
| 998 |
+
message += f"\nTo update, run: {' '.join(update_command)}"
|
| 999 |
+
out.hint(message)
|
| 1000 |
+
|
| 1001 |
+
|
| 1002 |
+
def _fetch_latest_pypi_version(library: str) -> str | None:
|
| 1003 |
+
"""Fetch the latest version of a library from PyPI. Returns None if the request fails."""
|
| 1004 |
+
try:
|
| 1005 |
+
response = get_session().get(f"https://pypi.org/pypi/{library}/json", timeout=2)
|
| 1006 |
+
hf_raise_for_status(response)
|
| 1007 |
+
return response.json()["info"]["version"]
|
| 1008 |
+
except Exception:
|
| 1009 |
+
logger.debug("Error while fetching latest version from PyPI.", exc_info=True)
|
| 1010 |
+
return None
|
| 1011 |
+
|
| 1012 |
+
|
| 1013 |
+
def _fetch_latest_brew_version() -> str | None:
|
| 1014 |
+
"""Fetch the latest version of the `hf` formula from the Homebrew registry. Returns None if the request fails."""
|
| 1015 |
+
try:
|
| 1016 |
+
response = get_session().get("https://formulae.brew.sh/api/formula/hf.json", timeout=2)
|
| 1017 |
+
hf_raise_for_status(response)
|
| 1018 |
+
return response.json()["versions"]["stable"]
|
| 1019 |
+
except Exception:
|
| 1020 |
+
logger.debug("Error while fetching latest version from Homebrew.", exc_info=True)
|
| 1021 |
+
return None
|
| 1022 |
+
|
| 1023 |
+
|
| 1024 |
+
def run_update() -> int:
|
| 1025 |
+
"""Run the install-method-appropriate update command for the `hf` CLI.
|
| 1026 |
+
|
| 1027 |
+
Raises CLIError if the installation method can't be determined.
|
| 1028 |
+
Returns the subprocess exit code on success/failure of the update itself.
|
| 1029 |
+
"""
|
| 1030 |
+
cmd = _get_huggingface_hub_update_command()
|
| 1031 |
+
if cmd is None:
|
| 1032 |
+
raise CLIError(
|
| 1033 |
+
"Cannot determine how to update huggingface_hub (unknown installation method). Please update manually."
|
| 1034 |
+
)
|
| 1035 |
+
return subprocess.call(cmd)
|
| 1036 |
+
|
| 1037 |
+
|
| 1038 |
+
def _get_huggingface_hub_update_command() -> list[str] | None:
|
| 1039 |
+
"""Return the command to update huggingface_hub as an argv list, or None if the installation method is unknown."""
|
| 1040 |
+
match installation_method():
|
| 1041 |
+
case "brew":
|
| 1042 |
+
return ["brew", "upgrade", "hf"]
|
| 1043 |
+
case "hf_installer" if os.name == "nt":
|
| 1044 |
+
return ["powershell", "-NoProfile", "-Command", "iwr -useb https://hf.co/cli/install.ps1 | iex"]
|
| 1045 |
+
case "hf_installer":
|
| 1046 |
+
return ["bash", "-c", "curl -LsSf https://hf.co/cli/install.sh | bash -"]
|
| 1047 |
+
case "pip":
|
| 1048 |
+
return [sys.executable, "-m", "pip", "install", "-U", "huggingface_hub"]
|
| 1049 |
+
case _:
|
| 1050 |
+
return None
|
| 1051 |
+
|
| 1052 |
+
|
| 1053 |
+
def _get_transformers_update_command() -> list[str] | None:
|
| 1054 |
+
"""Return the command to update transformers as an argv list, or None if the installation method is unknown."""
|
| 1055 |
+
match installation_method():
|
| 1056 |
+
case "hf_installer" if os.name == "nt":
|
| 1057 |
+
return [
|
| 1058 |
+
"powershell",
|
| 1059 |
+
"-NoProfile",
|
| 1060 |
+
"-Command",
|
| 1061 |
+
"iwr -useb https://hf.co/cli/install.ps1 | iex -WithTransformers",
|
| 1062 |
+
]
|
| 1063 |
+
case "hf_installer":
|
| 1064 |
+
return ["bash", "-c", "curl -LsSf https://hf.co/cli/install.sh | bash -s -- --with-transformers"]
|
| 1065 |
+
case "pip":
|
| 1066 |
+
return [sys.executable, "-m", "pip", "install", "-U", "transformers"]
|
| 1067 |
+
case _:
|
| 1068 |
+
return None
|
venv/Lib/site-packages/huggingface_hub/cli/_cp.py
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Shared ``cp`` command to copy files between local paths, repositories and buckets.
|
| 15 |
+
|
| 16 |
+
This single command backs three identical CLI entry points: ``hf cp`` (top-level),
|
| 17 |
+
``hf repos cp`` and ``hf buckets cp``. It supports any source/destination combination
|
| 18 |
+
of local file, repo/bucket ``hf://`` URI, and ``-`` (stdin/stdout), with two exceptions:
|
| 19 |
+
- bucket-to-repo copies are not supported (server limitation), and
|
| 20 |
+
- local-to-local copies (use a regular ``cp`` for that).
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
import os
|
| 24 |
+
import sys
|
| 25 |
+
from dataclasses import replace
|
| 26 |
+
from typing import Annotated, Literal
|
| 27 |
+
|
| 28 |
+
import typer
|
| 29 |
+
|
| 30 |
+
from huggingface_hub import HfApi
|
| 31 |
+
from huggingface_hub.errors import CLIError
|
| 32 |
+
from huggingface_hub.utils import HfUri, SoftTemporaryDirectory, disable_progress_bars, is_hf_uri, parse_hf_uri
|
| 33 |
+
|
| 34 |
+
from ._cli_utils import TokenOpt, get_hf_api
|
| 35 |
+
from ._output import out
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
CP_EXAMPLES = [
|
| 39 |
+
# Download (repo or bucket -> local / stdout)
|
| 40 |
+
"hf cp hf://username/my-model/config.json",
|
| 41 |
+
"hf cp hf://username/my-model/config.json ./config.json",
|
| 42 |
+
"hf cp hf://datasets/username/my-dataset/data.csv ./data/",
|
| 43 |
+
"hf cp hf://buckets/username/my-bucket/config.json -",
|
| 44 |
+
# Upload (local / stdin -> repo or bucket)
|
| 45 |
+
"hf cp ./model.safetensors hf://username/my-model/model.safetensors",
|
| 46 |
+
"hf cp ./config.json hf://buckets/username/my-bucket/logs/",
|
| 47 |
+
"hf cp - hf://buckets/username/my-bucket/config.json",
|
| 48 |
+
# Remote to remote (repo/bucket -> repo/bucket, server-side when possible)
|
| 49 |
+
"hf cp hf://username/source-model/ hf://username/dest-model/",
|
| 50 |
+
"hf cp hf://datasets/username/my-dataset/processed/ hf://buckets/username/my-bucket/processed/",
|
| 51 |
+
"hf cp hf://buckets/username/my-bucket/logs/ hf://buckets/username/archive-bucket/ # copies contents only",
|
| 52 |
+
]
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
# Which alias registered the command, used to restrict the remote endpoint type (see `_enforce_context`).
|
| 56 |
+
CpContext = Literal["repos", "buckets"]
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def make_cp(context: CpContext | None = None):
|
| 60 |
+
"""Build the ``cp`` command function for a given alias.
|
| 61 |
+
|
| 62 |
+
The three entry points (`hf cp`, `hf repos cp`, `hf buckets cp`) share the exact same logic;
|
| 63 |
+
'context' only adds a guardrail on the remote endpoint type (see `_enforce_context`).
|
| 64 |
+
"""
|
| 65 |
+
|
| 66 |
+
def cp(
|
| 67 |
+
src: Annotated[
|
| 68 |
+
str,
|
| 69 |
+
typer.Argument(help="Source: local file, hf:// URI (repo or bucket), or - for stdin."),
|
| 70 |
+
],
|
| 71 |
+
dst: Annotated[
|
| 72 |
+
str | None,
|
| 73 |
+
typer.Argument(help="Destination: local path, hf:// URI (repo or bucket), or - for stdout."),
|
| 74 |
+
] = None,
|
| 75 |
+
token: TokenOpt = None,
|
| 76 |
+
) -> None:
|
| 77 |
+
"""Copy files between local paths, repositories, and buckets.
|
| 78 |
+
|
| 79 |
+
Handles uploads (local/stdin -> repo/bucket), downloads (repo/bucket -> local/stdout) and
|
| 80 |
+
remote-to-remote copies (repo/bucket -> repo/bucket). Bucket-to-repo and local-to-local
|
| 81 |
+
copies are not supported. For directories, use `hf upload`/`hf download` (repos) or
|
| 82 |
+
`hf buckets sync` (buckets). Remote-to-remote copies only work within the same storage
|
| 83 |
+
region (https://huggingface.co/docs/hub/storage-regions).
|
| 84 |
+
"""
|
| 85 |
+
_enforce_context(context, src, dst)
|
| 86 |
+
_run_cp(src, dst, token)
|
| 87 |
+
|
| 88 |
+
return cp
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _enforce_context(context: CpContext | None, src: str, dst: str | None) -> None:
|
| 92 |
+
"""Guardrail for the `hf repos cp` / `hf buckets cp` aliases.
|
| 93 |
+
|
| 94 |
+
These aliases are exact duplicates of `hf cp`, so a bare `hf repos cp` could otherwise touch a
|
| 95 |
+
bucket (and vice versa). We validate the type of the remote side: the destination for uploads and
|
| 96 |
+
remote-to-remote copies, or the source when downloading to a local path / stdout. The top-level
|
| 97 |
+
`hf cp` (i.e. 'context' is None) accepts any combination.
|
| 98 |
+
"""
|
| 99 |
+
if context is None:
|
| 100 |
+
return
|
| 101 |
+
# The remote endpoint is the destination when it is an hf:// URI, otherwise the source (download).
|
| 102 |
+
remote = dst if (dst is not None and is_hf_uri(dst)) else src
|
| 103 |
+
if not is_hf_uri(remote):
|
| 104 |
+
return
|
| 105 |
+
if context == "repos" and parse_hf_uri(remote).is_bucket:
|
| 106 |
+
raise CLIError("`hf repos cp` only works with repositories. Use `hf cp` or `hf buckets cp` for buckets.")
|
| 107 |
+
if context == "buckets" and not parse_hf_uri(remote).is_bucket:
|
| 108 |
+
raise CLIError("`hf buckets cp` only works with buckets. Use `hf cp` or `hf repos cp` for repositories.")
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _run_cp(src: str, dst: str | None, token: str | None) -> None:
|
| 112 |
+
api = get_hf_api(token=token)
|
| 113 |
+
|
| 114 |
+
src_is_stdin = src == "-"
|
| 115 |
+
dst_is_stdout = dst == "-"
|
| 116 |
+
src_is_hf = is_hf_uri(src)
|
| 117 |
+
dst_is_hf = dst is not None and is_hf_uri(dst)
|
| 118 |
+
|
| 119 |
+
# --- Remote to remote: delegate to copy_files (repo/bucket -> repo/bucket) ---
|
| 120 |
+
if src_is_hf and dst_is_hf:
|
| 121 |
+
assert dst is not None # guaranteed by dst_is_hf
|
| 122 |
+
api.copy_files(src, dst)
|
| 123 |
+
out.result("Copied", src=src, dst=dst)
|
| 124 |
+
return
|
| 125 |
+
|
| 126 |
+
# --- At least one side must be a remote hf:// URI (rules out local->local, stdin->local, etc.) ---
|
| 127 |
+
if not src_is_hf and not dst_is_hf:
|
| 128 |
+
if dst is None:
|
| 129 |
+
raise typer.BadParameter("Missing destination. Provide a repo or bucket hf:// URI as DST.")
|
| 130 |
+
raise typer.BadParameter(
|
| 131 |
+
"One of SRC or DST must be a repo (hf://username/...) or bucket (hf://buckets/...) URI."
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
# --- Download: repo/bucket -> local file or stdout ---
|
| 135 |
+
if src_is_hf:
|
| 136 |
+
if dst_is_stdout:
|
| 137 |
+
_download_file_to_stdout(api, src)
|
| 138 |
+
return
|
| 139 |
+
_download_file_to_local(api, src, dst)
|
| 140 |
+
return
|
| 141 |
+
|
| 142 |
+
# --- Upload: local file or stdin -> repo/bucket ---
|
| 143 |
+
assert dst is not None # guaranteed: reaching here means dst_is_hf is True
|
| 144 |
+
_upload_file_to_remote(api, src, dst, src_is_stdin=src_is_stdin)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _download_file_to_stdout(api: HfApi, src: str) -> None:
|
| 148 |
+
uri = parse_hf_uri(src)
|
| 149 |
+
filename = _source_filename(uri, src)
|
| 150 |
+
# Suppress progress bars to avoid polluting the piped output.
|
| 151 |
+
with disable_progress_bars():
|
| 152 |
+
with SoftTemporaryDirectory() as tmp_dir:
|
| 153 |
+
tmp_path = os.path.join(tmp_dir, filename)
|
| 154 |
+
_download_single(api, uri, tmp_path)
|
| 155 |
+
with open(tmp_path, "rb") as f:
|
| 156 |
+
while chunk := f.read(32_000_000): # 32MB chunks
|
| 157 |
+
sys.stdout.buffer.write(chunk)
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _download_file_to_local(api: HfApi, src: str, dst: str | None) -> None:
|
| 161 |
+
uri = parse_hf_uri(src)
|
| 162 |
+
filename = _source_filename(uri, src)
|
| 163 |
+
|
| 164 |
+
if dst is None:
|
| 165 |
+
local_path = filename
|
| 166 |
+
elif os.path.isdir(dst) or dst.endswith(os.sep) or dst.endswith("/"):
|
| 167 |
+
local_path = os.path.join(dst, filename)
|
| 168 |
+
else:
|
| 169 |
+
local_path = dst
|
| 170 |
+
|
| 171 |
+
parent_dir = os.path.dirname(local_path)
|
| 172 |
+
if parent_dir:
|
| 173 |
+
os.makedirs(parent_dir, exist_ok=True)
|
| 174 |
+
|
| 175 |
+
_download_single(api, uri, local_path)
|
| 176 |
+
out.result("Downloaded", src=src, dst=local_path)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def _download_single(api: HfApi, uri: HfUri, local_path: str) -> None:
|
| 180 |
+
"""Download a single file (repo or bucket) to ``local_path``.
|
| 181 |
+
|
| 182 |
+
Used by `_download_file_to_local` and `_download_file_to_stdout`.
|
| 183 |
+
"""
|
| 184 |
+
if uri.is_bucket:
|
| 185 |
+
api.download_bucket_files(uri.id, [(uri.path_in_repo, local_path)], raise_on_missing_files=True)
|
| 186 |
+
else:
|
| 187 |
+
# Download into a temporary folder next to the destination (rather than the shared cache)
|
| 188 |
+
# so the final move stays on the same filesystem and is instant. The temp folder is
|
| 189 |
+
# cleaned up automatically once the move is complete.
|
| 190 |
+
parent_dir = os.path.dirname(local_path) or "."
|
| 191 |
+
with SoftTemporaryDirectory(prefix=".tmp", dir=parent_dir) as tmp_dir:
|
| 192 |
+
downloaded_path = api.hf_hub_download(
|
| 193 |
+
repo_id=uri.id,
|
| 194 |
+
repo_type=uri.type,
|
| 195 |
+
filename=uri.path_in_repo,
|
| 196 |
+
revision=uri.revision,
|
| 197 |
+
local_dir=tmp_dir,
|
| 198 |
+
)
|
| 199 |
+
os.replace(downloaded_path, local_path)
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def _source_filename(uri: HfUri, src: str) -> str:
|
| 203 |
+
if uri.path_in_repo == "" or src.endswith("/"):
|
| 204 |
+
raise typer.BadParameter(
|
| 205 |
+
"Source path must include a file name, not just a repo/bucket or directory path."
|
| 206 |
+
" Use `hf download` or `hf buckets sync` to copy directories."
|
| 207 |
+
)
|
| 208 |
+
return uri.path_in_repo.rsplit("/", 1)[-1]
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _upload_file_to_remote(api: HfApi, src: str, dst: str, *, src_is_stdin: bool) -> None:
|
| 212 |
+
uri = parse_hf_uri(dst)
|
| 213 |
+
|
| 214 |
+
if src_is_stdin:
|
| 215 |
+
if uri.path_in_repo == "" or dst.endswith("/"):
|
| 216 |
+
raise typer.BadParameter("Stdin upload requires a full destination path including filename.")
|
| 217 |
+
data = sys.stdin.buffer.read()
|
| 218 |
+
_upload_single(api, uri, data, uri.path_in_repo)
|
| 219 |
+
out.result("Uploaded", src="stdin", dst=uri.to_uri())
|
| 220 |
+
return
|
| 221 |
+
|
| 222 |
+
if os.path.isdir(src):
|
| 223 |
+
raise typer.BadParameter(
|
| 224 |
+
"Source must be a file, not a directory. Use `hf upload` or `hf buckets sync` for directories."
|
| 225 |
+
)
|
| 226 |
+
if not os.path.isfile(src):
|
| 227 |
+
raise typer.BadParameter(f"Source file not found: {src}")
|
| 228 |
+
|
| 229 |
+
prefix = uri.path_in_repo
|
| 230 |
+
if prefix == "":
|
| 231 |
+
remote_path = os.path.basename(src)
|
| 232 |
+
elif dst.endswith("/"):
|
| 233 |
+
remote_path = prefix + "/" + os.path.basename(src)
|
| 234 |
+
else:
|
| 235 |
+
remote_path = prefix
|
| 236 |
+
|
| 237 |
+
_upload_single(api, uri, src, remote_path)
|
| 238 |
+
out.result("Uploaded", src=src, dst=replace(uri, path_in_repo=remote_path).to_uri())
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def _upload_single(api: HfApi, uri: HfUri, source: str | bytes, remote_path: str) -> None:
|
| 242 |
+
"""Upload a single file or bytes (to a repo or bucket)."""
|
| 243 |
+
if uri.is_bucket:
|
| 244 |
+
api.batch_bucket_files(uri.id, add=[(source, remote_path)])
|
| 245 |
+
else:
|
| 246 |
+
api.upload_file(
|
| 247 |
+
path_or_fileobj=source,
|
| 248 |
+
path_in_repo=remote_path,
|
| 249 |
+
repo_id=uri.id,
|
| 250 |
+
repo_type=uri.type,
|
| 251 |
+
revision=uri.revision,
|
| 252 |
+
)
|
venv/Lib/site-packages/huggingface_hub/cli/_errors.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""CLI error handling utilities."""
|
| 15 |
+
|
| 16 |
+
import traceback
|
| 17 |
+
from collections.abc import Callable
|
| 18 |
+
|
| 19 |
+
from huggingface_hub.errors import (
|
| 20 |
+
BucketNotFoundError,
|
| 21 |
+
CLIError,
|
| 22 |
+
CLIExtensionInstallError,
|
| 23 |
+
EntryNotFoundError,
|
| 24 |
+
GatedRepoError,
|
| 25 |
+
HfHubHTTPError,
|
| 26 |
+
HfUriError,
|
| 27 |
+
LocalEntryNotFoundError,
|
| 28 |
+
LocalTokenNotFoundError,
|
| 29 |
+
OfflineModeIsEnabled,
|
| 30 |
+
OIDCError,
|
| 31 |
+
RemoteEntryNotFoundError,
|
| 32 |
+
RepositoryNotFoundError,
|
| 33 |
+
RevisionNotFoundError,
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _format_repo_not_found(error: RepositoryNotFoundError) -> str:
|
| 38 |
+
label = error.repo_type.capitalize() if error.repo_type else "Repository"
|
| 39 |
+
if error.repo_id:
|
| 40 |
+
msg = f"{label} '{error.repo_id}' not found."
|
| 41 |
+
else:
|
| 42 |
+
msg = f"{label} not found."
|
| 43 |
+
msg += " If the repo is private, make sure you are authenticated and your token has the required permissions."
|
| 44 |
+
return msg
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _format_gated_repo(error: GatedRepoError) -> str:
|
| 48 |
+
label = error.repo_type if error.repo_type else "repository"
|
| 49 |
+
if error.repo_id:
|
| 50 |
+
return f"Access denied. {label.capitalize()} '{error.repo_id}' requires approval."
|
| 51 |
+
return f"Access denied. This {label} requires approval."
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _format_bucket_not_found(error: BucketNotFoundError) -> str:
|
| 55 |
+
if error.bucket_id:
|
| 56 |
+
return f"Bucket '{error.bucket_id}' not found. If the bucket is private, make sure you are authenticated and your token has the required permissions."
|
| 57 |
+
return "Bucket not found. Check the bucket id (namespace/name). If the bucket is private, make sure you are authenticated and your token has the required permissions."
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _format_entry_not_found(error: RemoteEntryNotFoundError) -> str:
|
| 61 |
+
label = error.repo_type if error.repo_type else "repository"
|
| 62 |
+
url = str(error.response.url) if error.response else None
|
| 63 |
+
if error.repo_id:
|
| 64 |
+
msg = f"File not found in {label} '{error.repo_id}'."
|
| 65 |
+
else:
|
| 66 |
+
msg = f"File not found in {label}."
|
| 67 |
+
if url:
|
| 68 |
+
msg += f"\nURL: {url}"
|
| 69 |
+
return msg
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _format_local_entry_not_found(error: LocalEntryNotFoundError) -> str:
|
| 73 |
+
cause = error.__cause__
|
| 74 |
+
if cause is not None:
|
| 75 |
+
return f"Local entry not found. {cause}"
|
| 76 |
+
return f"Local entry not found. {error}"
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _format_revision_not_found(error: RevisionNotFoundError) -> str:
|
| 80 |
+
label = error.repo_type if error.repo_type else "repository"
|
| 81 |
+
if error.repo_id:
|
| 82 |
+
return f"Revision not found in {label} '{error.repo_id}'."
|
| 83 |
+
return f"Revision not found in {label}. Check the revision parameter."
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _format_cli_error(error: CLIError) -> str:
|
| 87 |
+
"""No traceback, just the error message."""
|
| 88 |
+
return str(error)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _format_cli_extension_install_error(error: CLIExtensionInstallError) -> str:
|
| 92 |
+
"""Format a CLI extension installation error.
|
| 93 |
+
|
| 94 |
+
The error is likely to be a tricky subprocess error to investigate. In this specific case we want to format the
|
| 95 |
+
traceback of the root cause while keeping the "nicely formatted" error message of the CLIExtensionInstallError
|
| 96 |
+
as a 1-line message.
|
| 97 |
+
"""
|
| 98 |
+
cause_tb = (
|
| 99 |
+
"".join(traceback.format_exception(type(error.__cause__), error.__cause__, error.__cause__.__traceback__))
|
| 100 |
+
if error.__cause__ is not None
|
| 101 |
+
else ""
|
| 102 |
+
)
|
| 103 |
+
return f"{cause_tb}\n{error}"
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
CLI_ERROR_MAPPINGS: dict[type[Exception], Callable[..., str]] = {
|
| 107 |
+
OfflineModeIsEnabled: lambda error: str(error),
|
| 108 |
+
# GatedRepoError must come before RepositoryNotFoundError (it's a subclass).
|
| 109 |
+
GatedRepoError: _format_gated_repo,
|
| 110 |
+
BucketNotFoundError: _format_bucket_not_found,
|
| 111 |
+
RepositoryNotFoundError: _format_repo_not_found,
|
| 112 |
+
RevisionNotFoundError: _format_revision_not_found,
|
| 113 |
+
LocalTokenNotFoundError: lambda _: "Not logged in. Run 'hf auth login' first.",
|
| 114 |
+
OIDCError: lambda error: f"OIDC Exchange failed. {error}",
|
| 115 |
+
RemoteEntryNotFoundError: _format_entry_not_found,
|
| 116 |
+
LocalEntryNotFoundError: _format_local_entry_not_found,
|
| 117 |
+
EntryNotFoundError: lambda error: str(error),
|
| 118 |
+
HfHubHTTPError: lambda error: str(error),
|
| 119 |
+
HfUriError: lambda error: f"Invalid HF URI: {error.uri}. {error.msg}",
|
| 120 |
+
ValueError: lambda error: f"Invalid value. {error}",
|
| 121 |
+
CLIExtensionInstallError: _format_cli_extension_install_error,
|
| 122 |
+
CLIError: _format_cli_error,
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def format_known_exception(error: Exception) -> str | None:
|
| 127 |
+
for exc_type, formatter in CLI_ERROR_MAPPINGS.items():
|
| 128 |
+
if isinstance(error, exc_type):
|
| 129 |
+
return formatter(error)
|
| 130 |
+
return None
|
venv/Lib/site-packages/huggingface_hub/cli/_file_listing.py
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Shared helpers for listing files in buckets and repos (tree view, flat view, formatting)."""
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
from datetime import datetime
|
| 18 |
+
from typing import Sequence
|
| 19 |
+
|
| 20 |
+
import typer
|
| 21 |
+
|
| 22 |
+
from huggingface_hub._buckets import BucketFile, BucketFolder
|
| 23 |
+
from huggingface_hub.hf_api import RepoFile, RepoFolder
|
| 24 |
+
|
| 25 |
+
from ._cli_utils import get_hf_api
|
| 26 |
+
from ._output import OutputFormat, _dataclass_to_dict, out
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
BucketItem = BucketFile | BucketFolder
|
| 30 |
+
RepoItem = RepoFile | RepoFolder
|
| 31 |
+
ListingItem = BucketItem | RepoItem
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def get_item_date(item: ListingItem) -> datetime | None:
|
| 35 |
+
"""Extract date from an item, supporting both repo items (last_commit.date) and bucket items (mtime/uploaded_at)."""
|
| 36 |
+
match item:
|
| 37 |
+
case BucketFile(mtime=mtime) if mtime is not None:
|
| 38 |
+
return mtime
|
| 39 |
+
case BucketFile(uploaded_at=uploaded_at) | BucketFolder(uploaded_at=uploaded_at) if uploaded_at is not None:
|
| 40 |
+
return uploaded_at
|
| 41 |
+
case RepoFile(last_commit=last_commit) | RepoFolder(last_commit=last_commit) if last_commit is not None:
|
| 42 |
+
return last_commit.date
|
| 43 |
+
case _:
|
| 44 |
+
return None
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def format_size(size: int | float, human_readable: bool = False) -> str:
|
| 48 |
+
"""Format a size in bytes."""
|
| 49 |
+
if not human_readable:
|
| 50 |
+
return str(size)
|
| 51 |
+
|
| 52 |
+
for unit in ["B", "KB", "MB", "GB", "TB"]:
|
| 53 |
+
if size < 1000:
|
| 54 |
+
if unit == "B":
|
| 55 |
+
return f"{size} {unit}"
|
| 56 |
+
return f"{size:.1f} {unit}"
|
| 57 |
+
size /= 1000
|
| 58 |
+
return f"{size:.1f} PB"
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def format_date(dt: datetime | None, human_readable: bool = False) -> str:
|
| 62 |
+
"""Format a datetime to a readable date string."""
|
| 63 |
+
if dt is None:
|
| 64 |
+
return ""
|
| 65 |
+
if human_readable:
|
| 66 |
+
return dt.strftime("%b %d %H:%M")
|
| 67 |
+
return dt.strftime("%Y-%m-%d %H:%M:%S")
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def build_tree(
|
| 71 |
+
items: Sequence[BucketItem] | Sequence[RepoItem],
|
| 72 |
+
human_readable: bool = False,
|
| 73 |
+
quiet: bool = False,
|
| 74 |
+
) -> list[str]:
|
| 75 |
+
"""Build a tree representation of files and directories.
|
| 76 |
+
|
| 77 |
+
Produces ASCII tree with size and date columns before the tree connector.
|
| 78 |
+
When quiet=True, only the tree structure is shown (no size/date).
|
| 79 |
+
"""
|
| 80 |
+
tree: dict = {}
|
| 81 |
+
|
| 82 |
+
for item in items:
|
| 83 |
+
parts = item.path.split("/")
|
| 84 |
+
current = tree
|
| 85 |
+
for part in parts[:-1]:
|
| 86 |
+
if part not in current:
|
| 87 |
+
current[part] = {"__children__": {}}
|
| 88 |
+
current = current[part]["__children__"]
|
| 89 |
+
|
| 90 |
+
final_part = parts[-1]
|
| 91 |
+
if isinstance(item, BucketFolder | RepoFolder):
|
| 92 |
+
if final_part not in current:
|
| 93 |
+
current[final_part] = {"__children__": {}}
|
| 94 |
+
else:
|
| 95 |
+
current[final_part] = {"__item__": item}
|
| 96 |
+
|
| 97 |
+
prefix_width = 0
|
| 98 |
+
max_size_width = 0
|
| 99 |
+
max_date_width = 0
|
| 100 |
+
if not quiet:
|
| 101 |
+
for item in items:
|
| 102 |
+
if isinstance(item, BucketFile | RepoFile):
|
| 103 |
+
size_str = format_size(item.size, human_readable)
|
| 104 |
+
max_size_width = max(max_size_width, len(size_str))
|
| 105 |
+
date_str = format_date(get_item_date(item), human_readable)
|
| 106 |
+
max_date_width = max(max_date_width, len(date_str))
|
| 107 |
+
if max_size_width > 0:
|
| 108 |
+
prefix_width = max_size_width + 2 + max_date_width
|
| 109 |
+
|
| 110 |
+
lines: list[str] = []
|
| 111 |
+
_render_tree(
|
| 112 |
+
tree,
|
| 113 |
+
lines,
|
| 114 |
+
"",
|
| 115 |
+
prefix_width=prefix_width,
|
| 116 |
+
max_size_width=max_size_width,
|
| 117 |
+
human_readable=human_readable,
|
| 118 |
+
)
|
| 119 |
+
return lines
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _render_tree(
|
| 123 |
+
node: dict,
|
| 124 |
+
lines: list[str],
|
| 125 |
+
indent: str,
|
| 126 |
+
prefix_width: int = 0,
|
| 127 |
+
max_size_width: int = 0,
|
| 128 |
+
human_readable: bool = False,
|
| 129 |
+
) -> None:
|
| 130 |
+
"""Recursively render a tree structure with size+date prefix."""
|
| 131 |
+
sorted_items = sorted(node.items())
|
| 132 |
+
for i, (name, value) in enumerate(sorted_items):
|
| 133 |
+
is_last = i == len(sorted_items) - 1
|
| 134 |
+
connector = "└── " if is_last else "├── "
|
| 135 |
+
|
| 136 |
+
is_dir = "__children__" in value
|
| 137 |
+
children = value.get("__children__", {})
|
| 138 |
+
|
| 139 |
+
if prefix_width > 0:
|
| 140 |
+
if is_dir:
|
| 141 |
+
prefix = " " * prefix_width
|
| 142 |
+
else:
|
| 143 |
+
item = value.get("__item__")
|
| 144 |
+
if item is not None:
|
| 145 |
+
size_str = format_size(item.size, human_readable)
|
| 146 |
+
date_str = format_date(get_item_date(item), human_readable)
|
| 147 |
+
prefix = f"{size_str:>{max_size_width}} {date_str}"
|
| 148 |
+
else:
|
| 149 |
+
prefix = " " * prefix_width
|
| 150 |
+
lines.append(f"{prefix} {indent}{connector}{name}{'/' if is_dir else ''}")
|
| 151 |
+
else:
|
| 152 |
+
lines.append(f"{indent}{connector}{name}{'/' if is_dir else ''}")
|
| 153 |
+
|
| 154 |
+
if children:
|
| 155 |
+
child_indent = indent + (" " if is_last else "│ ")
|
| 156 |
+
_render_tree(
|
| 157 |
+
children,
|
| 158 |
+
lines,
|
| 159 |
+
child_indent,
|
| 160 |
+
prefix_width=prefix_width,
|
| 161 |
+
max_size_width=max_size_width,
|
| 162 |
+
human_readable=human_readable,
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def list_repo_files_cmd(
|
| 167 |
+
repo_id: str,
|
| 168 |
+
repo_type: str,
|
| 169 |
+
human_readable: bool,
|
| 170 |
+
as_tree: bool,
|
| 171 |
+
recursive: bool,
|
| 172 |
+
revision: str | None,
|
| 173 |
+
token: str | None,
|
| 174 |
+
) -> None:
|
| 175 |
+
"""List files in a repo on the Hub. Used by models/datasets/spaces ls commands."""
|
| 176 |
+
if as_tree and out.mode == OutputFormat.json:
|
| 177 |
+
raise typer.BadParameter("Cannot use --tree with --format json.")
|
| 178 |
+
|
| 179 |
+
api = get_hf_api(token=token)
|
| 180 |
+
items = list(api.list_repo_tree(repo_id, recursive=recursive, revision=revision, repo_type=repo_type, expand=True))
|
| 181 |
+
print_file_listing(items, human_readable=human_readable, as_tree=as_tree, recursive=recursive)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def print_file_listing(
|
| 185 |
+
items: Sequence[BucketItem] | Sequence[RepoItem],
|
| 186 |
+
*,
|
| 187 |
+
human_readable: bool = False,
|
| 188 |
+
as_tree: bool = False,
|
| 189 |
+
recursive: bool = False,
|
| 190 |
+
) -> None:
|
| 191 |
+
"""Print a file listing in the appropriate format based on the current output mode.
|
| 192 |
+
|
| 193 |
+
Supports tree, json, quiet, and flat human-readable views. Works with both
|
| 194 |
+
BucketFile/BucketFolder and RepoFile/RepoFolder items.
|
| 195 |
+
"""
|
| 196 |
+
if not items:
|
| 197 |
+
out.text("(empty)")
|
| 198 |
+
return
|
| 199 |
+
|
| 200 |
+
has_directories = any(isinstance(item, BucketFolder | RepoFolder) for item in items)
|
| 201 |
+
|
| 202 |
+
if as_tree:
|
| 203 |
+
quiet = out.mode == OutputFormat.quiet
|
| 204 |
+
for line in build_tree(items, human_readable=human_readable, quiet=quiet):
|
| 205 |
+
print(line)
|
| 206 |
+
elif out.mode == OutputFormat.json:
|
| 207 |
+
print(json.dumps([_dataclass_to_dict(item) for item in items], indent=2))
|
| 208 |
+
elif out.mode == OutputFormat.quiet:
|
| 209 |
+
for item in items:
|
| 210 |
+
if isinstance(item, BucketFolder | RepoFolder):
|
| 211 |
+
print(f"{item.path}/")
|
| 212 |
+
else:
|
| 213 |
+
print(item.path)
|
| 214 |
+
else:
|
| 215 |
+
for item in items:
|
| 216 |
+
if isinstance(item, BucketFolder | RepoFolder):
|
| 217 |
+
date_str = format_date(get_item_date(item), human_readable)
|
| 218 |
+
print(f"{'':>12} {date_str:>19} {item.path}/")
|
| 219 |
+
else:
|
| 220 |
+
size_str = format_size(item.size, human_readable)
|
| 221 |
+
date_str = format_date(get_item_date(item), human_readable)
|
| 222 |
+
print(f"{size_str:>12} {date_str:>19} {item.path}")
|
| 223 |
+
|
| 224 |
+
if not recursive and has_directories:
|
| 225 |
+
out.hint("Use -R to list files recursively.")
|
venv/Lib/site-packages/huggingface_hub/cli/_help_formatter.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Pretty ANSI help formatter for the `hf` CLI."""
|
| 15 |
+
|
| 16 |
+
from collections.abc import Sequence
|
| 17 |
+
|
| 18 |
+
import click
|
| 19 |
+
|
| 20 |
+
from huggingface_hub.utils import ANSI
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class StyledHelpFormatter(click.HelpFormatter):
|
| 24 |
+
def write_heading(self, heading: str) -> None:
|
| 25 |
+
styled = ANSI.underline(heading + ":")
|
| 26 |
+
self.write(f"{'':>{self.current_indent}}{styled}\n")
|
| 27 |
+
|
| 28 |
+
def write_dl(self, rows: Sequence[tuple[str, str]], col_max: int = 30, col_spacing: int = 2) -> None:
|
| 29 |
+
rows = [(ANSI.bold(first), second) for first, second in rows]
|
| 30 |
+
super().write_dl(rows, col_max=col_max, col_spacing=col_spacing)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class StyledContext(click.Context):
|
| 34 |
+
formatter_class = StyledHelpFormatter
|
venv/Lib/site-packages/huggingface_hub/cli/_output.py
ADDED
|
@@ -0,0 +1,345 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Output framework for the `hf` CLI."""
|
| 15 |
+
|
| 16 |
+
import dataclasses
|
| 17 |
+
import datetime
|
| 18 |
+
import json
|
| 19 |
+
import re
|
| 20 |
+
import shutil
|
| 21 |
+
import sys
|
| 22 |
+
from collections.abc import Sequence
|
| 23 |
+
from enum import Enum
|
| 24 |
+
from typing import Any, cast
|
| 25 |
+
|
| 26 |
+
import typer
|
| 27 |
+
|
| 28 |
+
from huggingface_hub.errors import ConfirmationError
|
| 29 |
+
from huggingface_hub.utils import ANSI, StatusLine, disable_progress_bars, is_agent, tabulate
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class OutputFormat(str, Enum):
|
| 33 |
+
"""Output format for CLI commands with auto detection of agent/human mode."""
|
| 34 |
+
|
| 35 |
+
agent = "agent"
|
| 36 |
+
auto = "auto"
|
| 37 |
+
human = "human"
|
| 38 |
+
json = "json"
|
| 39 |
+
quiet = "quiet"
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class Output:
|
| 43 |
+
"""Output sink for the `hf` CLI.
|
| 44 |
+
|
| 45 |
+
Mode is resolved once at init time based on `is_agent()` auto-detection
|
| 46 |
+
and can be overridden per-command via `set_mode()`.
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
mode: OutputFormat
|
| 50 |
+
no_truncate: bool
|
| 51 |
+
|
| 52 |
+
def __init__(self) -> None:
|
| 53 |
+
self.no_truncate = False
|
| 54 |
+
self.set_mode()
|
| 55 |
+
|
| 56 |
+
def set_mode(self, mode: OutputFormat = OutputFormat.auto) -> None:
|
| 57 |
+
"""Override the output mode (called once at startup and again per '--format' flag)."""
|
| 58 |
+
if mode == OutputFormat.auto:
|
| 59 |
+
mode = OutputFormat.agent if is_agent() else OutputFormat.human
|
| 60 |
+
self.mode = mode
|
| 61 |
+
if mode != OutputFormat.human:
|
| 62 |
+
disable_progress_bars()
|
| 63 |
+
|
| 64 |
+
def set_no_truncate(self, no_truncate: bool) -> None:
|
| 65 |
+
"""Toggle off cell truncation for human table output."""
|
| 66 |
+
self.no_truncate = no_truncate
|
| 67 |
+
|
| 68 |
+
def is_quiet(self) -> bool:
|
| 69 |
+
return self.mode == OutputFormat.quiet
|
| 70 |
+
|
| 71 |
+
def text(self, msg: str | None = None, *, human: str | None = None, agent: str | None = None) -> None:
|
| 72 |
+
"""Print a free-form text message to stdout."""
|
| 73 |
+
if msg is not None:
|
| 74 |
+
if human is not None or agent is not None:
|
| 75 |
+
raise ValueError("Cannot mix 'msg' with 'human'/'agent'.")
|
| 76 |
+
human = msg
|
| 77 |
+
agent = _strip_ansi(msg)
|
| 78 |
+
|
| 79 |
+
match self.mode:
|
| 80 |
+
case OutputFormat.human:
|
| 81 |
+
if human is not None:
|
| 82 |
+
print(human)
|
| 83 |
+
case OutputFormat.agent:
|
| 84 |
+
if agent is not None:
|
| 85 |
+
print(agent)
|
| 86 |
+
# json/quiet: no-op
|
| 87 |
+
|
| 88 |
+
def table(
|
| 89 |
+
self,
|
| 90 |
+
items: Sequence[dict[str, Any]],
|
| 91 |
+
*,
|
| 92 |
+
headers: list[str] | None = None,
|
| 93 |
+
id_key: str | None = None,
|
| 94 |
+
alignments: dict[str, str] | None = None,
|
| 95 |
+
) -> None:
|
| 96 |
+
"""Print tabular data to stdout.
|
| 97 |
+
|
| 98 |
+
Args:
|
| 99 |
+
items: List of dicts. Headers are auto-detected from keys if not provided.
|
| 100 |
+
headers: Explicit column names. If None, derived from dict keys (all-None columns filtered).
|
| 101 |
+
id_key: Key to print in quiet mode. If None, uses the first header.
|
| 102 |
+
alignments: Optional mapping of header name to "left" or "right". Defaults to "left".
|
| 103 |
+
"""
|
| 104 |
+
if not items:
|
| 105 |
+
match self.mode:
|
| 106 |
+
case OutputFormat.agent | OutputFormat.human:
|
| 107 |
+
print("No results found.")
|
| 108 |
+
case OutputFormat.json:
|
| 109 |
+
print("[]")
|
| 110 |
+
return
|
| 111 |
+
|
| 112 |
+
if headers is None:
|
| 113 |
+
all_columns = list(items[0].keys())
|
| 114 |
+
headers = [col for col in all_columns if any(item.get(col) is not None for item in items)]
|
| 115 |
+
rows = [[item.get(h) for h in headers] for item in items]
|
| 116 |
+
|
| 117 |
+
match self.mode:
|
| 118 |
+
case OutputFormat.human: # padded table, adaptive truncation, SCREAMING_SNAKE headers
|
| 119 |
+
screaming_headers = [_to_header(h) for h in headers]
|
| 120 |
+
formatted_rows: list[list[str]] = [[_format_table_value_human(v) for v in row] for row in rows]
|
| 121 |
+
|
| 122 |
+
is_truncated = _truncate_columns(screaming_headers, formatted_rows, no_truncate=self.no_truncate)
|
| 123 |
+
|
| 124 |
+
inferred = {**_infer_alignments(headers, rows), **(alignments or {})}
|
| 125 |
+
screaming_alignments = {_to_header(k): v for k, v in inferred.items()}
|
| 126 |
+
print(
|
| 127 |
+
tabulate(
|
| 128 |
+
cast("list[list[str | int]]", formatted_rows),
|
| 129 |
+
headers=screaming_headers,
|
| 130 |
+
alignments=screaming_alignments,
|
| 131 |
+
)
|
| 132 |
+
)
|
| 133 |
+
if is_truncated:
|
| 134 |
+
self.hint("Use `--no-truncate` or `--format json` to display full values.")
|
| 135 |
+
case OutputFormat.agent: # TSV, no truncation, full timestamps
|
| 136 |
+
print("\t".join(headers))
|
| 137 |
+
for row in rows:
|
| 138 |
+
print("\t".join(_format_table_cell_agent(v) for v in row))
|
| 139 |
+
case OutputFormat.json: # compact JSON array
|
| 140 |
+
print(json.dumps(list(items), default=str))
|
| 141 |
+
case OutputFormat.quiet: # id_key column (or first column), one per line
|
| 142 |
+
quiet_key = id_key or headers[0]
|
| 143 |
+
for item in items:
|
| 144 |
+
print(item.get(quiet_key, ""))
|
| 145 |
+
|
| 146 |
+
def dict(self, data: Any, *, id_key: str | None = None) -> None:
|
| 147 |
+
"""Print structured data as JSON in all modes (indented for human, compact otherwise).
|
| 148 |
+
|
| 149 |
+
Accepts a dict or a dataclass.
|
| 150 |
+
"""
|
| 151 |
+
if dataclasses.is_dataclass(data) and not isinstance(data, type):
|
| 152 |
+
data = _dataclass_to_dict(data)
|
| 153 |
+
if self.mode == OutputFormat.quiet and id_key is not None:
|
| 154 |
+
print(data.get(id_key, ""))
|
| 155 |
+
return
|
| 156 |
+
indent = 2 if self.mode == OutputFormat.human else None
|
| 157 |
+
print(json.dumps(data, indent=indent, default=str))
|
| 158 |
+
|
| 159 |
+
def result(self, message: str, **data: Any) -> None:
|
| 160 |
+
"""Print a success summary to stdout."""
|
| 161 |
+
match self.mode:
|
| 162 |
+
case OutputFormat.human: # ✓ message + key: value lines
|
| 163 |
+
parts = [ANSI.green(f"✓ {message}")]
|
| 164 |
+
for k, v in data.items():
|
| 165 |
+
if v is not None:
|
| 166 |
+
parts.append(f" {k}: {v}")
|
| 167 |
+
print("\n".join(parts))
|
| 168 |
+
case OutputFormat.agent: # key=val pairs, space-separated
|
| 169 |
+
parts = [f"{k}={v}" for k, v in data.items() if v is not None]
|
| 170 |
+
print(" ".join(parts) if parts else message)
|
| 171 |
+
case OutputFormat.json: # json.dumps(data), message ignored
|
| 172 |
+
print(json.dumps(data, default=str) if data else "")
|
| 173 |
+
case OutputFormat.quiet: # first value only
|
| 174 |
+
values = list(data.values())
|
| 175 |
+
if values:
|
| 176 |
+
print(values[0])
|
| 177 |
+
|
| 178 |
+
def confirm(self, message: str, *, default: bool = False, yes: bool = False, confirm_param: str = "--yes") -> None:
|
| 179 |
+
"""
|
| 180 |
+
Ask for confirmation. Raises `ConfirmationError` in non-human modes.
|
| 181 |
+
"""
|
| 182 |
+
if yes:
|
| 183 |
+
return
|
| 184 |
+
if self.mode != OutputFormat.human:
|
| 185 |
+
raise ConfirmationError(f"{message} Use {confirm_param} to skip confirmation.")
|
| 186 |
+
typer.confirm(message, default=default, abort=True)
|
| 187 |
+
|
| 188 |
+
def status(self, message: str | None = None) -> StatusLine:
|
| 189 |
+
"""Return a status line that emits only in human mode (no-op otherwise)."""
|
| 190 |
+
status = StatusLine(enabled=self.mode == OutputFormat.human)
|
| 191 |
+
if message is not None:
|
| 192 |
+
status.update(message)
|
| 193 |
+
return status
|
| 194 |
+
|
| 195 |
+
def warning(self, message: str) -> None:
|
| 196 |
+
"""Print a non-fatal warning to stderr (all modes)."""
|
| 197 |
+
if self.mode == OutputFormat.human:
|
| 198 |
+
print(ANSI.yellow(f"Warning: {message}"), file=sys.stderr)
|
| 199 |
+
else:
|
| 200 |
+
print(f"Warning: {message}", file=sys.stderr)
|
| 201 |
+
|
| 202 |
+
def error(self, message: str) -> None:
|
| 203 |
+
"""Print an error to stderr (all modes)."""
|
| 204 |
+
if self.mode == OutputFormat.human:
|
| 205 |
+
print(ANSI.red(f"Error: {message}"), file=sys.stderr)
|
| 206 |
+
else:
|
| 207 |
+
print(f"Error: {message}", file=sys.stderr)
|
| 208 |
+
|
| 209 |
+
def hint(self, message: str) -> None:
|
| 210 |
+
"""Print a helpful hint to stderr (human: gray, json/agent: plain text).
|
| 211 |
+
|
| 212 |
+
Suppressed in quiet mode. Kept in json mode (like agent) since agents
|
| 213 |
+
commonly run with ``--format json`` and the next-command hints are useful
|
| 214 |
+
there; hints go to stderr so they never pollute the parsed stdout.
|
| 215 |
+
"""
|
| 216 |
+
if self.mode == OutputFormat.quiet:
|
| 217 |
+
return
|
| 218 |
+
if self.mode == OutputFormat.human:
|
| 219 |
+
print(ANSI.gray(f"Hint: {message}"), file=sys.stderr)
|
| 220 |
+
else:
|
| 221 |
+
print(f"Hint: {message}", file=sys.stderr)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
# HELPERS
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def _serialize_value(v: object) -> object:
|
| 228 |
+
"""Recursively serialize a value to be JSON-compatible."""
|
| 229 |
+
if isinstance(v, datetime.datetime):
|
| 230 |
+
return v.isoformat()
|
| 231 |
+
elif isinstance(v, dict):
|
| 232 |
+
return {key: _serialize_value(val) for key, val in v.items() if val is not None}
|
| 233 |
+
elif isinstance(v, list):
|
| 234 |
+
return [_serialize_value(item) for item in v]
|
| 235 |
+
return v
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def _dataclass_to_dict(info: Any) -> dict[str, Any]:
|
| 239 |
+
"""Convert a dataclass to a json-serializable dict."""
|
| 240 |
+
return {k: _serialize_value(v) for k, v in dataclasses.asdict(info).items() if v is not None}
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
_ANSI_RE = re.compile(r"\033\[[0-9;]*m")
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def _strip_ansi(text: str) -> str:
|
| 247 |
+
return _ANSI_RE.sub("", text)
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def _single_line(text: str) -> str:
|
| 251 |
+
return " ".join(text.split())
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def _to_header(name: str) -> str:
|
| 255 |
+
"""Convert a camelCase or PascalCase string to SCREAMING_SNAKE_CASE."""
|
| 256 |
+
s = re.sub(r"([a-z])([A-Z])", r"\1_\2", name)
|
| 257 |
+
return s.upper()
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _infer_alignments(headers: list[str], rows: list[list[Any]]) -> dict[str, str]:
|
| 261 |
+
"""Return ``{"col": "right"}`` for columns where every non-None value is numeric."""
|
| 262 |
+
result: dict[str, str] = {}
|
| 263 |
+
for c, h in enumerate(headers):
|
| 264 |
+
if all(row[c] is None or (isinstance(row[c], (int, float)) and not isinstance(row[c], bool)) for row in rows):
|
| 265 |
+
result[h] = "right"
|
| 266 |
+
return result
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def _format_table_value_human(value: Any) -> str:
|
| 270 |
+
"""Convert a value to string for terminal display."""
|
| 271 |
+
if value is None:
|
| 272 |
+
return ""
|
| 273 |
+
if isinstance(value, bool):
|
| 274 |
+
return "✔" if value else ""
|
| 275 |
+
if isinstance(value, datetime.datetime):
|
| 276 |
+
return value.strftime("%Y-%m-%d")
|
| 277 |
+
if isinstance(value, str) and re.match(r"^\d{4}-\d{2}-\d{2}T", value):
|
| 278 |
+
return value[:10]
|
| 279 |
+
if isinstance(value, str):
|
| 280 |
+
return _single_line(value)
|
| 281 |
+
if isinstance(value, list):
|
| 282 |
+
return ", ".join(_format_table_value_human(v) for v in value)
|
| 283 |
+
elif isinstance(value, dict):
|
| 284 |
+
if "name" in value: # Likely to be a user or org => print name
|
| 285 |
+
return _single_line(str(value["name"]))
|
| 286 |
+
return _single_line(json.dumps(value))
|
| 287 |
+
return _single_line(str(value))
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def _truncate_columns(
|
| 291 |
+
headers: list[str],
|
| 292 |
+
rows: list[list[str]],
|
| 293 |
+
*,
|
| 294 |
+
no_truncate: bool,
|
| 295 |
+
) -> bool:
|
| 296 |
+
"""Truncate cells in-place to fit the current terminal width.
|
| 297 |
+
|
| 298 |
+
Returns `True` if any cell was truncated, so the caller can emit a hint.
|
| 299 |
+
`shutil.get_terminal_size` is cross-platform: it honors `$COLUMNS`, then
|
| 300 |
+
queries the OS-native API, then falls back to `(80, 24)`.
|
| 301 |
+
"""
|
| 302 |
+
if no_truncate or not rows:
|
| 303 |
+
return False
|
| 304 |
+
|
| 305 |
+
n = len(headers)
|
| 306 |
+
# Per-column natural width: longest of header label and cell values.
|
| 307 |
+
natural = [max(len(headers[c]), *(len(rows[r][c]) for r in range(len(rows)))) for c in range(n)]
|
| 308 |
+
|
| 309 |
+
# `max(0, n - 1)` accounts for the single-space separator between columns.
|
| 310 |
+
budget = shutil.get_terminal_size().columns - max(0, n - 1)
|
| 311 |
+
if sum(natural) <= budget:
|
| 312 |
+
return False
|
| 313 |
+
|
| 314 |
+
# Shrink the widest column 1 char at a time. Floors keep the header label
|
| 315 |
+
# visible; the `4` is the smallest cap that still shows "x..." (one content
|
| 316 |
+
# char plus the "..." marker).
|
| 317 |
+
caps = natural.copy()
|
| 318 |
+
min_widths = [max(len(h), 4) for h in headers]
|
| 319 |
+
while sum(caps) > budget:
|
| 320 |
+
widest = max(
|
| 321 |
+
(i for i, w in enumerate(caps) if w > min_widths[i]),
|
| 322 |
+
key=lambda i: caps[i],
|
| 323 |
+
default=-1,
|
| 324 |
+
)
|
| 325 |
+
if widest < 0:
|
| 326 |
+
break # everything at floor — table wraps slightly
|
| 327 |
+
caps[widest] -= 1
|
| 328 |
+
|
| 329 |
+
truncated = False
|
| 330 |
+
for row in rows:
|
| 331 |
+
for c, cell in enumerate(row):
|
| 332 |
+
if len(cell) > caps[c]:
|
| 333 |
+
truncated = True
|
| 334 |
+
row[c] = cell[: caps[c] - 3] + "..."
|
| 335 |
+
return truncated
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
def _format_table_cell_agent(value: Any) -> str:
|
| 339 |
+
"""Format a cell value for agent TSV output (ISO timestamps, tabs escaped)."""
|
| 340 |
+
if isinstance(value, datetime.datetime):
|
| 341 |
+
return value.isoformat()
|
| 342 |
+
return _single_line(str(value))
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
out = Output()
|
venv/Lib/site-packages/huggingface_hub/cli/_skills.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Internal helpers for Hugging Face marketplace skill installation and upgrades."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import shutil
|
| 5 |
+
import tempfile
|
| 6 |
+
from dataclasses import dataclass, replace
|
| 7 |
+
from pathlib import Path, PurePosixPath
|
| 8 |
+
from typing import Any, Literal
|
| 9 |
+
|
| 10 |
+
from huggingface_hub._buckets import BucketFile
|
| 11 |
+
from huggingface_hub.errors import CLIError
|
| 12 |
+
|
| 13 |
+
from ..utils import disable_progress_bars
|
| 14 |
+
from ._cli_utils import get_hf_api
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
DEFAULT_SKILLS_BUCKET_ID = "huggingface/skills"
|
| 18 |
+
MARKETPLACE_PATH = "marketplace.json"
|
| 19 |
+
# Empty marker file dropped into managed skill installs so `hf skills update` knows
|
| 20 |
+
# to touch them and leave user-placed skill dirs alone. Filename is historical (used
|
| 21 |
+
# to be a JSON manifest with a revision); we keep it for backward compat with installs
|
| 22 |
+
# made by previous versions.
|
| 23 |
+
MANAGED_MARKER_FILENAME = ".hf-skill-manifest.json"
|
| 24 |
+
|
| 25 |
+
SkillUpdateStatus = Literal["up_to_date", "unmanaged", "source_unreachable"]
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@dataclass(frozen=True)
|
| 29 |
+
class MarketplaceSkill:
|
| 30 |
+
name: str
|
| 31 |
+
repo_path: str
|
| 32 |
+
description: str | None = None
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@dataclass(frozen=True)
|
| 36 |
+
class SkillUpdateInfo:
|
| 37 |
+
name: str
|
| 38 |
+
skill_dir: Path
|
| 39 |
+
status: SkillUpdateStatus
|
| 40 |
+
detail: str | None = None
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def add_skill(skill_name: str, destination_root: Path, force: bool = False) -> Path:
|
| 44 |
+
"""Resolve a marketplace skill by name and install it."""
|
| 45 |
+
api = get_hf_api()
|
| 46 |
+
with disable_progress_bars():
|
| 47 |
+
marketplace_skills = _load_marketplace_skills(api)
|
| 48 |
+
skill = _select_marketplace_skill(marketplace_skills, skill_name)
|
| 49 |
+
if skill is None:
|
| 50 |
+
raise CLIError(
|
| 51 |
+
f"Skill '{skill_name}' not found in {DEFAULT_SKILLS_BUCKET_ID}. "
|
| 52 |
+
"Try `hf skills add` to install `hf-cli` or use a known skill name."
|
| 53 |
+
)
|
| 54 |
+
return _install_marketplace_skill(api, skill, destination_root, force=force)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def update_skills(roots: list[Path], selector: str | None = None) -> list[SkillUpdateInfo]:
|
| 58 |
+
"""Re-sync managed marketplace skill installs from the bucket."""
|
| 59 |
+
skill_dirs = _iter_unique_skill_dirs(roots)
|
| 60 |
+
if selector is not None:
|
| 61 |
+
selector_lower = selector.strip().lower()
|
| 62 |
+
skill_dirs = [d for d in skill_dirs if d.name.lower() == selector_lower]
|
| 63 |
+
if not skill_dirs:
|
| 64 |
+
raise CLIError(f"No installed skill matches '{selector}'. Install it with `hf skills add {selector}`.")
|
| 65 |
+
|
| 66 |
+
api = get_hf_api()
|
| 67 |
+
with disable_progress_bars():
|
| 68 |
+
marketplace_skills = {skill.name.lower(): skill for skill in _load_marketplace_skills(api)}
|
| 69 |
+
return [_apply_single_update(api, skill_dir, marketplace_skills) for skill_dir in skill_dirs]
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _load_marketplace_skills(api) -> list[MarketplaceSkill]:
|
| 73 |
+
payload = _load_marketplace_payload(api)
|
| 74 |
+
plugins = payload.get("plugins")
|
| 75 |
+
if not isinstance(plugins, list):
|
| 76 |
+
raise CLIError("Invalid marketplace payload: expected a top-level 'plugins' list.")
|
| 77 |
+
|
| 78 |
+
skills: list[MarketplaceSkill] = []
|
| 79 |
+
for plugin in plugins:
|
| 80 |
+
if not isinstance(plugin, dict):
|
| 81 |
+
continue
|
| 82 |
+
name = plugin.get("name")
|
| 83 |
+
source = plugin.get("source")
|
| 84 |
+
if not isinstance(name, str) or not isinstance(source, str):
|
| 85 |
+
continue
|
| 86 |
+
description = plugin.get("description")
|
| 87 |
+
skills.append(
|
| 88 |
+
MarketplaceSkill(
|
| 89 |
+
name=name,
|
| 90 |
+
repo_path=_normalize_repo_path(source),
|
| 91 |
+
description=description if isinstance(description, str) else None,
|
| 92 |
+
)
|
| 93 |
+
)
|
| 94 |
+
return skills
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _install_marketplace_skill(api, skill: MarketplaceSkill, destination_root: Path, force: bool = False) -> Path:
|
| 98 |
+
"""Install a marketplace skill into a local skills directory."""
|
| 99 |
+
destination_root = destination_root.expanduser().resolve()
|
| 100 |
+
destination_root.mkdir(parents=True, exist_ok=True)
|
| 101 |
+
install_dir = destination_root / skill.name
|
| 102 |
+
already_exists = install_dir.exists()
|
| 103 |
+
|
| 104 |
+
if already_exists and not force:
|
| 105 |
+
raise FileExistsError(f"Skill already exists: {install_dir}")
|
| 106 |
+
|
| 107 |
+
if already_exists:
|
| 108 |
+
# Stage the new content in a sibling tempdir and atomically rename, so the
|
| 109 |
+
# existing install stays intact if the download fails halfway through.
|
| 110 |
+
with tempfile.TemporaryDirectory(dir=destination_root, prefix=f".{install_dir.name}.install-") as tmp_dir_str:
|
| 111 |
+
staged_dir = Path(tmp_dir_str) / install_dir.name
|
| 112 |
+
_populate_install_dir(api, skill=skill, install_dir=staged_dir)
|
| 113 |
+
_atomic_replace_directory(existing_dir=install_dir, staged_dir=staged_dir)
|
| 114 |
+
return install_dir
|
| 115 |
+
|
| 116 |
+
try:
|
| 117 |
+
_populate_install_dir(api, skill=skill, install_dir=install_dir)
|
| 118 |
+
except Exception:
|
| 119 |
+
if install_dir.exists():
|
| 120 |
+
shutil.rmtree(install_dir)
|
| 121 |
+
raise
|
| 122 |
+
return install_dir
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def _load_marketplace_payload(api) -> dict[str, Any]:
|
| 126 |
+
with tempfile.TemporaryDirectory() as tmp_dir:
|
| 127 |
+
local_path = Path(tmp_dir) / "marketplace.json"
|
| 128 |
+
api.download_bucket_files(
|
| 129 |
+
DEFAULT_SKILLS_BUCKET_ID,
|
| 130 |
+
[(MARKETPLACE_PATH, local_path)],
|
| 131 |
+
raise_on_missing_files=True,
|
| 132 |
+
)
|
| 133 |
+
parsed = json.loads(local_path.read_text(encoding="utf-8"))
|
| 134 |
+
|
| 135 |
+
if not isinstance(parsed, dict):
|
| 136 |
+
raise CLIError("Invalid marketplace payload: expected a JSON object.")
|
| 137 |
+
return parsed
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _select_marketplace_skill(skills: list[MarketplaceSkill], selector: str) -> MarketplaceSkill | None:
|
| 141 |
+
selector_lower = selector.strip().lower()
|
| 142 |
+
for skill in skills:
|
| 143 |
+
if skill.name.lower() == selector_lower:
|
| 144 |
+
return skill
|
| 145 |
+
return None
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _normalize_repo_path(path: str) -> str:
|
| 149 |
+
normalized = path.strip()
|
| 150 |
+
while normalized.startswith("./"):
|
| 151 |
+
normalized = normalized[2:]
|
| 152 |
+
normalized = normalized.strip("/")
|
| 153 |
+
if not normalized:
|
| 154 |
+
raise CLIError("Invalid marketplace entry: empty source path.")
|
| 155 |
+
return normalized
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def _populate_install_dir(api, skill: MarketplaceSkill, install_dir: Path) -> None:
|
| 159 |
+
install_dir.mkdir(parents=True, exist_ok=True)
|
| 160 |
+
bucket_files = _list_skill_files(api, skill)
|
| 161 |
+
_download_skill_files(api, skill, bucket_files, install_dir)
|
| 162 |
+
_validate_installed_skill_dir(install_dir)
|
| 163 |
+
(install_dir / MANAGED_MARKER_FILENAME).touch()
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def _validate_installed_skill_dir(skill_dir: Path) -> None:
|
| 167 |
+
skill_file = skill_dir / "SKILL.md"
|
| 168 |
+
if not skill_file.is_file():
|
| 169 |
+
raise RuntimeError(f"Installed skill is missing SKILL.md: {skill_file}")
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _list_skill_files(api, skill: MarketplaceSkill) -> list[BucketFile]:
|
| 173 |
+
"""List all files under `skill.repo_path` in the marketplace bucket."""
|
| 174 |
+
prefix = skill.repo_path.rstrip("/")
|
| 175 |
+
files: list[BucketFile] = [
|
| 176 |
+
item
|
| 177 |
+
for item in api.list_bucket_tree(DEFAULT_SKILLS_BUCKET_ID, prefix=prefix, recursive=True)
|
| 178 |
+
if isinstance(item, BucketFile)
|
| 179 |
+
]
|
| 180 |
+
if not files:
|
| 181 |
+
raise FileNotFoundError(f"Path '{prefix}' not found in bucket '{DEFAULT_SKILLS_BUCKET_ID}'.")
|
| 182 |
+
return files
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def _download_skill_files(api, skill: MarketplaceSkill, files: list[BucketFile], install_dir: Path) -> None:
|
| 186 |
+
"""Download bucket files into `install_dir`."""
|
| 187 |
+
prefix = skill.repo_path.rstrip("/")
|
| 188 |
+
prefix_with_slash = f"{prefix}/"
|
| 189 |
+
|
| 190 |
+
# `list_bucket_tree(prefix=...)` matches as a raw string prefix, so e.g. asking for
|
| 191 |
+
# "skills/gradio" can also return "skills/gradio-tools/...". Filter on the trailing
|
| 192 |
+
# slash to keep only files actually inside the directory, then strip it so files land
|
| 193 |
+
# directly under `install_dir` preserving any nested structure.
|
| 194 |
+
download_specs: list[tuple[str | BucketFile, str | Path]] = []
|
| 195 |
+
for bucket_file in files:
|
| 196 |
+
if not bucket_file.path.startswith(prefix_with_slash):
|
| 197 |
+
continue
|
| 198 |
+
relative = bucket_file.path[len(prefix_with_slash) :]
|
| 199 |
+
local_file = install_dir.joinpath(*PurePosixPath(relative).parts)
|
| 200 |
+
local_file.parent.mkdir(parents=True, exist_ok=True)
|
| 201 |
+
download_specs.append((bucket_file, local_file))
|
| 202 |
+
|
| 203 |
+
if not download_specs:
|
| 204 |
+
raise FileNotFoundError(f"No files found under '{prefix}' in bucket '{DEFAULT_SKILLS_BUCKET_ID}'.")
|
| 205 |
+
|
| 206 |
+
api.download_bucket_files(DEFAULT_SKILLS_BUCKET_ID, download_specs)
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def _atomic_replace_directory(existing_dir: Path, staged_dir: Path) -> None:
|
| 210 |
+
backup_dir = staged_dir.parent / f"{existing_dir.name}.backup"
|
| 211 |
+
try:
|
| 212 |
+
existing_dir.rename(backup_dir)
|
| 213 |
+
staged_dir.rename(existing_dir)
|
| 214 |
+
shutil.rmtree(backup_dir)
|
| 215 |
+
except Exception:
|
| 216 |
+
if backup_dir.exists() and not existing_dir.exists():
|
| 217 |
+
backup_dir.rename(existing_dir)
|
| 218 |
+
raise
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def _iter_unique_skill_dirs(roots: list[Path]) -> list[Path]:
|
| 222 |
+
seen: set[Path] = set()
|
| 223 |
+
discovered: list[Path] = []
|
| 224 |
+
for root in roots:
|
| 225 |
+
root = root.expanduser().resolve()
|
| 226 |
+
if not root.is_dir():
|
| 227 |
+
continue
|
| 228 |
+
for child in sorted(root.iterdir()):
|
| 229 |
+
if child.name.startswith("."):
|
| 230 |
+
continue
|
| 231 |
+
if not child.is_dir() and not child.is_symlink():
|
| 232 |
+
continue
|
| 233 |
+
resolved = child.resolve()
|
| 234 |
+
if resolved in seen or not resolved.is_dir():
|
| 235 |
+
continue
|
| 236 |
+
seen.add(resolved)
|
| 237 |
+
discovered.append(resolved)
|
| 238 |
+
return discovered
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def _apply_single_update(api, skill_dir: Path, marketplace_skills: dict[str, MarketplaceSkill]) -> SkillUpdateInfo:
|
| 242 |
+
base = SkillUpdateInfo(name=skill_dir.name, skill_dir=skill_dir, status="unmanaged")
|
| 243 |
+
|
| 244 |
+
if not (skill_dir / MANAGED_MARKER_FILENAME).exists():
|
| 245 |
+
return base
|
| 246 |
+
|
| 247 |
+
skill = marketplace_skills.get(skill_dir.name.lower())
|
| 248 |
+
if skill is None:
|
| 249 |
+
return replace(
|
| 250 |
+
base,
|
| 251 |
+
status="source_unreachable",
|
| 252 |
+
detail=f"Skill '{skill_dir.name}' is no longer available in {DEFAULT_SKILLS_BUCKET_ID}.",
|
| 253 |
+
)
|
| 254 |
+
|
| 255 |
+
try:
|
| 256 |
+
_install_marketplace_skill(api, skill, skill_dir.parent, force=True)
|
| 257 |
+
except Exception as exc:
|
| 258 |
+
return replace(base, status="source_unreachable", detail=str(exc))
|
| 259 |
+
|
| 260 |
+
return replace(base, status="up_to_date")
|
venv/Lib/site-packages/huggingface_hub/cli/auth.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2020 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains commands to authenticate to the Hugging Face Hub and interact with your repositories.
|
| 15 |
+
|
| 16 |
+
Usage:
|
| 17 |
+
# login and save token locally.
|
| 18 |
+
hf auth login --token=hf_*** --add-to-git-credential
|
| 19 |
+
|
| 20 |
+
# switch between tokens
|
| 21 |
+
hf auth switch
|
| 22 |
+
|
| 23 |
+
# list all tokens
|
| 24 |
+
hf auth list
|
| 25 |
+
|
| 26 |
+
# logout from all tokens
|
| 27 |
+
hf auth logout
|
| 28 |
+
|
| 29 |
+
# check which account you are logged in as
|
| 30 |
+
hf auth whoami
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
from typing import Annotated
|
| 34 |
+
|
| 35 |
+
import typer
|
| 36 |
+
|
| 37 |
+
from huggingface_hub.constants import ENDPOINT
|
| 38 |
+
from huggingface_hub.hf_api import whoami
|
| 39 |
+
|
| 40 |
+
from .._login import auth_list, auth_switch, login, logout
|
| 41 |
+
from ..utils import get_stored_tokens, get_token, logging
|
| 42 |
+
from ._cli_utils import TokenOpt, typer_factory
|
| 43 |
+
from ._output import out
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
logger = logging.get_logger(__name__)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
auth_cli = typer_factory(help="Manage authentication (login, logout, etc.).")
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
@auth_cli.command(
|
| 53 |
+
"login",
|
| 54 |
+
examples=[
|
| 55 |
+
"hf auth login",
|
| 56 |
+
"hf auth login --token $HF_TOKEN",
|
| 57 |
+
"hf auth login --token $HF_TOKEN --add-to-git-credential",
|
| 58 |
+
"hf auth login --force",
|
| 59 |
+
],
|
| 60 |
+
)
|
| 61 |
+
def auth_login(
|
| 62 |
+
token: TokenOpt = None,
|
| 63 |
+
add_to_git_credential: Annotated[
|
| 64 |
+
bool,
|
| 65 |
+
typer.Option(
|
| 66 |
+
help="Save to git credential helper. Useful only if you plan to run git commands directly.",
|
| 67 |
+
),
|
| 68 |
+
] = False,
|
| 69 |
+
force: Annotated[
|
| 70 |
+
bool,
|
| 71 |
+
typer.Option(
|
| 72 |
+
help="Force re-login even if already logged in.",
|
| 73 |
+
),
|
| 74 |
+
] = False,
|
| 75 |
+
) -> None:
|
| 76 |
+
"""Login using a token from huggingface.co/settings/tokens."""
|
| 77 |
+
login(token=token, add_to_git_credential=add_to_git_credential, skip_if_logged_in=not force)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
@auth_cli.command(
|
| 81 |
+
"logout",
|
| 82 |
+
examples=["hf auth logout", "hf auth logout --token-name my-token"],
|
| 83 |
+
)
|
| 84 |
+
def auth_logout(
|
| 85 |
+
token_name: Annotated[
|
| 86 |
+
str | None,
|
| 87 |
+
typer.Option(help="Name of token to logout"),
|
| 88 |
+
] = None,
|
| 89 |
+
) -> None:
|
| 90 |
+
"""Logout from a specific token."""
|
| 91 |
+
logout(token_name=token_name)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _select_token_name() -> str | None:
|
| 95 |
+
token_names = list(get_stored_tokens().keys())
|
| 96 |
+
|
| 97 |
+
if not token_names:
|
| 98 |
+
logger.error("No stored tokens found. Please login first.")
|
| 99 |
+
return None
|
| 100 |
+
|
| 101 |
+
print("Available stored tokens:")
|
| 102 |
+
for i, token_name in enumerate(token_names, 1):
|
| 103 |
+
print(f"{i}. {token_name}")
|
| 104 |
+
while True:
|
| 105 |
+
try:
|
| 106 |
+
choice = input("Enter the number of the token to switch to (or 'q' to quit): ")
|
| 107 |
+
if choice.lower() == "q":
|
| 108 |
+
return None
|
| 109 |
+
index = int(choice) - 1
|
| 110 |
+
if 0 <= index < len(token_names):
|
| 111 |
+
return token_names[index]
|
| 112 |
+
else:
|
| 113 |
+
print("Invalid selection. Please try again.")
|
| 114 |
+
except ValueError:
|
| 115 |
+
print("Invalid input. Please enter a number or 'q' to quit.")
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
@auth_cli.command(
|
| 119 |
+
"switch",
|
| 120 |
+
examples=["hf auth switch", "hf auth switch --token-name my-token"],
|
| 121 |
+
)
|
| 122 |
+
def auth_switch_cmd(
|
| 123 |
+
token_name: Annotated[
|
| 124 |
+
str | None,
|
| 125 |
+
typer.Option(
|
| 126 |
+
help="Name of the token to switch to",
|
| 127 |
+
),
|
| 128 |
+
] = None,
|
| 129 |
+
add_to_git_credential: Annotated[
|
| 130 |
+
bool,
|
| 131 |
+
typer.Option(
|
| 132 |
+
help="Save to git credential helper. Useful only if you plan to run git commands directly.",
|
| 133 |
+
),
|
| 134 |
+
] = False,
|
| 135 |
+
) -> None:
|
| 136 |
+
"""Switch between access tokens."""
|
| 137 |
+
if token_name is None:
|
| 138 |
+
token_name = _select_token_name()
|
| 139 |
+
if token_name is None:
|
| 140 |
+
print("No token name provided. Aborting.")
|
| 141 |
+
raise typer.Exit()
|
| 142 |
+
auth_switch(token_name, add_to_git_credential=add_to_git_credential)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
@auth_cli.command("list | ls", examples=["hf auth list"])
|
| 146 |
+
def auth_list_cmd() -> None:
|
| 147 |
+
"""List all stored access tokens."""
|
| 148 |
+
auth_list()
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
@auth_cli.command("token", examples=["hf auth token", "hf auth token | xargs curl -H 'Authorization: Bearer {}'"])
|
| 152 |
+
def auth_token() -> None:
|
| 153 |
+
"""Print the current access token to stdout."""
|
| 154 |
+
token = get_token()
|
| 155 |
+
if token is None:
|
| 156 |
+
out.error("Not logged in. Run `hf auth login` first.")
|
| 157 |
+
raise typer.Exit(code=1)
|
| 158 |
+
print(token)
|
| 159 |
+
out.hint("Run `hf auth whoami` to see which account this token belongs to.")
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
@auth_cli.command("whoami", examples=["hf auth whoami", "hf auth whoami --format json"])
|
| 163 |
+
def auth_whoami() -> None:
|
| 164 |
+
"""Find out which huggingface.co account you are logged in as."""
|
| 165 |
+
|
| 166 |
+
token = get_token()
|
| 167 |
+
if token is None:
|
| 168 |
+
out.error("Not logged in")
|
| 169 |
+
raise typer.Exit(code=1)
|
| 170 |
+
|
| 171 |
+
info = whoami(token)
|
| 172 |
+
orgs = ",".join(org["name"] for org in info["orgs"]) or None
|
| 173 |
+
endpoint = ENDPOINT if ENDPOINT != "https://huggingface.co" else None
|
| 174 |
+
out.result("Logged in", user=info["name"], orgs=orgs, endpoint=endpoint)
|
venv/Lib/site-packages/huggingface_hub/cli/buckets.py
ADDED
|
@@ -0,0 +1,678 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains commands to interact with buckets via the CLI."""
|
| 15 |
+
|
| 16 |
+
from typing import Annotated
|
| 17 |
+
|
| 18 |
+
import typer
|
| 19 |
+
|
| 20 |
+
from huggingface_hub import logging
|
| 21 |
+
from huggingface_hub._buckets import (
|
| 22 |
+
BUCKET_PREFIX,
|
| 23 |
+
BucketFile,
|
| 24 |
+
FilterMatcher,
|
| 25 |
+
_parse_bucket_uri,
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
from ..hf_api import REPO_REGIONS
|
| 29 |
+
from ._cli_utils import (
|
| 30 |
+
SearchOpt,
|
| 31 |
+
TokenOpt,
|
| 32 |
+
get_hf_api,
|
| 33 |
+
typer_factory,
|
| 34 |
+
)
|
| 35 |
+
from ._cp import make_cp
|
| 36 |
+
from ._file_listing import format_size, print_file_listing
|
| 37 |
+
from ._output import OutputFormat, out
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
logger = logging.get_logger(__name__)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
buckets_cli = typer_factory(help="Commands to interact with buckets.")
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@buckets_cli.command(
|
| 47 |
+
name="create",
|
| 48 |
+
examples=[
|
| 49 |
+
"hf buckets create my-bucket",
|
| 50 |
+
"hf buckets create user/my-bucket",
|
| 51 |
+
"hf buckets create hf://buckets/user/my-bucket",
|
| 52 |
+
"hf buckets create user/my-bucket --private",
|
| 53 |
+
"hf buckets create user/my-bucket --exist-ok",
|
| 54 |
+
"hf buckets create user/my-bucket --region us",
|
| 55 |
+
],
|
| 56 |
+
)
|
| 57 |
+
def create(
|
| 58 |
+
bucket_id: Annotated[
|
| 59 |
+
str,
|
| 60 |
+
typer.Argument(
|
| 61 |
+
help="Bucket ID: bucket_name, namespace/bucket_name, or hf://buckets/namespace/bucket_name",
|
| 62 |
+
),
|
| 63 |
+
],
|
| 64 |
+
private: Annotated[
|
| 65 |
+
bool,
|
| 66 |
+
typer.Option(
|
| 67 |
+
"--private",
|
| 68 |
+
help="Create a private bucket.",
|
| 69 |
+
),
|
| 70 |
+
] = False,
|
| 71 |
+
region: Annotated[
|
| 72 |
+
REPO_REGIONS | None,
|
| 73 |
+
typer.Option(
|
| 74 |
+
"--region",
|
| 75 |
+
help="Cloud region in which to create the bucket. Can be one of 'us' or 'eu'. Requires Team plan or above.",
|
| 76 |
+
),
|
| 77 |
+
] = None,
|
| 78 |
+
exist_ok: Annotated[
|
| 79 |
+
bool,
|
| 80 |
+
typer.Option(
|
| 81 |
+
"--exist-ok",
|
| 82 |
+
help="Do not raise an error if the bucket already exists.",
|
| 83 |
+
),
|
| 84 |
+
] = False,
|
| 85 |
+
token: TokenOpt = None,
|
| 86 |
+
) -> None:
|
| 87 |
+
"""Create a new bucket."""
|
| 88 |
+
api = get_hf_api(token=token)
|
| 89 |
+
|
| 90 |
+
if bucket_id.startswith(BUCKET_PREFIX):
|
| 91 |
+
parsed = _parse_bucket_uri(bucket_id)
|
| 92 |
+
if parsed.path_in_repo:
|
| 93 |
+
raise typer.BadParameter(
|
| 94 |
+
f"Cannot specify a prefix for bucket creation: {bucket_id}."
|
| 95 |
+
f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name."
|
| 96 |
+
)
|
| 97 |
+
bucket_id = parsed.id
|
| 98 |
+
|
| 99 |
+
bucket_url = api.create_bucket(
|
| 100 |
+
bucket_id,
|
| 101 |
+
private=private if private else None,
|
| 102 |
+
region=region,
|
| 103 |
+
exist_ok=exist_ok,
|
| 104 |
+
)
|
| 105 |
+
out.result("Bucket created", uri=bucket_url.uri.to_uri(), url=bucket_url.url)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _is_bucket_id(argument: str) -> bool:
|
| 109 |
+
"""Check if argument is a bucket ID (namespace/name) vs just a namespace."""
|
| 110 |
+
if argument.startswith(BUCKET_PREFIX):
|
| 111 |
+
path = argument[len(BUCKET_PREFIX) :]
|
| 112 |
+
else:
|
| 113 |
+
path = argument
|
| 114 |
+
return "/" in path
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
@buckets_cli.command(
|
| 118 |
+
name="list | ls",
|
| 119 |
+
examples=[
|
| 120 |
+
"hf buckets list",
|
| 121 |
+
"hf buckets list huggingface",
|
| 122 |
+
'hf buckets list --search "my-prefix"',
|
| 123 |
+
"hf buckets list user/my-bucket",
|
| 124 |
+
"hf buckets list user/my-bucket -R",
|
| 125 |
+
"hf buckets list user/my-bucket -h",
|
| 126 |
+
"hf buckets list user/my-bucket --tree",
|
| 127 |
+
"hf buckets list user/my-bucket --tree -h",
|
| 128 |
+
"hf buckets list hf://buckets/user/my-bucket",
|
| 129 |
+
"hf buckets list user/my-bucket/sub -R",
|
| 130 |
+
],
|
| 131 |
+
)
|
| 132 |
+
def list_cmd(
|
| 133 |
+
argument: Annotated[
|
| 134 |
+
str | None,
|
| 135 |
+
typer.Argument(
|
| 136 |
+
help=(
|
| 137 |
+
"Namespace (user or org) to list buckets, or bucket ID"
|
| 138 |
+
" (namespace/bucket_name(/prefix) or hf://buckets/...) to list files."
|
| 139 |
+
),
|
| 140 |
+
),
|
| 141 |
+
] = None,
|
| 142 |
+
human_readable: Annotated[
|
| 143 |
+
bool,
|
| 144 |
+
typer.Option(
|
| 145 |
+
"--human-readable",
|
| 146 |
+
"-h",
|
| 147 |
+
help="Show sizes in human readable format.",
|
| 148 |
+
),
|
| 149 |
+
] = False,
|
| 150 |
+
as_tree: Annotated[
|
| 151 |
+
bool,
|
| 152 |
+
typer.Option(
|
| 153 |
+
"--tree",
|
| 154 |
+
help="List files in tree format (only for listing files).",
|
| 155 |
+
),
|
| 156 |
+
] = False,
|
| 157 |
+
recursive: Annotated[
|
| 158 |
+
bool,
|
| 159 |
+
typer.Option(
|
| 160 |
+
"--recursive",
|
| 161 |
+
"-R",
|
| 162 |
+
help="List files recursively (only for listing files).",
|
| 163 |
+
),
|
| 164 |
+
] = False,
|
| 165 |
+
search: SearchOpt = None,
|
| 166 |
+
token: TokenOpt = None,
|
| 167 |
+
) -> None:
|
| 168 |
+
"""List buckets or files in a bucket.
|
| 169 |
+
|
| 170 |
+
When called with no argument or a namespace, lists buckets.
|
| 171 |
+
When called with a bucket ID (namespace/bucket_name), lists files in the bucket.
|
| 172 |
+
"""
|
| 173 |
+
# Determine mode: listing buckets or listing files
|
| 174 |
+
is_file_mode = argument is not None and _is_bucket_id(argument)
|
| 175 |
+
|
| 176 |
+
if is_file_mode:
|
| 177 |
+
if search is not None:
|
| 178 |
+
raise typer.BadParameter("Cannot use --search when listing files.")
|
| 179 |
+
_list_files(
|
| 180 |
+
argument=argument, # type: ignore
|
| 181 |
+
human_readable=human_readable,
|
| 182 |
+
as_tree=as_tree,
|
| 183 |
+
recursive=recursive,
|
| 184 |
+
token=token,
|
| 185 |
+
)
|
| 186 |
+
else:
|
| 187 |
+
_list_buckets(
|
| 188 |
+
namespace=argument,
|
| 189 |
+
search=search,
|
| 190 |
+
human_readable=human_readable,
|
| 191 |
+
as_tree=as_tree,
|
| 192 |
+
recursive=recursive,
|
| 193 |
+
token=token,
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _list_buckets(
|
| 198 |
+
namespace: str | None,
|
| 199 |
+
search: str | None,
|
| 200 |
+
human_readable: bool,
|
| 201 |
+
as_tree: bool,
|
| 202 |
+
recursive: bool,
|
| 203 |
+
token: str | None,
|
| 204 |
+
) -> None:
|
| 205 |
+
"""List buckets in a namespace."""
|
| 206 |
+
# Validate incompatible flags
|
| 207 |
+
if as_tree:
|
| 208 |
+
raise typer.BadParameter("Cannot use --tree when listing buckets.")
|
| 209 |
+
if recursive:
|
| 210 |
+
raise typer.BadParameter("Cannot use --recursive when listing buckets.")
|
| 211 |
+
|
| 212 |
+
# Handle hf://buckets/namespace format
|
| 213 |
+
if namespace is not None and namespace.startswith(BUCKET_PREFIX):
|
| 214 |
+
namespace = namespace[len(BUCKET_PREFIX) :]
|
| 215 |
+
# Strip trailing slash if any
|
| 216 |
+
namespace = namespace.rstrip("/")
|
| 217 |
+
|
| 218 |
+
api = get_hf_api(token=token)
|
| 219 |
+
items = [
|
| 220 |
+
{
|
| 221 |
+
"id": bucket.id,
|
| 222 |
+
"private": bucket.private,
|
| 223 |
+
"size": format_size(bucket.size, human_readable) if human_readable else bucket.size,
|
| 224 |
+
"total_files": bucket.total_files,
|
| 225 |
+
"created_at": bucket.created_at,
|
| 226 |
+
}
|
| 227 |
+
for bucket in api.list_buckets(namespace=namespace, search=search)
|
| 228 |
+
]
|
| 229 |
+
out.table(items, alignments={"size": "right"})
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _list_files(
|
| 233 |
+
argument: str,
|
| 234 |
+
human_readable: bool,
|
| 235 |
+
as_tree: bool,
|
| 236 |
+
recursive: bool,
|
| 237 |
+
token: str | None,
|
| 238 |
+
) -> None:
|
| 239 |
+
"""List files in a bucket."""
|
| 240 |
+
if as_tree and out.mode == OutputFormat.json:
|
| 241 |
+
raise typer.BadParameter("Cannot use --tree with --format json.")
|
| 242 |
+
|
| 243 |
+
api = get_hf_api(token=token)
|
| 244 |
+
parsed = _parse_bucket_uri(argument)
|
| 245 |
+
items = list(
|
| 246 |
+
api.list_bucket_tree(
|
| 247 |
+
parsed.id,
|
| 248 |
+
prefix=parsed.path_in_repo or None,
|
| 249 |
+
recursive=recursive,
|
| 250 |
+
)
|
| 251 |
+
)
|
| 252 |
+
|
| 253 |
+
print_file_listing(items, human_readable=human_readable, as_tree=as_tree, recursive=recursive)
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
@buckets_cli.command(
|
| 257 |
+
name="info",
|
| 258 |
+
examples=[
|
| 259 |
+
"hf buckets info user/my-bucket",
|
| 260 |
+
"hf buckets info hf://buckets/user/my-bucket",
|
| 261 |
+
],
|
| 262 |
+
)
|
| 263 |
+
def info(
|
| 264 |
+
bucket_id: Annotated[
|
| 265 |
+
str,
|
| 266 |
+
typer.Argument(
|
| 267 |
+
help="Bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name",
|
| 268 |
+
),
|
| 269 |
+
],
|
| 270 |
+
token: TokenOpt = None,
|
| 271 |
+
) -> None:
|
| 272 |
+
"""Get info about a bucket."""
|
| 273 |
+
api = get_hf_api(token=token)
|
| 274 |
+
parsed = _parse_bucket_uri(bucket_id)
|
| 275 |
+
bucket = api.bucket_info(parsed.id)
|
| 276 |
+
out.dict(bucket, id_key="id")
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
@buckets_cli.command(
|
| 280 |
+
name="delete",
|
| 281 |
+
examples=[
|
| 282 |
+
"hf buckets delete user/my-bucket",
|
| 283 |
+
"hf buckets delete hf://buckets/user/my-bucket",
|
| 284 |
+
"hf buckets delete user/my-bucket --yes",
|
| 285 |
+
"hf buckets delete user/my-bucket --missing-ok",
|
| 286 |
+
],
|
| 287 |
+
)
|
| 288 |
+
def delete(
|
| 289 |
+
bucket_id: Annotated[
|
| 290 |
+
str,
|
| 291 |
+
typer.Argument(
|
| 292 |
+
help="Bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name",
|
| 293 |
+
),
|
| 294 |
+
],
|
| 295 |
+
yes: Annotated[
|
| 296 |
+
bool,
|
| 297 |
+
typer.Option(
|
| 298 |
+
"--yes",
|
| 299 |
+
"-y",
|
| 300 |
+
help="Skip confirmation prompt.",
|
| 301 |
+
),
|
| 302 |
+
] = False,
|
| 303 |
+
missing_ok: Annotated[
|
| 304 |
+
bool,
|
| 305 |
+
typer.Option(
|
| 306 |
+
"--missing-ok",
|
| 307 |
+
help="Do not raise an error if the bucket does not exist.",
|
| 308 |
+
),
|
| 309 |
+
] = False,
|
| 310 |
+
token: TokenOpt = None,
|
| 311 |
+
) -> None:
|
| 312 |
+
"""Delete a bucket.
|
| 313 |
+
|
| 314 |
+
This deletes the entire bucket and all its contents. Use `hf buckets rm` to remove individual files.
|
| 315 |
+
"""
|
| 316 |
+
if bucket_id.startswith(BUCKET_PREFIX):
|
| 317 |
+
parsed = _parse_bucket_uri(bucket_id)
|
| 318 |
+
if parsed.path_in_repo:
|
| 319 |
+
raise typer.BadParameter(
|
| 320 |
+
f"Cannot specify a prefix for bucket deletion: {bucket_id}."
|
| 321 |
+
f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name."
|
| 322 |
+
)
|
| 323 |
+
bucket_id = parsed.id
|
| 324 |
+
elif "/" not in bucket_id:
|
| 325 |
+
raise typer.BadParameter(
|
| 326 |
+
f"Invalid bucket ID: {bucket_id}."
|
| 327 |
+
f" Must be in format namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name."
|
| 328 |
+
)
|
| 329 |
+
|
| 330 |
+
out.confirm(f"Are you sure you want to delete bucket '{bucket_id}'?", yes=yes)
|
| 331 |
+
|
| 332 |
+
api = get_hf_api(token=token)
|
| 333 |
+
api.delete_bucket(bucket_id, missing_ok=missing_ok)
|
| 334 |
+
out.result("Bucket deleted", bucket_id=bucket_id)
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
@buckets_cli.command(
|
| 338 |
+
name="remove | rm",
|
| 339 |
+
examples=[
|
| 340 |
+
"hf buckets remove user/my-bucket/file.txt",
|
| 341 |
+
"hf buckets rm hf://buckets/user/my-bucket/file.txt",
|
| 342 |
+
"hf buckets rm user/my-bucket/logs/ --recursive",
|
| 343 |
+
'hf buckets rm user/my-bucket --recursive --include "*.tmp"',
|
| 344 |
+
"hf buckets rm user/my-bucket/data/ --recursive --dry-run",
|
| 345 |
+
],
|
| 346 |
+
)
|
| 347 |
+
def remove(
|
| 348 |
+
argument: Annotated[
|
| 349 |
+
str,
|
| 350 |
+
typer.Argument(
|
| 351 |
+
help=(
|
| 352 |
+
"Bucket path: namespace/bucket_name/path or hf://buckets/namespace/bucket_name/path."
|
| 353 |
+
" With --recursive, namespace/bucket_name is also accepted to target all files."
|
| 354 |
+
),
|
| 355 |
+
),
|
| 356 |
+
],
|
| 357 |
+
recursive: Annotated[
|
| 358 |
+
bool,
|
| 359 |
+
typer.Option(
|
| 360 |
+
"--recursive",
|
| 361 |
+
"-R",
|
| 362 |
+
help="Remove files recursively under the given prefix.",
|
| 363 |
+
),
|
| 364 |
+
] = False,
|
| 365 |
+
yes: Annotated[
|
| 366 |
+
bool,
|
| 367 |
+
typer.Option(
|
| 368 |
+
"--yes",
|
| 369 |
+
"-y",
|
| 370 |
+
help="Skip confirmation prompt.",
|
| 371 |
+
),
|
| 372 |
+
] = False,
|
| 373 |
+
dry_run: Annotated[
|
| 374 |
+
bool,
|
| 375 |
+
typer.Option(
|
| 376 |
+
"--dry-run",
|
| 377 |
+
help="Preview what would be deleted without actually deleting.",
|
| 378 |
+
),
|
| 379 |
+
] = False,
|
| 380 |
+
include: Annotated[
|
| 381 |
+
list[str] | None,
|
| 382 |
+
typer.Option(
|
| 383 |
+
help="Include only files matching pattern (can specify multiple). Requires --recursive.",
|
| 384 |
+
),
|
| 385 |
+
] = None,
|
| 386 |
+
exclude: Annotated[
|
| 387 |
+
list[str] | None,
|
| 388 |
+
typer.Option(
|
| 389 |
+
help="Exclude files matching pattern (can specify multiple). Requires --recursive.",
|
| 390 |
+
),
|
| 391 |
+
] = None,
|
| 392 |
+
token: TokenOpt = None,
|
| 393 |
+
) -> None:
|
| 394 |
+
"""Remove files from a bucket.
|
| 395 |
+
|
| 396 |
+
To delete an entire bucket, use `hf buckets delete` instead.
|
| 397 |
+
"""
|
| 398 |
+
parsed = _parse_bucket_uri(argument)
|
| 399 |
+
bucket_id = parsed.id
|
| 400 |
+
prefix = parsed.path_in_repo
|
| 401 |
+
|
| 402 |
+
if prefix == "" and not recursive:
|
| 403 |
+
raise typer.BadParameter(
|
| 404 |
+
f"No file path specified. To remove files, provide a path"
|
| 405 |
+
f" (e.g. '{bucket_id}/FILE') or use --recursive to remove all files."
|
| 406 |
+
f" To delete the entire bucket, use `hf buckets delete {bucket_id}`."
|
| 407 |
+
)
|
| 408 |
+
|
| 409 |
+
if (include or exclude) and not recursive:
|
| 410 |
+
raise typer.BadParameter("--include and --exclude require --recursive.")
|
| 411 |
+
|
| 412 |
+
api = get_hf_api(token=token)
|
| 413 |
+
|
| 414 |
+
if recursive:
|
| 415 |
+
status = out.status("Listing files from remote")
|
| 416 |
+
|
| 417 |
+
all_files: list[BucketFile] = []
|
| 418 |
+
for item in api.list_bucket_tree(
|
| 419 |
+
bucket_id,
|
| 420 |
+
prefix=prefix or None,
|
| 421 |
+
recursive=True,
|
| 422 |
+
):
|
| 423 |
+
if isinstance(item, BucketFile):
|
| 424 |
+
all_files.append(item)
|
| 425 |
+
status.update(f"Listing files from remote ({len(all_files)} files)")
|
| 426 |
+
status.done(f"Listing files from remote ({len(all_files)} files)")
|
| 427 |
+
|
| 428 |
+
if include or exclude:
|
| 429 |
+
matcher = FilterMatcher(include_patterns=include, exclude_patterns=exclude)
|
| 430 |
+
matched_files = [f for f in all_files if matcher.matches(f.path)]
|
| 431 |
+
else:
|
| 432 |
+
matched_files = all_files
|
| 433 |
+
|
| 434 |
+
file_paths = [f.path for f in matched_files]
|
| 435 |
+
total_size = sum(f.size for f in matched_files)
|
| 436 |
+
size_str = format_size(total_size, human_readable=True)
|
| 437 |
+
|
| 438 |
+
if not file_paths:
|
| 439 |
+
out.text("No files to remove.")
|
| 440 |
+
return
|
| 441 |
+
|
| 442 |
+
count_label = f"{len(file_paths)} file(s) totaling {size_str}"
|
| 443 |
+
|
| 444 |
+
if not yes and not dry_run:
|
| 445 |
+
out.text("\n".join(f" {path}" for path in file_paths))
|
| 446 |
+
out.confirm(f"Remove {count_label} from '{bucket_id}'?", yes=False)
|
| 447 |
+
|
| 448 |
+
if dry_run:
|
| 449 |
+
out.text("\n".join(f"delete: {BUCKET_PREFIX}{bucket_id}/{path}" for path in file_paths))
|
| 450 |
+
out.text(f"(dry run) {count_label} would be removed.")
|
| 451 |
+
return
|
| 452 |
+
|
| 453 |
+
api.batch_bucket_files(bucket_id, delete=file_paths)
|
| 454 |
+
out.result(
|
| 455 |
+
f"Removed {count_label} from '{bucket_id}'",
|
| 456 |
+
bucket_id=bucket_id,
|
| 457 |
+
files_deleted=len(file_paths),
|
| 458 |
+
size=size_str,
|
| 459 |
+
)
|
| 460 |
+
|
| 461 |
+
else:
|
| 462 |
+
file_path = prefix
|
| 463 |
+
if not file_path:
|
| 464 |
+
raise typer.BadParameter("File path cannot be empty.")
|
| 465 |
+
|
| 466 |
+
if dry_run:
|
| 467 |
+
out.text(f"delete: {BUCKET_PREFIX}{bucket_id}/{file_path}")
|
| 468 |
+
out.text("(dry run) 1 file would be removed.")
|
| 469 |
+
return
|
| 470 |
+
|
| 471 |
+
out.confirm(f"Remove '{file_path}' from '{bucket_id}'?", yes=yes)
|
| 472 |
+
|
| 473 |
+
api.batch_bucket_files(bucket_id, delete=[file_path])
|
| 474 |
+
out.result("File removed", path=file_path, bucket_id=bucket_id)
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
@buckets_cli.command(
|
| 478 |
+
name="move",
|
| 479 |
+
examples=[
|
| 480 |
+
"hf buckets move user/old-bucket user/new-bucket",
|
| 481 |
+
"hf buckets move user/my-bucket my-org/my-bucket",
|
| 482 |
+
"hf buckets move hf://buckets/user/old-bucket hf://buckets/user/new-bucket",
|
| 483 |
+
],
|
| 484 |
+
)
|
| 485 |
+
def move(
|
| 486 |
+
from_id: Annotated[
|
| 487 |
+
str,
|
| 488 |
+
typer.Argument(
|
| 489 |
+
help="Source bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name",
|
| 490 |
+
),
|
| 491 |
+
],
|
| 492 |
+
to_id: Annotated[
|
| 493 |
+
str,
|
| 494 |
+
typer.Argument(
|
| 495 |
+
help="Destination bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name",
|
| 496 |
+
),
|
| 497 |
+
],
|
| 498 |
+
token: TokenOpt = None,
|
| 499 |
+
) -> None:
|
| 500 |
+
"""Move (rename) a bucket to a new name or namespace."""
|
| 501 |
+
# Parse from_id
|
| 502 |
+
parsed_from = _parse_bucket_uri(from_id)
|
| 503 |
+
if parsed_from.path_in_repo:
|
| 504 |
+
raise typer.BadParameter(
|
| 505 |
+
f"Cannot specify a prefix for bucket move: {from_id}."
|
| 506 |
+
f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name."
|
| 507 |
+
)
|
| 508 |
+
|
| 509 |
+
# Parse to_id
|
| 510 |
+
parsed_to = _parse_bucket_uri(to_id)
|
| 511 |
+
if parsed_to.path_in_repo:
|
| 512 |
+
raise typer.BadParameter(
|
| 513 |
+
f"Cannot specify a prefix for bucket move: {to_id}."
|
| 514 |
+
f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name."
|
| 515 |
+
)
|
| 516 |
+
|
| 517 |
+
api = get_hf_api(token=token)
|
| 518 |
+
api.move_bucket(from_id=parsed_from.id, to_id=parsed_to.id)
|
| 519 |
+
out.result("Bucket moved", from_id=parsed_from.id, to_id=parsed_to.id)
|
| 520 |
+
|
| 521 |
+
|
| 522 |
+
# =============================================================================
|
| 523 |
+
# Sync command
|
| 524 |
+
# =============================================================================
|
| 525 |
+
|
| 526 |
+
|
| 527 |
+
@buckets_cli.command(
|
| 528 |
+
name="sync",
|
| 529 |
+
examples=[
|
| 530 |
+
"hf buckets sync ./data hf://buckets/user/my-bucket",
|
| 531 |
+
"hf buckets sync hf://buckets/user/my-bucket ./data",
|
| 532 |
+
"hf buckets sync ./data hf://buckets/user/my-bucket --delete",
|
| 533 |
+
'hf buckets sync hf://buckets/user/my-bucket ./data --include "*.safetensors" --exclude "*.tmp"',
|
| 534 |
+
"hf buckets sync ./data hf://buckets/user/my-bucket --plan sync-plan.jsonl",
|
| 535 |
+
"hf buckets sync --apply sync-plan.jsonl",
|
| 536 |
+
"hf buckets sync ./data hf://buckets/user/my-bucket --dry-run",
|
| 537 |
+
"hf buckets sync ./data hf://buckets/user/my-bucket --dry-run | jq .",
|
| 538 |
+
],
|
| 539 |
+
)
|
| 540 |
+
def sync(
|
| 541 |
+
source: Annotated[
|
| 542 |
+
str | None,
|
| 543 |
+
typer.Argument(
|
| 544 |
+
help="Source path: local directory or hf://buckets/namespace/bucket_name(/prefix)",
|
| 545 |
+
),
|
| 546 |
+
] = None,
|
| 547 |
+
dest: Annotated[
|
| 548 |
+
str | None,
|
| 549 |
+
typer.Argument(
|
| 550 |
+
help="Destination path: local directory or hf://buckets/namespace/bucket_name(/prefix)",
|
| 551 |
+
),
|
| 552 |
+
] = None,
|
| 553 |
+
delete: Annotated[
|
| 554 |
+
bool,
|
| 555 |
+
typer.Option(
|
| 556 |
+
help="Delete destination files not present in source.",
|
| 557 |
+
),
|
| 558 |
+
] = False,
|
| 559 |
+
ignore_times: Annotated[
|
| 560 |
+
bool,
|
| 561 |
+
typer.Option(
|
| 562 |
+
"--ignore-times",
|
| 563 |
+
help="Skip files only based on size, ignoring modification times.",
|
| 564 |
+
),
|
| 565 |
+
] = False,
|
| 566 |
+
ignore_sizes: Annotated[
|
| 567 |
+
bool,
|
| 568 |
+
typer.Option(
|
| 569 |
+
"--ignore-sizes",
|
| 570 |
+
help="Skip files only based on modification times, ignoring sizes.",
|
| 571 |
+
),
|
| 572 |
+
] = False,
|
| 573 |
+
plan: Annotated[
|
| 574 |
+
str | None,
|
| 575 |
+
typer.Option(
|
| 576 |
+
help="Save sync plan to JSONL file for review instead of executing.",
|
| 577 |
+
),
|
| 578 |
+
] = None,
|
| 579 |
+
apply: Annotated[
|
| 580 |
+
str | None,
|
| 581 |
+
typer.Option(
|
| 582 |
+
help="Apply a previously saved plan file.",
|
| 583 |
+
),
|
| 584 |
+
] = None,
|
| 585 |
+
dry_run: Annotated[
|
| 586 |
+
bool,
|
| 587 |
+
typer.Option(
|
| 588 |
+
"--dry-run",
|
| 589 |
+
help="Print sync plan to stdout as JSONL without executing.",
|
| 590 |
+
),
|
| 591 |
+
] = False,
|
| 592 |
+
include: Annotated[
|
| 593 |
+
list[str] | None,
|
| 594 |
+
typer.Option(
|
| 595 |
+
help="Include files matching pattern (can specify multiple).",
|
| 596 |
+
),
|
| 597 |
+
] = None,
|
| 598 |
+
exclude: Annotated[
|
| 599 |
+
list[str] | None,
|
| 600 |
+
typer.Option(
|
| 601 |
+
help="Exclude files matching pattern (can specify multiple).",
|
| 602 |
+
),
|
| 603 |
+
] = None,
|
| 604 |
+
filter_from: Annotated[
|
| 605 |
+
str | None,
|
| 606 |
+
typer.Option(
|
| 607 |
+
help="Read include/exclude patterns from file.",
|
| 608 |
+
),
|
| 609 |
+
] = None,
|
| 610 |
+
existing: Annotated[
|
| 611 |
+
bool,
|
| 612 |
+
typer.Option(
|
| 613 |
+
"--existing",
|
| 614 |
+
help="Skip creating new files on receiver (only update existing files).",
|
| 615 |
+
),
|
| 616 |
+
] = False,
|
| 617 |
+
ignore_existing: Annotated[
|
| 618 |
+
bool,
|
| 619 |
+
typer.Option(
|
| 620 |
+
"--ignore-existing",
|
| 621 |
+
help="Skip updating files that exist on receiver (only create new files).",
|
| 622 |
+
),
|
| 623 |
+
] = False,
|
| 624 |
+
verbose: Annotated[
|
| 625 |
+
bool,
|
| 626 |
+
typer.Option(
|
| 627 |
+
"--verbose",
|
| 628 |
+
"-v",
|
| 629 |
+
help="Show detailed logging with reasoning.",
|
| 630 |
+
),
|
| 631 |
+
] = False,
|
| 632 |
+
token: TokenOpt = None,
|
| 633 |
+
) -> None:
|
| 634 |
+
"""Sync files between local directory and a bucket."""
|
| 635 |
+
api = get_hf_api(token=token)
|
| 636 |
+
api.sync_bucket(
|
| 637 |
+
source=source,
|
| 638 |
+
dest=dest,
|
| 639 |
+
delete=delete,
|
| 640 |
+
ignore_times=ignore_times,
|
| 641 |
+
ignore_sizes=ignore_sizes,
|
| 642 |
+
existing=existing,
|
| 643 |
+
ignore_existing=ignore_existing,
|
| 644 |
+
include=include,
|
| 645 |
+
exclude=exclude,
|
| 646 |
+
filter_from=filter_from,
|
| 647 |
+
plan=plan,
|
| 648 |
+
apply=apply,
|
| 649 |
+
dry_run=dry_run,
|
| 650 |
+
verbose=verbose,
|
| 651 |
+
quiet=out.is_quiet(),
|
| 652 |
+
)
|
| 653 |
+
if plan and not out.is_quiet():
|
| 654 |
+
out.hint(f"Run `hf buckets sync --apply {plan}` to execute this plan.")
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
# =============================================================================
|
| 658 |
+
# Cp command
|
| 659 |
+
# =============================================================================
|
| 660 |
+
|
| 661 |
+
|
| 662 |
+
# `hf buckets cp` is an alias for the top-level `hf cp` command (see `cli/_cp.py`).
|
| 663 |
+
buckets_cli.command(
|
| 664 |
+
name="cp",
|
| 665 |
+
examples=[
|
| 666 |
+
# Download (repo or bucket -> local / stdout)
|
| 667 |
+
"hf buckets cp hf://buckets/username/my-bucket/config.json config.json",
|
| 668 |
+
"hf buckets cp hf://buckets/username/my-bucket/data.csv data/",
|
| 669 |
+
"hf buckets cp hf://buckets/username/my-bucket/config.json -",
|
| 670 |
+
# Upload (local / stdin -> bucket)
|
| 671 |
+
"hf buckets cp model.safetensors hf://buckets/username/my-bucket/model.safetensors",
|
| 672 |
+
"hf buckets cp config.json hf://buckets/username/my-bucket/logs/",
|
| 673 |
+
"hf buckets cp - hf://buckets/username/my-bucket/config.json",
|
| 674 |
+
# Remote to remote (repo or bucket -> bucket)
|
| 675 |
+
"hf buckets cp hf://buckets/username/my-bucket/data.csv hf://buckets/username/dest-bucket/",
|
| 676 |
+
"hf buckets cp hf://buckets/username/source-bucket/logs/ hf://buckets/username/dest-bucket/logs/",
|
| 677 |
+
],
|
| 678 |
+
)(make_cp("buckets"))
|
venv/Lib/site-packages/huggingface_hub/cli/cache.py
ADDED
|
@@ -0,0 +1,775 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains the 'hf cache' command group with cache management subcommands."""
|
| 15 |
+
|
| 16 |
+
import re
|
| 17 |
+
import time
|
| 18 |
+
from collections import defaultdict
|
| 19 |
+
from collections.abc import Callable, Mapping
|
| 20 |
+
from dataclasses import dataclass
|
| 21 |
+
from enum import Enum
|
| 22 |
+
from typing import Annotated, Any
|
| 23 |
+
|
| 24 |
+
import typer
|
| 25 |
+
|
| 26 |
+
from huggingface_hub.errors import CLIError
|
| 27 |
+
|
| 28 |
+
from ..utils import (
|
| 29 |
+
ANSI,
|
| 30 |
+
CachedRepoInfo,
|
| 31 |
+
CachedRevisionInfo,
|
| 32 |
+
CacheNotFound,
|
| 33 |
+
HFCacheInfo,
|
| 34 |
+
_format_size,
|
| 35 |
+
parse_hf_uri,
|
| 36 |
+
scan_cache_dir,
|
| 37 |
+
)
|
| 38 |
+
from ..utils._parsing import parse_duration, parse_size
|
| 39 |
+
from ._cli_utils import RepoIdArg, RepoTypeOpt, RevisionOpt, TokenOpt, get_hf_api, typer_factory
|
| 40 |
+
from ._output import out
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
cache_cli = typer_factory(help="Manage local cache directory.")
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
#### Cache helper utilities
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
@dataclass(frozen=True)
|
| 50 |
+
class _DeletionResolution:
|
| 51 |
+
revisions: frozenset[str]
|
| 52 |
+
selected: dict[CachedRepoInfo, frozenset[CachedRevisionInfo]]
|
| 53 |
+
missing: tuple[str, ...]
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
_FILTER_PATTERN = re.compile(r"^(?P<key>[a-zA-Z_]+)\s*(?P<op>==|!=|>=|<=|>|<|=)\s*(?P<value>.+)$")
|
| 57 |
+
_ALLOWED_OPERATORS = {"=", "!=", ">", "<", ">=", "<="}
|
| 58 |
+
_FILTER_KEYS = {"accessed", "modified", "refs", "size", "type"}
|
| 59 |
+
_SORT_KEYS = {"accessed", "modified", "name", "size"}
|
| 60 |
+
_SORT_PATTERN = re.compile(r"^(?P<key>[a-zA-Z_]+)(?::(?P<order>asc|desc))?$")
|
| 61 |
+
_SORT_DEFAULT_ORDER = {
|
| 62 |
+
# Default ordering: accessed/modified/size are descending (newest/biggest first), name is ascending
|
| 63 |
+
"accessed": "desc",
|
| 64 |
+
"modified": "desc",
|
| 65 |
+
"size": "desc",
|
| 66 |
+
"name": "asc",
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
# Dynamically generate SortOptions enum from _SORT_KEYS
|
| 71 |
+
_sort_options_dict = {}
|
| 72 |
+
for key in sorted(_SORT_KEYS):
|
| 73 |
+
_sort_options_dict[key] = key
|
| 74 |
+
_sort_options_dict[f"{key}_asc"] = f"{key}:asc"
|
| 75 |
+
_sort_options_dict[f"{key}_desc"] = f"{key}:desc"
|
| 76 |
+
|
| 77 |
+
SortOptions = Enum("SortOptions", _sort_options_dict, type=str, module=__name__) # type: ignore
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
@dataclass(frozen=True)
|
| 81 |
+
class CacheDeletionCounts:
|
| 82 |
+
"""Simple counters summarizing cache deletions for CLI messaging."""
|
| 83 |
+
|
| 84 |
+
repo_count: int
|
| 85 |
+
partial_revision_count: int
|
| 86 |
+
total_revision_count: int
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
CacheEntry = tuple[CachedRepoInfo, CachedRevisionInfo | None]
|
| 90 |
+
RepoRefsMap = dict[CachedRepoInfo, frozenset[str]]
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def summarize_deletions(
|
| 94 |
+
selected_by_repo: Mapping[CachedRepoInfo, frozenset[CachedRevisionInfo]],
|
| 95 |
+
) -> CacheDeletionCounts:
|
| 96 |
+
"""Summarize deletions across repositories."""
|
| 97 |
+
repo_count = 0
|
| 98 |
+
total_revisions = 0
|
| 99 |
+
revisions_in_full_repos = 0
|
| 100 |
+
|
| 101 |
+
for repo, revisions in selected_by_repo.items():
|
| 102 |
+
total_revisions += len(revisions)
|
| 103 |
+
if len(revisions) == len(repo.revisions):
|
| 104 |
+
repo_count += 1
|
| 105 |
+
revisions_in_full_repos += len(revisions)
|
| 106 |
+
|
| 107 |
+
partial_revision_count = total_revisions - revisions_in_full_repos
|
| 108 |
+
return CacheDeletionCounts(repo_count, partial_revision_count, total_revisions)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def print_cache_selected_revisions(selected_by_repo: Mapping[CachedRepoInfo, frozenset[CachedRevisionInfo]]) -> None:
|
| 112 |
+
"""Pretty-print selected cache revisions during confirmation prompts."""
|
| 113 |
+
for repo in sorted(selected_by_repo.keys(), key=lambda repo: (repo.repo_type, repo.repo_id.lower())):
|
| 114 |
+
repo_key = f"{repo.repo_type}/{repo.repo_id}"
|
| 115 |
+
revisions = sorted(selected_by_repo[repo], key=lambda rev: rev.commit_hash)
|
| 116 |
+
if len(revisions) == len(repo.revisions):
|
| 117 |
+
out.text(f" - {repo_key} (entire repo)")
|
| 118 |
+
continue
|
| 119 |
+
|
| 120 |
+
out.text(f" - {repo_key}:")
|
| 121 |
+
for revision in revisions:
|
| 122 |
+
refs = " ".join(sorted(revision.refs)) or "(detached)"
|
| 123 |
+
out.text(f" {revision.commit_hash} [{refs}] {revision.size_on_disk_str}")
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def build_cache_index(
|
| 127 |
+
hf_cache_info: HFCacheInfo,
|
| 128 |
+
) -> tuple[
|
| 129 |
+
dict[str, CachedRepoInfo],
|
| 130 |
+
dict[str, tuple[CachedRepoInfo, CachedRevisionInfo]],
|
| 131 |
+
]:
|
| 132 |
+
"""Create lookup tables so CLI commands can resolve repo ids and revisions quickly."""
|
| 133 |
+
repo_lookup: dict[str, CachedRepoInfo] = {}
|
| 134 |
+
revision_lookup: dict[str, tuple[CachedRepoInfo, CachedRevisionInfo]] = {}
|
| 135 |
+
for repo in hf_cache_info.repos:
|
| 136 |
+
repo_key = repo.cache_id.lower()
|
| 137 |
+
repo_lookup[repo_key] = repo
|
| 138 |
+
for revision in repo.revisions:
|
| 139 |
+
revision_lookup[revision.commit_hash.lower()] = (repo, revision)
|
| 140 |
+
return repo_lookup, revision_lookup
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _repo_cache_id_from_target(target: str) -> str:
|
| 144 |
+
"""Return the cache id matching a repo target passed to `hf cache rm`."""
|
| 145 |
+
if not target.startswith("hf://"):
|
| 146 |
+
return target
|
| 147 |
+
|
| 148 |
+
uri = parse_hf_uri(target)
|
| 149 |
+
if not uri.is_repo:
|
| 150 |
+
raise CLIError("Only repository hf:// URIs are supported by `hf cache rm`.")
|
| 151 |
+
if uri.revision is not None or uri.path_in_repo:
|
| 152 |
+
raise CLIError("Only repo-level hf:// URIs are supported by `hf cache rm` for now.")
|
| 153 |
+
return f"{uri.type}/{uri.id}"
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def collect_cache_entries(
|
| 157 |
+
hf_cache_info: HFCacheInfo, *, include_revisions: bool
|
| 158 |
+
) -> tuple[list[CacheEntry], RepoRefsMap]:
|
| 159 |
+
"""Flatten cache metadata into rows consumed by `hf cache ls`."""
|
| 160 |
+
entries: list[CacheEntry] = []
|
| 161 |
+
repo_refs_map: RepoRefsMap = {}
|
| 162 |
+
sorted_repos = sorted(hf_cache_info.repos, key=lambda repo: (repo.repo_type, repo.repo_id.lower()))
|
| 163 |
+
for repo in sorted_repos:
|
| 164 |
+
repo_refs_map[repo] = frozenset({ref for revision in repo.revisions for ref in revision.refs})
|
| 165 |
+
if include_revisions:
|
| 166 |
+
for revision in sorted(repo.revisions, key=lambda rev: rev.commit_hash):
|
| 167 |
+
entries.append((repo, revision))
|
| 168 |
+
else:
|
| 169 |
+
entries.append((repo, None))
|
| 170 |
+
if include_revisions:
|
| 171 |
+
entries.sort(
|
| 172 |
+
key=lambda entry: (
|
| 173 |
+
entry[0].cache_id,
|
| 174 |
+
entry[1].commit_hash if entry[1] is not None else "",
|
| 175 |
+
)
|
| 176 |
+
)
|
| 177 |
+
else:
|
| 178 |
+
entries.sort(key=lambda entry: entry[0].cache_id)
|
| 179 |
+
return entries, repo_refs_map
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def compile_cache_filter(
|
| 183 |
+
expr: str, repo_refs_map: RepoRefsMap
|
| 184 |
+
) -> Callable[[CachedRepoInfo, CachedRevisionInfo | None, float], bool]:
|
| 185 |
+
"""Convert a `hf cache ls` filter expression into the yes/no test we apply to each cache entry before displaying it."""
|
| 186 |
+
match = _FILTER_PATTERN.match(expr.strip())
|
| 187 |
+
if not match:
|
| 188 |
+
raise ValueError(f"Invalid filter expression: '{expr}'.")
|
| 189 |
+
|
| 190 |
+
key = match.group("key").lower()
|
| 191 |
+
op = match.group("op")
|
| 192 |
+
value_raw = match.group("value").strip()
|
| 193 |
+
|
| 194 |
+
if op not in _ALLOWED_OPERATORS:
|
| 195 |
+
raise ValueError(f"Unsupported operator '{op}' in filter '{expr}'. Must be one of {list(_ALLOWED_OPERATORS)}.")
|
| 196 |
+
|
| 197 |
+
if key not in _FILTER_KEYS:
|
| 198 |
+
raise ValueError(f"Unsupported filter key '{key}' in '{expr}'. Must be one of {list(_FILTER_KEYS)}.")
|
| 199 |
+
# at this point we know that key is in `_FILTER_KEYS`
|
| 200 |
+
if key == "size":
|
| 201 |
+
size_threshold = parse_size(value_raw)
|
| 202 |
+
return lambda repo, revision, _: _compare_numeric(
|
| 203 |
+
revision.size_on_disk if revision is not None else repo.size_on_disk,
|
| 204 |
+
op,
|
| 205 |
+
size_threshold,
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
if key in {"modified", "accessed"}:
|
| 209 |
+
seconds = parse_duration(value_raw.strip())
|
| 210 |
+
|
| 211 |
+
def _time_filter(repo: CachedRepoInfo, revision: CachedRevisionInfo | None, now: float) -> bool:
|
| 212 |
+
timestamp = (
|
| 213 |
+
repo.last_accessed
|
| 214 |
+
if key == "accessed"
|
| 215 |
+
else revision.last_modified
|
| 216 |
+
if revision is not None
|
| 217 |
+
else repo.last_modified
|
| 218 |
+
)
|
| 219 |
+
if timestamp is None:
|
| 220 |
+
return False
|
| 221 |
+
return _compare_numeric(now - timestamp, op, seconds)
|
| 222 |
+
|
| 223 |
+
return _time_filter
|
| 224 |
+
|
| 225 |
+
if key == "type":
|
| 226 |
+
expected = value_raw.lower()
|
| 227 |
+
|
| 228 |
+
if op != "=":
|
| 229 |
+
raise ValueError(f"Only '=' is supported for 'type' filters. Got '{op}'.")
|
| 230 |
+
|
| 231 |
+
def _type_filter(repo: CachedRepoInfo, revision: CachedRevisionInfo | None, _: float) -> bool:
|
| 232 |
+
return repo.repo_type.lower() == expected
|
| 233 |
+
|
| 234 |
+
return _type_filter
|
| 235 |
+
|
| 236 |
+
else: # key == "refs"
|
| 237 |
+
if op != "=":
|
| 238 |
+
raise ValueError(f"Only '=' is supported for 'refs' filters. Got {op}.")
|
| 239 |
+
|
| 240 |
+
def _refs_filter(repo: CachedRepoInfo, revision: CachedRevisionInfo | None, _: float) -> bool:
|
| 241 |
+
refs = revision.refs if revision is not None else repo_refs_map.get(repo, frozenset())
|
| 242 |
+
return value_raw.lower() in [ref.lower() for ref in refs]
|
| 243 |
+
|
| 244 |
+
return _refs_filter
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def _compare_numeric(left: float | None, op: str, right: float) -> bool:
|
| 248 |
+
"""Evaluate numeric comparisons for filters."""
|
| 249 |
+
if left is None:
|
| 250 |
+
return False
|
| 251 |
+
|
| 252 |
+
comparisons = {
|
| 253 |
+
"=": left == right,
|
| 254 |
+
"!=": left != right,
|
| 255 |
+
">": left > right,
|
| 256 |
+
"<": left < right,
|
| 257 |
+
">=": left >= right,
|
| 258 |
+
"<=": left <= right,
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
if op not in comparisons:
|
| 262 |
+
raise ValueError(f"Unsupported numeric comparison operator: {op}")
|
| 263 |
+
|
| 264 |
+
return comparisons[op]
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def compile_cache_sort(sort_expr: str) -> tuple[Callable[[CacheEntry], tuple[Any, ...]], bool]:
|
| 268 |
+
"""Convert a `hf cache ls` sort expression into a key function for sorting entries.
|
| 269 |
+
|
| 270 |
+
Returns:
|
| 271 |
+
A tuple of (key_function, reverse_flag) where reverse_flag indicates whether
|
| 272 |
+
to sort in descending order (True) or ascending order (False).
|
| 273 |
+
"""
|
| 274 |
+
match = _SORT_PATTERN.match(sort_expr.strip().lower())
|
| 275 |
+
if not match:
|
| 276 |
+
raise ValueError(f"Invalid sort expression: '{sort_expr}'. Expected format: 'key' or 'key:asc' or 'key:desc'.")
|
| 277 |
+
|
| 278 |
+
key = match.group("key").lower()
|
| 279 |
+
explicit_order = match.group("order")
|
| 280 |
+
|
| 281 |
+
if key not in _SORT_KEYS:
|
| 282 |
+
raise ValueError(f"Unsupported sort key '{key}' in '{sort_expr}'. Must be one of {list(_SORT_KEYS)}.")
|
| 283 |
+
|
| 284 |
+
# Use explicit order if provided, otherwise use default for the key
|
| 285 |
+
order = explicit_order if explicit_order else _SORT_DEFAULT_ORDER[key]
|
| 286 |
+
reverse = order == "desc"
|
| 287 |
+
|
| 288 |
+
def _sort_key(entry: CacheEntry) -> tuple[Any, ...]:
|
| 289 |
+
repo, revision = entry
|
| 290 |
+
|
| 291 |
+
if key == "name":
|
| 292 |
+
# Sort by cache_id (repo type/id)
|
| 293 |
+
value: Any = repo.cache_id.lower()
|
| 294 |
+
return (value,)
|
| 295 |
+
|
| 296 |
+
if key == "size":
|
| 297 |
+
# Use revision size if available, otherwise repo size
|
| 298 |
+
value = revision.size_on_disk if revision is not None else repo.size_on_disk
|
| 299 |
+
return (value,)
|
| 300 |
+
|
| 301 |
+
if key == "accessed":
|
| 302 |
+
# For revisions, accessed is not available per-revision, use repo's last_accessed
|
| 303 |
+
# For repos, use repo's last_accessed
|
| 304 |
+
value = repo.last_accessed if repo.last_accessed is not None else 0.0
|
| 305 |
+
return (value,)
|
| 306 |
+
|
| 307 |
+
if key == "modified":
|
| 308 |
+
# Use revision's last_modified if available, otherwise repo's last_modified
|
| 309 |
+
if revision is not None:
|
| 310 |
+
value = revision.last_modified if revision.last_modified is not None else 0.0
|
| 311 |
+
else:
|
| 312 |
+
value = repo.last_modified if repo.last_modified is not None else 0.0
|
| 313 |
+
return (value,)
|
| 314 |
+
|
| 315 |
+
# Should never reach here due to validation above
|
| 316 |
+
raise ValueError(f"Unsupported sort key: {key}")
|
| 317 |
+
|
| 318 |
+
return _sort_key, reverse
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def _resolve_deletion_targets(hf_cache_info: HFCacheInfo, targets: list[str]) -> _DeletionResolution:
|
| 322 |
+
"""Resolve the deletion targets into a deletion resolution."""
|
| 323 |
+
repo_lookup, revision_lookup = build_cache_index(hf_cache_info)
|
| 324 |
+
|
| 325 |
+
selected: dict[CachedRepoInfo, set[CachedRevisionInfo]] = defaultdict(set)
|
| 326 |
+
revisions: set[str] = set()
|
| 327 |
+
missing: list[str] = []
|
| 328 |
+
|
| 329 |
+
for raw_target in targets:
|
| 330 |
+
target = raw_target.strip()
|
| 331 |
+
if not target:
|
| 332 |
+
continue
|
| 333 |
+
lowered = target.lower()
|
| 334 |
+
|
| 335 |
+
if re.fullmatch(r"[0-9a-fA-F]{40}", lowered):
|
| 336 |
+
match = revision_lookup.get(lowered)
|
| 337 |
+
if match is None:
|
| 338 |
+
missing.append(raw_target)
|
| 339 |
+
continue
|
| 340 |
+
repo, revision = match
|
| 341 |
+
selected[repo].add(revision)
|
| 342 |
+
revisions.add(revision.commit_hash)
|
| 343 |
+
continue
|
| 344 |
+
|
| 345 |
+
matched_repo = repo_lookup.get(_repo_cache_id_from_target(target).lower())
|
| 346 |
+
if matched_repo is None:
|
| 347 |
+
missing.append(raw_target)
|
| 348 |
+
continue
|
| 349 |
+
|
| 350 |
+
for revision in matched_repo.revisions:
|
| 351 |
+
selected[matched_repo].add(revision)
|
| 352 |
+
revisions.add(revision.commit_hash)
|
| 353 |
+
|
| 354 |
+
frozen_selected = {repo: frozenset(revs) for repo, revs in selected.items()}
|
| 355 |
+
return _DeletionResolution(
|
| 356 |
+
revisions=frozenset(revisions),
|
| 357 |
+
selected=frozen_selected,
|
| 358 |
+
missing=tuple(missing),
|
| 359 |
+
)
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
#### Cache CLI commands
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
@cache_cli.command(
|
| 366 |
+
"list | ls",
|
| 367 |
+
examples=[
|
| 368 |
+
"hf cache ls",
|
| 369 |
+
"hf cache ls --revisions",
|
| 370 |
+
'hf cache ls --filter "size>1GB" --limit 20',
|
| 371 |
+
"hf cache ls --format json",
|
| 372 |
+
],
|
| 373 |
+
)
|
| 374 |
+
def ls(
|
| 375 |
+
cache_dir: Annotated[
|
| 376 |
+
str | None,
|
| 377 |
+
typer.Option(
|
| 378 |
+
help="Cache directory to scan (defaults to Hugging Face cache).",
|
| 379 |
+
),
|
| 380 |
+
] = None,
|
| 381 |
+
revisions: Annotated[
|
| 382 |
+
bool,
|
| 383 |
+
typer.Option(
|
| 384 |
+
help="Include revisions in the output instead of aggregated repositories.",
|
| 385 |
+
),
|
| 386 |
+
] = False,
|
| 387 |
+
filter: Annotated[
|
| 388 |
+
list[str] | None,
|
| 389 |
+
typer.Option(
|
| 390 |
+
"-f",
|
| 391 |
+
"--filter",
|
| 392 |
+
help="Filter entries (e.g. 'size>1GB', 'type=model', 'accessed>7d'). Can be used multiple times.",
|
| 393 |
+
),
|
| 394 |
+
] = None,
|
| 395 |
+
sort: Annotated[
|
| 396 |
+
SortOptions | None,
|
| 397 |
+
typer.Option(
|
| 398 |
+
help="Sort entries by key. Supported keys: 'accessed', 'modified', 'name', 'size'. "
|
| 399 |
+
"Append ':asc' or ':desc' to explicitly set the order (e.g., 'modified:asc'). "
|
| 400 |
+
"Defaults: 'accessed', 'modified', 'size' default to 'desc' (newest/biggest first); "
|
| 401 |
+
"'name' defaults to 'asc' (alphabetical).",
|
| 402 |
+
),
|
| 403 |
+
] = None,
|
| 404 |
+
limit: Annotated[
|
| 405 |
+
int | None,
|
| 406 |
+
typer.Option(
|
| 407 |
+
help="Limit the number of results returned. Returns only the top N entries after sorting.",
|
| 408 |
+
),
|
| 409 |
+
] = None,
|
| 410 |
+
) -> None:
|
| 411 |
+
"""List cached repositories or revisions."""
|
| 412 |
+
try:
|
| 413 |
+
hf_cache_info = scan_cache_dir(cache_dir)
|
| 414 |
+
except CacheNotFound as exc:
|
| 415 |
+
raise CLIError(f"Cache directory not found: {exc.cache_dir}") from exc
|
| 416 |
+
|
| 417 |
+
filters = filter or []
|
| 418 |
+
|
| 419 |
+
entries, repo_refs_map = collect_cache_entries(hf_cache_info, include_revisions=revisions)
|
| 420 |
+
try:
|
| 421 |
+
filter_fns = [compile_cache_filter(expr, repo_refs_map) for expr in filters]
|
| 422 |
+
except ValueError as exc:
|
| 423 |
+
raise typer.BadParameter(str(exc)) from exc
|
| 424 |
+
|
| 425 |
+
now = time.time()
|
| 426 |
+
for fn in filter_fns:
|
| 427 |
+
entries = [entry for entry in entries if fn(entry[0], entry[1], now)]
|
| 428 |
+
|
| 429 |
+
# Apply sorting if requested
|
| 430 |
+
if sort:
|
| 431 |
+
try:
|
| 432 |
+
sort_key_fn, reverse = compile_cache_sort(sort.value)
|
| 433 |
+
entries.sort(key=sort_key_fn, reverse=reverse)
|
| 434 |
+
except ValueError as exc:
|
| 435 |
+
raise typer.BadParameter(str(exc)) from exc
|
| 436 |
+
|
| 437 |
+
# Apply limit if requested
|
| 438 |
+
if limit is not None:
|
| 439 |
+
if limit < 0:
|
| 440 |
+
raise typer.BadParameter(f"Limit must be a positive integer, got {limit}.")
|
| 441 |
+
entries = entries[:limit]
|
| 442 |
+
|
| 443 |
+
if revisions:
|
| 444 |
+
items = [
|
| 445 |
+
{
|
| 446 |
+
"id": repo.cache_id,
|
| 447 |
+
"repo_id": repo.repo_id,
|
| 448 |
+
"repo_type": repo.repo_type,
|
| 449 |
+
"revision": revision.commit_hash,
|
| 450 |
+
"snapshot_path": str(revision.snapshot_path),
|
| 451 |
+
"size": revision.size_on_disk_str,
|
| 452 |
+
"last_modified": revision.last_modified_str,
|
| 453 |
+
"refs": sorted(revision.refs),
|
| 454 |
+
}
|
| 455 |
+
for repo, revision in entries
|
| 456 |
+
if revision is not None
|
| 457 |
+
]
|
| 458 |
+
out.table(
|
| 459 |
+
items,
|
| 460 |
+
headers=["id", "revision", "size", "last_modified", "refs"],
|
| 461 |
+
id_key="revision",
|
| 462 |
+
alignments={"size": "right"},
|
| 463 |
+
)
|
| 464 |
+
else:
|
| 465 |
+
items = [
|
| 466 |
+
{
|
| 467 |
+
"id": repo.cache_id,
|
| 468 |
+
"repo_id": repo.repo_id,
|
| 469 |
+
"repo_type": repo.repo_type,
|
| 470 |
+
"size": repo.size_on_disk_str,
|
| 471 |
+
"last_accessed": repo.last_accessed_str or "",
|
| 472 |
+
"last_modified": repo.last_modified_str,
|
| 473 |
+
"refs": sorted(repo_refs_map.get(repo, frozenset())),
|
| 474 |
+
}
|
| 475 |
+
for repo, _ in entries
|
| 476 |
+
]
|
| 477 |
+
out.table(
|
| 478 |
+
items,
|
| 479 |
+
headers=["id", "size", "last_accessed", "last_modified", "refs"],
|
| 480 |
+
id_key="id",
|
| 481 |
+
alignments={"size": "right"},
|
| 482 |
+
)
|
| 483 |
+
|
| 484 |
+
if entries:
|
| 485 |
+
unique_repos = {repo for repo, _ in entries}
|
| 486 |
+
repo_count = len(unique_repos)
|
| 487 |
+
if revisions:
|
| 488 |
+
revision_count = sum(1 for _, rev in entries if rev is not None)
|
| 489 |
+
total_size = sum(rev.size_on_disk for _, rev in entries if rev is not None)
|
| 490 |
+
else:
|
| 491 |
+
revision_count = sum(len(repo.revisions) for repo in unique_repos)
|
| 492 |
+
total_size = sum(repo.size_on_disk for repo in unique_repos)
|
| 493 |
+
out.text(
|
| 494 |
+
ANSI.bold(
|
| 495 |
+
f"\nFound {repo_count} repo(s) for a total of {revision_count} revision(s)"
|
| 496 |
+
f" and {_format_size(total_size)} on disk."
|
| 497 |
+
)
|
| 498 |
+
)
|
| 499 |
+
|
| 500 |
+
|
| 501 |
+
@cache_cli.command(
|
| 502 |
+
examples=[
|
| 503 |
+
"hf cache rm model/gpt2",
|
| 504 |
+
"hf cache rm hf://models/openai-community/gpt2",
|
| 505 |
+
"hf cache rm <revision_hash>",
|
| 506 |
+
"hf cache rm model/gpt2 --dry-run",
|
| 507 |
+
"hf cache rm model/gpt2 --yes",
|
| 508 |
+
],
|
| 509 |
+
)
|
| 510 |
+
def rm(
|
| 511 |
+
targets: Annotated[
|
| 512 |
+
list[str],
|
| 513 |
+
typer.Argument(
|
| 514 |
+
help="One or more repo IDs (e.g. model/bert-base-uncased), repo-level hf:// URIs, or revision hashes to delete.",
|
| 515 |
+
),
|
| 516 |
+
],
|
| 517 |
+
cache_dir: Annotated[
|
| 518 |
+
str | None,
|
| 519 |
+
typer.Option(
|
| 520 |
+
help="Cache directory to scan (defaults to Hugging Face cache).",
|
| 521 |
+
),
|
| 522 |
+
] = None,
|
| 523 |
+
yes: Annotated[
|
| 524 |
+
bool,
|
| 525 |
+
typer.Option(
|
| 526 |
+
"-y",
|
| 527 |
+
"--yes",
|
| 528 |
+
help="Skip confirmation prompt.",
|
| 529 |
+
),
|
| 530 |
+
] = False,
|
| 531 |
+
dry_run: Annotated[
|
| 532 |
+
bool,
|
| 533 |
+
typer.Option(
|
| 534 |
+
help="Preview deletions without removing anything.",
|
| 535 |
+
),
|
| 536 |
+
] = False,
|
| 537 |
+
) -> None:
|
| 538 |
+
"""Remove cached repositories or revisions."""
|
| 539 |
+
try:
|
| 540 |
+
hf_cache_info = scan_cache_dir(cache_dir)
|
| 541 |
+
except CacheNotFound as exc:
|
| 542 |
+
raise CLIError(f"Cache directory not found: {exc.cache_dir}") from exc
|
| 543 |
+
|
| 544 |
+
resolution = _resolve_deletion_targets(hf_cache_info, targets)
|
| 545 |
+
|
| 546 |
+
if resolution.missing:
|
| 547 |
+
details = "\n".join(f" - {entry}" for entry in resolution.missing)
|
| 548 |
+
out.warning(f"Could not find in cache:\n{details}")
|
| 549 |
+
|
| 550 |
+
if len(resolution.revisions) == 0:
|
| 551 |
+
out.text("Nothing to delete.")
|
| 552 |
+
raise typer.Exit(code=0)
|
| 553 |
+
|
| 554 |
+
strategy = hf_cache_info.delete_revisions(*sorted(resolution.revisions))
|
| 555 |
+
counts = summarize_deletions(resolution.selected)
|
| 556 |
+
|
| 557 |
+
summary_parts: list[str] = []
|
| 558 |
+
if counts.repo_count:
|
| 559 |
+
summary_parts.append(f"{counts.repo_count} repo(s)")
|
| 560 |
+
if counts.partial_revision_count:
|
| 561 |
+
summary_parts.append(f"{counts.partial_revision_count} revision(s)")
|
| 562 |
+
if not summary_parts:
|
| 563 |
+
summary_parts.append(f"{counts.total_revision_count} revision(s)")
|
| 564 |
+
|
| 565 |
+
summary_text = " and ".join(summary_parts)
|
| 566 |
+
out.text(f"About to delete {summary_text} totalling {strategy.expected_freed_size_str}.")
|
| 567 |
+
print_cache_selected_revisions(resolution.selected)
|
| 568 |
+
|
| 569 |
+
if dry_run:
|
| 570 |
+
out.result(
|
| 571 |
+
"Dry run: no files were deleted.",
|
| 572 |
+
dry_run=True,
|
| 573 |
+
repos=counts.repo_count,
|
| 574 |
+
revisions=counts.total_revision_count,
|
| 575 |
+
size=strategy.expected_freed_size_str,
|
| 576 |
+
)
|
| 577 |
+
return
|
| 578 |
+
|
| 579 |
+
out.confirm("Proceed with deletion?", yes=yes)
|
| 580 |
+
|
| 581 |
+
strategy.execute()
|
| 582 |
+
counts = summarize_deletions(resolution.selected)
|
| 583 |
+
out.result(
|
| 584 |
+
f"Deleted {counts.repo_count} repo(s) and {counts.total_revision_count} revision(s);"
|
| 585 |
+
f" freed {strategy.expected_freed_size_str}.",
|
| 586 |
+
repos_deleted=counts.repo_count,
|
| 587 |
+
revisions_deleted=counts.total_revision_count,
|
| 588 |
+
freed=strategy.expected_freed_size_str,
|
| 589 |
+
)
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
@cache_cli.command(examples=["hf cache prune", "hf cache prune --dry-run"])
|
| 593 |
+
def prune(
|
| 594 |
+
cache_dir: Annotated[
|
| 595 |
+
str | None,
|
| 596 |
+
typer.Option(
|
| 597 |
+
help="Cache directory to scan (defaults to Hugging Face cache).",
|
| 598 |
+
),
|
| 599 |
+
] = None,
|
| 600 |
+
yes: Annotated[
|
| 601 |
+
bool,
|
| 602 |
+
typer.Option(
|
| 603 |
+
"-y",
|
| 604 |
+
"--yes",
|
| 605 |
+
help="Skip confirmation prompt.",
|
| 606 |
+
),
|
| 607 |
+
] = False,
|
| 608 |
+
dry_run: Annotated[
|
| 609 |
+
bool,
|
| 610 |
+
typer.Option(
|
| 611 |
+
help="Preview deletions without removing anything.",
|
| 612 |
+
),
|
| 613 |
+
] = False,
|
| 614 |
+
) -> None:
|
| 615 |
+
"""Remove detached revisions from the cache."""
|
| 616 |
+
try:
|
| 617 |
+
hf_cache_info = scan_cache_dir(cache_dir)
|
| 618 |
+
except CacheNotFound as exc:
|
| 619 |
+
raise CLIError(f"Cache directory not found: {exc.cache_dir}") from exc
|
| 620 |
+
|
| 621 |
+
selected: dict[CachedRepoInfo, frozenset[CachedRevisionInfo]] = {}
|
| 622 |
+
revisions: set[str] = set()
|
| 623 |
+
for repo in hf_cache_info.repos:
|
| 624 |
+
detached = frozenset(revision for revision in repo.revisions if len(revision.refs) == 0)
|
| 625 |
+
if not detached:
|
| 626 |
+
continue
|
| 627 |
+
selected[repo] = detached
|
| 628 |
+
revisions.update(revision.commit_hash for revision in detached)
|
| 629 |
+
|
| 630 |
+
if len(revisions) == 0:
|
| 631 |
+
out.text("No unreferenced revisions found. Nothing to prune.")
|
| 632 |
+
return
|
| 633 |
+
|
| 634 |
+
resolution = _DeletionResolution(
|
| 635 |
+
revisions=frozenset(revisions),
|
| 636 |
+
selected=selected,
|
| 637 |
+
missing=(),
|
| 638 |
+
)
|
| 639 |
+
strategy = hf_cache_info.delete_revisions(*sorted(resolution.revisions))
|
| 640 |
+
counts = summarize_deletions(selected)
|
| 641 |
+
|
| 642 |
+
out.text(
|
| 643 |
+
f"About to delete {counts.total_revision_count} unreferenced revision(s) ({strategy.expected_freed_size_str} total)."
|
| 644 |
+
)
|
| 645 |
+
print_cache_selected_revisions(selected)
|
| 646 |
+
|
| 647 |
+
if dry_run:
|
| 648 |
+
out.result(
|
| 649 |
+
"Dry run: no files were deleted.",
|
| 650 |
+
dry_run=True,
|
| 651 |
+
revisions=counts.total_revision_count,
|
| 652 |
+
size=strategy.expected_freed_size_str,
|
| 653 |
+
)
|
| 654 |
+
return
|
| 655 |
+
|
| 656 |
+
out.confirm("Proceed?", yes=yes)
|
| 657 |
+
|
| 658 |
+
strategy.execute()
|
| 659 |
+
out.result(
|
| 660 |
+
f"Deleted {counts.total_revision_count} unreferenced revision(s); freed {strategy.expected_freed_size_str}.",
|
| 661 |
+
revisions_deleted=counts.total_revision_count,
|
| 662 |
+
freed=strategy.expected_freed_size_str,
|
| 663 |
+
)
|
| 664 |
+
|
| 665 |
+
|
| 666 |
+
@cache_cli.command(
|
| 667 |
+
examples=[
|
| 668 |
+
"hf cache verify gpt2",
|
| 669 |
+
"hf cache verify gpt2 --revision refs/pr/1",
|
| 670 |
+
"hf cache verify my-dataset --repo-type dataset",
|
| 671 |
+
],
|
| 672 |
+
)
|
| 673 |
+
def verify(
|
| 674 |
+
repo_id: RepoIdArg,
|
| 675 |
+
repo_type: RepoTypeOpt = RepoTypeOpt.model,
|
| 676 |
+
revision: RevisionOpt = None,
|
| 677 |
+
cache_dir: Annotated[
|
| 678 |
+
str | None,
|
| 679 |
+
typer.Option(
|
| 680 |
+
help="Cache directory to use when verifying files from cache (defaults to Hugging Face cache).",
|
| 681 |
+
),
|
| 682 |
+
] = None,
|
| 683 |
+
local_dir: Annotated[
|
| 684 |
+
str | None,
|
| 685 |
+
typer.Option(
|
| 686 |
+
help="If set, verify files under this directory instead of the cache.",
|
| 687 |
+
),
|
| 688 |
+
] = None,
|
| 689 |
+
fail_on_missing_files: Annotated[
|
| 690 |
+
bool,
|
| 691 |
+
typer.Option(
|
| 692 |
+
"--fail-on-missing-files",
|
| 693 |
+
help="Fail if some files exist on the remote but are missing locally.",
|
| 694 |
+
),
|
| 695 |
+
] = False,
|
| 696 |
+
fail_on_extra_files: Annotated[
|
| 697 |
+
bool,
|
| 698 |
+
typer.Option(
|
| 699 |
+
"--fail-on-extra-files",
|
| 700 |
+
help="Fail if some files exist locally but are not present on the remote revision.",
|
| 701 |
+
),
|
| 702 |
+
] = False,
|
| 703 |
+
token: TokenOpt = None,
|
| 704 |
+
) -> None:
|
| 705 |
+
"""Verify checksums for a single repo revision from cache or a local directory.
|
| 706 |
+
|
| 707 |
+
Examples:
|
| 708 |
+
- Verify main revision in cache: `hf cache verify gpt2`
|
| 709 |
+
- Verify specific revision: `hf cache verify gpt2 --revision refs/pr/1`
|
| 710 |
+
- Verify dataset: `hf cache verify karpathy/fineweb-edu-100b-shuffle --repo-type dataset`
|
| 711 |
+
- Verify local dir: `hf cache verify deepseek-ai/DeepSeek-OCR --local-dir /path/to/repo`
|
| 712 |
+
"""
|
| 713 |
+
|
| 714 |
+
if local_dir is not None and cache_dir is not None:
|
| 715 |
+
out.error("Cannot pass both --local-dir and --cache-dir. Use one or the other.")
|
| 716 |
+
raise typer.Exit(code=2)
|
| 717 |
+
|
| 718 |
+
api = get_hf_api(token=token)
|
| 719 |
+
|
| 720 |
+
result = api.verify_repo_checksums(
|
| 721 |
+
repo_id=repo_id,
|
| 722 |
+
repo_type=repo_type.value if hasattr(repo_type, "value") else str(repo_type),
|
| 723 |
+
revision=revision,
|
| 724 |
+
local_dir=local_dir,
|
| 725 |
+
cache_dir=cache_dir,
|
| 726 |
+
token=token,
|
| 727 |
+
)
|
| 728 |
+
|
| 729 |
+
exit_code = 0
|
| 730 |
+
|
| 731 |
+
if result.mismatches:
|
| 732 |
+
details = "\n".join(
|
| 733 |
+
f" - {m['path']}: expected {m['expected']} ({m['algorithm']}), got {m['actual']}"
|
| 734 |
+
for m in result.mismatches
|
| 735 |
+
)
|
| 736 |
+
out.text(f"❌ Checksum verification failed for the following file(s):\n{details}")
|
| 737 |
+
exit_code = 1
|
| 738 |
+
|
| 739 |
+
if result.missing_paths:
|
| 740 |
+
if fail_on_missing_files:
|
| 741 |
+
details = "\n".join(f" - {p}" for p in result.missing_paths)
|
| 742 |
+
out.text(f"❌ Missing files (present remotely, absent locally):\n{details}")
|
| 743 |
+
exit_code = 1
|
| 744 |
+
else:
|
| 745 |
+
out.warning(
|
| 746 |
+
f"{len(result.missing_paths)} remote file(s) are missing locally. "
|
| 747 |
+
"Use --fail-on-missing-files for details."
|
| 748 |
+
)
|
| 749 |
+
|
| 750 |
+
if result.extra_paths:
|
| 751 |
+
if fail_on_extra_files:
|
| 752 |
+
details = "\n".join(f" - {p}" for p in result.extra_paths)
|
| 753 |
+
out.text(f"❌ Extra files (present locally, absent remotely):\n{details}")
|
| 754 |
+
exit_code = 1
|
| 755 |
+
else:
|
| 756 |
+
out.warning(
|
| 757 |
+
f"{len(result.extra_paths)} local file(s) do not exist on the remote repo. "
|
| 758 |
+
"Use --fail-on-extra-files for details."
|
| 759 |
+
)
|
| 760 |
+
|
| 761 |
+
verified_location = result.verified_path
|
| 762 |
+
|
| 763 |
+
if exit_code != 0:
|
| 764 |
+
out.error(
|
| 765 |
+
f"Verification failed for '{repo_id}' ({repo_type.value}) in {verified_location}.\n Revision: {result.revision}"
|
| 766 |
+
)
|
| 767 |
+
raise typer.Exit(code=exit_code)
|
| 768 |
+
|
| 769 |
+
out.result(
|
| 770 |
+
f"Verified {result.checked_count} file(s) for {repo_type.value} '{repo_id}'. All checksums match.",
|
| 771 |
+
repo_id=repo_id,
|
| 772 |
+
repo_type=repo_type.value,
|
| 773 |
+
checked=result.checked_count,
|
| 774 |
+
path=str(verified_location),
|
| 775 |
+
)
|
venv/Lib/site-packages/huggingface_hub/cli/collections.py
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains commands to interact with collections on the Hugging Face Hub.
|
| 15 |
+
|
| 16 |
+
Usage:
|
| 17 |
+
# list collections on the Hub
|
| 18 |
+
hf collections ls
|
| 19 |
+
|
| 20 |
+
# list collections for a specific user
|
| 21 |
+
hf collections ls --owner username
|
| 22 |
+
|
| 23 |
+
# get info about a collection
|
| 24 |
+
hf collections info username/collection-slug
|
| 25 |
+
|
| 26 |
+
# create a new collection
|
| 27 |
+
hf collections create "My Collection" --description "A collection of models"
|
| 28 |
+
|
| 29 |
+
# add an item to a collection
|
| 30 |
+
hf collections add-item username/collection-slug username/model-name model
|
| 31 |
+
|
| 32 |
+
# delete a collection
|
| 33 |
+
hf collections delete username/collection-slug
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
import enum
|
| 37 |
+
from typing import Annotated, get_args
|
| 38 |
+
|
| 39 |
+
import typer
|
| 40 |
+
|
| 41 |
+
from huggingface_hub.hf_api import CollectionItemType_T, CollectionSort_T
|
| 42 |
+
|
| 43 |
+
from ._cli_utils import LimitOpt, TokenOpt, get_hf_api, typer_factory
|
| 44 |
+
from ._output import _dataclass_to_dict, out
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
# Build enums dynamically from Literal types to avoid duplication
|
| 48 |
+
_COLLECTION_ITEM_TYPES = get_args(CollectionItemType_T)
|
| 49 |
+
CollectionItemType = enum.Enum("CollectionItemType", {t: t for t in _COLLECTION_ITEM_TYPES}, type=str) # type: ignore[misc]
|
| 50 |
+
|
| 51 |
+
_COLLECTION_SORT_OPTIONS = get_args(CollectionSort_T)
|
| 52 |
+
CollectionSort = enum.Enum("CollectionSort", {s: s for s in _COLLECTION_SORT_OPTIONS}, type=str) # type: ignore[misc]
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
collections_cli = typer_factory(help="Interact with collections on the Hub.")
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@collections_cli.command(
|
| 59 |
+
"list | ls",
|
| 60 |
+
examples=[
|
| 61 |
+
"hf collections ls",
|
| 62 |
+
"hf collections ls --owner nvidia",
|
| 63 |
+
"hf collections ls --item models/teknium/OpenHermes-2.5-Mistral-7B --limit 10",
|
| 64 |
+
],
|
| 65 |
+
)
|
| 66 |
+
def collections_ls(
|
| 67 |
+
owner: Annotated[
|
| 68 |
+
str | None,
|
| 69 |
+
typer.Option(help="Filter by owner username or organization."),
|
| 70 |
+
] = None,
|
| 71 |
+
item: Annotated[
|
| 72 |
+
str | None,
|
| 73 |
+
typer.Option(
|
| 74 |
+
help='Filter collections containing a specific item (e.g., "models/gpt2", "datasets/squad", "papers/2311.12983").'
|
| 75 |
+
),
|
| 76 |
+
] = None,
|
| 77 |
+
sort: Annotated[
|
| 78 |
+
CollectionSort | None,
|
| 79 |
+
typer.Option(help="Sort results by last modified, trending, or upvotes."),
|
| 80 |
+
] = None,
|
| 81 |
+
limit: LimitOpt = 10,
|
| 82 |
+
token: TokenOpt = None,
|
| 83 |
+
) -> None:
|
| 84 |
+
"""List collections on the Hub."""
|
| 85 |
+
api = get_hf_api(token=token)
|
| 86 |
+
sort_key = sort.value if sort else None
|
| 87 |
+
results = [
|
| 88 |
+
_dataclass_to_dict(collection)
|
| 89 |
+
for collection in api.list_collections(
|
| 90 |
+
owner=owner,
|
| 91 |
+
item=item,
|
| 92 |
+
sort=sort_key, # type: ignore[arg-type]
|
| 93 |
+
limit=limit,
|
| 94 |
+
)
|
| 95 |
+
]
|
| 96 |
+
out.table(results)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
@collections_cli.command(
|
| 100 |
+
"info",
|
| 101 |
+
examples=[
|
| 102 |
+
"hf collections info username/my-collection-slug",
|
| 103 |
+
],
|
| 104 |
+
)
|
| 105 |
+
def collections_info(
|
| 106 |
+
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
| 107 |
+
token: TokenOpt = None,
|
| 108 |
+
) -> None:
|
| 109 |
+
"""Get info about a collection on the Hub."""
|
| 110 |
+
api = get_hf_api(token=token)
|
| 111 |
+
collection = api.get_collection(collection_slug)
|
| 112 |
+
out.dict(collection)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
@collections_cli.command(
|
| 116 |
+
"create",
|
| 117 |
+
examples=[
|
| 118 |
+
'hf collections create "My Models"',
|
| 119 |
+
'hf collections create "My Models" --description "A collection of my favorite models" --private',
|
| 120 |
+
'hf collections create "Org Collection" --namespace my-org',
|
| 121 |
+
],
|
| 122 |
+
)
|
| 123 |
+
def collections_create(
|
| 124 |
+
title: Annotated[str, typer.Argument(help="The title of the collection.")],
|
| 125 |
+
namespace: Annotated[
|
| 126 |
+
str | None,
|
| 127 |
+
typer.Option(help="The namespace (username or organization). Defaults to the authenticated user."),
|
| 128 |
+
] = None,
|
| 129 |
+
description: Annotated[
|
| 130 |
+
str | None,
|
| 131 |
+
typer.Option(help="A description for the collection."),
|
| 132 |
+
] = None,
|
| 133 |
+
private: Annotated[
|
| 134 |
+
bool,
|
| 135 |
+
typer.Option(help="Create a private collection."),
|
| 136 |
+
] = False,
|
| 137 |
+
exists_ok: Annotated[
|
| 138 |
+
bool,
|
| 139 |
+
typer.Option(help="Do not raise an error if the collection already exists."),
|
| 140 |
+
] = False,
|
| 141 |
+
token: TokenOpt = None,
|
| 142 |
+
) -> None:
|
| 143 |
+
"""Create a new collection on the Hub."""
|
| 144 |
+
api = get_hf_api(token=token)
|
| 145 |
+
collection = api.create_collection(
|
| 146 |
+
title=title,
|
| 147 |
+
namespace=namespace,
|
| 148 |
+
description=description,
|
| 149 |
+
private=private,
|
| 150 |
+
exists_ok=exists_ok,
|
| 151 |
+
)
|
| 152 |
+
out.result("Collection created", slug=collection.slug, url=collection.url)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
@collections_cli.command(
|
| 156 |
+
"update",
|
| 157 |
+
examples=[
|
| 158 |
+
'hf collections update username/my-collection --title "New Title"',
|
| 159 |
+
'hf collections update username/my-collection --description "Updated description"',
|
| 160 |
+
"hf collections update username/my-collection --private --theme green",
|
| 161 |
+
],
|
| 162 |
+
)
|
| 163 |
+
def collections_update(
|
| 164 |
+
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
| 165 |
+
title: Annotated[
|
| 166 |
+
str | None,
|
| 167 |
+
typer.Option(help="The new title for the collection."),
|
| 168 |
+
] = None,
|
| 169 |
+
description: Annotated[
|
| 170 |
+
str | None,
|
| 171 |
+
typer.Option(help="The new description for the collection."),
|
| 172 |
+
] = None,
|
| 173 |
+
position: Annotated[
|
| 174 |
+
int | None,
|
| 175 |
+
typer.Option(help="The new position of the collection in the owner's list."),
|
| 176 |
+
] = None,
|
| 177 |
+
private: Annotated[
|
| 178 |
+
bool | None,
|
| 179 |
+
typer.Option(help="Whether the collection should be private."),
|
| 180 |
+
] = None,
|
| 181 |
+
theme: Annotated[
|
| 182 |
+
str | None,
|
| 183 |
+
typer.Option(help="The theme color for the collection (e.g., 'green', 'blue')."),
|
| 184 |
+
] = None,
|
| 185 |
+
token: TokenOpt = None,
|
| 186 |
+
) -> None:
|
| 187 |
+
"""Update a collection's metadata on the Hub."""
|
| 188 |
+
api = get_hf_api(token=token)
|
| 189 |
+
collection = api.update_collection_metadata(
|
| 190 |
+
collection_slug=collection_slug,
|
| 191 |
+
title=title,
|
| 192 |
+
description=description,
|
| 193 |
+
position=position,
|
| 194 |
+
private=private,
|
| 195 |
+
theme=theme,
|
| 196 |
+
)
|
| 197 |
+
out.result("Collection updated", slug=collection.slug, url=collection.url)
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
@collections_cli.command(
|
| 201 |
+
"delete",
|
| 202 |
+
examples=[
|
| 203 |
+
"hf collections delete username/my-collection",
|
| 204 |
+
"hf collections delete username/my-collection --missing-ok",
|
| 205 |
+
],
|
| 206 |
+
)
|
| 207 |
+
def collections_delete(
|
| 208 |
+
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
| 209 |
+
missing_ok: Annotated[
|
| 210 |
+
bool,
|
| 211 |
+
typer.Option(help="Do not raise an error if the collection doesn't exist."),
|
| 212 |
+
] = False,
|
| 213 |
+
token: TokenOpt = None,
|
| 214 |
+
) -> None:
|
| 215 |
+
"""Delete a collection from the Hub."""
|
| 216 |
+
api = get_hf_api(token=token)
|
| 217 |
+
api.delete_collection(collection_slug, missing_ok=missing_ok)
|
| 218 |
+
out.result("Collection deleted", slug=collection_slug)
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
@collections_cli.command(
|
| 222 |
+
"add-item",
|
| 223 |
+
examples=[
|
| 224 |
+
"hf collections add-item username/my-collection moonshotai/kimi-k2 model",
|
| 225 |
+
'hf collections add-item username/my-collection Qwen/DeepPlanning dataset --note "Useful dataset"',
|
| 226 |
+
"hf collections add-item username/my-collection Tongyi-MAI/Z-Image space",
|
| 227 |
+
],
|
| 228 |
+
)
|
| 229 |
+
def collections_add_item(
|
| 230 |
+
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
| 231 |
+
item_id: Annotated[
|
| 232 |
+
str, typer.Argument(help="The ID of the item to add (repo_id for repos, paper ID for papers).")
|
| 233 |
+
],
|
| 234 |
+
item_type: Annotated[
|
| 235 |
+
CollectionItemType,
|
| 236 |
+
typer.Argument(help="The type of item (model, dataset, space, paper, collection, or bucket)."),
|
| 237 |
+
],
|
| 238 |
+
note: Annotated[
|
| 239 |
+
str | None,
|
| 240 |
+
typer.Option(help="A note to attach to the item (max 500 characters)."),
|
| 241 |
+
] = None,
|
| 242 |
+
exists_ok: Annotated[
|
| 243 |
+
bool,
|
| 244 |
+
typer.Option(help="Do not raise an error if the item is already in the collection."),
|
| 245 |
+
] = False,
|
| 246 |
+
token: TokenOpt = None,
|
| 247 |
+
) -> None:
|
| 248 |
+
"""Add an item to a collection."""
|
| 249 |
+
api = get_hf_api(token=token)
|
| 250 |
+
collection = api.add_collection_item(
|
| 251 |
+
collection_slug=collection_slug,
|
| 252 |
+
item_id=item_id,
|
| 253 |
+
item_type=item_type.value, # type: ignore[arg-type]
|
| 254 |
+
note=note,
|
| 255 |
+
exists_ok=exists_ok,
|
| 256 |
+
)
|
| 257 |
+
out.result("Item added to collection", slug=collection_slug, url=collection.url)
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
@collections_cli.command(
|
| 261 |
+
"update-item",
|
| 262 |
+
examples=[
|
| 263 |
+
'hf collections update-item username/my-collection ITEM_OBJECT_ID --note "Updated note"',
|
| 264 |
+
"hf collections update-item username/my-collection ITEM_OBJECT_ID --position 0",
|
| 265 |
+
],
|
| 266 |
+
)
|
| 267 |
+
def collections_update_item(
|
| 268 |
+
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
| 269 |
+
item_object_id: Annotated[
|
| 270 |
+
str,
|
| 271 |
+
typer.Argument(help="The ID of the item in the collection (from 'item_object_id' field, not the repo_id)."),
|
| 272 |
+
],
|
| 273 |
+
note: Annotated[
|
| 274 |
+
str | None,
|
| 275 |
+
typer.Option(help="A new note for the item (max 500 characters)."),
|
| 276 |
+
] = None,
|
| 277 |
+
position: Annotated[
|
| 278 |
+
int | None,
|
| 279 |
+
typer.Option(help="The new position of the item in the collection."),
|
| 280 |
+
] = None,
|
| 281 |
+
token: TokenOpt = None,
|
| 282 |
+
) -> None:
|
| 283 |
+
"""Update an item in a collection."""
|
| 284 |
+
api = get_hf_api(token=token)
|
| 285 |
+
api.update_collection_item(
|
| 286 |
+
collection_slug=collection_slug,
|
| 287 |
+
item_object_id=item_object_id,
|
| 288 |
+
note=note,
|
| 289 |
+
position=position,
|
| 290 |
+
)
|
| 291 |
+
out.result("Item updated in collection", slug=collection_slug)
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
@collections_cli.command("delete-item")
|
| 295 |
+
def collections_delete_item(
|
| 296 |
+
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
| 297 |
+
item_object_id: Annotated[
|
| 298 |
+
str,
|
| 299 |
+
typer.Argument(
|
| 300 |
+
help="The ID of the item in the collection (retrieved from `item_object_id` field returned by 'hf collections info'."
|
| 301 |
+
),
|
| 302 |
+
],
|
| 303 |
+
missing_ok: Annotated[
|
| 304 |
+
bool,
|
| 305 |
+
typer.Option(help="Do not raise an error if the item doesn't exist."),
|
| 306 |
+
] = False,
|
| 307 |
+
token: TokenOpt = None,
|
| 308 |
+
) -> None:
|
| 309 |
+
"""Delete an item from a collection."""
|
| 310 |
+
api = get_hf_api(token=token)
|
| 311 |
+
api.delete_collection_item(
|
| 312 |
+
collection_slug=collection_slug,
|
| 313 |
+
item_object_id=item_object_id,
|
| 314 |
+
missing_ok=missing_ok,
|
| 315 |
+
)
|
| 316 |
+
out.result("Item deleted from collection", slug=collection_slug)
|
venv/Lib/site-packages/huggingface_hub/cli/datasets.py
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains commands to interact with datasets on the Hugging Face Hub.
|
| 15 |
+
|
| 16 |
+
Usage:
|
| 17 |
+
# list datasets on the Hub
|
| 18 |
+
hf datasets ls
|
| 19 |
+
|
| 20 |
+
# list datasets with a search query
|
| 21 |
+
hf datasets ls --search "code"
|
| 22 |
+
|
| 23 |
+
# get info about a dataset
|
| 24 |
+
hf datasets info HuggingFaceFW/fineweb
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
import enum
|
| 28 |
+
from typing import Annotated, get_args
|
| 29 |
+
|
| 30 |
+
import typer
|
| 31 |
+
|
| 32 |
+
from huggingface_hub._dataset_viewer import execute_raw_sql_query
|
| 33 |
+
from huggingface_hub.errors import CLIError, RepositoryNotFoundError, RevisionNotFoundError
|
| 34 |
+
from huggingface_hub.hf_api import DatasetSort_T, ExpandDatasetProperty_T
|
| 35 |
+
from huggingface_hub.repocard import DatasetCard
|
| 36 |
+
|
| 37 |
+
from ._cli_utils import (
|
| 38 |
+
REPO_LIST_DEFAULT_LIMIT,
|
| 39 |
+
AuthorOpt,
|
| 40 |
+
FilterOpt,
|
| 41 |
+
LimitOpt,
|
| 42 |
+
RevisionOpt,
|
| 43 |
+
SearchOpt,
|
| 44 |
+
TokenOpt,
|
| 45 |
+
get_hf_api,
|
| 46 |
+
make_expand_properties_parser,
|
| 47 |
+
typer_factory,
|
| 48 |
+
)
|
| 49 |
+
from ._file_listing import list_repo_files_cmd
|
| 50 |
+
from ._output import _dataclass_to_dict, out
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
_EXPAND_PROPERTIES = sorted(get_args(ExpandDatasetProperty_T))
|
| 54 |
+
_SORT_OPTIONS = get_args(DatasetSort_T)
|
| 55 |
+
DatasetSortEnum = enum.Enum("DatasetSortEnum", {s: s for s in _SORT_OPTIONS}, type=str) # type: ignore[misc]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
ExpandOpt = Annotated[
|
| 59 |
+
str | None,
|
| 60 |
+
typer.Option(
|
| 61 |
+
help=f"Comma-separated properties to return. When used, only the listed properties (and id) are returned. Example: '--expand=downloads,likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.",
|
| 62 |
+
callback=make_expand_properties_parser(_EXPAND_PROPERTIES),
|
| 63 |
+
),
|
| 64 |
+
]
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
datasets_cli = typer_factory(help="Interact with datasets on the Hub.")
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
@datasets_cli.command(
|
| 71 |
+
"list | ls",
|
| 72 |
+
examples=[
|
| 73 |
+
"hf datasets ls",
|
| 74 |
+
"hf datasets ls --sort downloads --limit 10",
|
| 75 |
+
'hf datasets ls --search "code"',
|
| 76 |
+
"hf datasets ls --filter benchmark:official",
|
| 77 |
+
"hf datasets ls HuggingFaceFW/fineweb",
|
| 78 |
+
"hf datasets ls HuggingFaceFW/fineweb -R",
|
| 79 |
+
"hf datasets ls HuggingFaceFW/fineweb --tree -h",
|
| 80 |
+
],
|
| 81 |
+
)
|
| 82 |
+
def datasets_ls(
|
| 83 |
+
repo_id: Annotated[
|
| 84 |
+
str | None,
|
| 85 |
+
typer.Argument(help="Dataset ID (e.g. `username/repo-name`) to list files from. If omitted, lists datasets."),
|
| 86 |
+
] = None,
|
| 87 |
+
search: SearchOpt = None,
|
| 88 |
+
author: AuthorOpt = None,
|
| 89 |
+
filter: FilterOpt = None,
|
| 90 |
+
sort: Annotated[
|
| 91 |
+
DatasetSortEnum | None,
|
| 92 |
+
typer.Option(help="Sort results."),
|
| 93 |
+
] = None,
|
| 94 |
+
limit: LimitOpt = REPO_LIST_DEFAULT_LIMIT,
|
| 95 |
+
expand: ExpandOpt = None,
|
| 96 |
+
human_readable: Annotated[
|
| 97 |
+
bool,
|
| 98 |
+
typer.Option("--human-readable", "-h", help="Show sizes in human readable format (only for listing files)."),
|
| 99 |
+
] = False,
|
| 100 |
+
as_tree: Annotated[
|
| 101 |
+
bool,
|
| 102 |
+
typer.Option("--tree", help="List files in tree format (only for listing files)."),
|
| 103 |
+
] = False,
|
| 104 |
+
recursive: Annotated[
|
| 105 |
+
bool,
|
| 106 |
+
typer.Option("--recursive", "-R", help="List files recursively (only for listing files)."),
|
| 107 |
+
] = False,
|
| 108 |
+
revision: RevisionOpt = None,
|
| 109 |
+
token: TokenOpt = None,
|
| 110 |
+
) -> None:
|
| 111 |
+
"""List datasets on the Hub, or files in a dataset repo.
|
| 112 |
+
|
| 113 |
+
When called with no argument, lists datasets on the Hub.
|
| 114 |
+
When called with a dataset ID, lists files in that dataset repo.
|
| 115 |
+
"""
|
| 116 |
+
if repo_id is not None:
|
| 117 |
+
if search is not None:
|
| 118 |
+
raise typer.BadParameter("Cannot use --search when listing files.")
|
| 119 |
+
if author is not None:
|
| 120 |
+
raise typer.BadParameter("Cannot use --author when listing files.")
|
| 121 |
+
if filter is not None:
|
| 122 |
+
raise typer.BadParameter("Cannot use --filter when listing files.")
|
| 123 |
+
if sort is not None:
|
| 124 |
+
raise typer.BadParameter("Cannot use --sort when listing files.")
|
| 125 |
+
if limit != REPO_LIST_DEFAULT_LIMIT:
|
| 126 |
+
raise typer.BadParameter("Cannot use --limit when listing files.")
|
| 127 |
+
if expand is not None:
|
| 128 |
+
raise typer.BadParameter("Cannot use --expand when listing files.")
|
| 129 |
+
return list_repo_files_cmd(
|
| 130 |
+
repo_id=repo_id,
|
| 131 |
+
repo_type="dataset",
|
| 132 |
+
human_readable=human_readable,
|
| 133 |
+
as_tree=as_tree,
|
| 134 |
+
recursive=recursive,
|
| 135 |
+
revision=revision,
|
| 136 |
+
token=token,
|
| 137 |
+
)
|
| 138 |
+
|
| 139 |
+
if as_tree:
|
| 140 |
+
raise typer.BadParameter("Cannot use --tree when listing datasets.")
|
| 141 |
+
if recursive:
|
| 142 |
+
raise typer.BadParameter("Cannot use --recursive when listing datasets.")
|
| 143 |
+
if human_readable:
|
| 144 |
+
raise typer.BadParameter("Cannot use --human-readable when listing datasets.")
|
| 145 |
+
if revision is not None:
|
| 146 |
+
raise typer.BadParameter("Cannot use --revision when listing datasets.")
|
| 147 |
+
|
| 148 |
+
api = get_hf_api(token=token)
|
| 149 |
+
sort_key = sort.value if sort else None
|
| 150 |
+
results = [
|
| 151 |
+
_dataclass_to_dict(dataset_info)
|
| 152 |
+
for dataset_info in api.list_datasets(
|
| 153 |
+
filter=filter,
|
| 154 |
+
author=author,
|
| 155 |
+
search=search,
|
| 156 |
+
sort=sort_key,
|
| 157 |
+
limit=limit,
|
| 158 |
+
expand=expand, # type: ignore
|
| 159 |
+
)
|
| 160 |
+
]
|
| 161 |
+
out.table(results)
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
@datasets_cli.command(
|
| 165 |
+
"leaderboard",
|
| 166 |
+
examples=[
|
| 167 |
+
"hf datasets leaderboard SWE-bench/SWE-bench_Verified",
|
| 168 |
+
"hf datasets leaderboard SWE-bench/SWE-bench_Verified --limit 5 --format json",
|
| 169 |
+
"hf datasets ls --filter benchmark:official # list available leaderboards",
|
| 170 |
+
],
|
| 171 |
+
)
|
| 172 |
+
def datasets_leaderboard(
|
| 173 |
+
dataset_id: Annotated[str, typer.Argument(help="The benchmark dataset ID (e.g. `SWE-bench/SWE-bench_Verified`).")],
|
| 174 |
+
limit: LimitOpt = 20,
|
| 175 |
+
token: TokenOpt = None,
|
| 176 |
+
) -> None:
|
| 177 |
+
"""List model scores from a dataset leaderboard. This command helps find the best models for a task or compare models by benchmark scores. Use 'hf datasets ls --filter benchmark:official' to list available leaderboards."""
|
| 178 |
+
api = get_hf_api(token=token)
|
| 179 |
+
leaderboard = api.get_dataset_leaderboard(repo_id=dataset_id)
|
| 180 |
+
results = [_dataclass_to_dict(entry) for entry in leaderboard[:limit]]
|
| 181 |
+
out.table(
|
| 182 |
+
results,
|
| 183 |
+
headers=["rank", "model_id", "value", "source"],
|
| 184 |
+
id_key="model_id",
|
| 185 |
+
)
|
| 186 |
+
out.hint("Use 'hf datasets ls --filter benchmark:official' to list available leaderboards.")
|
| 187 |
+
if leaderboard:
|
| 188 |
+
out.hint(f"Use 'hf models info {leaderboard[0].model_id}' to get details about a model.")
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
@datasets_cli.command(
|
| 192 |
+
"info",
|
| 193 |
+
examples=[
|
| 194 |
+
"hf datasets info HuggingFaceFW/fineweb",
|
| 195 |
+
"hf datasets info my-dataset --expand downloads,likes,tags",
|
| 196 |
+
],
|
| 197 |
+
)
|
| 198 |
+
def datasets_info(
|
| 199 |
+
dataset_id: Annotated[str, typer.Argument(help="The dataset ID (e.g. `username/repo-name`).")],
|
| 200 |
+
revision: RevisionOpt = None,
|
| 201 |
+
expand: ExpandOpt = None,
|
| 202 |
+
token: TokenOpt = None,
|
| 203 |
+
) -> None:
|
| 204 |
+
"""Get info about a dataset on the Hub."""
|
| 205 |
+
api = get_hf_api(token=token)
|
| 206 |
+
try:
|
| 207 |
+
info = api.dataset_info(repo_id=dataset_id, revision=revision, expand=expand) # type: ignore
|
| 208 |
+
except RepositoryNotFoundError as e:
|
| 209 |
+
raise CLIError(f"Dataset '{dataset_id}' not found.") from e
|
| 210 |
+
except RevisionNotFoundError as e:
|
| 211 |
+
raise CLIError(f"Revision '{revision}' not found on '{dataset_id}'.") from e
|
| 212 |
+
out.dict(info)
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
@datasets_cli.command(
|
| 216 |
+
"parquet",
|
| 217 |
+
examples=[
|
| 218 |
+
"hf datasets parquet cfahlgren1/hub-stats",
|
| 219 |
+
"hf datasets parquet cfahlgren1/hub-stats --subset models",
|
| 220 |
+
"hf datasets parquet cfahlgren1/hub-stats --split train",
|
| 221 |
+
"hf datasets parquet cfahlgren1/hub-stats --format json",
|
| 222 |
+
],
|
| 223 |
+
)
|
| 224 |
+
def datasets_parquet(
|
| 225 |
+
dataset_id: Annotated[str, typer.Argument(help="The dataset ID (e.g. `username/repo-name`).")],
|
| 226 |
+
subset: Annotated[str | None, typer.Option("--subset", help="Filter parquet entries by subset/config.")] = None,
|
| 227 |
+
split: Annotated[str | None, typer.Option(help="Filter parquet entries by split.")] = None,
|
| 228 |
+
token: TokenOpt = None,
|
| 229 |
+
) -> None:
|
| 230 |
+
"""List parquet file URLs available for a dataset."""
|
| 231 |
+
api = get_hf_api(token=token)
|
| 232 |
+
entries = api.list_dataset_parquet_files(repo_id=dataset_id, config=subset)
|
| 233 |
+
filtered = [entry for entry in entries if split is None or entry.split == split]
|
| 234 |
+
results = [
|
| 235 |
+
{"subset": entry.config, "split": entry.split, "url": entry.url, "size": entry.size} for entry in filtered
|
| 236 |
+
]
|
| 237 |
+
out.table(results, headers=["subset", "split", "url", "size"], id_key="url")
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
@datasets_cli.command(
|
| 241 |
+
"sql",
|
| 242 |
+
examples=[
|
| 243 |
+
"hf datasets sql \"SELECT COUNT(*) AS rows FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet')\"",
|
| 244 |
+
"hf datasets sql \"SELECT * FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet') LIMIT 5\" --format json",
|
| 245 |
+
],
|
| 246 |
+
)
|
| 247 |
+
def datasets_sql(
|
| 248 |
+
sql: Annotated[str, typer.Argument(help="Raw SQL query to execute.")],
|
| 249 |
+
token: TokenOpt = None,
|
| 250 |
+
) -> None:
|
| 251 |
+
"""Execute a raw SQL query with DuckDB against dataset parquet URLs."""
|
| 252 |
+
try:
|
| 253 |
+
result = execute_raw_sql_query(sql_query=sql, token=token)
|
| 254 |
+
except ImportError as e:
|
| 255 |
+
raise CLIError(str(e)) from e
|
| 256 |
+
out.table(result)
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
@datasets_cli.command(
|
| 260 |
+
"card",
|
| 261 |
+
examples=[
|
| 262 |
+
"hf datasets card HuggingFaceFW/fineweb",
|
| 263 |
+
"hf datasets card HuggingFaceFW/fineweb --metadata",
|
| 264 |
+
"hf datasets card HuggingFaceFW/fineweb --metadata --format json",
|
| 265 |
+
"hf datasets card HuggingFaceFW/fineweb --text",
|
| 266 |
+
],
|
| 267 |
+
)
|
| 268 |
+
def datasets_card(
|
| 269 |
+
dataset_id: Annotated[str, typer.Argument(help="The dataset ID (e.g. `username/repo-name`).")],
|
| 270 |
+
metadata: Annotated[bool, typer.Option("--metadata", help="Output only the metadata from the card.")] = False,
|
| 271 |
+
text: Annotated[bool, typer.Option("--text", help="Output only the text body (no metadata).")] = False,
|
| 272 |
+
token: TokenOpt = None,
|
| 273 |
+
) -> None:
|
| 274 |
+
"""Get the dataset card (README) for a dataset on the Hub."""
|
| 275 |
+
if metadata and text:
|
| 276 |
+
raise CLIError("--metadata and --text are mutually exclusive.")
|
| 277 |
+
card = DatasetCard.load(dataset_id, token=token)
|
| 278 |
+
if metadata:
|
| 279 |
+
out.dict(card.data.to_dict())
|
| 280 |
+
elif text:
|
| 281 |
+
out.text(card.text)
|
| 282 |
+
else:
|
| 283 |
+
out.text(card.content)
|
| 284 |
+
out.hint(f"Use `hf datasets card {dataset_id} --metadata` to extract only the card metadata.")
|
venv/Lib/site-packages/huggingface_hub/cli/deprecated_cli.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deprecated `huggingface-cli` entry point. Warns and exits."""
|
| 2 |
+
|
| 3 |
+
import shutil
|
| 4 |
+
import sys
|
| 5 |
+
|
| 6 |
+
from ._output import out
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def main() -> None:
|
| 10 |
+
out.warning("`huggingface-cli` is deprecated and no longer works. Use `hf` instead.\n")
|
| 11 |
+
|
| 12 |
+
if shutil.which("hf"):
|
| 13 |
+
from huggingface_hub.cli._cli_utils import check_cli_update
|
| 14 |
+
|
| 15 |
+
check_cli_update("huggingface_hub")
|
| 16 |
+
out.hint("`hf` is already installed! Use it directly.\n")
|
| 17 |
+
else:
|
| 18 |
+
out.hint(
|
| 19 |
+
"Install `hf`:\n"
|
| 20 |
+
" Standalone (recommended): curl -LsSf https://hf.co/cli/install.sh | bash\n"
|
| 21 |
+
" Using Homebrew: brew install hf\n"
|
| 22 |
+
" Using pip: pip install huggingface_hub\n",
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
out.hint(
|
| 26 |
+
"Examples:\n"
|
| 27 |
+
" hf auth login\n"
|
| 28 |
+
" hf download unsloth/gemma-4-31B-it-GGUF\n"
|
| 29 |
+
" hf upload my-cool-model . .\n"
|
| 30 |
+
' hf models ls --search "gemma"\n'
|
| 31 |
+
" hf repos ls --format json\n"
|
| 32 |
+
" hf jobs run python:3.12 python -c 'print(\"Hello!\")'\n"
|
| 33 |
+
" hf --help\n",
|
| 34 |
+
)
|
| 35 |
+
sys.exit(1)
|
venv/Lib/site-packages/huggingface_hub/cli/discussions.py
ADDED
|
@@ -0,0 +1,448 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains commands to interact with discussions and pull requests on the Hugging Face Hub."""
|
| 15 |
+
|
| 16 |
+
import enum
|
| 17 |
+
import sys
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
from typing import Annotated
|
| 20 |
+
|
| 21 |
+
import typer
|
| 22 |
+
|
| 23 |
+
from huggingface_hub import constants
|
| 24 |
+
|
| 25 |
+
from ._cli_utils import (
|
| 26 |
+
AuthorOpt,
|
| 27 |
+
LimitOpt,
|
| 28 |
+
RepoIdArg,
|
| 29 |
+
RepoType,
|
| 30 |
+
RepoTypeOpt,
|
| 31 |
+
TokenOpt,
|
| 32 |
+
get_hf_api,
|
| 33 |
+
typer_factory,
|
| 34 |
+
)
|
| 35 |
+
from ._output import _dataclass_to_dict, out
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class DiscussionStatus(str, enum.Enum):
|
| 39 |
+
open = "open"
|
| 40 |
+
closed = "closed"
|
| 41 |
+
merged = "merged"
|
| 42 |
+
draft = "draft"
|
| 43 |
+
all = "all"
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class DiscussionKind(str, enum.Enum):
|
| 47 |
+
all = "all"
|
| 48 |
+
discussion = "discussion"
|
| 49 |
+
pull_request = "pull_request"
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
# "merged" and "draft" are valid Discussion statuses but the Hub API filter
|
| 53 |
+
# (DiscussionStatusFilter) only accepts "all", "open", "closed". When the user
|
| 54 |
+
# asks for merged/draft we fetch with api_status=None (i.e. all) and filter
|
| 55 |
+
# client-side.
|
| 56 |
+
_CLIENT_SIDE_STATUSES = {"merged", "draft"}
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
DiscussionNumArg = Annotated[
|
| 60 |
+
int,
|
| 61 |
+
typer.Argument(
|
| 62 |
+
help="The discussion or pull request number.",
|
| 63 |
+
min=1,
|
| 64 |
+
),
|
| 65 |
+
]
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _read_body(body: str | None, body_file: Path | None) -> str | None:
|
| 69 |
+
"""Resolve body text from --body or --body-file (supports '-' for stdin)."""
|
| 70 |
+
if body is not None and body_file is not None:
|
| 71 |
+
raise typer.BadParameter("Cannot use both --body and --body-file.")
|
| 72 |
+
if body_file is not None:
|
| 73 |
+
if str(body_file) == "-":
|
| 74 |
+
return sys.stdin.read()
|
| 75 |
+
return body_file.read_text(encoding="utf-8")
|
| 76 |
+
return body
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
discussions_cli = typer_factory(help="Manage discussions and pull requests on the Hub.")
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
@discussions_cli.command(
|
| 83 |
+
"list | ls",
|
| 84 |
+
examples=[
|
| 85 |
+
"hf discussions list username/my-model",
|
| 86 |
+
"hf discussions list username/my-model --kind pull_request --status merged",
|
| 87 |
+
"hf discussions list username/my-dataset --type dataset --status closed",
|
| 88 |
+
"hf discussions list username/my-model --author alice --format json",
|
| 89 |
+
],
|
| 90 |
+
)
|
| 91 |
+
def discussion_list(
|
| 92 |
+
repo_id: RepoIdArg,
|
| 93 |
+
status: Annotated[
|
| 94 |
+
DiscussionStatus,
|
| 95 |
+
typer.Option(
|
| 96 |
+
"-s",
|
| 97 |
+
"--status",
|
| 98 |
+
help="Filter by status (open, closed, merged, draft, all).",
|
| 99 |
+
),
|
| 100 |
+
] = DiscussionStatus.open,
|
| 101 |
+
kind: Annotated[
|
| 102 |
+
DiscussionKind,
|
| 103 |
+
typer.Option(
|
| 104 |
+
"-k",
|
| 105 |
+
"--kind",
|
| 106 |
+
help="Filter by kind (discussion, pull_request, all).",
|
| 107 |
+
),
|
| 108 |
+
] = DiscussionKind.all,
|
| 109 |
+
author: AuthorOpt = None,
|
| 110 |
+
limit: LimitOpt = 30,
|
| 111 |
+
repo_type: RepoTypeOpt = RepoType.model,
|
| 112 |
+
token: TokenOpt = None,
|
| 113 |
+
) -> None:
|
| 114 |
+
"""List discussions and pull requests on a repo."""
|
| 115 |
+
api = get_hf_api(token=token)
|
| 116 |
+
|
| 117 |
+
api_status: constants.DiscussionStatusFilter | None
|
| 118 |
+
if status == DiscussionStatus.open:
|
| 119 |
+
api_status = "open"
|
| 120 |
+
elif status == DiscussionStatus.closed:
|
| 121 |
+
api_status = "closed"
|
| 122 |
+
else:
|
| 123 |
+
api_status = None
|
| 124 |
+
|
| 125 |
+
api_discussion_type: constants.DiscussionTypeFilter | None
|
| 126 |
+
if kind == DiscussionKind.all:
|
| 127 |
+
api_discussion_type = None
|
| 128 |
+
else:
|
| 129 |
+
api_discussion_type = kind.value # type: ignore[assignment]
|
| 130 |
+
|
| 131 |
+
discussions = []
|
| 132 |
+
for d in api.get_repo_discussions(
|
| 133 |
+
repo_id=repo_id,
|
| 134 |
+
author=author,
|
| 135 |
+
discussion_type=api_discussion_type,
|
| 136 |
+
discussion_status=api_status,
|
| 137 |
+
repo_type=repo_type.value,
|
| 138 |
+
):
|
| 139 |
+
if status.value in _CLIENT_SIDE_STATUSES and d.status != status.value:
|
| 140 |
+
continue
|
| 141 |
+
discussions.append(d)
|
| 142 |
+
if len(discussions) >= limit:
|
| 143 |
+
break
|
| 144 |
+
|
| 145 |
+
items = [_dataclass_to_dict(d) for d in discussions]
|
| 146 |
+
out.table(
|
| 147 |
+
items,
|
| 148 |
+
headers=["num", "title", "is_pull_request", "status", "author", "created_at"],
|
| 149 |
+
id_key="num",
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
@discussions_cli.command(
|
| 154 |
+
"info",
|
| 155 |
+
examples=[
|
| 156 |
+
"hf discussions info username/my-model 5",
|
| 157 |
+
"hf discussions info username/my-model 5 --format json",
|
| 158 |
+
],
|
| 159 |
+
)
|
| 160 |
+
def discussion_info(
|
| 161 |
+
repo_id: RepoIdArg,
|
| 162 |
+
num: DiscussionNumArg,
|
| 163 |
+
repo_type: RepoTypeOpt = RepoType.model,
|
| 164 |
+
token: TokenOpt = None,
|
| 165 |
+
) -> None:
|
| 166 |
+
"""Get info about a discussion or pull request."""
|
| 167 |
+
api = get_hf_api(token=token)
|
| 168 |
+
details = api.get_discussion_details(
|
| 169 |
+
repo_id=repo_id,
|
| 170 |
+
discussion_num=num,
|
| 171 |
+
repo_type=repo_type.value,
|
| 172 |
+
)
|
| 173 |
+
out.dict(details)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
@discussions_cli.command(
|
| 177 |
+
"create",
|
| 178 |
+
examples=[
|
| 179 |
+
'hf discussions create username/my-model --title "Bug report"',
|
| 180 |
+
'hf discussions create username/my-model --title "Feature request" --body "Please add X"',
|
| 181 |
+
'hf discussions create username/my-model --title "Fix typo" --pull-request',
|
| 182 |
+
'hf discussions create username/my-dataset --type dataset --title "Data quality issue"',
|
| 183 |
+
],
|
| 184 |
+
)
|
| 185 |
+
def discussion_create(
|
| 186 |
+
repo_id: RepoIdArg,
|
| 187 |
+
title: Annotated[
|
| 188 |
+
str,
|
| 189 |
+
typer.Option(
|
| 190 |
+
"--title",
|
| 191 |
+
help="The title of the discussion or pull request.",
|
| 192 |
+
),
|
| 193 |
+
],
|
| 194 |
+
body: Annotated[
|
| 195 |
+
str | None,
|
| 196 |
+
typer.Option(
|
| 197 |
+
"--body",
|
| 198 |
+
help="The description (supports Markdown).",
|
| 199 |
+
),
|
| 200 |
+
] = None,
|
| 201 |
+
body_file: Annotated[
|
| 202 |
+
Path | None,
|
| 203 |
+
typer.Option(
|
| 204 |
+
"--body-file",
|
| 205 |
+
help="Read the description from a file. Use '-' for stdin.",
|
| 206 |
+
),
|
| 207 |
+
] = None,
|
| 208 |
+
pull_request: Annotated[
|
| 209 |
+
bool,
|
| 210 |
+
typer.Option(
|
| 211 |
+
"--pull-request",
|
| 212 |
+
"--pr",
|
| 213 |
+
help="Create a pull request instead of a discussion.",
|
| 214 |
+
),
|
| 215 |
+
] = False,
|
| 216 |
+
repo_type: RepoTypeOpt = RepoType.model,
|
| 217 |
+
token: TokenOpt = None,
|
| 218 |
+
) -> None:
|
| 219 |
+
"""Create a new discussion or pull request on a repo."""
|
| 220 |
+
description = _read_body(body, body_file)
|
| 221 |
+
api = get_hf_api(token=token)
|
| 222 |
+
discussion = api.create_discussion(
|
| 223 |
+
repo_id=repo_id,
|
| 224 |
+
title=title,
|
| 225 |
+
description=description,
|
| 226 |
+
repo_type=repo_type.value,
|
| 227 |
+
pull_request=pull_request,
|
| 228 |
+
)
|
| 229 |
+
kind = "pull request" if pull_request else "discussion"
|
| 230 |
+
ref = f"refs/pr/{discussion.num}" if pull_request else None
|
| 231 |
+
out.result(f"Created {kind} #{discussion.num} on {repo_id}", num=discussion.num, url=discussion.url, ref=ref)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
@discussions_cli.command(
|
| 235 |
+
"comment",
|
| 236 |
+
examples=[
|
| 237 |
+
'hf discussions comment username/my-model 5 --body "Thanks for reporting!"',
|
| 238 |
+
'hf discussions comment username/my-model 5 --body "LGTM!"',
|
| 239 |
+
],
|
| 240 |
+
)
|
| 241 |
+
def discussion_comment(
|
| 242 |
+
repo_id: RepoIdArg,
|
| 243 |
+
num: DiscussionNumArg,
|
| 244 |
+
body: Annotated[
|
| 245 |
+
str | None,
|
| 246 |
+
typer.Option(
|
| 247 |
+
"--body",
|
| 248 |
+
help="The comment text (supports Markdown).",
|
| 249 |
+
),
|
| 250 |
+
] = None,
|
| 251 |
+
body_file: Annotated[
|
| 252 |
+
Path | None,
|
| 253 |
+
typer.Option(
|
| 254 |
+
"--body-file",
|
| 255 |
+
help="Read the comment from a file. Use '-' for stdin.",
|
| 256 |
+
),
|
| 257 |
+
] = None,
|
| 258 |
+
repo_type: RepoTypeOpt = RepoType.model,
|
| 259 |
+
token: TokenOpt = None,
|
| 260 |
+
) -> None:
|
| 261 |
+
"""Comment on a discussion or pull request."""
|
| 262 |
+
comment = _read_body(body, body_file)
|
| 263 |
+
if comment is None:
|
| 264 |
+
raise typer.BadParameter("Either --body or --body-file is required.")
|
| 265 |
+
api = get_hf_api(token=token)
|
| 266 |
+
api.comment_discussion(
|
| 267 |
+
repo_id=repo_id,
|
| 268 |
+
discussion_num=num,
|
| 269 |
+
comment=comment,
|
| 270 |
+
repo_type=repo_type.value,
|
| 271 |
+
)
|
| 272 |
+
out.result(f"Commented on #{num} in {repo_id}", num=num, repo=repo_id)
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
@discussions_cli.command(
|
| 276 |
+
"close",
|
| 277 |
+
examples=[
|
| 278 |
+
"hf discussions close username/my-model 5",
|
| 279 |
+
'hf discussions close username/my-model 5 --comment "Closing as resolved."',
|
| 280 |
+
],
|
| 281 |
+
)
|
| 282 |
+
def discussion_close(
|
| 283 |
+
repo_id: RepoIdArg,
|
| 284 |
+
num: DiscussionNumArg,
|
| 285 |
+
comment: Annotated[
|
| 286 |
+
str | None,
|
| 287 |
+
typer.Option(
|
| 288 |
+
"--comment",
|
| 289 |
+
help="An optional comment to post when closing.",
|
| 290 |
+
),
|
| 291 |
+
] = None,
|
| 292 |
+
yes: Annotated[
|
| 293 |
+
bool,
|
| 294 |
+
typer.Option(
|
| 295 |
+
"--yes",
|
| 296 |
+
"-y",
|
| 297 |
+
help="Skip confirmation prompt.",
|
| 298 |
+
),
|
| 299 |
+
] = False,
|
| 300 |
+
repo_type: RepoTypeOpt = RepoType.model,
|
| 301 |
+
token: TokenOpt = None,
|
| 302 |
+
) -> None:
|
| 303 |
+
"""Close a discussion or pull request."""
|
| 304 |
+
out.confirm(f"Close #{num} on '{repo_id}'?", yes=yes)
|
| 305 |
+
api = get_hf_api(token=token)
|
| 306 |
+
api.change_discussion_status(
|
| 307 |
+
repo_id=repo_id,
|
| 308 |
+
discussion_num=num,
|
| 309 |
+
new_status="closed",
|
| 310 |
+
comment=comment,
|
| 311 |
+
repo_type=repo_type.value,
|
| 312 |
+
)
|
| 313 |
+
out.result(f"Closed #{num} in {repo_id}", num=num, repo=repo_id)
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
@discussions_cli.command(
|
| 317 |
+
"reopen",
|
| 318 |
+
examples=[
|
| 319 |
+
"hf discussions reopen username/my-model 5",
|
| 320 |
+
'hf discussions reopen username/my-model 5 --comment "Reopening for further investigation."',
|
| 321 |
+
],
|
| 322 |
+
)
|
| 323 |
+
def discussion_reopen(
|
| 324 |
+
repo_id: RepoIdArg,
|
| 325 |
+
num: DiscussionNumArg,
|
| 326 |
+
comment: Annotated[
|
| 327 |
+
str | None,
|
| 328 |
+
typer.Option(
|
| 329 |
+
"--comment",
|
| 330 |
+
help="An optional comment to post when reopening.",
|
| 331 |
+
),
|
| 332 |
+
] = None,
|
| 333 |
+
yes: Annotated[
|
| 334 |
+
bool,
|
| 335 |
+
typer.Option(
|
| 336 |
+
"--yes",
|
| 337 |
+
"-y",
|
| 338 |
+
help="Skip confirmation prompt.",
|
| 339 |
+
),
|
| 340 |
+
] = False,
|
| 341 |
+
repo_type: RepoTypeOpt = RepoType.model,
|
| 342 |
+
token: TokenOpt = None,
|
| 343 |
+
) -> None:
|
| 344 |
+
"""Reopen a closed discussion or pull request."""
|
| 345 |
+
out.confirm(f"Reopen #{num} on '{repo_id}'?", yes=yes)
|
| 346 |
+
api = get_hf_api(token=token)
|
| 347 |
+
api.change_discussion_status(
|
| 348 |
+
repo_id=repo_id,
|
| 349 |
+
discussion_num=num,
|
| 350 |
+
new_status="open",
|
| 351 |
+
comment=comment,
|
| 352 |
+
repo_type=repo_type.value,
|
| 353 |
+
)
|
| 354 |
+
out.result(f"Reopened #{num} in {repo_id}", num=num, repo=repo_id)
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
@discussions_cli.command(
|
| 358 |
+
"rename",
|
| 359 |
+
examples=[
|
| 360 |
+
'hf discussions rename username/my-model 5 "Updated title"',
|
| 361 |
+
],
|
| 362 |
+
)
|
| 363 |
+
def discussion_rename(
|
| 364 |
+
repo_id: RepoIdArg,
|
| 365 |
+
num: DiscussionNumArg,
|
| 366 |
+
new_title: Annotated[
|
| 367 |
+
str,
|
| 368 |
+
typer.Argument(
|
| 369 |
+
help="The new title.",
|
| 370 |
+
),
|
| 371 |
+
],
|
| 372 |
+
repo_type: RepoTypeOpt = RepoType.model,
|
| 373 |
+
token: TokenOpt = None,
|
| 374 |
+
) -> None:
|
| 375 |
+
"""Rename a discussion or pull request."""
|
| 376 |
+
api = get_hf_api(token=token)
|
| 377 |
+
api.rename_discussion(
|
| 378 |
+
repo_id=repo_id,
|
| 379 |
+
discussion_num=num,
|
| 380 |
+
new_title=new_title,
|
| 381 |
+
repo_type=repo_type.value,
|
| 382 |
+
)
|
| 383 |
+
out.result(f"Renamed #{num} in {repo_id}", num=num, repo=repo_id, title=new_title)
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
@discussions_cli.command(
|
| 387 |
+
"merge",
|
| 388 |
+
examples=[
|
| 389 |
+
"hf discussions merge username/my-model 5",
|
| 390 |
+
'hf discussions merge username/my-model 5 --comment "Merging, thanks!"',
|
| 391 |
+
],
|
| 392 |
+
)
|
| 393 |
+
def discussion_merge(
|
| 394 |
+
repo_id: RepoIdArg,
|
| 395 |
+
num: DiscussionNumArg,
|
| 396 |
+
comment: Annotated[
|
| 397 |
+
str | None,
|
| 398 |
+
typer.Option(
|
| 399 |
+
"--comment",
|
| 400 |
+
help="An optional comment to post when merging.",
|
| 401 |
+
),
|
| 402 |
+
] = None,
|
| 403 |
+
yes: Annotated[
|
| 404 |
+
bool,
|
| 405 |
+
typer.Option(
|
| 406 |
+
"--yes",
|
| 407 |
+
"-y",
|
| 408 |
+
help="Skip confirmation prompt.",
|
| 409 |
+
),
|
| 410 |
+
] = False,
|
| 411 |
+
repo_type: RepoTypeOpt = RepoType.model,
|
| 412 |
+
token: TokenOpt = None,
|
| 413 |
+
) -> None:
|
| 414 |
+
"""Merge a pull request."""
|
| 415 |
+
out.confirm(f"Merge #{num} on '{repo_id}'?", yes=yes)
|
| 416 |
+
api = get_hf_api(token=token)
|
| 417 |
+
api.merge_pull_request(
|
| 418 |
+
repo_id=repo_id,
|
| 419 |
+
discussion_num=num,
|
| 420 |
+
comment=comment,
|
| 421 |
+
repo_type=repo_type.value,
|
| 422 |
+
)
|
| 423 |
+
out.result(f"Merged #{num} in {repo_id}", num=num, repo=repo_id)
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
@discussions_cli.command(
|
| 427 |
+
"diff",
|
| 428 |
+
examples=[
|
| 429 |
+
"hf discussions diff username/my-model 5",
|
| 430 |
+
],
|
| 431 |
+
)
|
| 432 |
+
def discussion_diff(
|
| 433 |
+
repo_id: RepoIdArg,
|
| 434 |
+
num: DiscussionNumArg,
|
| 435 |
+
repo_type: RepoTypeOpt = RepoType.model,
|
| 436 |
+
token: TokenOpt = None,
|
| 437 |
+
) -> None:
|
| 438 |
+
"""Show the diff of a pull request."""
|
| 439 |
+
api = get_hf_api(token=token)
|
| 440 |
+
details = api.get_discussion_details(
|
| 441 |
+
repo_id=repo_id,
|
| 442 |
+
discussion_num=num,
|
| 443 |
+
repo_type=repo_type.value,
|
| 444 |
+
)
|
| 445 |
+
if details.diff:
|
| 446 |
+
out.text(details.diff)
|
| 447 |
+
else:
|
| 448 |
+
out.text("No diff available.")
|
venv/Lib/site-packages/huggingface_hub/cli/download.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 202-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains command to download files from the Hub with the CLI.
|
| 15 |
+
|
| 16 |
+
Usage:
|
| 17 |
+
hf download --help
|
| 18 |
+
|
| 19 |
+
# Download file
|
| 20 |
+
hf download gpt2 config.json
|
| 21 |
+
|
| 22 |
+
# Download entire repo
|
| 23 |
+
hf download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78
|
| 24 |
+
|
| 25 |
+
# Download repo with filters
|
| 26 |
+
hf download gpt2 --include="*.safetensors"
|
| 27 |
+
|
| 28 |
+
# Download with token
|
| 29 |
+
hf download Wauplin/private-model --token=hf_***
|
| 30 |
+
|
| 31 |
+
# Download quietly (no progress bar, no warnings, only the returned path)
|
| 32 |
+
hf download gpt2 config.json --quiet
|
| 33 |
+
|
| 34 |
+
# Download to local dir
|
| 35 |
+
hf download gpt2 --local-dir=./models/gpt2
|
| 36 |
+
|
| 37 |
+
# Download a subfolder
|
| 38 |
+
hf download HuggingFaceM4/FineVision art/ --repo-type=dataset
|
| 39 |
+
|
| 40 |
+
# Download using an hf:// URI (repo type, revision and file path are read from the URI)
|
| 41 |
+
hf download hf://datasets/HuggingFaceM4/FineVision@refs/pr/1/data/train.parquet
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
import warnings
|
| 45 |
+
from typing import Annotated
|
| 46 |
+
|
| 47 |
+
import typer
|
| 48 |
+
|
| 49 |
+
from huggingface_hub import constants
|
| 50 |
+
from huggingface_hub._snapshot_download import snapshot_download
|
| 51 |
+
from huggingface_hub.errors import CLIError
|
| 52 |
+
from huggingface_hub.file_download import DryRunFileInfo, hf_hub_download
|
| 53 |
+
from huggingface_hub.utils import _format_size, parse_hf_uri
|
| 54 |
+
|
| 55 |
+
from ._cli_utils import RepoIdArg, RepoType, RepoTypeOptionalOpt, RevisionOpt, TokenOpt
|
| 56 |
+
from ._output import out
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
DOWNLOAD_EXAMPLES = [
|
| 60 |
+
"hf download meta-llama/Llama-3.2-1B-Instruct",
|
| 61 |
+
"hf download meta-llama/Llama-3.2-1B-Instruct config.json tokenizer.json",
|
| 62 |
+
'hf download meta-llama/Llama-3.2-1B-Instruct --include "*.safetensors" --exclude "*.bin"',
|
| 63 |
+
"hf download meta-llama/Llama-3.2-1B-Instruct --local-dir ./models/llama",
|
| 64 |
+
"hf download HuggingFaceM4/FineVision art/ --repo-type dataset",
|
| 65 |
+
"hf download hf://datasets/HuggingFaceH4/ultrachat_200k",
|
| 66 |
+
]
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def download(
|
| 70 |
+
repo_id: RepoIdArg,
|
| 71 |
+
filenames: Annotated[
|
| 72 |
+
list[str] | None,
|
| 73 |
+
typer.Argument(
|
| 74 |
+
help="Files to download (e.g. `config.json`, `data/metadata.jsonl`).",
|
| 75 |
+
),
|
| 76 |
+
] = None,
|
| 77 |
+
repo_type: RepoTypeOptionalOpt = None,
|
| 78 |
+
revision: RevisionOpt = None,
|
| 79 |
+
include: Annotated[
|
| 80 |
+
list[str] | None,
|
| 81 |
+
typer.Option(
|
| 82 |
+
help="Glob patterns to include from files to download. eg: *.json",
|
| 83 |
+
),
|
| 84 |
+
] = None,
|
| 85 |
+
exclude: Annotated[
|
| 86 |
+
list[str] | None,
|
| 87 |
+
typer.Option(
|
| 88 |
+
help="Glob patterns to exclude from files to download.",
|
| 89 |
+
),
|
| 90 |
+
] = None,
|
| 91 |
+
cache_dir: Annotated[
|
| 92 |
+
str | None,
|
| 93 |
+
typer.Option(
|
| 94 |
+
help="Directory where to save files.",
|
| 95 |
+
),
|
| 96 |
+
] = None,
|
| 97 |
+
local_dir: Annotated[
|
| 98 |
+
str | None,
|
| 99 |
+
typer.Option(
|
| 100 |
+
help="If set, the downloaded file will be placed under this directory. Check out https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-a-local-folder for more details.",
|
| 101 |
+
),
|
| 102 |
+
] = None,
|
| 103 |
+
force_download: Annotated[
|
| 104 |
+
bool,
|
| 105 |
+
typer.Option(
|
| 106 |
+
help="If True, the files will be downloaded even if they are already cached.",
|
| 107 |
+
),
|
| 108 |
+
] = False,
|
| 109 |
+
dry_run: Annotated[
|
| 110 |
+
bool,
|
| 111 |
+
typer.Option(
|
| 112 |
+
help="If True, perform a dry run without actually downloading the file.",
|
| 113 |
+
),
|
| 114 |
+
] = False,
|
| 115 |
+
token: TokenOpt = None,
|
| 116 |
+
max_workers: Annotated[
|
| 117 |
+
int,
|
| 118 |
+
typer.Option(
|
| 119 |
+
help="Maximum number of workers to use for downloading files. Default is 8.",
|
| 120 |
+
),
|
| 121 |
+
] = 8,
|
| 122 |
+
) -> None:
|
| 123 |
+
"""Download files from the Hub."""
|
| 124 |
+
if local_dir is not None and cache_dir is not None:
|
| 125 |
+
raise CLIError(
|
| 126 |
+
"Cannot use both `--local-dir` and `--cache-dir` at the same time. "
|
| 127 |
+
"Use `--cache-dir` (or set the HF_HOME environment variable) for shared caching, "
|
| 128 |
+
"or `--local-dir` for a one-off download to a specific directory."
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
# `repo_id` may be a plain repo id or an `hf://` URI (e.g. `hf://datasets/my-org/my-dataset@v1.0/data/`).
|
| 132 |
+
# When a URI is provided, it is authoritative for the repo type, revision and (optionally) file path,
|
| 133 |
+
# so explicit `--repo-type` / `--revision` options are forbidden alongside it.
|
| 134 |
+
# We branch on the `hf://` prefix (the user's *intent*) rather than on whether the string parses as a
|
| 135 |
+
# valid URI: a malformed URI then surfaces a precise `HfUriError` (formatted globally in `cli/_errors.py`)
|
| 136 |
+
# instead of silently falling through to the plain-repo-id path and failing later with an opaque error.
|
| 137 |
+
if repo_id.startswith(constants.HF_PROTOCOL):
|
| 138 |
+
if repo_type is not None:
|
| 139 |
+
raise CLIError(f"'--repo-type' cannot be used with an 'hf://' URI ('{repo_id}').")
|
| 140 |
+
if revision is not None:
|
| 141 |
+
raise CLIError(f"'--revision' cannot be used with an 'hf://' URI ('{repo_id}').")
|
| 142 |
+
uri = parse_hf_uri(repo_id)
|
| 143 |
+
if uri.is_bucket:
|
| 144 |
+
raise CLIError("Buckets are not supported by `hf download`. Use `hf sync` instead.")
|
| 145 |
+
# The URI parser strips trailing slashes, but `hf download` uses a trailing '/' to denote a subfolder
|
| 146 |
+
# download (e.g. `data/` -> `data/**`). Re-append it when the URI explicitly ended with '/' so a folder
|
| 147 |
+
# URI keeps routing through the subfolder code path below.
|
| 148 |
+
path_in_repo = uri.path_in_repo
|
| 149 |
+
if path_in_repo and repo_id.endswith("/"):
|
| 150 |
+
path_in_repo += "/"
|
| 151 |
+
repo_id, repo_type_str, revision = uri.id, uri.type, uri.revision
|
| 152 |
+
if path_in_repo:
|
| 153 |
+
if filenames:
|
| 154 |
+
raise CLIError(
|
| 155 |
+
f"Cannot combine a file path in the hf:// URI ('{path_in_repo}') with positional filenames {filenames}."
|
| 156 |
+
)
|
| 157 |
+
filenames = [path_in_repo]
|
| 158 |
+
else:
|
| 159 |
+
repo_type_str = (repo_type or RepoType.model).value
|
| 160 |
+
|
| 161 |
+
def run_download() -> str | DryRunFileInfo | list[DryRunFileInfo]:
|
| 162 |
+
filenames_list = filenames if filenames is not None else []
|
| 163 |
+
|
| 164 |
+
# Separate subfolder patterns (ending with '/') from regular filenames
|
| 165 |
+
# Subfolders like "art/" are converted to include patterns like "art/**"
|
| 166 |
+
subfolders = [f for f in filenames_list if f.endswith("/")]
|
| 167 |
+
subfolder_patterns = [f"{f.rstrip('/')}/**" for f in subfolders]
|
| 168 |
+
regular_filenames = [f for f in filenames_list if not f.endswith("/")]
|
| 169 |
+
|
| 170 |
+
# Error if subfolder patterns are combined with --include/--exclude
|
| 171 |
+
# Guide user to use --include instead of subfolder argument
|
| 172 |
+
if len(subfolder_patterns) > 0:
|
| 173 |
+
if include is not None and len(include) > 0:
|
| 174 |
+
raise CLIError(
|
| 175 |
+
f"Cannot combine subfolder argument ('{subfolders[0]}') with `--include`. "
|
| 176 |
+
f'Please use `--include "{subfolders[0]}*"` instead.'
|
| 177 |
+
)
|
| 178 |
+
if exclude is not None and len(exclude) > 0:
|
| 179 |
+
raise CLIError(
|
| 180 |
+
f"Cannot combine subfolder argument ('{subfolders[0]}') with `--exclude`. "
|
| 181 |
+
f'Please use `--include "{subfolders[0]}*"` with `--exclude` instead.'
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
# Warn user if patterns are ignored (only if regular filenames are provided)
|
| 185 |
+
if len(regular_filenames) > 0:
|
| 186 |
+
if include is not None and len(include) > 0:
|
| 187 |
+
warnings.warn("Ignoring `--include` since filenames have been explicitly set.")
|
| 188 |
+
if exclude is not None and len(exclude) > 0:
|
| 189 |
+
warnings.warn("Ignoring `--exclude` since filenames have been explicitly set.")
|
| 190 |
+
|
| 191 |
+
# Single file to download (not a subfolder): use `hf_hub_download`
|
| 192 |
+
if len(regular_filenames) == 1 and len(subfolder_patterns) == 0:
|
| 193 |
+
return hf_hub_download(
|
| 194 |
+
repo_id=repo_id,
|
| 195 |
+
repo_type=repo_type_str,
|
| 196 |
+
revision=revision,
|
| 197 |
+
filename=regular_filenames[0],
|
| 198 |
+
cache_dir=cache_dir,
|
| 199 |
+
force_download=force_download,
|
| 200 |
+
token=token,
|
| 201 |
+
local_dir=local_dir,
|
| 202 |
+
library_name="huggingface-cli",
|
| 203 |
+
dry_run=dry_run,
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
# Otherwise: use `snapshot_download` to ensure all files comes from same revision
|
| 207 |
+
if len(regular_filenames) == 0 and len(subfolder_patterns) == 0:
|
| 208 |
+
# No filenames provided: use include/exclude patterns
|
| 209 |
+
allow_patterns = include
|
| 210 |
+
ignore_patterns = exclude
|
| 211 |
+
else:
|
| 212 |
+
# Combine regular filenames and subfolder patterns as allow_patterns
|
| 213 |
+
allow_patterns = regular_filenames + subfolder_patterns
|
| 214 |
+
ignore_patterns = None
|
| 215 |
+
|
| 216 |
+
return snapshot_download(
|
| 217 |
+
repo_id=repo_id,
|
| 218 |
+
repo_type=repo_type_str,
|
| 219 |
+
revision=revision,
|
| 220 |
+
allow_patterns=allow_patterns,
|
| 221 |
+
ignore_patterns=ignore_patterns,
|
| 222 |
+
force_download=force_download,
|
| 223 |
+
cache_dir=cache_dir,
|
| 224 |
+
token=token,
|
| 225 |
+
local_dir=local_dir,
|
| 226 |
+
library_name="huggingface-cli",
|
| 227 |
+
max_workers=max_workers,
|
| 228 |
+
dry_run=dry_run,
|
| 229 |
+
)
|
| 230 |
+
|
| 231 |
+
def _print_result(result: str | DryRunFileInfo | list[DryRunFileInfo]) -> None:
|
| 232 |
+
if isinstance(result, str):
|
| 233 |
+
out.result("Downloaded", path=result)
|
| 234 |
+
return
|
| 235 |
+
|
| 236 |
+
# Print dry run info
|
| 237 |
+
if isinstance(result, DryRunFileInfo):
|
| 238 |
+
result = [result]
|
| 239 |
+
will_download = [r for r in result if r.will_download]
|
| 240 |
+
out.text(
|
| 241 |
+
f"[dry-run] Will download {len(will_download)} files"
|
| 242 |
+
f" (out of {len(result)})"
|
| 243 |
+
f" totalling {_format_size(sum(r.file_size for r in will_download))}."
|
| 244 |
+
)
|
| 245 |
+
items = [
|
| 246 |
+
{
|
| 247 |
+
"file": info.filename,
|
| 248 |
+
"size": _format_size(info.file_size) if info.will_download else "-",
|
| 249 |
+
}
|
| 250 |
+
for info in sorted(result, key=lambda x: x.filename)
|
| 251 |
+
]
|
| 252 |
+
out.table(items)
|
| 253 |
+
|
| 254 |
+
_print_result(run_download())
|
venv/Lib/site-packages/huggingface_hub/cli/extensions.py
ADDED
|
@@ -0,0 +1,627 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains helper utilities for hf CLI extensions."""
|
| 15 |
+
|
| 16 |
+
import errno
|
| 17 |
+
import json
|
| 18 |
+
import os
|
| 19 |
+
import re
|
| 20 |
+
import shutil
|
| 21 |
+
import subprocess
|
| 22 |
+
import venv
|
| 23 |
+
from dataclasses import asdict, dataclass
|
| 24 |
+
from datetime import datetime, timezone
|
| 25 |
+
from pathlib import Path
|
| 26 |
+
from typing import Annotated, Literal
|
| 27 |
+
|
| 28 |
+
import typer
|
| 29 |
+
|
| 30 |
+
from huggingface_hub.errors import CLIError, CLIExtensionInstallError, ConfirmationError
|
| 31 |
+
from huggingface_hub.utils import get_session, logging
|
| 32 |
+
|
| 33 |
+
from ._cli_utils import typer_factory
|
| 34 |
+
from ._output import out
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
DEFAULT_EXTENSION_OWNER = "huggingface"
|
| 38 |
+
EXTENSIONS_ROOT = Path("~/.local/share/hf/extensions")
|
| 39 |
+
MANIFEST_FILENAME = "manifest.json"
|
| 40 |
+
EXTENSIONS_HELP = (
|
| 41 |
+
"Manage hf CLI extensions.\n\n"
|
| 42 |
+
"Security Warning: extensions are third-party executables or Python packages. "
|
| 43 |
+
"Install only from sources you trust."
|
| 44 |
+
)
|
| 45 |
+
extensions_cli = typer_factory(help=EXTENSIONS_HELP)
|
| 46 |
+
_EXTENSIONS_DEFAULT_BRANCH = "main" # Fallback when the GitHub API is unreachable.
|
| 47 |
+
_EXTENSIONS_GITHUB_TOPIC = "hf-extension"
|
| 48 |
+
_EXTENSIONS_DOWNLOAD_TIMEOUT = 10
|
| 49 |
+
_EXTENSIONS_PIP_INSTALL_TIMEOUT = 300
|
| 50 |
+
|
| 51 |
+
logger = logging.get_logger(__name__)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@dataclass
|
| 55 |
+
class ExtensionManifest:
|
| 56 |
+
owner: str
|
| 57 |
+
repo: str
|
| 58 |
+
repo_id: str
|
| 59 |
+
short_name: str
|
| 60 |
+
executable_name: str
|
| 61 |
+
executable_path: str
|
| 62 |
+
type: Literal["binary", "python"]
|
| 63 |
+
installed_at: datetime
|
| 64 |
+
source: str
|
| 65 |
+
description: str | None = None
|
| 66 |
+
|
| 67 |
+
@classmethod
|
| 68 |
+
def load(cls, path: Path) -> "ExtensionManifest":
|
| 69 |
+
manifest_path = path / MANIFEST_FILENAME
|
| 70 |
+
if not manifest_path.is_file():
|
| 71 |
+
raise CLIError(f"Manifest file not found at {manifest_path}. Your extension may be corrupted.")
|
| 72 |
+
data = json.loads(manifest_path.read_text())
|
| 73 |
+
data["installed_at"] = datetime.fromisoformat(data["installed_at"])
|
| 74 |
+
return ExtensionManifest(**data)
|
| 75 |
+
|
| 76 |
+
def save(self, path: Path) -> None:
|
| 77 |
+
manifest_path = path / MANIFEST_FILENAME
|
| 78 |
+
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
| 79 |
+
data = asdict(self)
|
| 80 |
+
data["installed_at"] = self.installed_at.isoformat()
|
| 81 |
+
manifest_path.write_text(json.dumps(data, indent=2, sort_keys=True))
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
@extensions_cli.command(
|
| 85 |
+
"install",
|
| 86 |
+
examples=[
|
| 87 |
+
"hf extensions install hf-claude",
|
| 88 |
+
"hf extensions install hanouticelina/hf-claude",
|
| 89 |
+
"hf extensions install alvarobartt/hf-mem",
|
| 90 |
+
],
|
| 91 |
+
)
|
| 92 |
+
def extension_install(
|
| 93 |
+
ctx: typer.Context,
|
| 94 |
+
repo_id: Annotated[
|
| 95 |
+
str,
|
| 96 |
+
typer.Argument(help="GitHub extension repository in `[OWNER/]hf-<name>` format."),
|
| 97 |
+
],
|
| 98 |
+
force: Annotated[bool, typer.Option("--force", help="Overwrite if already installed.")] = False,
|
| 99 |
+
) -> None:
|
| 100 |
+
"""Install an extension from a public GitHub repository.
|
| 101 |
+
|
| 102 |
+
Security warning: this installs a third-party executable or Python package.
|
| 103 |
+
Install only from sources you trust.
|
| 104 |
+
"""
|
| 105 |
+
owner, repo_name, short_name = _normalize_repo_id(repo_id)
|
| 106 |
+
root_ctx = ctx.find_root()
|
| 107 |
+
reserved_commands = set(getattr(root_ctx.command, "commands", {}).keys())
|
| 108 |
+
if short_name in reserved_commands:
|
| 109 |
+
raise CLIError(
|
| 110 |
+
f"Cannot install extension '{short_name}' because it conflicts with an existing `hf {short_name}` command."
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
extension_dir = _get_extension_dir(short_name)
|
| 114 |
+
extension_exists = extension_dir.exists()
|
| 115 |
+
if extension_exists and not force:
|
| 116 |
+
raise CLIError(f"Extension '{short_name}' is already installed. Use --force to overwrite.")
|
| 117 |
+
|
| 118 |
+
branch, description = _resolve_github_repo_info(owner=owner, repo_name=repo_name)
|
| 119 |
+
|
| 120 |
+
if extension_exists:
|
| 121 |
+
shutil.rmtree(extension_dir)
|
| 122 |
+
|
| 123 |
+
manifest = _install_extension_from_github(
|
| 124 |
+
owner=owner,
|
| 125 |
+
repo_name=repo_name,
|
| 126 |
+
short_name=short_name,
|
| 127 |
+
extension_dir=extension_dir,
|
| 128 |
+
branch=branch,
|
| 129 |
+
description=description,
|
| 130 |
+
)
|
| 131 |
+
ext_type = manifest.type.capitalize()
|
| 132 |
+
out.result(
|
| 133 |
+
f"{ext_type} extension installed",
|
| 134 |
+
source=f"{owner}/{repo_name}",
|
| 135 |
+
command=f"hf {short_name}",
|
| 136 |
+
)
|
| 137 |
+
out.hint(f"Run it with: hf {short_name}")
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
@extensions_cli.command(
|
| 141 |
+
"exec",
|
| 142 |
+
context_settings={"allow_extra_args": True, "allow_interspersed_args": False, "ignore_unknown_options": True},
|
| 143 |
+
examples=[
|
| 144 |
+
"hf extensions exec claude -- --help",
|
| 145 |
+
"hf extensions exec claude --model zai-org/GLM-5",
|
| 146 |
+
],
|
| 147 |
+
)
|
| 148 |
+
def extension_exec(
|
| 149 |
+
ctx: typer.Context,
|
| 150 |
+
name: Annotated[
|
| 151 |
+
str,
|
| 152 |
+
typer.Argument(help="Extension name (with or without `hf-` prefix)."),
|
| 153 |
+
],
|
| 154 |
+
) -> None:
|
| 155 |
+
"""Execute an installed extension."""
|
| 156 |
+
short_name = _normalize_extension_name(name)
|
| 157 |
+
executable_path = _resolve_installed_executable_path(short_name)
|
| 158 |
+
|
| 159 |
+
if not executable_path.is_file():
|
| 160 |
+
raise CLIError(f"Extension '{short_name}' is not installed.")
|
| 161 |
+
|
| 162 |
+
exit_code = _execute_extension_binary(executable_path=executable_path, args=list(ctx.args))
|
| 163 |
+
raise typer.Exit(code=exit_code)
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
@extensions_cli.command("list | ls", examples=["hf extensions list"])
|
| 167 |
+
def extension_list() -> None:
|
| 168 |
+
"""List installed extension commands."""
|
| 169 |
+
rows = [
|
| 170 |
+
{
|
| 171 |
+
"command": f"hf {manifest.short_name}",
|
| 172 |
+
"source": str(manifest.repo_id),
|
| 173 |
+
"type": str(manifest.type),
|
| 174 |
+
"installed": manifest.installed_at.strftime("%Y-%m-%d"),
|
| 175 |
+
"description": manifest.description,
|
| 176 |
+
}
|
| 177 |
+
for manifest in _list_installed_extensions()
|
| 178 |
+
]
|
| 179 |
+
out.table(rows, id_key="command")
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
@extensions_cli.command("search", examples=["hf extensions search"])
|
| 183 |
+
def extension_search() -> None:
|
| 184 |
+
"""Search extensions available on GitHub (tagged with 'hf-extension' topic)."""
|
| 185 |
+
response = get_session().get(
|
| 186 |
+
"https://api.github.com/search/repositories",
|
| 187 |
+
params={"q": f"topic:{_EXTENSIONS_GITHUB_TOPIC}", "sort": "stars", "order": "desc", "per_page": 100},
|
| 188 |
+
follow_redirects=True,
|
| 189 |
+
timeout=_EXTENSIONS_DOWNLOAD_TIMEOUT,
|
| 190 |
+
)
|
| 191 |
+
response.raise_for_status()
|
| 192 |
+
data = response.json()
|
| 193 |
+
|
| 194 |
+
installed = {m.short_name for m in _list_installed_extensions()}
|
| 195 |
+
|
| 196 |
+
rows = []
|
| 197 |
+
for repo in data.get("items", []):
|
| 198 |
+
repo_name = repo["name"]
|
| 199 |
+
short_name = repo_name[3:] if repo_name.startswith("hf-") else repo_name
|
| 200 |
+
rows.append(
|
| 201 |
+
{
|
| 202 |
+
"name": short_name,
|
| 203 |
+
"repo": repo["full_name"],
|
| 204 |
+
"stars": repo.get("stargazers_count", 0),
|
| 205 |
+
"description": repo.get("description") or "",
|
| 206 |
+
"installed": "yes" if short_name in installed else "",
|
| 207 |
+
}
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
out.table(rows, id_key="repo")
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
@extensions_cli.command("remove | rm", examples=["hf extensions remove claude"])
|
| 214 |
+
def extension_remove(
|
| 215 |
+
name: Annotated[
|
| 216 |
+
str,
|
| 217 |
+
typer.Argument(help="Extension name to remove (with or without `hf-` prefix)."),
|
| 218 |
+
],
|
| 219 |
+
) -> None:
|
| 220 |
+
"""Remove an installed extension."""
|
| 221 |
+
short_name = _normalize_extension_name(name)
|
| 222 |
+
extension_dir = _get_extension_dir(short_name)
|
| 223 |
+
|
| 224 |
+
if not extension_dir.is_dir():
|
| 225 |
+
raise CLIError(f"Extension '{short_name}' is not installed.")
|
| 226 |
+
|
| 227 |
+
shutil.rmtree(extension_dir)
|
| 228 |
+
out.result("Extension removed", name=short_name)
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
### HELPER FUNCTIONS
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def _list_installed_extensions() -> list[ExtensionManifest]:
|
| 235 |
+
"""Return manifests for all validly-installed extensions, sorted by directory name."""
|
| 236 |
+
root_dir = EXTENSIONS_ROOT.expanduser()
|
| 237 |
+
if not root_dir.is_dir():
|
| 238 |
+
return []
|
| 239 |
+
manifests = []
|
| 240 |
+
for extension_dir in sorted(root_dir.iterdir()):
|
| 241 |
+
if not extension_dir.is_dir() or not extension_dir.name.startswith("hf-"):
|
| 242 |
+
continue
|
| 243 |
+
try:
|
| 244 |
+
manifests.append(ExtensionManifest.load(extension_dir))
|
| 245 |
+
except Exception as e:
|
| 246 |
+
logger.debug(f"Failed to load manifest for extension '{extension_dir.name}': {e}")
|
| 247 |
+
continue
|
| 248 |
+
return manifests
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def list_installed_extensions_for_help() -> list[tuple[str, str]]:
|
| 252 |
+
entries = []
|
| 253 |
+
for manifest in _list_installed_extensions():
|
| 254 |
+
tag = f"[extension {manifest.repo_id}]"
|
| 255 |
+
help_text = f"{manifest.description} {tag}" if manifest.description is not None else tag
|
| 256 |
+
entries.append((manifest.short_name, help_text))
|
| 257 |
+
return entries
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def dispatch_unknown_top_level_extension(args: list[str], known_commands: set[str]) -> int | None:
|
| 261 |
+
if not args:
|
| 262 |
+
return None
|
| 263 |
+
|
| 264 |
+
command_name = args[0]
|
| 265 |
+
if command_name.startswith("-"):
|
| 266 |
+
return None
|
| 267 |
+
all_known = {a.strip() for cmd in known_commands for a in cmd.split("|")}
|
| 268 |
+
if command_name in all_known:
|
| 269 |
+
return None
|
| 270 |
+
|
| 271 |
+
short_name = command_name[3:] if command_name.startswith("hf-") else command_name
|
| 272 |
+
if not short_name:
|
| 273 |
+
return None
|
| 274 |
+
|
| 275 |
+
executable_path: Path | None = None
|
| 276 |
+
try:
|
| 277 |
+
executable_path = _resolve_installed_executable_path(short_name)
|
| 278 |
+
except Exception:
|
| 279 |
+
executable_path = _auto_install_official_extension(short_name)
|
| 280 |
+
|
| 281 |
+
if executable_path is None or not executable_path.is_file():
|
| 282 |
+
return None
|
| 283 |
+
|
| 284 |
+
return _execute_extension_binary(executable_path=executable_path, args=list(args[1:]))
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def _auto_install_official_extension(short_name: str) -> Path | None:
|
| 288 |
+
"""Try to auto-install huggingface/hf-<name>. Returns executable path or None."""
|
| 289 |
+
owner, repo_name = DEFAULT_EXTENSION_OWNER, f"hf-{short_name}"
|
| 290 |
+
try:
|
| 291 |
+
extension_dir = _get_extension_dir(short_name)
|
| 292 |
+
except Exception:
|
| 293 |
+
return None
|
| 294 |
+
if extension_dir.exists():
|
| 295 |
+
return None
|
| 296 |
+
try:
|
| 297 |
+
response = get_session().get(
|
| 298 |
+
f"https://api.github.com/repos/{owner}/{repo_name}",
|
| 299 |
+
follow_redirects=True,
|
| 300 |
+
timeout=_EXTENSIONS_DOWNLOAD_TIMEOUT,
|
| 301 |
+
)
|
| 302 |
+
if response.status_code == 404:
|
| 303 |
+
return None
|
| 304 |
+
response.raise_for_status()
|
| 305 |
+
branch = response.json()["default_branch"]
|
| 306 |
+
except Exception:
|
| 307 |
+
return None
|
| 308 |
+
try:
|
| 309 |
+
out.confirm(f"'{short_name}' is an official Hugging Face extension ({owner}/{repo_name}). Install it?")
|
| 310 |
+
except ConfirmationError:
|
| 311 |
+
return None
|
| 312 |
+
try:
|
| 313 |
+
manifest = _install_extension_from_github(
|
| 314 |
+
owner=owner, repo_name=repo_name, short_name=short_name, extension_dir=extension_dir, branch=branch
|
| 315 |
+
)
|
| 316 |
+
return Path(manifest.executable_path).expanduser()
|
| 317 |
+
except Exception:
|
| 318 |
+
shutil.rmtree(extension_dir, ignore_errors=True)
|
| 319 |
+
return None
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def _install_extension_from_github(
|
| 323 |
+
*,
|
| 324 |
+
owner: str,
|
| 325 |
+
repo_name: str,
|
| 326 |
+
short_name: str,
|
| 327 |
+
extension_dir: Path,
|
| 328 |
+
branch: str,
|
| 329 |
+
description: str | None = None,
|
| 330 |
+
) -> ExtensionManifest:
|
| 331 |
+
"""Fetch, install (binary or Python), and save manifest for a GitHub extension."""
|
| 332 |
+
try:
|
| 333 |
+
binary = _fetch_remote_binary(owner=owner, repo_name=repo_name, branch=branch, short_name=short_name)
|
| 334 |
+
except Exception:
|
| 335 |
+
binary = None
|
| 336 |
+
if binary is not None:
|
| 337 |
+
manifest = _install_binary_extension(
|
| 338 |
+
owner=owner, repo_name=repo_name, short_name=short_name, extension_dir=extension_dir, binary=binary
|
| 339 |
+
)
|
| 340 |
+
else:
|
| 341 |
+
manifest = _install_python_extension(
|
| 342 |
+
owner=owner, repo_name=repo_name, short_name=short_name, extension_dir=extension_dir, branch=branch
|
| 343 |
+
)
|
| 344 |
+
manifest.description = _try_fetch_remote_description(
|
| 345 |
+
owner=owner, repo_name=repo_name, branch=branch, candidate_description=description
|
| 346 |
+
)
|
| 347 |
+
manifest.save(extension_dir)
|
| 348 |
+
return manifest
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def _fetch_remote_binary(owner: str, repo_name: str, branch: str, short_name: str) -> bytes:
|
| 352 |
+
executable_name = _get_executable_name(short_name)
|
| 353 |
+
raw_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/refs/heads/{branch}/{executable_name}"
|
| 354 |
+
response = get_session().get(raw_url, follow_redirects=True, timeout=_EXTENSIONS_DOWNLOAD_TIMEOUT)
|
| 355 |
+
response.raise_for_status()
|
| 356 |
+
return response.content
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
def _install_binary_extension(
|
| 360 |
+
*, owner: str, repo_name: str, short_name: str, extension_dir: Path, binary: bytes
|
| 361 |
+
) -> ExtensionManifest:
|
| 362 |
+
# Save extension binary
|
| 363 |
+
executable_name = _get_executable_name(short_name)
|
| 364 |
+
extension_dir.mkdir(parents=True, exist_ok=False)
|
| 365 |
+
executable_path = extension_dir / executable_name
|
| 366 |
+
executable_path.write_bytes(binary)
|
| 367 |
+
|
| 368 |
+
# Make it executable
|
| 369 |
+
if os.name != "nt":
|
| 370 |
+
os.chmod(executable_path, 0o755)
|
| 371 |
+
|
| 372 |
+
# Create manifest
|
| 373 |
+
return ExtensionManifest(
|
| 374 |
+
owner=owner,
|
| 375 |
+
repo=repo_name,
|
| 376 |
+
repo_id=f"{owner}/{repo_name}",
|
| 377 |
+
short_name=short_name,
|
| 378 |
+
executable_name=executable_name,
|
| 379 |
+
executable_path=str(executable_path),
|
| 380 |
+
type="binary",
|
| 381 |
+
installed_at=datetime.now(timezone.utc),
|
| 382 |
+
source=f"https://github.com/{owner}/{repo_name}",
|
| 383 |
+
)
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
def _install_python_extension(
|
| 387 |
+
*, owner: str, repo_name: str, short_name: str, extension_dir: Path, branch: str
|
| 388 |
+
) -> ExtensionManifest:
|
| 389 |
+
source_url = f"https://github.com/{owner}/{repo_name}/archive/refs/heads/{branch}.zip"
|
| 390 |
+
venv_dir = extension_dir / "venv"
|
| 391 |
+
installed = False
|
| 392 |
+
|
| 393 |
+
status = out.status()
|
| 394 |
+
try:
|
| 395 |
+
status.update(f"Creating virtual environment in {venv_dir}")
|
| 396 |
+
if extension_dir.exists():
|
| 397 |
+
shutil.rmtree(extension_dir, ignore_errors=True)
|
| 398 |
+
extension_dir.mkdir(parents=True, exist_ok=False)
|
| 399 |
+
|
| 400 |
+
uv_path = shutil.which("uv")
|
| 401 |
+
venv_python = _get_venv_python_path(venv_dir)
|
| 402 |
+
if uv_path:
|
| 403 |
+
subprocess.run([uv_path, "venv", str(venv_dir)], check=True)
|
| 404 |
+
status.done(f"Virtual environment created in {venv_dir}")
|
| 405 |
+
|
| 406 |
+
status.update(f"Installing package from {source_url}")
|
| 407 |
+
subprocess.run(
|
| 408 |
+
[uv_path, "pip", "install", "--python", str(venv_python), source_url],
|
| 409 |
+
check=True,
|
| 410 |
+
timeout=_EXTENSIONS_PIP_INSTALL_TIMEOUT,
|
| 411 |
+
)
|
| 412 |
+
else:
|
| 413 |
+
venv.EnvBuilder(with_pip=True).create(str(venv_dir))
|
| 414 |
+
status.done(f"Virtual environment created in {venv_dir}")
|
| 415 |
+
|
| 416 |
+
status.update(f"Installing package from {source_url}")
|
| 417 |
+
subprocess.run(
|
| 418 |
+
[
|
| 419 |
+
str(venv_python),
|
| 420 |
+
"-m",
|
| 421 |
+
"pip",
|
| 422 |
+
"install",
|
| 423 |
+
"--disable-pip-version-check",
|
| 424 |
+
"--no-input",
|
| 425 |
+
source_url,
|
| 426 |
+
],
|
| 427 |
+
check=True,
|
| 428 |
+
timeout=_EXTENSIONS_PIP_INSTALL_TIMEOUT,
|
| 429 |
+
)
|
| 430 |
+
status.done(f"Package installed from {source_url}")
|
| 431 |
+
|
| 432 |
+
executable_name = _get_executable_name(short_name)
|
| 433 |
+
venv_executable = _get_venv_extension_executable_path(venv_dir, short_name)
|
| 434 |
+
if not venv_executable.is_file():
|
| 435 |
+
raise CLIError(
|
| 436 |
+
f"Installed package from '{owner}/{repo_name}' does not expose the required console script "
|
| 437 |
+
f"'{executable_name}'."
|
| 438 |
+
)
|
| 439 |
+
|
| 440 |
+
manifest = ExtensionManifest(
|
| 441 |
+
owner=owner,
|
| 442 |
+
repo=repo_name,
|
| 443 |
+
repo_id=f"{owner}/{repo_name}",
|
| 444 |
+
short_name=short_name,
|
| 445 |
+
executable_name=executable_name,
|
| 446 |
+
executable_path=str(venv_executable.resolve()),
|
| 447 |
+
type="python",
|
| 448 |
+
installed_at=datetime.now(timezone.utc),
|
| 449 |
+
source=f"https://github.com/{owner}/{repo_name}",
|
| 450 |
+
)
|
| 451 |
+
installed = True
|
| 452 |
+
return manifest
|
| 453 |
+
except CLIError:
|
| 454 |
+
raise
|
| 455 |
+
except subprocess.TimeoutExpired as e:
|
| 456 |
+
raise CLIExtensionInstallError(
|
| 457 |
+
f"Pip install timed out after {_EXTENSIONS_PIP_INSTALL_TIMEOUT}s for '{owner}/{repo_name}'. "
|
| 458 |
+
"See pip output above for details."
|
| 459 |
+
) from e
|
| 460 |
+
except subprocess.CalledProcessError as e:
|
| 461 |
+
raise CLIExtensionInstallError(
|
| 462 |
+
f"Failed to install pip package from '{owner}/{repo_name}' (exit code {e.returncode}). "
|
| 463 |
+
"See pip output above for details."
|
| 464 |
+
) from e
|
| 465 |
+
except Exception as e:
|
| 466 |
+
raise CLIExtensionInstallError(f"Failed to set up pip extension from '{owner}/{repo_name}': {e}") from e
|
| 467 |
+
finally:
|
| 468 |
+
if not installed:
|
| 469 |
+
shutil.rmtree(extension_dir, ignore_errors=True)
|
| 470 |
+
|
| 471 |
+
|
| 472 |
+
def _try_fetch_remote_description(
|
| 473 |
+
owner: str, repo_name: str, branch: str, candidate_description: str | None
|
| 474 |
+
) -> str | None:
|
| 475 |
+
"""Try to fetch project description either from:
|
| 476 |
+
- manifest.json
|
| 477 |
+
- pyproject.toml
|
| 478 |
+
|
| 479 |
+
Only best effort, no error handling.
|
| 480 |
+
"""
|
| 481 |
+
# from manifest.json
|
| 482 |
+
try:
|
| 483 |
+
response = get_session().get(
|
| 484 |
+
f"https://raw.githubusercontent.com/{owner}/{repo_name}/refs/heads/{branch}/{MANIFEST_FILENAME}",
|
| 485 |
+
follow_redirects=True,
|
| 486 |
+
)
|
| 487 |
+
response.raise_for_status()
|
| 488 |
+
data = response.json()
|
| 489 |
+
description = data.get("description")
|
| 490 |
+
if isinstance(description, str):
|
| 491 |
+
return description
|
| 492 |
+
except Exception:
|
| 493 |
+
pass
|
| 494 |
+
|
| 495 |
+
# from pyproject.toml
|
| 496 |
+
try:
|
| 497 |
+
response = get_session().get(
|
| 498 |
+
f"https://raw.githubusercontent.com/{owner}/{repo_name}/refs/heads/{branch}/pyproject.toml",
|
| 499 |
+
follow_redirects=True,
|
| 500 |
+
)
|
| 501 |
+
response.raise_for_status()
|
| 502 |
+
|
| 503 |
+
# Weak parser but ok for "best effort"
|
| 504 |
+
for line in response.text.splitlines():
|
| 505 |
+
line = line.strip()
|
| 506 |
+
if line.startswith("description"):
|
| 507 |
+
_, _, value = line.partition("=")
|
| 508 |
+
return value.strip().strip("\"'")
|
| 509 |
+
except Exception:
|
| 510 |
+
pass
|
| 511 |
+
|
| 512 |
+
# fallback to value fetched from GH API directly
|
| 513 |
+
return candidate_description
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
def _get_extensions_root() -> Path:
|
| 517 |
+
root_dir = EXTENSIONS_ROOT.expanduser()
|
| 518 |
+
root_dir.mkdir(parents=True, exist_ok=True)
|
| 519 |
+
return root_dir
|
| 520 |
+
|
| 521 |
+
|
| 522 |
+
def _get_extension_dir(short_name: str) -> Path:
|
| 523 |
+
safe_name = _validate_extension_short_name(short_name, original_input=short_name)
|
| 524 |
+
root = _get_extensions_root().resolve()
|
| 525 |
+
target = (root / f"hf-{safe_name}").resolve()
|
| 526 |
+
if root not in target.parents:
|
| 527 |
+
raise CLIError(f"Invalid extension name '{short_name}'.")
|
| 528 |
+
return target
|
| 529 |
+
|
| 530 |
+
|
| 531 |
+
def _resolve_github_repo_info(owner: str, repo_name: str) -> tuple[str, str | None]:
|
| 532 |
+
try:
|
| 533 |
+
response = get_session().get(
|
| 534 |
+
f"https://api.github.com/repos/{owner}/{repo_name}",
|
| 535 |
+
follow_redirects=True,
|
| 536 |
+
timeout=_EXTENSIONS_DOWNLOAD_TIMEOUT,
|
| 537 |
+
)
|
| 538 |
+
response.raise_for_status()
|
| 539 |
+
data = response.json()
|
| 540 |
+
return data["default_branch"], data.get("description")
|
| 541 |
+
except Exception:
|
| 542 |
+
return _EXTENSIONS_DEFAULT_BRANCH, None
|
| 543 |
+
|
| 544 |
+
|
| 545 |
+
def _get_executable_name(short_name: str) -> str:
|
| 546 |
+
name = f"hf-{short_name}"
|
| 547 |
+
if os.name == "nt":
|
| 548 |
+
name += ".exe"
|
| 549 |
+
return name
|
| 550 |
+
|
| 551 |
+
|
| 552 |
+
def _resolve_installed_executable_path(short_name: str) -> Path:
|
| 553 |
+
extension_dir = _get_extension_dir(short_name)
|
| 554 |
+
manifest = ExtensionManifest.load(extension_dir)
|
| 555 |
+
return Path(manifest.executable_path).expanduser()
|
| 556 |
+
|
| 557 |
+
|
| 558 |
+
def _get_venv_python_path(venv_dir: Path) -> Path:
|
| 559 |
+
if os.name == "nt":
|
| 560 |
+
return venv_dir / "Scripts" / "python.exe"
|
| 561 |
+
return venv_dir / "bin" / "python"
|
| 562 |
+
|
| 563 |
+
|
| 564 |
+
def _get_venv_extension_executable_path(venv_dir: Path, short_name: str) -> Path:
|
| 565 |
+
executable_name = _get_executable_name(short_name)
|
| 566 |
+
if os.name == "nt":
|
| 567 |
+
return venv_dir / "Scripts" / executable_name
|
| 568 |
+
return venv_dir / "bin" / executable_name
|
| 569 |
+
|
| 570 |
+
|
| 571 |
+
_ALLOWED_EXTENSION_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
def _validate_extension_short_name(short_name: str, *, original_input: str) -> str:
|
| 575 |
+
name = short_name.strip()
|
| 576 |
+
if not name:
|
| 577 |
+
raise CLIError("Extension name cannot be empty.")
|
| 578 |
+
if any(sep in name for sep in ("/", "\\")):
|
| 579 |
+
raise CLIError(f"Invalid extension name '{original_input}'.")
|
| 580 |
+
if ".." in name or ":" in name:
|
| 581 |
+
raise CLIError(f"Invalid extension name '{original_input}'.")
|
| 582 |
+
if not _ALLOWED_EXTENSION_NAME.fullmatch(name):
|
| 583 |
+
raise CLIError(
|
| 584 |
+
f"Invalid extension name '{original_input}'. Allowed characters: letters, digits, '.', '_' and '-'."
|
| 585 |
+
)
|
| 586 |
+
return name
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
def _normalize_repo_id(repo_id: str) -> tuple[str, str, str]:
|
| 590 |
+
if "://" in repo_id:
|
| 591 |
+
raise CLIError("Only GitHub repositories in `[OWNER/]hf-<name>` format are supported.")
|
| 592 |
+
|
| 593 |
+
parts = repo_id.split("/")
|
| 594 |
+
if len(parts) == 1:
|
| 595 |
+
owner = DEFAULT_EXTENSION_OWNER
|
| 596 |
+
repo_name = parts[0]
|
| 597 |
+
elif len(parts) == 2 and all(parts):
|
| 598 |
+
owner, repo_name = parts
|
| 599 |
+
else:
|
| 600 |
+
raise CLIError(f"Expected `[OWNER/]REPO` format, got '{repo_id}'.")
|
| 601 |
+
|
| 602 |
+
if not repo_name.startswith("hf-"):
|
| 603 |
+
raise CLIError(f"Extension repository name must start with 'hf-', got '{repo_name}'.")
|
| 604 |
+
|
| 605 |
+
short_name = repo_name[3:]
|
| 606 |
+
if not short_name:
|
| 607 |
+
raise CLIError("Invalid extension repository name 'hf-'.")
|
| 608 |
+
_validate_extension_short_name(short_name, original_input=repo_id)
|
| 609 |
+
|
| 610 |
+
return owner, repo_name, short_name
|
| 611 |
+
|
| 612 |
+
|
| 613 |
+
def _normalize_extension_name(name: str) -> str:
|
| 614 |
+
candidate = name.strip()
|
| 615 |
+
if not candidate:
|
| 616 |
+
raise CLIError("Extension name cannot be empty.")
|
| 617 |
+
normalized = candidate[3:] if candidate.startswith("hf-") else candidate
|
| 618 |
+
return _validate_extension_short_name(normalized, original_input=name)
|
| 619 |
+
|
| 620 |
+
|
| 621 |
+
def _execute_extension_binary(executable_path: Path, args: list[str]) -> int:
|
| 622 |
+
try:
|
| 623 |
+
return subprocess.call([str(executable_path)] + args)
|
| 624 |
+
except OSError as e:
|
| 625 |
+
if os.name == "nt" or e.errno != errno.ENOEXEC:
|
| 626 |
+
raise
|
| 627 |
+
return subprocess.call(["sh", str(executable_path)] + args)
|
venv/Lib/site-packages/huggingface_hub/cli/hf.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2020 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
import sys
|
| 16 |
+
import traceback
|
| 17 |
+
from typing import Annotated
|
| 18 |
+
|
| 19 |
+
import typer
|
| 20 |
+
|
| 21 |
+
from huggingface_hub import __version__, constants
|
| 22 |
+
from huggingface_hub.cli._cli_utils import check_cli_update, fallback_typer_group_factory, typer_factory
|
| 23 |
+
from huggingface_hub.cli._cp import CP_EXAMPLES, make_cp
|
| 24 |
+
from huggingface_hub.cli._errors import format_known_exception
|
| 25 |
+
from huggingface_hub.cli.auth import auth_cli
|
| 26 |
+
from huggingface_hub.cli.buckets import buckets_cli, sync
|
| 27 |
+
from huggingface_hub.cli.cache import cache_cli
|
| 28 |
+
from huggingface_hub.cli.collections import collections_cli
|
| 29 |
+
from huggingface_hub.cli.datasets import datasets_cli
|
| 30 |
+
from huggingface_hub.cli.discussions import discussions_cli
|
| 31 |
+
from huggingface_hub.cli.download import DOWNLOAD_EXAMPLES, download
|
| 32 |
+
from huggingface_hub.cli.extensions import (
|
| 33 |
+
dispatch_unknown_top_level_extension,
|
| 34 |
+
extensions_cli,
|
| 35 |
+
list_installed_extensions_for_help,
|
| 36 |
+
)
|
| 37 |
+
from huggingface_hub.cli.inference_endpoints import ie_cli
|
| 38 |
+
from huggingface_hub.cli.jobs import jobs_cli
|
| 39 |
+
from huggingface_hub.cli.lfs import lfs_enable_largefiles, lfs_multipart_upload
|
| 40 |
+
from huggingface_hub.cli.models import models_cli
|
| 41 |
+
from huggingface_hub.cli.papers import papers_cli
|
| 42 |
+
from huggingface_hub.cli.repo_files import repo_files_cli
|
| 43 |
+
from huggingface_hub.cli.repos import repos_cli
|
| 44 |
+
from huggingface_hub.cli.skills import skills_cli
|
| 45 |
+
from huggingface_hub.cli.spaces import spaces_cli
|
| 46 |
+
from huggingface_hub.cli.system import env, update, version
|
| 47 |
+
from huggingface_hub.cli.upload import UPLOAD_EXAMPLES, upload
|
| 48 |
+
from huggingface_hub.cli.upload_large_folder import UPLOAD_LARGE_FOLDER_EXAMPLES, upload_large_folder
|
| 49 |
+
from huggingface_hub.cli.webhooks import webhooks_cli
|
| 50 |
+
from huggingface_hub.utils import ANSI, logging
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
app = typer_factory(
|
| 54 |
+
help="Hugging Face Hub CLI",
|
| 55 |
+
cls=fallback_typer_group_factory(
|
| 56 |
+
dispatch_unknown_top_level_extension,
|
| 57 |
+
extra_commands_provider=list_installed_extensions_for_help,
|
| 58 |
+
),
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _version_callback(value: bool) -> None:
|
| 63 |
+
if value:
|
| 64 |
+
print(__version__)
|
| 65 |
+
raise typer.Exit()
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
@app.callback(invoke_without_command=True)
|
| 69 |
+
def app_callback(
|
| 70 |
+
version: Annotated[
|
| 71 |
+
bool | None, typer.Option("-v", "--version", callback=_version_callback, is_eager=True, hidden=True)
|
| 72 |
+
] = None,
|
| 73 |
+
) -> None:
|
| 74 |
+
pass
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
# top level single commands (defined in their respective files)
|
| 78 |
+
app.command(examples=CP_EXAMPLES)(make_cp())
|
| 79 |
+
app.command()(sync)
|
| 80 |
+
app.command(examples=DOWNLOAD_EXAMPLES)(download)
|
| 81 |
+
app.command(examples=UPLOAD_EXAMPLES)(upload)
|
| 82 |
+
app.command(examples=UPLOAD_LARGE_FOLDER_EXAMPLES)(upload_large_folder)
|
| 83 |
+
|
| 84 |
+
app.command(topic="help")(env)
|
| 85 |
+
app.command(topic="help")(update)
|
| 86 |
+
app.command(topic="help")(version)
|
| 87 |
+
|
| 88 |
+
app.command(hidden=True)(lfs_enable_largefiles)
|
| 89 |
+
app.command(hidden=True)(lfs_multipart_upload)
|
| 90 |
+
|
| 91 |
+
# command groups
|
| 92 |
+
app.add_typer(auth_cli, name="auth")
|
| 93 |
+
app.add_typer(buckets_cli, name="buckets")
|
| 94 |
+
app.add_typer(cache_cli, name="cache")
|
| 95 |
+
app.add_typer(collections_cli, name="collections")
|
| 96 |
+
app.add_typer(datasets_cli, name="datasets")
|
| 97 |
+
app.add_typer(discussions_cli, name="discussions")
|
| 98 |
+
app.add_typer(jobs_cli, name="jobs")
|
| 99 |
+
app.add_typer(models_cli, name="models")
|
| 100 |
+
app.add_typer(papers_cli, name="papers")
|
| 101 |
+
app.add_typer(repos_cli, name="repos | repo")
|
| 102 |
+
app.add_typer(repo_files_cli, name="repo-files", hidden=True)
|
| 103 |
+
app.add_typer(skills_cli, name="skills")
|
| 104 |
+
app.add_typer(spaces_cli, name="spaces")
|
| 105 |
+
app.add_typer(webhooks_cli, name="webhooks")
|
| 106 |
+
app.add_typer(ie_cli, name="endpoints")
|
| 107 |
+
app.add_typer(extensions_cli, name="extensions | ext")
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def main():
|
| 111 |
+
if not constants.HF_DEBUG:
|
| 112 |
+
logging.set_verbosity_info()
|
| 113 |
+
check_cli_update("huggingface_hub")
|
| 114 |
+
|
| 115 |
+
try:
|
| 116 |
+
app()
|
| 117 |
+
except Exception as e:
|
| 118 |
+
message = format_known_exception(e)
|
| 119 |
+
if message:
|
| 120 |
+
print(f"Error: {message}", file=sys.stderr)
|
| 121 |
+
if constants.HF_DEBUG:
|
| 122 |
+
traceback.print_exc()
|
| 123 |
+
else:
|
| 124 |
+
print(ANSI.gray("Set HF_DEBUG=1 as environment variable for full traceback."))
|
| 125 |
+
sys.exit(1)
|
| 126 |
+
raise
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
if __name__ == "__main__":
|
| 130 |
+
main()
|
venv/Lib/site-packages/huggingface_hub/cli/inference_endpoints.py
ADDED
|
@@ -0,0 +1,438 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CLI commands for Hugging Face Inference Endpoints."""
|
| 2 |
+
|
| 3 |
+
from typing import Annotated
|
| 4 |
+
|
| 5 |
+
import typer
|
| 6 |
+
|
| 7 |
+
from huggingface_hub._inference_endpoints import InferenceEndpointScalingMetric
|
| 8 |
+
from huggingface_hub.errors import HfHubHTTPError
|
| 9 |
+
|
| 10 |
+
from ._cli_utils import TokenOpt, get_hf_api, typer_factory
|
| 11 |
+
from ._output import out
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
ie_cli = typer_factory(help="Manage Hugging Face Inference Endpoints.")
|
| 15 |
+
|
| 16 |
+
catalog_app = typer_factory(help="Interact with the Inference Endpoints catalog.")
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
NameArg = Annotated[
|
| 20 |
+
str,
|
| 21 |
+
typer.Argument(help="Endpoint name."),
|
| 22 |
+
]
|
| 23 |
+
NameOpt = Annotated[
|
| 24 |
+
str | None,
|
| 25 |
+
typer.Option(help="Endpoint name."),
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
NamespaceOpt = Annotated[
|
| 29 |
+
str | None,
|
| 30 |
+
typer.Option(
|
| 31 |
+
help="The namespace associated with the Inference Endpoint. Defaults to the current user's namespace.",
|
| 32 |
+
),
|
| 33 |
+
]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
@ie_cli.command("list | ls", examples=["hf endpoints ls", "hf endpoints ls --namespace my-org"])
|
| 37 |
+
def ls(
|
| 38 |
+
namespace: NamespaceOpt = None,
|
| 39 |
+
token: TokenOpt = None,
|
| 40 |
+
) -> None:
|
| 41 |
+
"""Lists all Inference Endpoints for the given namespace."""
|
| 42 |
+
api = get_hf_api(token=token)
|
| 43 |
+
try:
|
| 44 |
+
endpoints = api.list_inference_endpoints(namespace=namespace, token=token)
|
| 45 |
+
except HfHubHTTPError as error:
|
| 46 |
+
out.error(f"Listing failed: {error}")
|
| 47 |
+
raise typer.Exit(code=error.response.status_code) from error
|
| 48 |
+
|
| 49 |
+
results = []
|
| 50 |
+
for endpoint in endpoints:
|
| 51 |
+
raw = endpoint.raw
|
| 52 |
+
status = raw.get("status", {})
|
| 53 |
+
model = raw.get("model", {})
|
| 54 |
+
compute = raw.get("compute", {})
|
| 55 |
+
provider = raw.get("provider", {})
|
| 56 |
+
results.append(
|
| 57 |
+
{
|
| 58 |
+
"name": raw.get("name", ""),
|
| 59 |
+
"model": model.get("repository", "") if isinstance(model, dict) else "",
|
| 60 |
+
"status": status.get("state", "") if isinstance(status, dict) else "",
|
| 61 |
+
"task": model.get("task", "") if isinstance(model, dict) else "",
|
| 62 |
+
"framework": model.get("framework", "") if isinstance(model, dict) else "",
|
| 63 |
+
"instance": compute.get("instanceType", "") if isinstance(compute, dict) else "",
|
| 64 |
+
"vendor": provider.get("vendor", "") if isinstance(provider, dict) else "",
|
| 65 |
+
"region": provider.get("region", "") if isinstance(provider, dict) else "",
|
| 66 |
+
}
|
| 67 |
+
)
|
| 68 |
+
out.table(results, id_key="name")
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
@ie_cli.command(name="deploy", examples=["hf endpoints deploy my-endpoint --repo gpt2 --framework pytorch ..."])
|
| 72 |
+
def deploy(
|
| 73 |
+
name: NameArg,
|
| 74 |
+
repo: Annotated[
|
| 75 |
+
str,
|
| 76 |
+
typer.Option(
|
| 77 |
+
help="The name of the model repository associated with the Inference Endpoint (e.g. 'openai/gpt-oss-120b').",
|
| 78 |
+
),
|
| 79 |
+
],
|
| 80 |
+
framework: Annotated[
|
| 81 |
+
str,
|
| 82 |
+
typer.Option(
|
| 83 |
+
help="The machine learning framework used for the model (e.g. 'vllm').",
|
| 84 |
+
),
|
| 85 |
+
],
|
| 86 |
+
accelerator: Annotated[
|
| 87 |
+
str,
|
| 88 |
+
typer.Option(
|
| 89 |
+
help="The hardware accelerator to be used for inference (e.g. 'cpu').",
|
| 90 |
+
),
|
| 91 |
+
],
|
| 92 |
+
instance_size: Annotated[
|
| 93 |
+
str,
|
| 94 |
+
typer.Option(
|
| 95 |
+
help="The size or type of the instance to be used for hosting the model (e.g. 'x4').",
|
| 96 |
+
),
|
| 97 |
+
],
|
| 98 |
+
instance_type: Annotated[
|
| 99 |
+
str,
|
| 100 |
+
typer.Option(
|
| 101 |
+
help="The cloud instance type where the Inference Endpoint will be deployed (e.g. 'intel-icl').",
|
| 102 |
+
),
|
| 103 |
+
],
|
| 104 |
+
region: Annotated[
|
| 105 |
+
str,
|
| 106 |
+
typer.Option(
|
| 107 |
+
help="The cloud region in which the Inference Endpoint will be created (e.g. 'us-east-1').",
|
| 108 |
+
),
|
| 109 |
+
],
|
| 110 |
+
vendor: Annotated[
|
| 111 |
+
str,
|
| 112 |
+
typer.Option(
|
| 113 |
+
help="The cloud provider or vendor where the Inference Endpoint will be hosted (e.g. 'aws').",
|
| 114 |
+
),
|
| 115 |
+
],
|
| 116 |
+
*,
|
| 117 |
+
namespace: NamespaceOpt = None,
|
| 118 |
+
task: Annotated[
|
| 119 |
+
str | None,
|
| 120 |
+
typer.Option(
|
| 121 |
+
help="The task on which to deploy the model (e.g. 'text-classification').",
|
| 122 |
+
),
|
| 123 |
+
] = None,
|
| 124 |
+
token: TokenOpt = None,
|
| 125 |
+
min_replica: Annotated[
|
| 126 |
+
int,
|
| 127 |
+
typer.Option(
|
| 128 |
+
help="The minimum number of replicas (instances) to keep running for the Inference Endpoint.",
|
| 129 |
+
),
|
| 130 |
+
] = 1,
|
| 131 |
+
max_replica: Annotated[
|
| 132 |
+
int,
|
| 133 |
+
typer.Option(
|
| 134 |
+
help="The maximum number of replicas (instances) to scale to for the Inference Endpoint.",
|
| 135 |
+
),
|
| 136 |
+
] = 1,
|
| 137 |
+
scale_to_zero_timeout: Annotated[
|
| 138 |
+
int | None,
|
| 139 |
+
typer.Option(
|
| 140 |
+
help="The duration in minutes before an inactive endpoint is scaled to zero.",
|
| 141 |
+
),
|
| 142 |
+
] = None,
|
| 143 |
+
scaling_metric: Annotated[
|
| 144 |
+
InferenceEndpointScalingMetric | None,
|
| 145 |
+
typer.Option(
|
| 146 |
+
help="The metric reference for scaling.",
|
| 147 |
+
),
|
| 148 |
+
] = None,
|
| 149 |
+
scaling_threshold: Annotated[
|
| 150 |
+
float | None,
|
| 151 |
+
typer.Option(
|
| 152 |
+
help="The scaling metric threshold used to trigger a scale up. Ignored when scaling metric is not provided.",
|
| 153 |
+
),
|
| 154 |
+
] = None,
|
| 155 |
+
) -> None:
|
| 156 |
+
"""Deploy an Inference Endpoint from a Hub repository."""
|
| 157 |
+
api = get_hf_api(token=token)
|
| 158 |
+
endpoint = api.create_inference_endpoint(
|
| 159 |
+
name=name,
|
| 160 |
+
repository=repo,
|
| 161 |
+
framework=framework,
|
| 162 |
+
accelerator=accelerator,
|
| 163 |
+
instance_size=instance_size,
|
| 164 |
+
instance_type=instance_type,
|
| 165 |
+
region=region,
|
| 166 |
+
vendor=vendor,
|
| 167 |
+
namespace=namespace,
|
| 168 |
+
task=task,
|
| 169 |
+
token=token,
|
| 170 |
+
min_replica=min_replica,
|
| 171 |
+
max_replica=max_replica,
|
| 172 |
+
scaling_metric=scaling_metric,
|
| 173 |
+
scaling_threshold=scaling_threshold,
|
| 174 |
+
scale_to_zero_timeout=scale_to_zero_timeout,
|
| 175 |
+
)
|
| 176 |
+
out.dict(endpoint.raw)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
@catalog_app.command(name="deploy", examples=["hf endpoints catalog deploy --repo meta-llama/Llama-3.2-1B-Instruct"])
|
| 180 |
+
def deploy_from_catalog(
|
| 181 |
+
repo: Annotated[
|
| 182 |
+
str,
|
| 183 |
+
typer.Option(
|
| 184 |
+
help="The name of the model repository associated with the Inference Endpoint (e.g. 'openai/gpt-oss-120b').",
|
| 185 |
+
),
|
| 186 |
+
],
|
| 187 |
+
name: NameOpt = None,
|
| 188 |
+
accelerator: Annotated[
|
| 189 |
+
str | None,
|
| 190 |
+
typer.Option(
|
| 191 |
+
help="The hardware accelerator to be used for inference (e.g. 'cpu', 'gpu', 'neuron').",
|
| 192 |
+
),
|
| 193 |
+
] = None,
|
| 194 |
+
namespace: NamespaceOpt = None,
|
| 195 |
+
token: TokenOpt = None,
|
| 196 |
+
) -> None:
|
| 197 |
+
"""Deploy an Inference Endpoint from the Model Catalog."""
|
| 198 |
+
api = get_hf_api(token=token)
|
| 199 |
+
try:
|
| 200 |
+
endpoint = api.create_inference_endpoint_from_catalog(
|
| 201 |
+
repo_id=repo,
|
| 202 |
+
name=name,
|
| 203 |
+
accelerator=accelerator,
|
| 204 |
+
namespace=namespace,
|
| 205 |
+
token=token,
|
| 206 |
+
)
|
| 207 |
+
except HfHubHTTPError as error:
|
| 208 |
+
out.error(f"Deployment failed: {error}")
|
| 209 |
+
raise typer.Exit(code=error.response.status_code) from error
|
| 210 |
+
|
| 211 |
+
out.dict(endpoint.raw)
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def list_catalog(
|
| 215 |
+
token: TokenOpt = None,
|
| 216 |
+
) -> None:
|
| 217 |
+
"""List available Catalog models."""
|
| 218 |
+
api = get_hf_api(token=token)
|
| 219 |
+
try:
|
| 220 |
+
models = api.list_inference_catalog(token=token)
|
| 221 |
+
except HfHubHTTPError as error:
|
| 222 |
+
out.error(f"Catalog fetch failed: {error}")
|
| 223 |
+
raise typer.Exit(code=error.response.status_code) from error
|
| 224 |
+
|
| 225 |
+
out.dict({"models": models})
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
catalog_app.command(name="list | ls", examples=["hf endpoints catalog ls"])(list_catalog)
|
| 229 |
+
ie_cli.command(name="list-catalog", hidden=True)(list_catalog)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
ie_cli.add_typer(catalog_app, name="catalog")
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
@ie_cli.command(examples=["hf endpoints describe my-endpoint"])
|
| 236 |
+
def describe(
|
| 237 |
+
name: NameArg,
|
| 238 |
+
namespace: NamespaceOpt = None,
|
| 239 |
+
token: TokenOpt = None,
|
| 240 |
+
) -> None:
|
| 241 |
+
"""Get information about an existing endpoint."""
|
| 242 |
+
api = get_hf_api(token=token)
|
| 243 |
+
try:
|
| 244 |
+
endpoint = api.get_inference_endpoint(name=name, namespace=namespace, token=token)
|
| 245 |
+
except HfHubHTTPError as error:
|
| 246 |
+
out.error(f"Fetch failed: {error}")
|
| 247 |
+
raise typer.Exit(code=error.response.status_code) from error
|
| 248 |
+
|
| 249 |
+
out.dict(endpoint.raw)
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
@ie_cli.command(examples=["hf endpoints update my-endpoint --min-replica 2"])
|
| 253 |
+
def update(
|
| 254 |
+
name: NameArg,
|
| 255 |
+
namespace: NamespaceOpt = None,
|
| 256 |
+
repo: Annotated[
|
| 257 |
+
str | None,
|
| 258 |
+
typer.Option(
|
| 259 |
+
help="The name of the model repository associated with the Inference Endpoint (e.g. 'openai/gpt-oss-120b').",
|
| 260 |
+
),
|
| 261 |
+
] = None,
|
| 262 |
+
accelerator: Annotated[
|
| 263 |
+
str | None,
|
| 264 |
+
typer.Option(
|
| 265 |
+
help="The hardware accelerator to be used for inference (e.g. 'cpu').",
|
| 266 |
+
),
|
| 267 |
+
] = None,
|
| 268 |
+
instance_size: Annotated[
|
| 269 |
+
str | None,
|
| 270 |
+
typer.Option(
|
| 271 |
+
help="The size or type of the instance to be used for hosting the model (e.g. 'x4').",
|
| 272 |
+
),
|
| 273 |
+
] = None,
|
| 274 |
+
instance_type: Annotated[
|
| 275 |
+
str | None,
|
| 276 |
+
typer.Option(
|
| 277 |
+
help="The cloud instance type where the Inference Endpoint will be deployed (e.g. 'intel-icl').",
|
| 278 |
+
),
|
| 279 |
+
] = None,
|
| 280 |
+
framework: Annotated[
|
| 281 |
+
str | None,
|
| 282 |
+
typer.Option(
|
| 283 |
+
help="The machine learning framework used for the model (e.g. 'custom').",
|
| 284 |
+
),
|
| 285 |
+
] = None,
|
| 286 |
+
revision: Annotated[
|
| 287 |
+
str | None,
|
| 288 |
+
typer.Option(
|
| 289 |
+
help="The specific model revision to deploy on the Inference Endpoint (e.g. '6c0e6080953db56375760c0471a8c5f2929baf11').",
|
| 290 |
+
),
|
| 291 |
+
] = None,
|
| 292 |
+
task: Annotated[
|
| 293 |
+
str | None,
|
| 294 |
+
typer.Option(
|
| 295 |
+
help="The task on which to deploy the model (e.g. 'text-classification').",
|
| 296 |
+
),
|
| 297 |
+
] = None,
|
| 298 |
+
min_replica: Annotated[
|
| 299 |
+
int | None,
|
| 300 |
+
typer.Option(
|
| 301 |
+
help="The minimum number of replicas (instances) to keep running for the Inference Endpoint.",
|
| 302 |
+
),
|
| 303 |
+
] = None,
|
| 304 |
+
max_replica: Annotated[
|
| 305 |
+
int | None,
|
| 306 |
+
typer.Option(
|
| 307 |
+
help="The maximum number of replicas (instances) to scale to for the Inference Endpoint.",
|
| 308 |
+
),
|
| 309 |
+
] = None,
|
| 310 |
+
scale_to_zero_timeout: Annotated[
|
| 311 |
+
int | None,
|
| 312 |
+
typer.Option(
|
| 313 |
+
help="The duration in minutes before an inactive endpoint is scaled to zero.",
|
| 314 |
+
),
|
| 315 |
+
] = None,
|
| 316 |
+
scaling_metric: Annotated[
|
| 317 |
+
InferenceEndpointScalingMetric | None,
|
| 318 |
+
typer.Option(
|
| 319 |
+
help="The metric reference for scaling.",
|
| 320 |
+
),
|
| 321 |
+
] = None,
|
| 322 |
+
scaling_threshold: Annotated[
|
| 323 |
+
float | None,
|
| 324 |
+
typer.Option(
|
| 325 |
+
help="The scaling metric threshold used to trigger a scale up. Ignored when scaling metric is not provided.",
|
| 326 |
+
),
|
| 327 |
+
] = None,
|
| 328 |
+
token: TokenOpt = None,
|
| 329 |
+
) -> None:
|
| 330 |
+
"""Update an existing endpoint."""
|
| 331 |
+
api = get_hf_api(token=token)
|
| 332 |
+
try:
|
| 333 |
+
endpoint = api.update_inference_endpoint(
|
| 334 |
+
name=name,
|
| 335 |
+
namespace=namespace,
|
| 336 |
+
repository=repo,
|
| 337 |
+
framework=framework,
|
| 338 |
+
revision=revision,
|
| 339 |
+
task=task,
|
| 340 |
+
accelerator=accelerator,
|
| 341 |
+
instance_size=instance_size,
|
| 342 |
+
instance_type=instance_type,
|
| 343 |
+
min_replica=min_replica,
|
| 344 |
+
max_replica=max_replica,
|
| 345 |
+
scale_to_zero_timeout=scale_to_zero_timeout,
|
| 346 |
+
scaling_metric=scaling_metric,
|
| 347 |
+
scaling_threshold=scaling_threshold,
|
| 348 |
+
token=token,
|
| 349 |
+
)
|
| 350 |
+
except HfHubHTTPError as error:
|
| 351 |
+
out.error(f"Update failed: {error}")
|
| 352 |
+
raise typer.Exit(code=error.response.status_code) from error
|
| 353 |
+
out.dict(endpoint.raw)
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
@ie_cli.command(examples=["hf endpoints delete my-endpoint"])
|
| 357 |
+
def delete(
|
| 358 |
+
name: NameArg,
|
| 359 |
+
namespace: NamespaceOpt = None,
|
| 360 |
+
yes: Annotated[
|
| 361 |
+
bool,
|
| 362 |
+
typer.Option("--yes", help="Skip confirmation prompts."),
|
| 363 |
+
] = False,
|
| 364 |
+
token: TokenOpt = None,
|
| 365 |
+
) -> None:
|
| 366 |
+
"""Delete an Inference Endpoint permanently."""
|
| 367 |
+
out.confirm(f"Delete endpoint '{name}'?", yes=yes)
|
| 368 |
+
|
| 369 |
+
api = get_hf_api(token=token)
|
| 370 |
+
try:
|
| 371 |
+
api.delete_inference_endpoint(name=name, namespace=namespace, token=token)
|
| 372 |
+
except HfHubHTTPError as error:
|
| 373 |
+
out.error(f"Delete failed: {error}")
|
| 374 |
+
raise typer.Exit(code=error.response.status_code) from error
|
| 375 |
+
|
| 376 |
+
out.result(f"Deleted '{name}'.", name=name)
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
@ie_cli.command(examples=["hf endpoints pause my-endpoint"])
|
| 380 |
+
def pause(
|
| 381 |
+
name: NameArg,
|
| 382 |
+
namespace: NamespaceOpt = None,
|
| 383 |
+
token: TokenOpt = None,
|
| 384 |
+
) -> None:
|
| 385 |
+
"""Pause an Inference Endpoint."""
|
| 386 |
+
api = get_hf_api(token=token)
|
| 387 |
+
try:
|
| 388 |
+
endpoint = api.pause_inference_endpoint(name=name, namespace=namespace, token=token)
|
| 389 |
+
except HfHubHTTPError as error:
|
| 390 |
+
out.error(f"Pause failed: {error}")
|
| 391 |
+
raise typer.Exit(code=error.response.status_code) from error
|
| 392 |
+
|
| 393 |
+
out.dict(endpoint.raw)
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
@ie_cli.command(examples=["hf endpoints resume my-endpoint"])
|
| 397 |
+
def resume(
|
| 398 |
+
name: NameArg,
|
| 399 |
+
namespace: NamespaceOpt = None,
|
| 400 |
+
fail_if_already_running: Annotated[
|
| 401 |
+
bool,
|
| 402 |
+
typer.Option(
|
| 403 |
+
"--fail-if-already-running",
|
| 404 |
+
help="If `True`, the method will raise an error if the Inference Endpoint is already running.",
|
| 405 |
+
),
|
| 406 |
+
] = False,
|
| 407 |
+
token: TokenOpt = None,
|
| 408 |
+
) -> None:
|
| 409 |
+
"""Resume an Inference Endpoint."""
|
| 410 |
+
api = get_hf_api(token=token)
|
| 411 |
+
try:
|
| 412 |
+
endpoint = api.resume_inference_endpoint(
|
| 413 |
+
name=name,
|
| 414 |
+
namespace=namespace,
|
| 415 |
+
token=token,
|
| 416 |
+
running_ok=not fail_if_already_running,
|
| 417 |
+
)
|
| 418 |
+
except HfHubHTTPError as error:
|
| 419 |
+
out.error(f"Resume failed: {error}")
|
| 420 |
+
raise typer.Exit(code=error.response.status_code) from error
|
| 421 |
+
out.dict(endpoint.raw)
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
@ie_cli.command(examples=["hf endpoints scale-to-zero my-endpoint"])
|
| 425 |
+
def scale_to_zero(
|
| 426 |
+
name: NameArg,
|
| 427 |
+
namespace: NamespaceOpt = None,
|
| 428 |
+
token: TokenOpt = None,
|
| 429 |
+
) -> None:
|
| 430 |
+
"""Scale an Inference Endpoint to zero."""
|
| 431 |
+
api = get_hf_api(token=token)
|
| 432 |
+
try:
|
| 433 |
+
endpoint = api.scale_to_zero_inference_endpoint(name=name, namespace=namespace, token=token)
|
| 434 |
+
except HfHubHTTPError as error:
|
| 435 |
+
out.error(f"Scale To Zero failed: {error}")
|
| 436 |
+
raise typer.Exit(code=error.response.status_code) from error
|
| 437 |
+
|
| 438 |
+
out.dict(endpoint.raw)
|
venv/Lib/site-packages/huggingface_hub/cli/jobs.py
ADDED
|
@@ -0,0 +1,1172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains commands to interact with jobs on the Hugging Face Hub.
|
| 15 |
+
|
| 16 |
+
Usage:
|
| 17 |
+
# run a job
|
| 18 |
+
hf jobs run <image> <command>
|
| 19 |
+
|
| 20 |
+
# List running or completed jobs
|
| 21 |
+
hf jobs ps [-a] [-f key=value]
|
| 22 |
+
|
| 23 |
+
# Print logs from a job (non-blocking)
|
| 24 |
+
hf jobs logs <job-id>
|
| 25 |
+
|
| 26 |
+
# Stream logs from a job (blocking, like `docker logs -f`)
|
| 27 |
+
hf jobs logs -f <job-id>
|
| 28 |
+
|
| 29 |
+
# Stream resources usage stats and metrics from a job
|
| 30 |
+
hf jobs stats <job-id>
|
| 31 |
+
|
| 32 |
+
# Inspect detailed information about a job
|
| 33 |
+
hf jobs inspect <job-id>
|
| 34 |
+
|
| 35 |
+
# Cancel a running job
|
| 36 |
+
hf jobs cancel <job-id>
|
| 37 |
+
|
| 38 |
+
# List available hardware options
|
| 39 |
+
hf jobs hardware
|
| 40 |
+
|
| 41 |
+
# Run a UV script
|
| 42 |
+
hf jobs uv run <script>
|
| 43 |
+
|
| 44 |
+
# Schedule a job
|
| 45 |
+
hf jobs scheduled run <schedule> <image> <command>
|
| 46 |
+
|
| 47 |
+
# List scheduled jobs
|
| 48 |
+
hf jobs scheduled ps [-a] [-f key=value]
|
| 49 |
+
|
| 50 |
+
# Inspect a scheduled job
|
| 51 |
+
hf jobs scheduled inspect <scheduled_job_id>
|
| 52 |
+
|
| 53 |
+
# Suspend a scheduled job
|
| 54 |
+
hf jobs scheduled suspend <scheduled_job_id>
|
| 55 |
+
|
| 56 |
+
# Resume a scheduled job
|
| 57 |
+
hf jobs scheduled resume <scheduled_job_id>
|
| 58 |
+
|
| 59 |
+
# Delete a scheduled job
|
| 60 |
+
hf jobs scheduled delete <scheduled_job_id>
|
| 61 |
+
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
import multiprocessing
|
| 65 |
+
import multiprocessing.pool
|
| 66 |
+
import shutil
|
| 67 |
+
import time
|
| 68 |
+
from collections.abc import Callable, Iterable
|
| 69 |
+
from fnmatch import fnmatch
|
| 70 |
+
from queue import Empty, Queue
|
| 71 |
+
from typing import Annotated, Any, TypeVar
|
| 72 |
+
|
| 73 |
+
import typer
|
| 74 |
+
|
| 75 |
+
from huggingface_hub import JobHardware
|
| 76 |
+
from huggingface_hub.errors import CLIError, HfHubHTTPError
|
| 77 |
+
from huggingface_hub.utils import logging
|
| 78 |
+
from huggingface_hub.utils._cache_manager import _format_size
|
| 79 |
+
from huggingface_hub.utils._parsing import format_duration
|
| 80 |
+
|
| 81 |
+
from ._cli_utils import (
|
| 82 |
+
EnvFileOpt,
|
| 83 |
+
EnvOpt,
|
| 84 |
+
SecretsFileOpt,
|
| 85 |
+
SecretsOpt,
|
| 86 |
+
SoftChoice,
|
| 87 |
+
TokenOpt,
|
| 88 |
+
VolumesOpt,
|
| 89 |
+
get_hf_api,
|
| 90 |
+
parse_env_map,
|
| 91 |
+
parse_volumes,
|
| 92 |
+
typer_factory,
|
| 93 |
+
)
|
| 94 |
+
from ._output import _dataclass_to_dict, out
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
logger = logging.get_logger(__name__)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _parse_namespace_from_job_id(job_id: str, namespace: str | None) -> tuple[str, str | None]:
|
| 101 |
+
"""Extract namespace from job_id if provided in 'namespace/job_id' format.
|
| 102 |
+
|
| 103 |
+
Allows users to pass job IDs copied from the Hub UI (e.g. 'username/job_id')
|
| 104 |
+
instead of only bare job IDs. If the namespace is also provided explicitly via
|
| 105 |
+
--namespace and conflicts, a CLIError is raised.
|
| 106 |
+
"""
|
| 107 |
+
if not job_id:
|
| 108 |
+
raise CLIError("Job ID cannot be empty.")
|
| 109 |
+
|
| 110 |
+
if job_id.count("/") > 1:
|
| 111 |
+
raise CLIError(f"Job ID must be in the form 'job_id' or 'namespace/job_id': '{job_id}'.")
|
| 112 |
+
|
| 113 |
+
if "/" not in job_id:
|
| 114 |
+
return job_id, namespace
|
| 115 |
+
|
| 116 |
+
extracted_namespace, parsed_job_id = job_id.split("/", 1)
|
| 117 |
+
if not extracted_namespace or not parsed_job_id:
|
| 118 |
+
raise CLIError(f"Job ID must be in the form 'job_id' or 'namespace/job_id': '{job_id}'.")
|
| 119 |
+
|
| 120 |
+
if namespace is not None and namespace != extracted_namespace:
|
| 121 |
+
raise CLIError(
|
| 122 |
+
f"Conflicting namespace: got --namespace='{namespace}' but job ID implies namespace='{extracted_namespace}'"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
return parsed_job_id, extracted_namespace
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
STATS_UPDATE_MIN_INTERVAL = 0.1 # we set a limit here since there is one update per second per job
|
| 129 |
+
|
| 130 |
+
# Common job-related options
|
| 131 |
+
ImageArg = Annotated[
|
| 132 |
+
str,
|
| 133 |
+
typer.Argument(
|
| 134 |
+
help="The Docker image to use.",
|
| 135 |
+
),
|
| 136 |
+
]
|
| 137 |
+
|
| 138 |
+
ImageOpt = Annotated[
|
| 139 |
+
str | None,
|
| 140 |
+
typer.Option(
|
| 141 |
+
help="Use a custom Docker image with `uv` installed.",
|
| 142 |
+
),
|
| 143 |
+
]
|
| 144 |
+
|
| 145 |
+
FlavorOpt = Annotated[
|
| 146 |
+
str | None,
|
| 147 |
+
typer.Option(
|
| 148 |
+
help="Flavor for the hardware. Run 'hf jobs hardware' to list available flavors. Defaults to `cpu-basic`.",
|
| 149 |
+
click_type=SoftChoice(JobHardware),
|
| 150 |
+
),
|
| 151 |
+
]
|
| 152 |
+
|
| 153 |
+
LabelsOpt = Annotated[
|
| 154 |
+
list[str] | None,
|
| 155 |
+
typer.Option(
|
| 156 |
+
"-l",
|
| 157 |
+
"--label",
|
| 158 |
+
help="Set labels. E.g. --label KEY=VALUE or --label LABEL",
|
| 159 |
+
),
|
| 160 |
+
]
|
| 161 |
+
|
| 162 |
+
TimeoutOpt = Annotated[
|
| 163 |
+
str | None,
|
| 164 |
+
typer.Option(
|
| 165 |
+
help="Max duration: int/float with s (seconds, default), m (minutes), h (hours) or d (days).",
|
| 166 |
+
),
|
| 167 |
+
]
|
| 168 |
+
|
| 169 |
+
DetachOpt = Annotated[
|
| 170 |
+
bool,
|
| 171 |
+
typer.Option(
|
| 172 |
+
"-d",
|
| 173 |
+
"--detach",
|
| 174 |
+
help="Run the Job in the background and print the Job ID.",
|
| 175 |
+
),
|
| 176 |
+
]
|
| 177 |
+
|
| 178 |
+
NamespaceOpt = Annotated[
|
| 179 |
+
str | None,
|
| 180 |
+
typer.Option(
|
| 181 |
+
help="The namespace where the job will be running. Defaults to the current user's namespace.",
|
| 182 |
+
),
|
| 183 |
+
]
|
| 184 |
+
|
| 185 |
+
ExposeOpt = Annotated[
|
| 186 |
+
list[int] | None,
|
| 187 |
+
typer.Option(
|
| 188 |
+
"--expose",
|
| 189 |
+
help="Expose a container port through the jobs proxy. Repeat the flag for multiple ports (e.g. `--expose 8000 --expose 8001`). Each exposed port is reachable on the public jobs domain; access requires an HF token with read access to the job's namespace.",
|
| 190 |
+
),
|
| 191 |
+
]
|
| 192 |
+
|
| 193 |
+
WithOpt = Annotated[
|
| 194 |
+
list[str] | None,
|
| 195 |
+
typer.Option(
|
| 196 |
+
"--with",
|
| 197 |
+
help="Run with the given packages installed",
|
| 198 |
+
),
|
| 199 |
+
]
|
| 200 |
+
|
| 201 |
+
PythonOpt = Annotated[
|
| 202 |
+
str | None,
|
| 203 |
+
typer.Option(
|
| 204 |
+
"-p",
|
| 205 |
+
"--python",
|
| 206 |
+
help="The Python interpreter to use for the run environment",
|
| 207 |
+
),
|
| 208 |
+
]
|
| 209 |
+
|
| 210 |
+
SuspendOpt = Annotated[
|
| 211 |
+
bool | None,
|
| 212 |
+
typer.Option(
|
| 213 |
+
help="Suspend (pause) the scheduled Job",
|
| 214 |
+
),
|
| 215 |
+
]
|
| 216 |
+
|
| 217 |
+
ConcurrencyOpt = Annotated[
|
| 218 |
+
bool | None,
|
| 219 |
+
typer.Option(
|
| 220 |
+
help="Allow multiple instances of this Job to run concurrently",
|
| 221 |
+
),
|
| 222 |
+
]
|
| 223 |
+
|
| 224 |
+
ScheduleArg = Annotated[
|
| 225 |
+
str,
|
| 226 |
+
typer.Argument(
|
| 227 |
+
help="One of annually, yearly, monthly, weekly, daily, hourly, or a CRON schedule expression.",
|
| 228 |
+
),
|
| 229 |
+
]
|
| 230 |
+
|
| 231 |
+
ScriptArg = Annotated[
|
| 232 |
+
str,
|
| 233 |
+
typer.Argument(
|
| 234 |
+
help="UV script to run (local file or URL)",
|
| 235 |
+
),
|
| 236 |
+
]
|
| 237 |
+
|
| 238 |
+
ScriptArgsArg = Annotated[
|
| 239 |
+
list[str] | None,
|
| 240 |
+
typer.Argument(
|
| 241 |
+
help="Arguments for the script",
|
| 242 |
+
),
|
| 243 |
+
]
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
CommandArg = Annotated[
|
| 247 |
+
list[str],
|
| 248 |
+
typer.Argument(
|
| 249 |
+
help="The command to run.",
|
| 250 |
+
),
|
| 251 |
+
]
|
| 252 |
+
|
| 253 |
+
JobIdArg = Annotated[
|
| 254 |
+
str,
|
| 255 |
+
typer.Argument(
|
| 256 |
+
help="Job ID (or 'namespace/job_id')",
|
| 257 |
+
),
|
| 258 |
+
]
|
| 259 |
+
|
| 260 |
+
JobIdsArg = Annotated[
|
| 261 |
+
list[str] | None,
|
| 262 |
+
typer.Argument(
|
| 263 |
+
help="Job IDs (or 'namespace/job_id')",
|
| 264 |
+
),
|
| 265 |
+
]
|
| 266 |
+
|
| 267 |
+
ScheduledJobIdArg = Annotated[
|
| 268 |
+
str,
|
| 269 |
+
typer.Argument(
|
| 270 |
+
help="Scheduled Job ID (or 'namespace/scheduled_job_id')",
|
| 271 |
+
),
|
| 272 |
+
]
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
jobs_cli = typer_factory(help="Run and manage Jobs on the Hub.")
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
@jobs_cli.command(
|
| 279 |
+
"run",
|
| 280 |
+
context_settings={"ignore_unknown_options": True},
|
| 281 |
+
examples=[
|
| 282 |
+
"hf jobs run python:3.12 python -c 'print(\"Hello!\")'",
|
| 283 |
+
"hf jobs run --detach python:3.12 python script.py",
|
| 284 |
+
"hf jobs run -e FOO=foo python:3.12 python script.py",
|
| 285 |
+
"hf jobs run --secrets HF_TOKEN python:3.12 python script.py",
|
| 286 |
+
"hf jobs run -v hf://org/my-model:/data -v hf://buckets/org/b:/mnt python:3.12 python script.py",
|
| 287 |
+
],
|
| 288 |
+
)
|
| 289 |
+
def jobs_run(
|
| 290 |
+
image: ImageArg,
|
| 291 |
+
command: CommandArg,
|
| 292 |
+
env: EnvOpt = None,
|
| 293 |
+
secrets: SecretsOpt = None,
|
| 294 |
+
label: LabelsOpt = None,
|
| 295 |
+
volume: VolumesOpt = None,
|
| 296 |
+
env_file: EnvFileOpt = None,
|
| 297 |
+
secrets_file: SecretsFileOpt = None,
|
| 298 |
+
flavor: FlavorOpt = None,
|
| 299 |
+
timeout: TimeoutOpt = None,
|
| 300 |
+
detach: DetachOpt = False,
|
| 301 |
+
expose: ExposeOpt = None,
|
| 302 |
+
namespace: NamespaceOpt = None,
|
| 303 |
+
token: TokenOpt = None,
|
| 304 |
+
) -> None:
|
| 305 |
+
"""Run a Job."""
|
| 306 |
+
env_map = parse_env_map(env, env_file)
|
| 307 |
+
secrets_map = parse_env_map(secrets, secrets_file)
|
| 308 |
+
|
| 309 |
+
api = get_hf_api(token=token)
|
| 310 |
+
job = api.run_job(
|
| 311 |
+
image=image,
|
| 312 |
+
command=command,
|
| 313 |
+
env=env_map,
|
| 314 |
+
secrets=secrets_map,
|
| 315 |
+
labels=_parse_labels_map(label),
|
| 316 |
+
volumes=parse_volumes(volume),
|
| 317 |
+
flavor=flavor,
|
| 318 |
+
timeout=timeout,
|
| 319 |
+
expose=expose,
|
| 320 |
+
namespace=namespace,
|
| 321 |
+
)
|
| 322 |
+
out.result("Job started", id=job.id, url=job.url)
|
| 323 |
+
if isinstance(job.status.expose_urls, list):
|
| 324 |
+
urls = "\n".join(f" {url}" for url in job.status.expose_urls)
|
| 325 |
+
out.hint(f"Exposed ports are reachable at (requires an HF token with read access to the job):\n{urls}")
|
| 326 |
+
if detach:
|
| 327 |
+
job_ref = f"{job.owner.name}/{job.id}"
|
| 328 |
+
out.hint(f"Use `hf jobs logs -f {job_ref}` to stream logs, or `hf jobs inspect {job_ref}` to check status.")
|
| 329 |
+
return
|
| 330 |
+
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True):
|
| 331 |
+
out.text(log)
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
@jobs_cli.command(
|
| 335 |
+
"logs",
|
| 336 |
+
examples=[
|
| 337 |
+
"hf jobs logs <job_id>",
|
| 338 |
+
"hf jobs logs -f <job_id>",
|
| 339 |
+
"hf jobs logs --tail 20 <job_id>",
|
| 340 |
+
"hf jobs logs -f --tail 100 <job_id>",
|
| 341 |
+
],
|
| 342 |
+
)
|
| 343 |
+
def jobs_logs(
|
| 344 |
+
job_id: JobIdArg,
|
| 345 |
+
follow: Annotated[
|
| 346 |
+
bool,
|
| 347 |
+
typer.Option(
|
| 348 |
+
"-f",
|
| 349 |
+
"--follow",
|
| 350 |
+
help="Follow log output (stream until the job completes). Without this flag, only currently available logs are printed.",
|
| 351 |
+
),
|
| 352 |
+
] = False,
|
| 353 |
+
tail: Annotated[
|
| 354 |
+
int | None,
|
| 355 |
+
typer.Option(
|
| 356 |
+
"-n",
|
| 357 |
+
"--tail",
|
| 358 |
+
help="Number of lines to show from the end of the logs. When combined with --follow, starts streaming from the last N lines.",
|
| 359 |
+
),
|
| 360 |
+
] = None,
|
| 361 |
+
namespace: NamespaceOpt = None,
|
| 362 |
+
token: TokenOpt = None,
|
| 363 |
+
) -> None:
|
| 364 |
+
"""Fetch the logs of a Job.
|
| 365 |
+
|
| 366 |
+
By default, prints currently available logs and exits (non-blocking).
|
| 367 |
+
Use --follow/-f to stream logs in real-time until the job completes.
|
| 368 |
+
Use --tail/-n to limit the number of lines returned (server-side when supported).
|
| 369 |
+
|
| 370 |
+
Note: following exits when the log stream ends, regardless of whether the Job
|
| 371 |
+
succeeded or failed. Run `hf jobs inspect <job_id>` to check the final status.
|
| 372 |
+
"""
|
| 373 |
+
job_id, namespace = _parse_namespace_from_job_id(job_id, namespace)
|
| 374 |
+
|
| 375 |
+
api = get_hf_api(token=token)
|
| 376 |
+
try:
|
| 377 |
+
logs = api.fetch_job_logs(job_id=job_id, namespace=namespace, follow=follow, tail=tail)
|
| 378 |
+
for log in logs:
|
| 379 |
+
out.text(log)
|
| 380 |
+
if follow:
|
| 381 |
+
job_ref = f"{namespace}/{job_id}" if namespace else job_id
|
| 382 |
+
out.hint(
|
| 383 |
+
f"Stream ended. Run `hf jobs inspect {job_ref}` to check the final status (e.g. COMPLETED or ERROR)."
|
| 384 |
+
)
|
| 385 |
+
except HfHubHTTPError as e:
|
| 386 |
+
status = e.response.status_code if e.response is not None else None
|
| 387 |
+
if status == 404:
|
| 388 |
+
raise CLIError("Job not found. Please check the job ID.") from e
|
| 389 |
+
elif status == 403:
|
| 390 |
+
raise CLIError("Access denied. You may not have permission to view this job.") from e
|
| 391 |
+
else:
|
| 392 |
+
raise CLIError(f"Failed to fetch job logs: {e}") from e
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
def _matches_filters(job_properties: dict[str, str], filters: list[tuple[str, str, str]]) -> bool:
|
| 396 |
+
"""Check if scheduled job matches all specified filters."""
|
| 397 |
+
for key, op_str, pattern in filters:
|
| 398 |
+
value = job_properties.get(key)
|
| 399 |
+
if value is None:
|
| 400 |
+
if op_str == "!=":
|
| 401 |
+
continue
|
| 402 |
+
return False
|
| 403 |
+
match = fnmatch(value.lower(), pattern.lower())
|
| 404 |
+
if (op_str == "=" and not match) or (op_str == "!=" and match):
|
| 405 |
+
return False
|
| 406 |
+
return True
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
def _clear_line(n: int) -> None:
|
| 410 |
+
LINE_UP = "\033[1A"
|
| 411 |
+
LINE_CLEAR = "\x1b[2K"
|
| 412 |
+
for i in range(n):
|
| 413 |
+
print(LINE_UP, end=LINE_CLEAR)
|
| 414 |
+
|
| 415 |
+
|
| 416 |
+
def _get_jobs_stats_rows(
|
| 417 |
+
job_id: str, metrics_stream: Iterable[dict[str, Any]], table_headers: list[str]
|
| 418 |
+
) -> Iterable[tuple[bool, str, list[list[str | int]]]]:
|
| 419 |
+
for metrics in metrics_stream:
|
| 420 |
+
row = [
|
| 421 |
+
job_id,
|
| 422 |
+
f"{metrics['cpu_usage_pct']}%",
|
| 423 |
+
round(metrics["cpu_millicores"] / 1000.0, 1),
|
| 424 |
+
f"{round(100 * metrics['memory_used_bytes'] / metrics['memory_total_bytes'], 2)}%",
|
| 425 |
+
f"{_format_size(metrics['memory_used_bytes'])}B / {_format_size(metrics['memory_total_bytes'])}B",
|
| 426 |
+
f"{_format_size(metrics['rx_bps'])}bps / {_format_size(metrics['tx_bps'])}bps",
|
| 427 |
+
]
|
| 428 |
+
if metrics["gpus"] and isinstance(metrics["gpus"], dict):
|
| 429 |
+
rows = [row] + [[""] * len(row)] * (len(metrics["gpus"]) - 1)
|
| 430 |
+
for row, gpu_id in zip(rows, sorted(metrics["gpus"])):
|
| 431 |
+
gpu = metrics["gpus"][gpu_id]
|
| 432 |
+
row += [
|
| 433 |
+
f"{gpu['utilization']}%",
|
| 434 |
+
f"{round(100 * gpu['memory_used_bytes'] / gpu['memory_total_bytes'], 2)}%",
|
| 435 |
+
f"{_format_size(gpu['memory_used_bytes'])}B / {_format_size(gpu['memory_total_bytes'])}B",
|
| 436 |
+
]
|
| 437 |
+
else:
|
| 438 |
+
row += ["N/A"] * (len(table_headers) - len(row))
|
| 439 |
+
rows = [row]
|
| 440 |
+
yield False, job_id, rows
|
| 441 |
+
yield True, job_id, []
|
| 442 |
+
|
| 443 |
+
|
| 444 |
+
@jobs_cli.command("stats", examples=["hf jobs stats <job_id>"])
|
| 445 |
+
def jobs_stats(
|
| 446 |
+
job_ids: JobIdsArg = None,
|
| 447 |
+
namespace: NamespaceOpt = None,
|
| 448 |
+
token: TokenOpt = None,
|
| 449 |
+
) -> None:
|
| 450 |
+
"""Fetch the resource usage statistics and metrics of Jobs"""
|
| 451 |
+
if job_ids is not None:
|
| 452 |
+
parsed_ids = []
|
| 453 |
+
for job_id in job_ids:
|
| 454 |
+
job_id, namespace = _parse_namespace_from_job_id(job_id, namespace)
|
| 455 |
+
parsed_ids.append(job_id)
|
| 456 |
+
job_ids = parsed_ids
|
| 457 |
+
api = get_hf_api(token=token)
|
| 458 |
+
if namespace is None:
|
| 459 |
+
namespace = api.whoami()["name"]
|
| 460 |
+
if job_ids is None:
|
| 461 |
+
job_ids = [
|
| 462 |
+
job.id
|
| 463 |
+
for job in api.list_jobs(namespace=namespace)
|
| 464 |
+
if (job.status.stage if job.status else "UNKNOWN") in ("RUNNING", "UPDATING")
|
| 465 |
+
]
|
| 466 |
+
if len(job_ids) == 0:
|
| 467 |
+
out.text("No running jobs found")
|
| 468 |
+
return
|
| 469 |
+
table_headers = [
|
| 470 |
+
"JOB ID",
|
| 471 |
+
"CPU %",
|
| 472 |
+
"NUM CPU",
|
| 473 |
+
"MEM %",
|
| 474 |
+
"MEM USAGE",
|
| 475 |
+
"NET I/O",
|
| 476 |
+
"GPU UTIL %",
|
| 477 |
+
"GPU MEM %",
|
| 478 |
+
"GPU MEM USAGE",
|
| 479 |
+
]
|
| 480 |
+
try:
|
| 481 |
+
with multiprocessing.pool.ThreadPool(len(job_ids)) as pool:
|
| 482 |
+
rows_per_job_id: dict[str, list[list[str | int]]] = {}
|
| 483 |
+
for job_id in job_ids:
|
| 484 |
+
row: list[str | int] = [job_id]
|
| 485 |
+
row += ["-- / --" if ("/" in header or "USAGE" in header) else "--" for header in table_headers[1:]]
|
| 486 |
+
rows_per_job_id[job_id] = [row]
|
| 487 |
+
last_update_time = time.time()
|
| 488 |
+
total_rows = [row for job_id in rows_per_job_id for row in rows_per_job_id[job_id]]
|
| 489 |
+
# In-place refresh (cursor-up + clear) requires a fixed line count and layout —
|
| 490 |
+
# `out.table`'s mode-dependent formatting would break it.
|
| 491 |
+
print(_tabulate(total_rows, headers=table_headers))
|
| 492 |
+
|
| 493 |
+
kwargs_list = [
|
| 494 |
+
{
|
| 495 |
+
"job_id": job_id,
|
| 496 |
+
"metrics_stream": api.fetch_job_metrics(job_id=job_id, namespace=namespace),
|
| 497 |
+
"table_headers": table_headers,
|
| 498 |
+
}
|
| 499 |
+
for job_id in job_ids
|
| 500 |
+
]
|
| 501 |
+
for done, job_id, rows in iflatmap_unordered(pool, _get_jobs_stats_rows, kwargs_list=kwargs_list):
|
| 502 |
+
if done:
|
| 503 |
+
rows_per_job_id.pop(job_id, None)
|
| 504 |
+
else:
|
| 505 |
+
rows_per_job_id[job_id] = rows
|
| 506 |
+
now = time.time()
|
| 507 |
+
if now - last_update_time >= STATS_UPDATE_MIN_INTERVAL:
|
| 508 |
+
_clear_line(2 + len(total_rows))
|
| 509 |
+
total_rows = [row for job_id in rows_per_job_id for row in rows_per_job_id[job_id]]
|
| 510 |
+
print(_tabulate(total_rows, headers=table_headers))
|
| 511 |
+
last_update_time = now
|
| 512 |
+
except HfHubHTTPError as e:
|
| 513 |
+
status = e.response.status_code if e.response is not None else None
|
| 514 |
+
if status == 404:
|
| 515 |
+
raise CLIError("Job not found. Please check the job ID.") from e
|
| 516 |
+
elif status == 403:
|
| 517 |
+
raise CLIError("Access denied. You may not have permission to view this job.") from e
|
| 518 |
+
else:
|
| 519 |
+
raise CLIError(f"Failed to fetch job stats: {e}") from e
|
| 520 |
+
|
| 521 |
+
|
| 522 |
+
@jobs_cli.command("ps", examples=["hf jobs ps", "hf jobs ps -a"])
|
| 523 |
+
def jobs_ps(
|
| 524 |
+
all: Annotated[
|
| 525 |
+
bool,
|
| 526 |
+
typer.Option(
|
| 527 |
+
"-a",
|
| 528 |
+
"--all",
|
| 529 |
+
help="Show all Jobs (default shows just running)",
|
| 530 |
+
),
|
| 531 |
+
] = False,
|
| 532 |
+
namespace: NamespaceOpt = None,
|
| 533 |
+
token: TokenOpt = None,
|
| 534 |
+
filter: Annotated[
|
| 535 |
+
list[str] | None,
|
| 536 |
+
typer.Option(
|
| 537 |
+
"-f",
|
| 538 |
+
"--filter",
|
| 539 |
+
help="Filter output based on conditions provided (format: key=value)",
|
| 540 |
+
),
|
| 541 |
+
] = None,
|
| 542 |
+
) -> None:
|
| 543 |
+
"""List Jobs."""
|
| 544 |
+
api = get_hf_api(token=token)
|
| 545 |
+
jobs = api.list_jobs(namespace=namespace)
|
| 546 |
+
|
| 547 |
+
filters: list[tuple[str, str, str]] = []
|
| 548 |
+
labels_filters: list[tuple[str, str, str]] = []
|
| 549 |
+
for f in filter or []:
|
| 550 |
+
if f.startswith("label!=") or f.startswith("label="):
|
| 551 |
+
if f.startswith("label!="):
|
| 552 |
+
label_part = f[len("label!=") :]
|
| 553 |
+
if "=" in label_part:
|
| 554 |
+
out.warning(f"Ignoring invalid label filter format 'label!={label_part}'. Use label!=key format.")
|
| 555 |
+
continue
|
| 556 |
+
label_key, op, label_value = label_part, "!=", "*"
|
| 557 |
+
else:
|
| 558 |
+
label_part = f[len("label=") :]
|
| 559 |
+
if "=" in label_part:
|
| 560 |
+
label_key, label_value = label_part.split("=", 1)
|
| 561 |
+
else:
|
| 562 |
+
label_key, label_value = label_part, "*"
|
| 563 |
+
# Negate predicate in case of key!=value
|
| 564 |
+
if label_key.endswith("!"):
|
| 565 |
+
op = "!="
|
| 566 |
+
label_key = label_key[:-1]
|
| 567 |
+
else:
|
| 568 |
+
op = "="
|
| 569 |
+
labels_filters.append((label_key.lower(), op, label_value.lower()))
|
| 570 |
+
elif "=" in f:
|
| 571 |
+
key, value = f.split("=", 1)
|
| 572 |
+
# Negate predicate in case of key!=value
|
| 573 |
+
if key.endswith("!"):
|
| 574 |
+
op = "!="
|
| 575 |
+
key = key[:-1]
|
| 576 |
+
else:
|
| 577 |
+
op = "="
|
| 578 |
+
filters.append((key.lower(), op, value.lower()))
|
| 579 |
+
else:
|
| 580 |
+
out.warning(f"Ignoring invalid filter format '{f}'. Use key=value format.")
|
| 581 |
+
|
| 582 |
+
# Filter jobs (operating on JobInfo objects to preserve existing filter behavior)
|
| 583 |
+
filtered_jobs = []
|
| 584 |
+
for job in jobs:
|
| 585 |
+
status = job.status.stage if job.status else "UNKNOWN"
|
| 586 |
+
if not all and status not in ("RUNNING", "UPDATING"):
|
| 587 |
+
continue
|
| 588 |
+
image_or_space = job.docker_image or "N/A"
|
| 589 |
+
cmd = job.command or []
|
| 590 |
+
command_str = " ".join(cmd) if cmd else "N/A"
|
| 591 |
+
props = {"id": job.id, "image": image_or_space, "status": status.lower(), "command": command_str}
|
| 592 |
+
if not _matches_filters(props, filters):
|
| 593 |
+
continue
|
| 594 |
+
if not _matches_filters(job.labels or {}, labels_filters):
|
| 595 |
+
continue
|
| 596 |
+
filtered_jobs.append(job)
|
| 597 |
+
|
| 598 |
+
# Build display items. Augment the raw api dict with curated, table-friendly columns.
|
| 599 |
+
items: list[dict[str, Any]] = []
|
| 600 |
+
for job in filtered_jobs:
|
| 601 |
+
item = _dataclass_to_dict(job)
|
| 602 |
+
durations = item.get("durations") or {}
|
| 603 |
+
cmd = item.get("command") or []
|
| 604 |
+
item["job_id"] = item.get("id", "")
|
| 605 |
+
item["image/space"] = item.get("docker_image") or "N/A"
|
| 606 |
+
item["command"] = " ".join(cmd) if cmd else "N/A"
|
| 607 |
+
item["created"] = item["created_at"][:19].replace("T", " ") if item.get("created_at") else "N/A"
|
| 608 |
+
item["status"] = (item.get("status") or {}).get("stage", "UNKNOWN")
|
| 609 |
+
item["runtime"] = format_duration(durations.get("running_secs"))
|
| 610 |
+
items.append(item)
|
| 611 |
+
|
| 612 |
+
out.table(
|
| 613 |
+
items,
|
| 614 |
+
headers=["job_id", "image/space", "command", "created", "status", "runtime"],
|
| 615 |
+
id_key="job_id",
|
| 616 |
+
)
|
| 617 |
+
if not items:
|
| 618 |
+
if filters:
|
| 619 |
+
filters_msg = ", ".join(f"{k}{o}{v}" for k, o, v in filters)
|
| 620 |
+
out.text(f"No jobs matched filters: {filters_msg}")
|
| 621 |
+
elif not all and not labels_filters:
|
| 622 |
+
out.hint("No running jobs. Use `-a`/`--all` to include finished (and failed) jobs.")
|
| 623 |
+
|
| 624 |
+
|
| 625 |
+
@jobs_cli.command("hardware", examples=["hf jobs hardware"])
|
| 626 |
+
def jobs_hardware() -> None:
|
| 627 |
+
"""List available hardware options for Jobs"""
|
| 628 |
+
api = get_hf_api()
|
| 629 |
+
hardware_list = api.list_jobs_hardware()
|
| 630 |
+
items = []
|
| 631 |
+
for hw in hardware_list:
|
| 632 |
+
accelerator_info = ""
|
| 633 |
+
if hw.accelerator:
|
| 634 |
+
accelerator_info = f"{hw.accelerator.quantity}x {hw.accelerator.model} ({hw.accelerator.vram})"
|
| 635 |
+
cost_min = f"${hw.unit_cost_usd:.4f}" if hw.unit_cost_usd else "free"
|
| 636 |
+
cost_hour = f"${hw.unit_cost_usd * 60:.2f}" if hw.unit_cost_usd else "free"
|
| 637 |
+
items.append(
|
| 638 |
+
{
|
| 639 |
+
"name": hw.name,
|
| 640 |
+
"pretty name": hw.pretty_name,
|
| 641 |
+
"cpu": hw.cpu,
|
| 642 |
+
"ram": hw.ram,
|
| 643 |
+
"storage": hw.ephemeral_storage,
|
| 644 |
+
"accelerator": accelerator_info,
|
| 645 |
+
"cost/min": cost_min,
|
| 646 |
+
"cost/hour": cost_hour,
|
| 647 |
+
}
|
| 648 |
+
)
|
| 649 |
+
out.table(items)
|
| 650 |
+
out.hint("Use `hf jobs run --flavor <name> ...` to request a specific hardware flavor.")
|
| 651 |
+
|
| 652 |
+
|
| 653 |
+
@jobs_cli.command("inspect", examples=["hf jobs inspect <job_id>"])
|
| 654 |
+
def jobs_inspect(
|
| 655 |
+
job_ids: Annotated[
|
| 656 |
+
list[str],
|
| 657 |
+
typer.Argument(
|
| 658 |
+
help="Job IDs to inspect (or 'namespace/job_id')",
|
| 659 |
+
),
|
| 660 |
+
],
|
| 661 |
+
namespace: NamespaceOpt = None,
|
| 662 |
+
token: TokenOpt = None,
|
| 663 |
+
) -> None:
|
| 664 |
+
"""Display detailed information on one or more Jobs"""
|
| 665 |
+
parsed_ids = []
|
| 666 |
+
for job_id in job_ids:
|
| 667 |
+
job_id, namespace = _parse_namespace_from_job_id(job_id, namespace)
|
| 668 |
+
parsed_ids.append(job_id)
|
| 669 |
+
job_ids = parsed_ids
|
| 670 |
+
api = get_hf_api(token=token)
|
| 671 |
+
try:
|
| 672 |
+
jobs = [api.inspect_job(job_id=job_id, namespace=namespace) for job_id in job_ids]
|
| 673 |
+
except HfHubHTTPError as e:
|
| 674 |
+
status = e.response.status_code if e.response is not None else None
|
| 675 |
+
if status == 404:
|
| 676 |
+
raise CLIError("Job not found. Please check the job ID.") from e
|
| 677 |
+
elif status == 403:
|
| 678 |
+
raise CLIError("Access denied. You may not have permission to view this job.") from e
|
| 679 |
+
else:
|
| 680 |
+
raise CLIError(f"Failed to inspect job: {e}") from e
|
| 681 |
+
out.table([_dataclass_to_dict(job) for job in jobs])
|
| 682 |
+
|
| 683 |
+
|
| 684 |
+
@jobs_cli.command("cancel", examples=["hf jobs cancel <job_id>"])
|
| 685 |
+
def jobs_cancel(
|
| 686 |
+
job_id: JobIdArg,
|
| 687 |
+
namespace: NamespaceOpt = None,
|
| 688 |
+
token: TokenOpt = None,
|
| 689 |
+
) -> None:
|
| 690 |
+
"""Cancel a Job"""
|
| 691 |
+
job_id, namespace = _parse_namespace_from_job_id(job_id, namespace)
|
| 692 |
+
api = get_hf_api(token=token)
|
| 693 |
+
try:
|
| 694 |
+
api.cancel_job(job_id=job_id, namespace=namespace)
|
| 695 |
+
except HfHubHTTPError as e:
|
| 696 |
+
status = e.response.status_code if e.response is not None else None
|
| 697 |
+
if status == 404:
|
| 698 |
+
raise CLIError("Job not found. Please check the job ID.") from e
|
| 699 |
+
elif status == 403:
|
| 700 |
+
raise CLIError("Access denied. You may not have permission to cancel this job.") from e
|
| 701 |
+
else:
|
| 702 |
+
raise CLIError(f"Failed to cancel job: {e}") from e
|
| 703 |
+
out.result("Job cancelled", id=job_id)
|
| 704 |
+
|
| 705 |
+
|
| 706 |
+
@jobs_cli.command(
|
| 707 |
+
"labels",
|
| 708 |
+
examples=[
|
| 709 |
+
"hf jobs labels <job_id> --label env=prod --label team=ml",
|
| 710 |
+
"hf jobs labels <job_id> --clear",
|
| 711 |
+
],
|
| 712 |
+
)
|
| 713 |
+
def jobs_labels(
|
| 714 |
+
job_id: JobIdArg,
|
| 715 |
+
label: LabelsOpt = None,
|
| 716 |
+
clear: Annotated[bool, typer.Option("--clear", help="Remove all labels from the job.")] = False,
|
| 717 |
+
namespace: NamespaceOpt = None,
|
| 718 |
+
token: TokenOpt = None,
|
| 719 |
+
) -> None:
|
| 720 |
+
"""Update labels on a Job. Replaces all existing labels."""
|
| 721 |
+
if not label and not clear:
|
| 722 |
+
raise CLIError("Please set at least one label with --label. To remove all labels, pass --clear.")
|
| 723 |
+
if label and clear:
|
| 724 |
+
raise CLIError(
|
| 725 |
+
"Cannot set labels and clear them at the same time. Please use either --label or --clear, not both."
|
| 726 |
+
)
|
| 727 |
+
job_id, namespace = _parse_namespace_from_job_id(job_id, namespace)
|
| 728 |
+
labels = _parse_labels_map(label) or {}
|
| 729 |
+
api = get_hf_api(token=token)
|
| 730 |
+
job = api.update_job_labels(job_id=job_id, labels=labels, namespace=namespace)
|
| 731 |
+
out.result("Labels updated", id=job.id)
|
| 732 |
+
|
| 733 |
+
|
| 734 |
+
uv_app = typer_factory(help="Run UV scripts (Python with inline dependencies) on HF infrastructure.")
|
| 735 |
+
jobs_cli.add_typer(uv_app, name="uv")
|
| 736 |
+
|
| 737 |
+
|
| 738 |
+
@uv_app.command(
|
| 739 |
+
"run",
|
| 740 |
+
context_settings={"ignore_unknown_options": True},
|
| 741 |
+
examples=[
|
| 742 |
+
"hf jobs uv run my_script.py",
|
| 743 |
+
"hf jobs uv run --detach my_script.py",
|
| 744 |
+
"hf jobs uv run ml_training.py --flavor a10g-small",
|
| 745 |
+
"hf jobs uv run --with transformers train.py",
|
| 746 |
+
"hf jobs uv run -v hf://org/my-model:/data -v hf://buckets/org/b:/mnt script.py",
|
| 747 |
+
],
|
| 748 |
+
)
|
| 749 |
+
def jobs_uv_run(
|
| 750 |
+
script: ScriptArg,
|
| 751 |
+
script_args: ScriptArgsArg = None,
|
| 752 |
+
image: ImageOpt = None,
|
| 753 |
+
flavor: FlavorOpt = None,
|
| 754 |
+
env: EnvOpt = None,
|
| 755 |
+
secrets: SecretsOpt = None,
|
| 756 |
+
label: LabelsOpt = None,
|
| 757 |
+
volume: VolumesOpt = None,
|
| 758 |
+
env_file: EnvFileOpt = None,
|
| 759 |
+
secrets_file: SecretsFileOpt = None,
|
| 760 |
+
timeout: TimeoutOpt = None,
|
| 761 |
+
detach: DetachOpt = False,
|
| 762 |
+
expose: ExposeOpt = None,
|
| 763 |
+
namespace: NamespaceOpt = None,
|
| 764 |
+
token: TokenOpt = None,
|
| 765 |
+
with_: WithOpt = None,
|
| 766 |
+
python: PythonOpt = None,
|
| 767 |
+
) -> None:
|
| 768 |
+
"""Run a UV script (local file or URL) on HF infrastructure"""
|
| 769 |
+
env_map = parse_env_map(env, env_file)
|
| 770 |
+
secrets_map = parse_env_map(secrets, secrets_file)
|
| 771 |
+
|
| 772 |
+
api = get_hf_api(token=token)
|
| 773 |
+
job = api.run_uv_job(
|
| 774 |
+
script=script,
|
| 775 |
+
script_args=script_args or [],
|
| 776 |
+
dependencies=with_,
|
| 777 |
+
python=python,
|
| 778 |
+
image=image,
|
| 779 |
+
env=env_map,
|
| 780 |
+
secrets=secrets_map,
|
| 781 |
+
labels=_parse_labels_map(label),
|
| 782 |
+
volumes=parse_volumes(volume),
|
| 783 |
+
flavor=flavor,
|
| 784 |
+
timeout=timeout,
|
| 785 |
+
expose=expose,
|
| 786 |
+
namespace=namespace,
|
| 787 |
+
)
|
| 788 |
+
out.result("Job started", id=job.id, url=job.url)
|
| 789 |
+
if isinstance(job.status.expose_urls, list):
|
| 790 |
+
urls = "\n".join(f" {url}" for url in job.status.expose_urls)
|
| 791 |
+
out.hint(f"Exposed ports are reachable at (requires an HF token with read access to the job):\n{urls}")
|
| 792 |
+
if detach:
|
| 793 |
+
job_ref = f"{job.owner.name}/{job.id}"
|
| 794 |
+
out.hint(f"Use `hf jobs logs -f {job_ref}` to stream logs, or `hf jobs inspect {job_ref}` to check status.")
|
| 795 |
+
return
|
| 796 |
+
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True):
|
| 797 |
+
out.text(log)
|
| 798 |
+
|
| 799 |
+
|
| 800 |
+
scheduled_app = typer_factory(help="Create and manage scheduled Jobs on the Hub.")
|
| 801 |
+
jobs_cli.add_typer(scheduled_app, name="scheduled")
|
| 802 |
+
|
| 803 |
+
|
| 804 |
+
@scheduled_app.command(
|
| 805 |
+
"run",
|
| 806 |
+
context_settings={"ignore_unknown_options": True},
|
| 807 |
+
examples=['hf jobs scheduled run "0 0 * * *" python:3.12 python script.py'],
|
| 808 |
+
)
|
| 809 |
+
def scheduled_run(
|
| 810 |
+
schedule: ScheduleArg,
|
| 811 |
+
image: ImageArg,
|
| 812 |
+
command: CommandArg,
|
| 813 |
+
suspend: SuspendOpt = None,
|
| 814 |
+
concurrency: ConcurrencyOpt = None,
|
| 815 |
+
env: EnvOpt = None,
|
| 816 |
+
secrets: SecretsOpt = None,
|
| 817 |
+
label: LabelsOpt = None,
|
| 818 |
+
volume: VolumesOpt = None,
|
| 819 |
+
env_file: EnvFileOpt = None,
|
| 820 |
+
secrets_file: SecretsFileOpt = None,
|
| 821 |
+
flavor: FlavorOpt = None,
|
| 822 |
+
timeout: TimeoutOpt = None,
|
| 823 |
+
expose: ExposeOpt = None,
|
| 824 |
+
namespace: NamespaceOpt = None,
|
| 825 |
+
token: TokenOpt = None,
|
| 826 |
+
) -> None:
|
| 827 |
+
"""Schedule a Job."""
|
| 828 |
+
env_map = parse_env_map(env, env_file)
|
| 829 |
+
secrets_map = parse_env_map(secrets, secrets_file)
|
| 830 |
+
|
| 831 |
+
api = get_hf_api(token=token)
|
| 832 |
+
scheduled_job = api.create_scheduled_job(
|
| 833 |
+
image=image,
|
| 834 |
+
command=command,
|
| 835 |
+
schedule=schedule,
|
| 836 |
+
suspend=suspend,
|
| 837 |
+
concurrency=concurrency,
|
| 838 |
+
env=env_map,
|
| 839 |
+
secrets=secrets_map,
|
| 840 |
+
labels=_parse_labels_map(label),
|
| 841 |
+
volumes=parse_volumes(volume),
|
| 842 |
+
flavor=flavor,
|
| 843 |
+
timeout=timeout,
|
| 844 |
+
expose=expose,
|
| 845 |
+
namespace=namespace,
|
| 846 |
+
)
|
| 847 |
+
out.result("Scheduled Job created", id=scheduled_job.id)
|
| 848 |
+
out.hint(f"Use `hf jobs scheduled inspect {scheduled_job.id}` to view its details.")
|
| 849 |
+
|
| 850 |
+
|
| 851 |
+
@scheduled_app.command("ps", examples=["hf jobs scheduled ps"])
|
| 852 |
+
def scheduled_ps(
|
| 853 |
+
all: Annotated[
|
| 854 |
+
bool,
|
| 855 |
+
typer.Option(
|
| 856 |
+
"-a",
|
| 857 |
+
"--all",
|
| 858 |
+
help="Show all scheduled Jobs (default hides suspended)",
|
| 859 |
+
),
|
| 860 |
+
] = False,
|
| 861 |
+
namespace: NamespaceOpt = None,
|
| 862 |
+
token: TokenOpt = None,
|
| 863 |
+
filter: Annotated[
|
| 864 |
+
list[str] | None,
|
| 865 |
+
typer.Option(
|
| 866 |
+
"-f",
|
| 867 |
+
"--filter",
|
| 868 |
+
help="Filter output based on conditions provided (format: key=value)",
|
| 869 |
+
),
|
| 870 |
+
] = None,
|
| 871 |
+
) -> None:
|
| 872 |
+
"""List scheduled Jobs"""
|
| 873 |
+
api = get_hf_api(token=token)
|
| 874 |
+
scheduled_jobs = api.list_scheduled_jobs(namespace=namespace)
|
| 875 |
+
filters: list[tuple[str, str, str]] = []
|
| 876 |
+
for f in filter or []:
|
| 877 |
+
if "=" in f:
|
| 878 |
+
key, value = f.split("=", 1)
|
| 879 |
+
# Negate predicate in case of key!=value
|
| 880 |
+
if key.endswith("!"):
|
| 881 |
+
op = "!="
|
| 882 |
+
key = key[:-1]
|
| 883 |
+
else:
|
| 884 |
+
op = "="
|
| 885 |
+
filters.append((key.lower(), op, value.lower()))
|
| 886 |
+
else:
|
| 887 |
+
out.warning(f"Ignoring invalid filter format '{f}'. Use key=value format.")
|
| 888 |
+
|
| 889 |
+
# Filter scheduled jobs (operating on ScheduledJobInfo objects to preserve existing filter behavior)
|
| 890 |
+
filtered_jobs = []
|
| 891 |
+
for scheduled_job in scheduled_jobs:
|
| 892 |
+
suspend = scheduled_job.suspend or False
|
| 893 |
+
if not all and suspend:
|
| 894 |
+
continue
|
| 895 |
+
image_or_space = scheduled_job.job_spec.docker_image or "N/A"
|
| 896 |
+
cmd = scheduled_job.job_spec.command or []
|
| 897 |
+
command_str = " ".join(cmd) if cmd else "N/A"
|
| 898 |
+
props = {"id": scheduled_job.id, "image": image_or_space, "suspend": str(suspend), "command": command_str}
|
| 899 |
+
if not _matches_filters(props, filters):
|
| 900 |
+
continue
|
| 901 |
+
filtered_jobs.append(scheduled_job)
|
| 902 |
+
|
| 903 |
+
# Build display items. Augment with curated columns.
|
| 904 |
+
items: list[dict[str, Any]] = []
|
| 905 |
+
for sj in filtered_jobs:
|
| 906 |
+
item = _dataclass_to_dict(sj)
|
| 907 |
+
job_spec = item.get("job_spec") or {}
|
| 908 |
+
status_dict = item.get("status") or {}
|
| 909 |
+
last_job = status_dict.get("last_job")
|
| 910 |
+
cmd = job_spec.get("command") or []
|
| 911 |
+
item["image/space"] = job_spec.get("docker_image") or "N/A"
|
| 912 |
+
item["command"] = " ".join(cmd) if cmd else "N/A"
|
| 913 |
+
item["last_run"] = last_job["at"][:19].replace("T", " ") if last_job and last_job.get("at") else "N/A"
|
| 914 |
+
item["next_run"] = (
|
| 915 |
+
status_dict["next_job_run_at"][:19].replace("T", " ") if status_dict.get("next_job_run_at") else "N/A"
|
| 916 |
+
)
|
| 917 |
+
item["suspend"] = item.get("suspend") or False
|
| 918 |
+
items.append(item)
|
| 919 |
+
|
| 920 |
+
out.table(
|
| 921 |
+
items,
|
| 922 |
+
headers=["id", "schedule", "image/space", "command", "last_run", "next_run", "suspend"],
|
| 923 |
+
id_key="id",
|
| 924 |
+
)
|
| 925 |
+
if not items and filters:
|
| 926 |
+
filters_msg = ", ".join(f"{k}{o}{v}" for k, o, v in filters)
|
| 927 |
+
out.text(f"No scheduled jobs matched filters: {filters_msg}")
|
| 928 |
+
|
| 929 |
+
|
| 930 |
+
@scheduled_app.command("inspect", examples=["hf jobs scheduled inspect <id>"])
|
| 931 |
+
def scheduled_inspect(
|
| 932 |
+
scheduled_job_ids: Annotated[
|
| 933 |
+
list[str],
|
| 934 |
+
typer.Argument(
|
| 935 |
+
help="Scheduled Job IDs to inspect (or 'namespace/scheduled_job_id')",
|
| 936 |
+
),
|
| 937 |
+
],
|
| 938 |
+
namespace: NamespaceOpt = None,
|
| 939 |
+
token: TokenOpt = None,
|
| 940 |
+
) -> None:
|
| 941 |
+
"""Display detailed information on one or more scheduled Jobs"""
|
| 942 |
+
parsed_ids = []
|
| 943 |
+
for job_id in scheduled_job_ids:
|
| 944 |
+
job_id, namespace = _parse_namespace_from_job_id(job_id, namespace)
|
| 945 |
+
parsed_ids.append(job_id)
|
| 946 |
+
scheduled_job_ids = parsed_ids
|
| 947 |
+
api = get_hf_api(token=token)
|
| 948 |
+
scheduled_jobs = [
|
| 949 |
+
api.inspect_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=namespace)
|
| 950 |
+
for scheduled_job_id in scheduled_job_ids
|
| 951 |
+
]
|
| 952 |
+
out.table([_dataclass_to_dict(scheduled_job) for scheduled_job in scheduled_jobs])
|
| 953 |
+
|
| 954 |
+
|
| 955 |
+
@scheduled_app.command("delete", examples=["hf jobs scheduled delete <id>"])
|
| 956 |
+
def scheduled_delete(
|
| 957 |
+
scheduled_job_id: ScheduledJobIdArg,
|
| 958 |
+
namespace: NamespaceOpt = None,
|
| 959 |
+
token: TokenOpt = None,
|
| 960 |
+
) -> None:
|
| 961 |
+
"""Delete a scheduled Job."""
|
| 962 |
+
scheduled_job_id, namespace = _parse_namespace_from_job_id(scheduled_job_id, namespace)
|
| 963 |
+
api = get_hf_api(token=token)
|
| 964 |
+
api.delete_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=namespace)
|
| 965 |
+
out.result("Scheduled Job deleted", id=scheduled_job_id)
|
| 966 |
+
|
| 967 |
+
|
| 968 |
+
@scheduled_app.command("suspend", examples=["hf jobs scheduled suspend <id>"])
|
| 969 |
+
def scheduled_suspend(
|
| 970 |
+
scheduled_job_id: ScheduledJobIdArg,
|
| 971 |
+
namespace: NamespaceOpt = None,
|
| 972 |
+
token: TokenOpt = None,
|
| 973 |
+
) -> None:
|
| 974 |
+
"""Suspend (pause) a scheduled Job."""
|
| 975 |
+
scheduled_job_id, namespace = _parse_namespace_from_job_id(scheduled_job_id, namespace)
|
| 976 |
+
api = get_hf_api(token=token)
|
| 977 |
+
api.suspend_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=namespace)
|
| 978 |
+
out.result("Scheduled Job suspended", id=scheduled_job_id)
|
| 979 |
+
out.hint(f"Use `hf jobs scheduled resume {scheduled_job_id}` to resume it.")
|
| 980 |
+
|
| 981 |
+
|
| 982 |
+
@scheduled_app.command("resume", examples=["hf jobs scheduled resume <id>"])
|
| 983 |
+
def scheduled_resume(
|
| 984 |
+
scheduled_job_id: ScheduledJobIdArg,
|
| 985 |
+
namespace: NamespaceOpt = None,
|
| 986 |
+
token: TokenOpt = None,
|
| 987 |
+
) -> None:
|
| 988 |
+
"""Resume (unpause) a scheduled Job."""
|
| 989 |
+
scheduled_job_id, namespace = _parse_namespace_from_job_id(scheduled_job_id, namespace)
|
| 990 |
+
api = get_hf_api(token=token)
|
| 991 |
+
api.resume_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=namespace)
|
| 992 |
+
out.result("Scheduled Job resumed", id=scheduled_job_id)
|
| 993 |
+
|
| 994 |
+
|
| 995 |
+
@scheduled_app.command(
|
| 996 |
+
"labels",
|
| 997 |
+
examples=[
|
| 998 |
+
"hf jobs scheduled labels <id> --label env=prod --label team=ml",
|
| 999 |
+
"hf jobs scheduled labels <id> --clear",
|
| 1000 |
+
],
|
| 1001 |
+
)
|
| 1002 |
+
def scheduled_labels(
|
| 1003 |
+
scheduled_job_id: ScheduledJobIdArg,
|
| 1004 |
+
label: LabelsOpt = None,
|
| 1005 |
+
clear: Annotated[bool, typer.Option("--clear", help="Remove all labels from the scheduled job.")] = False,
|
| 1006 |
+
namespace: NamespaceOpt = None,
|
| 1007 |
+
token: TokenOpt = None,
|
| 1008 |
+
) -> None:
|
| 1009 |
+
"""Update labels on a scheduled Job. Replaces all existing labels."""
|
| 1010 |
+
if not label and not clear:
|
| 1011 |
+
raise CLIError("Please set at least one label with --label. To remove all labels, pass --clear.")
|
| 1012 |
+
if label and clear:
|
| 1013 |
+
raise CLIError(
|
| 1014 |
+
"Cannot set labels and clear them at the same time. Please use either --label or --clear, not both."
|
| 1015 |
+
)
|
| 1016 |
+
scheduled_job_id, namespace = _parse_namespace_from_job_id(scheduled_job_id, namespace)
|
| 1017 |
+
labels = _parse_labels_map(label) or {}
|
| 1018 |
+
api = get_hf_api(token=token)
|
| 1019 |
+
scheduled_job = api.update_scheduled_job_labels(
|
| 1020 |
+
scheduled_job_id=scheduled_job_id, labels=labels, namespace=namespace
|
| 1021 |
+
)
|
| 1022 |
+
out.result("Labels updated", id=scheduled_job.id)
|
| 1023 |
+
|
| 1024 |
+
|
| 1025 |
+
scheduled_uv_app = typer_factory(help="Schedule UV scripts on HF infrastructure.")
|
| 1026 |
+
scheduled_app.add_typer(scheduled_uv_app, name="uv")
|
| 1027 |
+
|
| 1028 |
+
|
| 1029 |
+
@scheduled_uv_app.command(
|
| 1030 |
+
"run",
|
| 1031 |
+
context_settings={"ignore_unknown_options": True},
|
| 1032 |
+
examples=[
|
| 1033 |
+
'hf jobs scheduled uv run "0 0 * * *" script.py',
|
| 1034 |
+
'hf jobs scheduled uv run "0 0 * * *" script.py --with pandas',
|
| 1035 |
+
],
|
| 1036 |
+
)
|
| 1037 |
+
def scheduled_uv_run(
|
| 1038 |
+
schedule: ScheduleArg,
|
| 1039 |
+
script: ScriptArg,
|
| 1040 |
+
script_args: ScriptArgsArg = None,
|
| 1041 |
+
suspend: SuspendOpt = None,
|
| 1042 |
+
concurrency: ConcurrencyOpt = None,
|
| 1043 |
+
image: ImageOpt = None,
|
| 1044 |
+
flavor: FlavorOpt = None,
|
| 1045 |
+
env: EnvOpt = None,
|
| 1046 |
+
secrets: SecretsOpt = None,
|
| 1047 |
+
label: LabelsOpt = None,
|
| 1048 |
+
volume: VolumesOpt = None,
|
| 1049 |
+
env_file: EnvFileOpt = None,
|
| 1050 |
+
secrets_file: SecretsFileOpt = None,
|
| 1051 |
+
timeout: TimeoutOpt = None,
|
| 1052 |
+
expose: ExposeOpt = None,
|
| 1053 |
+
namespace: NamespaceOpt = None,
|
| 1054 |
+
token: TokenOpt = None,
|
| 1055 |
+
with_: WithOpt = None,
|
| 1056 |
+
python: PythonOpt = None,
|
| 1057 |
+
) -> None:
|
| 1058 |
+
"""Run a UV script (local file or URL) on HF infrastructure"""
|
| 1059 |
+
env_map = parse_env_map(env, env_file)
|
| 1060 |
+
secrets_map = parse_env_map(secrets, secrets_file)
|
| 1061 |
+
|
| 1062 |
+
api = get_hf_api(token=token)
|
| 1063 |
+
job = api.create_scheduled_uv_job(
|
| 1064 |
+
script=script,
|
| 1065 |
+
script_args=script_args or [],
|
| 1066 |
+
schedule=schedule,
|
| 1067 |
+
suspend=suspend,
|
| 1068 |
+
concurrency=concurrency,
|
| 1069 |
+
dependencies=with_,
|
| 1070 |
+
python=python,
|
| 1071 |
+
image=image,
|
| 1072 |
+
env=env_map,
|
| 1073 |
+
secrets=secrets_map,
|
| 1074 |
+
labels=_parse_labels_map(label),
|
| 1075 |
+
volumes=parse_volumes(volume),
|
| 1076 |
+
flavor=flavor,
|
| 1077 |
+
timeout=timeout,
|
| 1078 |
+
expose=expose,
|
| 1079 |
+
namespace=namespace,
|
| 1080 |
+
)
|
| 1081 |
+
out.result("Scheduled Job created", id=job.id)
|
| 1082 |
+
out.hint(f"Use `hf jobs scheduled inspect {job.id}` to view its details.")
|
| 1083 |
+
|
| 1084 |
+
|
| 1085 |
+
### UTILS
|
| 1086 |
+
|
| 1087 |
+
|
| 1088 |
+
def _parse_labels_map(labels: list[str] | None) -> dict[str, str] | None:
|
| 1089 |
+
"""Parse label key-value pairs from CLI arguments.
|
| 1090 |
+
|
| 1091 |
+
Args:
|
| 1092 |
+
labels: List of label strings in KEY=VALUE format. If KEY only, then VALUE is set to empty string.
|
| 1093 |
+
|
| 1094 |
+
Returns:
|
| 1095 |
+
Dictionary mapping label keys to values, or None if no labels provided.
|
| 1096 |
+
"""
|
| 1097 |
+
if not labels:
|
| 1098 |
+
return None
|
| 1099 |
+
labels_map: dict[str, str] = {}
|
| 1100 |
+
for label_var in labels:
|
| 1101 |
+
key, value = label_var.split("=", 1) if "=" in label_var else (label_var, "")
|
| 1102 |
+
labels_map[key] = value
|
| 1103 |
+
return labels_map
|
| 1104 |
+
|
| 1105 |
+
|
| 1106 |
+
def _tabulate(rows: list[list[str | int]], headers: list[str]) -> str:
|
| 1107 |
+
"""
|
| 1108 |
+
Inspired by:
|
| 1109 |
+
|
| 1110 |
+
- stackoverflow.com/a/8356620/593036
|
| 1111 |
+
- stackoverflow.com/questions/9535954/printing-lists-as-tabular-data
|
| 1112 |
+
"""
|
| 1113 |
+
col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)]
|
| 1114 |
+
terminal_width = max(shutil.get_terminal_size().columns, len(headers) * 12)
|
| 1115 |
+
while len(headers) + sum(col_widths) > terminal_width:
|
| 1116 |
+
col_to_minimize = col_widths.index(max(col_widths))
|
| 1117 |
+
col_widths[col_to_minimize] //= 2
|
| 1118 |
+
if len(headers) + sum(col_widths) <= terminal_width:
|
| 1119 |
+
col_widths[col_to_minimize] = terminal_width - sum(col_widths) - len(headers) + col_widths[col_to_minimize]
|
| 1120 |
+
row_format = ("{{:{}}} " * len(headers)).format(*col_widths)
|
| 1121 |
+
lines = []
|
| 1122 |
+
lines.append(row_format.format(*headers))
|
| 1123 |
+
lines.append(row_format.format(*["-" * w for w in col_widths]))
|
| 1124 |
+
for row in rows:
|
| 1125 |
+
row_format_args = [
|
| 1126 |
+
str(x)[: col_width - 3] + "..." if len(str(x)) > col_width else str(x)
|
| 1127 |
+
for x, col_width in zip(row, col_widths)
|
| 1128 |
+
]
|
| 1129 |
+
lines.append(row_format.format(*row_format_args))
|
| 1130 |
+
return "\n".join(lines)
|
| 1131 |
+
|
| 1132 |
+
|
| 1133 |
+
T = TypeVar("T")
|
| 1134 |
+
|
| 1135 |
+
|
| 1136 |
+
def _write_generator_to_queue(queue: Queue[T], func: Callable[..., Iterable[T]], kwargs: dict) -> None:
|
| 1137 |
+
for result in func(**kwargs):
|
| 1138 |
+
queue.put(result)
|
| 1139 |
+
|
| 1140 |
+
|
| 1141 |
+
def iflatmap_unordered(
|
| 1142 |
+
pool: multiprocessing.pool.ThreadPool,
|
| 1143 |
+
func: Callable[..., Iterable[T]],
|
| 1144 |
+
*,
|
| 1145 |
+
kwargs_list: list[dict],
|
| 1146 |
+
) -> Iterable[T]:
|
| 1147 |
+
"""
|
| 1148 |
+
Takes a function that returns an iterable of items, and run it in parallel using threads to return the flattened iterable of items as they arrive.
|
| 1149 |
+
|
| 1150 |
+
This is inspired by those three `map()` variants, and is the mix of all three:
|
| 1151 |
+
|
| 1152 |
+
* `imap()`: like `map()` but returns an iterable instead of a list of results
|
| 1153 |
+
* `imap_unordered()`: like `imap()` but the output is sorted by time of arrival
|
| 1154 |
+
* `flatmap()`: like `map()` but given a function which returns a list, `flatmap()` returns the flattened list that is the concatenation of all the output lists
|
| 1155 |
+
"""
|
| 1156 |
+
queue: Queue[T] = Queue()
|
| 1157 |
+
async_results = [pool.apply_async(_write_generator_to_queue, (queue, func, kwargs)) for kwargs in kwargs_list]
|
| 1158 |
+
try:
|
| 1159 |
+
while True:
|
| 1160 |
+
try:
|
| 1161 |
+
yield queue.get(timeout=0.05)
|
| 1162 |
+
except Empty:
|
| 1163 |
+
if all(async_result.ready() for async_result in async_results) and queue.empty():
|
| 1164 |
+
break
|
| 1165 |
+
except KeyboardInterrupt:
|
| 1166 |
+
pass
|
| 1167 |
+
finally:
|
| 1168 |
+
# we get the result in case there's an error to raise
|
| 1169 |
+
try:
|
| 1170 |
+
[async_result.get(timeout=0.05) for async_result in async_results]
|
| 1171 |
+
except multiprocessing.TimeoutError:
|
| 1172 |
+
pass
|
venv/Lib/site-packages/huggingface_hub/cli/lfs.py
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Implementation of a custom transfer agent for the transfer type "multipart" for
|
| 3 |
+
git-lfs.
|
| 4 |
+
|
| 5 |
+
Inspired by:
|
| 6 |
+
github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py
|
| 7 |
+
|
| 8 |
+
Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
To launch debugger while developing:
|
| 12 |
+
|
| 13 |
+
``` [lfs "customtransfer.multipart"]
|
| 14 |
+
path = /path/to/huggingface_hub/.venv/bin/python args = -m debugpy --listen 5678
|
| 15 |
+
--wait-for-client
|
| 16 |
+
/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
|
| 17 |
+
lfs-multipart-upload ```"""
|
| 18 |
+
|
| 19 |
+
import json
|
| 20 |
+
import os
|
| 21 |
+
import subprocess
|
| 22 |
+
import sys
|
| 23 |
+
from typing import Annotated
|
| 24 |
+
|
| 25 |
+
import typer
|
| 26 |
+
|
| 27 |
+
from huggingface_hub.errors import CLIError
|
| 28 |
+
from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND
|
| 29 |
+
|
| 30 |
+
from ..utils import get_session, hf_raise_for_status, logging
|
| 31 |
+
from ..utils._lfs import SliceFileObj
|
| 32 |
+
from ._output import out
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
logger = logging.get_logger(__name__)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def lfs_enable_largefiles(
|
| 39 |
+
path: Annotated[
|
| 40 |
+
str,
|
| 41 |
+
typer.Argument(
|
| 42 |
+
help="Local path to repository you want to configure.",
|
| 43 |
+
),
|
| 44 |
+
],
|
| 45 |
+
) -> None:
|
| 46 |
+
"""
|
| 47 |
+
Configure your repository to enable upload of files > 5GB.
|
| 48 |
+
|
| 49 |
+
This command sets up git-lfs to use the custom multipart transfer agent
|
| 50 |
+
which enables efficient uploading of large files in chunks.
|
| 51 |
+
"""
|
| 52 |
+
local_path = os.path.abspath(path)
|
| 53 |
+
if not os.path.isdir(local_path):
|
| 54 |
+
raise CLIError("This does not look like a valid git repo.")
|
| 55 |
+
subprocess.run(
|
| 56 |
+
"git config lfs.customtransfer.multipart.path hf".split(),
|
| 57 |
+
check=True,
|
| 58 |
+
cwd=local_path,
|
| 59 |
+
)
|
| 60 |
+
subprocess.run(
|
| 61 |
+
f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
|
| 62 |
+
check=True,
|
| 63 |
+
cwd=local_path,
|
| 64 |
+
)
|
| 65 |
+
out.result("Local repo set up for largefiles", path=local_path)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def write_msg(msg: dict):
|
| 69 |
+
"""Write out the message in Line delimited JSON."""
|
| 70 |
+
msg_str = json.dumps(msg) + "\n"
|
| 71 |
+
sys.stdout.write(msg_str)
|
| 72 |
+
sys.stdout.flush()
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def read_msg() -> dict | None:
|
| 76 |
+
"""Read Line delimited JSON from stdin."""
|
| 77 |
+
msg = json.loads(sys.stdin.readline().strip())
|
| 78 |
+
|
| 79 |
+
if "terminate" in (msg.get("type"), msg.get("event")):
|
| 80 |
+
# terminate message received
|
| 81 |
+
return None
|
| 82 |
+
|
| 83 |
+
if msg.get("event") not in ("download", "upload"):
|
| 84 |
+
logger.critical("Received unexpected message")
|
| 85 |
+
sys.exit(1)
|
| 86 |
+
|
| 87 |
+
return msg
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def lfs_multipart_upload() -> None:
|
| 91 |
+
"""Internal git-lfs custom transfer agent for multipart uploads.
|
| 92 |
+
|
| 93 |
+
This function implements the custom transfer protocol for git-lfs multipart uploads.
|
| 94 |
+
Handles chunked uploads of large files to Hugging Face Hub.
|
| 95 |
+
"""
|
| 96 |
+
# Immediately after invoking a custom transfer process, git-lfs
|
| 97 |
+
# sends initiation data to the process over stdin.
|
| 98 |
+
# This tells the process useful information about the configuration.
|
| 99 |
+
init_msg = json.loads(sys.stdin.readline().strip())
|
| 100 |
+
if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"):
|
| 101 |
+
write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}})
|
| 102 |
+
sys.exit(1)
|
| 103 |
+
|
| 104 |
+
# The transfer process should use the information it needs from the
|
| 105 |
+
# initiation structure, and also perform any one-off setup tasks it
|
| 106 |
+
# needs to do. It should then respond on stdout with a simple empty
|
| 107 |
+
# confirmation structure, as follows:
|
| 108 |
+
write_msg({})
|
| 109 |
+
|
| 110 |
+
# After the initiation exchange, git-lfs will send any number of
|
| 111 |
+
# transfer requests to the stdin of the transfer process, in a serial sequence.
|
| 112 |
+
while True:
|
| 113 |
+
msg = read_msg()
|
| 114 |
+
if msg is None:
|
| 115 |
+
# When all transfers have been processed, git-lfs will send
|
| 116 |
+
# a terminate event to the stdin of the transfer process.
|
| 117 |
+
# On receiving this message the transfer process should
|
| 118 |
+
# clean up and terminate. No response is expected.
|
| 119 |
+
sys.exit(0)
|
| 120 |
+
|
| 121 |
+
oid = msg["oid"]
|
| 122 |
+
filepath = msg["path"]
|
| 123 |
+
completion_url = msg["action"]["href"]
|
| 124 |
+
header = msg["action"]["header"]
|
| 125 |
+
chunk_size = int(header.pop("chunk_size"))
|
| 126 |
+
presigned_urls: list[str] = list(header.values())
|
| 127 |
+
|
| 128 |
+
# Send a "started" progress event to allow other workers to start.
|
| 129 |
+
# Otherwise they're delayed until first "progress" event is reported,
|
| 130 |
+
# i.e. after the first 5GB by default (!)
|
| 131 |
+
write_msg(
|
| 132 |
+
{
|
| 133 |
+
"event": "progress",
|
| 134 |
+
"oid": oid,
|
| 135 |
+
"bytesSoFar": 1,
|
| 136 |
+
"bytesSinceLast": 0,
|
| 137 |
+
}
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
parts = []
|
| 141 |
+
with open(filepath, "rb") as file:
|
| 142 |
+
for i, presigned_url in enumerate(presigned_urls):
|
| 143 |
+
with SliceFileObj(
|
| 144 |
+
file,
|
| 145 |
+
seek_from=i * chunk_size,
|
| 146 |
+
read_limit=chunk_size,
|
| 147 |
+
) as data:
|
| 148 |
+
r = get_session().put(presigned_url, data=data)
|
| 149 |
+
hf_raise_for_status(r)
|
| 150 |
+
parts.append(
|
| 151 |
+
{
|
| 152 |
+
"etag": r.headers.get("etag"),
|
| 153 |
+
"partNumber": i + 1,
|
| 154 |
+
}
|
| 155 |
+
)
|
| 156 |
+
# In order to support progress reporting while data is uploading / downloading,
|
| 157 |
+
# the transfer process should post messages to stdout
|
| 158 |
+
write_msg(
|
| 159 |
+
{
|
| 160 |
+
"event": "progress",
|
| 161 |
+
"oid": oid,
|
| 162 |
+
"bytesSoFar": (i + 1) * chunk_size,
|
| 163 |
+
"bytesSinceLast": chunk_size,
|
| 164 |
+
}
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
r = get_session().post(
|
| 168 |
+
completion_url,
|
| 169 |
+
json={
|
| 170 |
+
"oid": oid,
|
| 171 |
+
"parts": parts,
|
| 172 |
+
},
|
| 173 |
+
)
|
| 174 |
+
hf_raise_for_status(r)
|
| 175 |
+
|
| 176 |
+
write_msg({"event": "complete", "oid": oid})
|
venv/Lib/site-packages/huggingface_hub/cli/models.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains commands to interact with models on the Hugging Face Hub.
|
| 15 |
+
|
| 16 |
+
Usage:
|
| 17 |
+
# list models on the Hub
|
| 18 |
+
hf models ls
|
| 19 |
+
|
| 20 |
+
# list models with a search query
|
| 21 |
+
hf models ls --search "llama"
|
| 22 |
+
|
| 23 |
+
# get info about a model
|
| 24 |
+
hf models info Lightricks/LTX-2
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
import enum
|
| 28 |
+
from typing import Annotated, get_args
|
| 29 |
+
|
| 30 |
+
import typer
|
| 31 |
+
|
| 32 |
+
from huggingface_hub.errors import CLIError, RepositoryNotFoundError, RevisionNotFoundError
|
| 33 |
+
from huggingface_hub.hf_api import ExpandModelProperty_T, ModelSort_T
|
| 34 |
+
from huggingface_hub.repocard import ModelCard
|
| 35 |
+
|
| 36 |
+
from ._cli_utils import (
|
| 37 |
+
REPO_LIST_DEFAULT_LIMIT,
|
| 38 |
+
AuthorOpt,
|
| 39 |
+
FilterOpt,
|
| 40 |
+
LimitOpt,
|
| 41 |
+
RevisionOpt,
|
| 42 |
+
SearchOpt,
|
| 43 |
+
TokenOpt,
|
| 44 |
+
get_hf_api,
|
| 45 |
+
make_expand_properties_parser,
|
| 46 |
+
typer_factory,
|
| 47 |
+
)
|
| 48 |
+
from ._file_listing import list_repo_files_cmd
|
| 49 |
+
from ._output import _dataclass_to_dict, out
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
_EXPAND_PROPERTIES = sorted(get_args(ExpandModelProperty_T))
|
| 53 |
+
_SORT_OPTIONS = get_args(ModelSort_T)
|
| 54 |
+
ModelSortEnum = enum.Enum("ModelSortEnum", {s: s for s in _SORT_OPTIONS}, type=str) # type: ignore[misc]
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
ExpandOpt = Annotated[
|
| 58 |
+
str | None,
|
| 59 |
+
typer.Option(
|
| 60 |
+
help=f"Comma-separated properties to return. When used, only the listed properties (and id) are returned. Example: '--expand=downloads,likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.",
|
| 61 |
+
callback=make_expand_properties_parser(_EXPAND_PROPERTIES),
|
| 62 |
+
),
|
| 63 |
+
]
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
models_cli = typer_factory(help="Interact with models on the Hub.")
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
@models_cli.command(
|
| 70 |
+
"list | ls",
|
| 71 |
+
examples=[
|
| 72 |
+
"hf models ls --sort downloads --limit 10",
|
| 73 |
+
'hf models ls --search "llama" --author meta-llama',
|
| 74 |
+
"hf models ls --num-parameters min:6B,max:128B --sort likes",
|
| 75 |
+
"hf models ls meta-llama/Llama-3.2-1B-Instruct",
|
| 76 |
+
"hf models ls meta-llama/Llama-3.2-1B-Instruct -R",
|
| 77 |
+
"hf models ls meta-llama/Llama-3.2-1B-Instruct --tree -h",
|
| 78 |
+
],
|
| 79 |
+
)
|
| 80 |
+
def models_ls(
|
| 81 |
+
repo_id: Annotated[
|
| 82 |
+
str | None,
|
| 83 |
+
typer.Argument(help="Model ID (e.g. `username/repo-name`) to list files from. If omitted, lists models."),
|
| 84 |
+
] = None,
|
| 85 |
+
search: SearchOpt = None,
|
| 86 |
+
author: AuthorOpt = None,
|
| 87 |
+
filter: FilterOpt = None,
|
| 88 |
+
num_parameters: Annotated[
|
| 89 |
+
str | None,
|
| 90 |
+
typer.Option(help="Filter by parameter count, e.g. 'min:6B,max:128B'."),
|
| 91 |
+
] = None,
|
| 92 |
+
sort: Annotated[
|
| 93 |
+
ModelSortEnum | None,
|
| 94 |
+
typer.Option(help="Sort results."),
|
| 95 |
+
] = None,
|
| 96 |
+
limit: LimitOpt = REPO_LIST_DEFAULT_LIMIT,
|
| 97 |
+
expand: ExpandOpt = None,
|
| 98 |
+
human_readable: Annotated[
|
| 99 |
+
bool,
|
| 100 |
+
typer.Option("--human-readable", "-h", help="Show sizes in human readable format (only for listing files)."),
|
| 101 |
+
] = False,
|
| 102 |
+
as_tree: Annotated[
|
| 103 |
+
bool,
|
| 104 |
+
typer.Option("--tree", help="List files in tree format (only for listing files)."),
|
| 105 |
+
] = False,
|
| 106 |
+
recursive: Annotated[
|
| 107 |
+
bool,
|
| 108 |
+
typer.Option("--recursive", "-R", help="List files recursively (only for listing files)."),
|
| 109 |
+
] = False,
|
| 110 |
+
revision: RevisionOpt = None,
|
| 111 |
+
token: TokenOpt = None,
|
| 112 |
+
) -> None:
|
| 113 |
+
"""List models on the Hub, or files in a model repo.
|
| 114 |
+
|
| 115 |
+
When called with no argument, lists models on the Hub.
|
| 116 |
+
When called with a model ID, lists files in that model repo.
|
| 117 |
+
"""
|
| 118 |
+
if repo_id is not None:
|
| 119 |
+
if search is not None:
|
| 120 |
+
raise typer.BadParameter("Cannot use --search when listing files.")
|
| 121 |
+
if author is not None:
|
| 122 |
+
raise typer.BadParameter("Cannot use --author when listing files.")
|
| 123 |
+
if filter is not None:
|
| 124 |
+
raise typer.BadParameter("Cannot use --filter when listing files.")
|
| 125 |
+
if num_parameters is not None:
|
| 126 |
+
raise typer.BadParameter("Cannot use --num-parameters when listing files.")
|
| 127 |
+
if sort is not None:
|
| 128 |
+
raise typer.BadParameter("Cannot use --sort when listing files.")
|
| 129 |
+
if limit != REPO_LIST_DEFAULT_LIMIT:
|
| 130 |
+
raise typer.BadParameter("Cannot use --limit when listing files.")
|
| 131 |
+
if expand is not None:
|
| 132 |
+
raise typer.BadParameter("Cannot use --expand when listing files.")
|
| 133 |
+
return list_repo_files_cmd(
|
| 134 |
+
repo_id=repo_id,
|
| 135 |
+
repo_type="model",
|
| 136 |
+
human_readable=human_readable,
|
| 137 |
+
as_tree=as_tree,
|
| 138 |
+
recursive=recursive,
|
| 139 |
+
revision=revision,
|
| 140 |
+
token=token,
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
if as_tree:
|
| 144 |
+
raise typer.BadParameter("Cannot use --tree when listing models.")
|
| 145 |
+
if recursive:
|
| 146 |
+
raise typer.BadParameter("Cannot use --recursive when listing models.")
|
| 147 |
+
if human_readable:
|
| 148 |
+
raise typer.BadParameter("Cannot use --human-readable when listing models.")
|
| 149 |
+
if revision is not None:
|
| 150 |
+
raise typer.BadParameter("Cannot use --revision when listing models.")
|
| 151 |
+
api = get_hf_api(token=token)
|
| 152 |
+
sort_key = sort.value if sort else None
|
| 153 |
+
results = [
|
| 154 |
+
_dataclass_to_dict(model_info)
|
| 155 |
+
for model_info in api.list_models(
|
| 156 |
+
filter=filter,
|
| 157 |
+
author=author,
|
| 158 |
+
search=search,
|
| 159 |
+
num_parameters=num_parameters,
|
| 160 |
+
sort=sort_key,
|
| 161 |
+
limit=limit,
|
| 162 |
+
expand=expand, # type: ignore
|
| 163 |
+
)
|
| 164 |
+
]
|
| 165 |
+
out.table(results)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
@models_cli.command(
|
| 169 |
+
"info",
|
| 170 |
+
examples=[
|
| 171 |
+
"hf models info meta-llama/Llama-3.2-1B-Instruct",
|
| 172 |
+
"hf models info Qwen/Qwen3.5-9B --expand downloads,likes,tags",
|
| 173 |
+
],
|
| 174 |
+
)
|
| 175 |
+
def models_info(
|
| 176 |
+
model_id: Annotated[str, typer.Argument(help="The model ID (e.g. `username/repo-name`).")],
|
| 177 |
+
revision: RevisionOpt = None,
|
| 178 |
+
expand: ExpandOpt = None,
|
| 179 |
+
token: TokenOpt = None,
|
| 180 |
+
) -> None:
|
| 181 |
+
"""Get info about a model on the Hub."""
|
| 182 |
+
api = get_hf_api(token=token)
|
| 183 |
+
try:
|
| 184 |
+
info = api.model_info(repo_id=model_id, revision=revision, expand=expand) # type: ignore
|
| 185 |
+
except RepositoryNotFoundError as e:
|
| 186 |
+
raise CLIError(f"Model '{model_id}' not found.") from e
|
| 187 |
+
except RevisionNotFoundError as e:
|
| 188 |
+
raise CLIError(f"Revision '{revision}' not found on '{model_id}'.") from e
|
| 189 |
+
out.dict(info)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
@models_cli.command(
|
| 193 |
+
"card",
|
| 194 |
+
examples=[
|
| 195 |
+
"hf models card google/gemma-4-31B-it",
|
| 196 |
+
"hf models card google/gemma-4-31B-it --metadata",
|
| 197 |
+
"hf models card google/gemma-4-31B-it --metadata --format json",
|
| 198 |
+
"hf models card google/gemma-4-31B-it --text",
|
| 199 |
+
],
|
| 200 |
+
)
|
| 201 |
+
def models_card(
|
| 202 |
+
model_id: Annotated[str, typer.Argument(help="The model ID (e.g. `username/repo-name`).")],
|
| 203 |
+
metadata: Annotated[bool, typer.Option("--metadata", help="Output only the metadata from the card.")] = False,
|
| 204 |
+
text: Annotated[bool, typer.Option("--text", help="Output only the text body (no metadata).")] = False,
|
| 205 |
+
token: TokenOpt = None,
|
| 206 |
+
) -> None:
|
| 207 |
+
"""Get the model card (README) for a model on the Hub."""
|
| 208 |
+
if metadata and text:
|
| 209 |
+
raise CLIError("--metadata and --text are mutually exclusive.")
|
| 210 |
+
card = ModelCard.load(model_id, token=token)
|
| 211 |
+
if metadata:
|
| 212 |
+
out.dict(card.data.to_dict())
|
| 213 |
+
elif text:
|
| 214 |
+
out.text(card.text)
|
| 215 |
+
else:
|
| 216 |
+
out.text(card.content)
|
| 217 |
+
out.hint(f"Use `hf models card {model_id} --metadata` to extract only the card metadata.")
|
venv/Lib/site-packages/huggingface_hub/cli/papers.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains commands to interact with papers on the Hugging Face Hub.
|
| 15 |
+
|
| 16 |
+
Usage:
|
| 17 |
+
# list daily papers (most recently submitted)
|
| 18 |
+
hf papers ls
|
| 19 |
+
|
| 20 |
+
# list trending papers
|
| 21 |
+
hf papers ls --sort=trending
|
| 22 |
+
|
| 23 |
+
# list papers from a specific date, ordered by upvotes
|
| 24 |
+
hf papers ls --date=2025-01-23
|
| 25 |
+
|
| 26 |
+
# list today's papers, ordered by upvotes
|
| 27 |
+
hf papers ls --date=today
|
| 28 |
+
|
| 29 |
+
# list papers from a specific week
|
| 30 |
+
hf papers ls --week=2025-W09
|
| 31 |
+
|
| 32 |
+
# list papers by a specific submitter
|
| 33 |
+
hf papers ls --submitter=someuser
|
| 34 |
+
|
| 35 |
+
# search papers
|
| 36 |
+
hf papers search "vision language"
|
| 37 |
+
|
| 38 |
+
# get info about a paper
|
| 39 |
+
hf papers info 2502.08025
|
| 40 |
+
|
| 41 |
+
# read a paper as markdown
|
| 42 |
+
hf papers read 2502.08025
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
import datetime
|
| 46 |
+
import enum
|
| 47 |
+
from typing import Annotated, get_args
|
| 48 |
+
|
| 49 |
+
import typer
|
| 50 |
+
|
| 51 |
+
from huggingface_hub.errors import CLIError, HfHubHTTPError
|
| 52 |
+
from huggingface_hub.hf_api import DailyPapersSort_T
|
| 53 |
+
|
| 54 |
+
from ._cli_utils import (
|
| 55 |
+
LimitOpt,
|
| 56 |
+
TokenOpt,
|
| 57 |
+
get_hf_api,
|
| 58 |
+
typer_factory,
|
| 59 |
+
)
|
| 60 |
+
from ._output import _dataclass_to_dict, out
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
_SORT_OPTIONS = get_args(DailyPapersSort_T)
|
| 64 |
+
PaperSortEnum = enum.Enum("PaperSortEnum", {s: s for s in _SORT_OPTIONS}, type=str) # type: ignore[misc]
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _parse_date(value: str | None) -> str | None:
|
| 68 |
+
"""Parse date option, converting 'today' to current date."""
|
| 69 |
+
if value is None:
|
| 70 |
+
return None
|
| 71 |
+
if value.lower() == "today":
|
| 72 |
+
return datetime.date.today().isoformat()
|
| 73 |
+
return value
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
papers_cli = typer_factory(help="Interact with papers on the Hub.")
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
@papers_cli.command(
|
| 80 |
+
"list | ls",
|
| 81 |
+
examples=[
|
| 82 |
+
"hf papers ls",
|
| 83 |
+
"hf papers ls --sort trending",
|
| 84 |
+
"hf papers ls --date 2025-01-23",
|
| 85 |
+
"hf papers ls --week 2025-W09",
|
| 86 |
+
"hf papers ls --submitter akhaliq",
|
| 87 |
+
"hf papers ls --format json",
|
| 88 |
+
],
|
| 89 |
+
)
|
| 90 |
+
def papers_ls(
|
| 91 |
+
date: Annotated[
|
| 92 |
+
str | None,
|
| 93 |
+
typer.Option(
|
| 94 |
+
help="Date in ISO format (YYYY-MM-DD) or 'today'.",
|
| 95 |
+
callback=_parse_date,
|
| 96 |
+
),
|
| 97 |
+
] = None,
|
| 98 |
+
week: Annotated[
|
| 99 |
+
str | None,
|
| 100 |
+
typer.Option(help="ISO week to filter by, e.g. '2025-W09'."),
|
| 101 |
+
] = None,
|
| 102 |
+
month: Annotated[
|
| 103 |
+
str | None,
|
| 104 |
+
typer.Option(help="Month to filter by in ISO format (YYYY-MM), e.g. '2025-02'."),
|
| 105 |
+
] = None,
|
| 106 |
+
submitter: Annotated[
|
| 107 |
+
str | None,
|
| 108 |
+
typer.Option(help="Filter by username of the submitter."),
|
| 109 |
+
] = None,
|
| 110 |
+
sort: Annotated[
|
| 111 |
+
PaperSortEnum | None,
|
| 112 |
+
typer.Option(help="Sort results."),
|
| 113 |
+
] = None,
|
| 114 |
+
limit: LimitOpt = 50,
|
| 115 |
+
token: TokenOpt = None,
|
| 116 |
+
) -> None:
|
| 117 |
+
"""List daily papers on the Hub."""
|
| 118 |
+
api = get_hf_api(token=token)
|
| 119 |
+
sort_key = sort.value if sort else None
|
| 120 |
+
results = []
|
| 121 |
+
for paper_info in api.list_daily_papers(
|
| 122 |
+
date=date,
|
| 123 |
+
week=week,
|
| 124 |
+
month=month,
|
| 125 |
+
submitter=submitter,
|
| 126 |
+
sort=sort_key,
|
| 127 |
+
limit=limit,
|
| 128 |
+
):
|
| 129 |
+
item = _dataclass_to_dict(paper_info)
|
| 130 |
+
submitted_by = item.get("submitted_by") or {}
|
| 131 |
+
item["submitted_by_name"] = submitted_by.get("fullname") or submitted_by.get("username") or ""
|
| 132 |
+
results.append(item)
|
| 133 |
+
out.table(
|
| 134 |
+
results,
|
| 135 |
+
headers=["id", "title", "upvotes", "comments", "published_at", "submitted_by_name"],
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
@papers_cli.command(
|
| 140 |
+
"search",
|
| 141 |
+
examples=[
|
| 142 |
+
'hf papers search "vision language"',
|
| 143 |
+
'hf papers search "attention mechanism" --limit 10',
|
| 144 |
+
'hf papers search "diffusion" --format json',
|
| 145 |
+
],
|
| 146 |
+
)
|
| 147 |
+
def papers_search(
|
| 148 |
+
query: Annotated[str, typer.Argument(help="Search query string.")],
|
| 149 |
+
limit: LimitOpt = 20,
|
| 150 |
+
token: TokenOpt = None,
|
| 151 |
+
) -> None:
|
| 152 |
+
"""Search papers on the Hub."""
|
| 153 |
+
api = get_hf_api(token=token)
|
| 154 |
+
results = [_dataclass_to_dict(paper_info) for paper_info in api.list_papers(query=query, limit=limit)]
|
| 155 |
+
out.table(results, headers=["id", "title", "summary", "upvotes", "published_at"])
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
@papers_cli.command(
|
| 159 |
+
"info",
|
| 160 |
+
examples=[
|
| 161 |
+
"hf papers info 2601.15621",
|
| 162 |
+
],
|
| 163 |
+
)
|
| 164 |
+
def papers_info(
|
| 165 |
+
paper_id: Annotated[str, typer.Argument(help="The arXiv paper ID (e.g. '2502.08025').")],
|
| 166 |
+
token: TokenOpt = None,
|
| 167 |
+
) -> None:
|
| 168 |
+
"""Get info about a paper on the Hub."""
|
| 169 |
+
api = get_hf_api(token=token)
|
| 170 |
+
try:
|
| 171 |
+
info = api.paper_info(id=paper_id)
|
| 172 |
+
except HfHubHTTPError as e:
|
| 173 |
+
if e.response.status_code == 404:
|
| 174 |
+
raise CLIError(f"Paper '{paper_id}' not found on the Hub.") from e
|
| 175 |
+
raise
|
| 176 |
+
out.dict(info)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
@papers_cli.command(
|
| 180 |
+
"read",
|
| 181 |
+
examples=[
|
| 182 |
+
"hf papers read 2601.15621",
|
| 183 |
+
],
|
| 184 |
+
)
|
| 185 |
+
def papers_read(
|
| 186 |
+
paper_id: Annotated[str, typer.Argument(help="The arXiv paper ID (e.g. '2502.08025').")],
|
| 187 |
+
token: TokenOpt = None,
|
| 188 |
+
) -> None:
|
| 189 |
+
"""Read a paper as markdown."""
|
| 190 |
+
api = get_hf_api(token=token)
|
| 191 |
+
try:
|
| 192 |
+
content = api.read_paper(id=paper_id)
|
| 193 |
+
except HfHubHTTPError as e:
|
| 194 |
+
if e.response.status_code == 404:
|
| 195 |
+
raise CLIError(f"Paper '{paper_id}' not found on the Hub.") from e
|
| 196 |
+
raise
|
| 197 |
+
out.text(content)
|
venv/Lib/site-packages/huggingface_hub/cli/repo_files.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2023-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Legacy `hf repo-files` command.
|
| 15 |
+
|
| 16 |
+
Kept for backward compatibility. Users are nudged to use `hf repos delete-files` instead.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
from typing import Annotated
|
| 20 |
+
|
| 21 |
+
import typer
|
| 22 |
+
|
| 23 |
+
from ._cli_utils import (
|
| 24 |
+
RepoIdArg,
|
| 25 |
+
RepoType,
|
| 26 |
+
RepoTypeOpt,
|
| 27 |
+
RevisionOpt,
|
| 28 |
+
TokenOpt,
|
| 29 |
+
get_hf_api,
|
| 30 |
+
typer_factory,
|
| 31 |
+
)
|
| 32 |
+
from ._output import out
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
repo_files_cli = typer_factory(
|
| 36 |
+
help="(Deprecated) Manage files in a repo on the Hub. Use `hf repos delete-files` instead."
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@repo_files_cli.command(
|
| 41 |
+
"delete",
|
| 42 |
+
)
|
| 43 |
+
def repo_files_delete(
|
| 44 |
+
repo_id: RepoIdArg,
|
| 45 |
+
patterns: Annotated[
|
| 46 |
+
list[str],
|
| 47 |
+
typer.Argument(
|
| 48 |
+
help="Glob patterns to match files to delete. Based on fnmatch, '*' matches files recursively.",
|
| 49 |
+
),
|
| 50 |
+
],
|
| 51 |
+
repo_type: RepoTypeOpt = RepoType.model,
|
| 52 |
+
revision: RevisionOpt = None,
|
| 53 |
+
commit_message: Annotated[
|
| 54 |
+
str | None,
|
| 55 |
+
typer.Option(
|
| 56 |
+
help="The summary / title / first line of the generated commit.",
|
| 57 |
+
),
|
| 58 |
+
] = None,
|
| 59 |
+
commit_description: Annotated[
|
| 60 |
+
str | None,
|
| 61 |
+
typer.Option(
|
| 62 |
+
help="The description of the generated commit.",
|
| 63 |
+
),
|
| 64 |
+
] = None,
|
| 65 |
+
create_pr: Annotated[
|
| 66 |
+
bool,
|
| 67 |
+
typer.Option(
|
| 68 |
+
help="Whether to create a new Pull Request for these changes.",
|
| 69 |
+
),
|
| 70 |
+
] = False,
|
| 71 |
+
token: TokenOpt = None,
|
| 72 |
+
) -> None:
|
| 73 |
+
out.warning("`hf repo-files delete` is deprecated. Use `hf repos delete-files` instead.")
|
| 74 |
+
api = get_hf_api(token=token)
|
| 75 |
+
url = api.delete_files(
|
| 76 |
+
delete_patterns=patterns,
|
| 77 |
+
repo_id=repo_id,
|
| 78 |
+
repo_type=repo_type.value,
|
| 79 |
+
revision=revision,
|
| 80 |
+
commit_message=commit_message,
|
| 81 |
+
commit_description=commit_description,
|
| 82 |
+
create_pr=create_pr,
|
| 83 |
+
)
|
| 84 |
+
out.result("Files deleted", repo_id=repo_id, commit_url=url)
|