Add files using upload-large-folder tool

.venv/lib/python3.11/site-packages/huggingface_hub/commands/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from argparse import _SubParsersAction
+
+
+class BaseHuggingfaceCLICommand(ABC):
+    @staticmethod
+    @abstractmethod
+    def register_subcommand(parser: _SubParsersAction):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def run(self):
+        raise NotImplementedError()

.venv/lib/python3.11/site-packages/huggingface_hub/commands/_cli_utils.py

@@ -0,0 +1,69 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains a utility for good-looking prints."""
+
+import os
+from typing import List, Union
+
+
+class ANSI:
+    """
+    Helper for en.wikipedia.org/wiki/ANSI_escape_code
+    """
+
+    _bold = "\u001b[1m"
+    _gray = "\u001b[90m"
+    _red = "\u001b[31m"
+    _reset = "\u001b[0m"
+    _yellow = "\u001b[33m"
+
+    @classmethod
+    def bold(cls, s: str) -> str:
+        return cls._format(s, cls._bold)
+
+    @classmethod
+    def gray(cls, s: str) -> str:
+        return cls._format(s, cls._gray)
+
+    @classmethod
+    def red(cls, s: str) -> str:
+        return cls._format(s, cls._bold + cls._red)
+
+    @classmethod
+    def yellow(cls, s: str) -> str:
+        return cls._format(s, cls._yellow)
+
+    @classmethod
+    def _format(cls, s: str, code: str) -> str:
+        if os.environ.get("NO_COLOR"):
+            # See https://no-color.org/
+            return s
+        return f"{code}{s}{cls._reset}"
+
+
+def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str:
+    """
+    Inspired by:
+
+    - stackoverflow.com/a/8356620/593036
+    - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data
+    """
+    col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)]
+    row_format = ("{{:{}}} " * len(headers)).format(*col_widths)
+    lines = []
+    lines.append(row_format.format(*headers))
+    lines.append(row_format.format(*["-" * w for w in col_widths]))
+    for row in rows:
+        lines.append(row_format.format(*row))
+    return "\n".join(lines)

.venv/lib/python3.11/site-packages/huggingface_hub/commands/delete_cache.py

@@ -0,0 +1,428 @@
+# coding=utf-8
+# Copyright 2022-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to delete some revisions from the HF cache directory.
+
+Usage:
+    huggingface-cli delete-cache
+    huggingface-cli delete-cache --disable-tui
+    huggingface-cli delete-cache --dir ~/.cache/huggingface/hub
+
+NOTE:
+    This command is based on `InquirerPy` to build the multiselect menu in the terminal.
+    This dependency has to be installed with `pip install huggingface_hub[cli]`. Since
+    we want to avoid as much as possible cross-platform issues, I chose a library that
+    is built on top of `python-prompt-toolkit` which seems to be a reference in terminal
+    GUI (actively maintained on both Unix and Windows, 7.9k stars).
+
+    For the moment, the TUI feature is in beta.
+
+    See:
+    - https://github.com/kazhala/InquirerPy
+    - https://inquirerpy.readthedocs.io/en/latest/
+    - https://github.com/prompt-toolkit/python-prompt-toolkit
+
+    Other solutions could have been:
+    - `simple_term_menu`: would be good as well for our use case but some issues suggest
+      that Windows is less supported.
+      See: https://github.com/IngoMeyer441/simple-term-menu
+    - `PyInquirer`: very similar to `InquirerPy` but older and not maintained anymore.
+      In particular, no support of Python3.10.
+      See: https://github.com/CITGuru/PyInquirer
+    - `pick` (or `pickpack`): easy to use and flexible but built on top of Python's
+      standard library `curses` that is specific to Unix (not implemented on Windows).
+      See https://github.com/wong2/pick and https://github.com/anafvana/pickpack.
+    - `inquirer`: lot of traction (700 stars) but explicitly states "experimental
+      support of Windows". Not built on top of `python-prompt-toolkit`.
+      See https://github.com/magmax/python-inquirer
+
+TODO: add support for `huggingface-cli delete-cache aaaaaa bbbbbb cccccc (...)` ?
+TODO: add "--keep-last" arg to delete revisions that are not on `main` ref
+TODO: add "--filter" arg to filter repositories by name ?
+TODO: add "--sort" arg to sort by size ?
+TODO: add "--limit" arg to limit to X repos ?
+TODO: add "-y" arg for immediate deletion ?
+See discussions in https://github.com/huggingface/huggingface_hub/issues/1025.
+"""
+
+import os
+from argparse import Namespace, _SubParsersAction
+from functools import wraps
+from tempfile import mkstemp
+from typing import Any, Callable, Iterable, List, Optional, Union
+
+from ..utils import CachedRepoInfo, CachedRevisionInfo, HFCacheInfo, scan_cache_dir
+from . import BaseHuggingfaceCLICommand
+from ._cli_utils import ANSI
+
+
+try:
+    from InquirerPy import inquirer
+    from InquirerPy.base.control import Choice
+    from InquirerPy.separator import Separator
+
+    _inquirer_py_available = True
+except ImportError:
+    _inquirer_py_available = False
+
+
+def require_inquirer_py(fn: Callable) -> Callable:
+    """Decorator to flag methods that require `InquirerPy`."""
+
+    # TODO: refactor this + imports in a unified pattern across codebase
+    @wraps(fn)
+    def _inner(*args, **kwargs):
+        if not _inquirer_py_available:
+            raise ImportError(
+                "The `delete-cache` command requires extra dependencies to work with"
+                " the TUI.\nPlease run `pip install huggingface_hub[cli]` to install"
+                " them.\nOtherwise, disable TUI using the `--disable-tui` flag."
+            )
+
+        return fn(*args, **kwargs)
+
+    return _inner
+
+
+# Possibility for the user to cancel deletion
+_CANCEL_DELETION_STR = "CANCEL_DELETION"
+
+
+class DeleteCacheCommand(BaseHuggingfaceCLICommand):
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        delete_cache_parser = parser.add_parser("delete-cache", help="Delete revisions from the cache directory.")
+
+        delete_cache_parser.add_argument(
+            "--dir",
+            type=str,
+            default=None,
+            help="cache directory (optional). Default to the default HuggingFace cache.",
+        )
+
+        delete_cache_parser.add_argument(
+            "--disable-tui",
+            action="store_true",
+            help=(
+                "Disable Terminal User Interface (TUI) mode. Useful if your"
+                " platform/terminal doesn't support the multiselect menu."
+            ),
+        )
+
+        delete_cache_parser.set_defaults(func=DeleteCacheCommand)
+
+    def __init__(self, args: Namespace) -> None:
+        self.cache_dir: Optional[str] = args.dir
+        self.disable_tui: bool = args.disable_tui
+
+    def run(self):
+        """Run `delete-cache` command with or without TUI."""
+        # Scan cache directory
+        hf_cache_info = scan_cache_dir(self.cache_dir)
+
+        # Manual review from the user
+        if self.disable_tui:
+            selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[])
+        else:
+            selected_hashes = _manual_review_tui(hf_cache_info, preselected=[])
+
+        # If deletion is not cancelled
+        if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes:
+            confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?"
+
+            # Confirm deletion
+            if self.disable_tui:
+                confirmed = _ask_for_confirmation_no_tui(confirm_message)
+            else:
+                confirmed = _ask_for_confirmation_tui(confirm_message)
+
+            # Deletion is confirmed
+            if confirmed:
+                strategy = hf_cache_info.delete_revisions(*selected_hashes)
+                print("Start deletion.")
+                strategy.execute()
+                print(
+                    f"Done. Deleted {len(strategy.repos)} repo(s) and"
+                    f" {len(strategy.snapshots)} revision(s) for a total of"
+                    f" {strategy.expected_freed_size_str}."
+                )
+                return
+
+        # Deletion is cancelled
+        print("Deletion is cancelled. Do nothing.")
+
+
+@require_inquirer_py
+def _manual_review_tui(hf_cache_info: HFCacheInfo, preselected: List[str]) -> List[str]:
+    """Ask the user for a manual review of the revisions to delete.
+
+    Displays a multi-select menu in the terminal (TUI).
+    """
+    # Define multiselect list
+    choices = _get_tui_choices_from_scan(repos=hf_cache_info.repos, preselected=preselected)
+    checkbox = inquirer.checkbox(
+        message="Select revisions to delete:",
+        choices=choices,  # List of revisions with some pre-selection
+        cycle=False,  # No loop between top and bottom
+        height=100,  # Large list if possible
+        # We use the instruction to display to the user the expected effect of the
+        # deletion.
+        instruction=_get_expectations_str(
+            hf_cache_info,
+            selected_hashes=[c.value for c in choices if isinstance(c, Choice) and c.enabled],
+        ),
+        # We use the long instruction to should keybindings instructions to the user
+        long_instruction="Press <space> to select, <enter> to validate and <ctrl+c> to quit without modification.",
+        # Message that is displayed once the user validates its selection.
+        transformer=lambda result: f"{len(result)} revision(s) selected.",
+    )
+
+    # Add a callback to update the information line when a revision is
+    # selected/unselected
+    def _update_expectations(_) -> None:
+        # Hacky way to dynamically set an instruction message to the checkbox when
+        # a revision hash is selected/unselected.
+        checkbox._instruction = _get_expectations_str(
+            hf_cache_info,
+            selected_hashes=[choice["value"] for choice in checkbox.content_control.choices if choice["enabled"]],
+        )
+
+    checkbox.kb_func_lookup["toggle"].append({"func": _update_expectations})
+
+    # Finally display the form to the user.
+    try:
+        return checkbox.execute()
+    except KeyboardInterrupt:
+        return []  # Quit without deletion
+
+
+@require_inquirer_py
+def _ask_for_confirmation_tui(message: str, default: bool = True) -> bool:
+    """Ask for confirmation using Inquirer."""
+    return inquirer.confirm(message, default=default).execute()
+
+
+def _get_tui_choices_from_scan(repos: Iterable[CachedRepoInfo], preselected: List[str]) -> List:
+    """Build a list of choices from the scanned repos.
+
+    Args:
+        repos (*Iterable[`CachedRepoInfo`]*):
+            List of scanned repos on which we want to delete revisions.
+        preselected (*List[`str`]*):
+            List of revision hashes that will be preselected.
+
+    Return:
+        The list of choices to pass to `inquirer.checkbox`.
+    """
+    choices: List[Union[Choice, Separator]] = []
+
+    # First choice is to cancel the deletion. If selected, nothing will be deleted,
+    # no matter the other selected items.
+    choices.append(
+        Choice(
+            _CANCEL_DELETION_STR,
+            name="None of the following (if selected, nothing will be deleted).",
+            enabled=False,
+        )
+    )
+
+    # Display a separator per repo and a Choice for each revisions of the repo
+    for repo in sorted(repos, key=_repo_sorting_order):
+        # Repo as separator
+        choices.append(
+            Separator(
+                f"\n{repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str},"
+                f" used {repo.last_accessed_str})"
+            )
+        )
+        for revision in sorted(repo.revisions, key=_revision_sorting_order):
+            # Revision as choice
+            choices.append(
+                Choice(
+                    revision.commit_hash,
+                    name=(
+                        f"{revision.commit_hash[:8]}:"
+                        f" {', '.join(sorted(revision.refs)) or '(detached)'} #"
+                        f" modified {revision.last_modified_str}"
+                    ),
+                    enabled=revision.commit_hash in preselected,
+                )
+            )
+
+    # Return choices
+    return choices
+
+
+def _manual_review_no_tui(hf_cache_info: HFCacheInfo, preselected: List[str]) -> List[str]:
+    """Ask the user for a manual review of the revisions to delete.
+
+    Used when TUI is disabled. Manual review happens in a separate tmp file that the
+    user can manually edit.
+    """
+    # 1. Generate temporary file with delete commands.
+    fd, tmp_path = mkstemp(suffix=".txt")  # suffix to make it easier to find by editors
+    os.close(fd)
+
+    lines = []
+    for repo in sorted(hf_cache_info.repos, key=_repo_sorting_order):
+        lines.append(
+            f"\n# {repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str},"
+            f" used {repo.last_accessed_str})"
+        )
+        for revision in sorted(repo.revisions, key=_revision_sorting_order):
+            lines.append(
+                # Deselect by prepending a '#'
+                f"{'' if revision.commit_hash in preselected else '#'}   "
+                f" {revision.commit_hash} # Refs:"
+                # Print `refs` as comment on same line
+                f" {', '.join(sorted(revision.refs)) or '(detached)'} # modified"
+                # Print `last_modified` as comment on same line
+                f" {revision.last_modified_str}"
+            )
+
+    with open(tmp_path, "w") as f:
+        f.write(_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS)
+        f.write("\n".join(lines))
+
+    # 2. Prompt instructions to user.
+    instructions = f"""
+    TUI is disabled. In order to select which revisions you want to delete, please edit
+    the following file using the text editor of your choice. Instructions for manual
+    editing are located at the beginning of the file. Edit the file, save it and confirm
+    to continue.
+    File to edit: {ANSI.bold(tmp_path)}
+    """
+    print("\n".join(line.strip() for line in instructions.strip().split("\n")))
+
+    # 3. Wait for user confirmation.
+    while True:
+        selected_hashes = _read_manual_review_tmp_file(tmp_path)
+        if _ask_for_confirmation_no_tui(
+            _get_expectations_str(hf_cache_info, selected_hashes) + " Continue ?",
+            default=False,
+        ):
+            break
+
+    # 4. Return selected_hashes
+    os.remove(tmp_path)
+    return selected_hashes
+
+
+def _ask_for_confirmation_no_tui(message: str, default: bool = True) -> bool:
+    """Ask for confirmation using pure-python."""
+    YES = ("y", "yes", "1")
+    NO = ("n", "no", "0")
+    DEFAULT = ""
+    ALL = YES + NO + (DEFAULT,)
+    full_message = message + (" (Y/n) " if default else " (y/N) ")
+    while True:
+        answer = input(full_message).lower()
+        if answer == DEFAULT:
+            return default
+        if answer in YES:
+            return True
+        if answer in NO:
+            return False
+        print(f"Invalid input. Must be one of {ALL}")
+
+
+def _get_expectations_str(hf_cache_info: HFCacheInfo, selected_hashes: List[str]) -> str:
+    """Format a string to display to the user how much space would be saved.
+
+    Example:
+    ```
+    >>> _get_expectations_str(hf_cache_info, selected_hashes)
+    '7 revisions selected counting for 4.3G.'
+    ```
+    """
+    if _CANCEL_DELETION_STR in selected_hashes:
+        return "Nothing will be deleted."
+    strategy = hf_cache_info.delete_revisions(*selected_hashes)
+    return f"{len(selected_hashes)} revisions selected counting for {strategy.expected_freed_size_str}."
+
+
+def _read_manual_review_tmp_file(tmp_path: str) -> List[str]:
+    """Read the manually reviewed instruction file and return a list of revision hash.
+
+    Example:
+        ```txt
+        # This is the tmp file content
+        ###
+
+        # Commented out line
+        123456789 # revision hash
+
+        # Something else
+        #      a_newer_hash # 2 days ago
+            an_older_hash # 3 days ago
+        ```
+
+        ```py
+        >>> _read_manual_review_tmp_file(tmp_path)
+        ['123456789', 'an_older_hash']
+        ```
+    """
+    with open(tmp_path) as f:
+        content = f.read()
+
+    # Split lines
+    lines = [line.strip() for line in content.split("\n")]
+
+    # Filter commented lines
+    selected_lines = [line for line in lines if not line.startswith("#")]
+
+    # Select only before comment
+    selected_hashes = [line.split("#")[0].strip() for line in selected_lines]
+
+    # Return revision hashes
+    return [hash for hash in selected_hashes if len(hash) > 0]
+
+
+_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS = f"""
+# INSTRUCTIONS
+# ------------
+# This is a temporary file created by running `huggingface-cli delete-cache` with the
+# `--disable-tui` option. It contains a set of revisions that can be deleted from your
+# local cache directory.
+#
+# Please manually review the revisions you want to delete:
+#   - Revision hashes can be commented out with '#'.
+#   - Only non-commented revisions in this file will be deleted.
+#   - Revision hashes that are removed from this file are ignored as well.
+#   - If `{_CANCEL_DELETION_STR}` line is uncommented, the all cache deletion is cancelled and
+#     no changes will be applied.
+#
+# Once you've manually reviewed this file, please confirm deletion in the terminal. This
+# file will be automatically removed once done.
+# ------------
+
+# KILL SWITCH
+# ------------
+# Un-comment following line to completely cancel the deletion process
+# {_CANCEL_DELETION_STR}
+# ------------
+
+# REVISIONS
+# ------------
+""".strip()
+
+
+def _repo_sorting_order(repo: CachedRepoInfo) -> Any:
+    # First split by Dataset/Model, then sort by last accessed (oldest first)
+    return (repo.repo_type, repo.last_accessed)
+
+
+def _revision_sorting_order(revision: CachedRevisionInfo) -> Any:
+    # Sort by last modified (oldest first)
+    return revision.last_modified

.venv/lib/python3.11/site-packages/huggingface_hub/commands/download.py

@@ -0,0 +1,200 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to download files from the Hub with the CLI.
+
+Usage:
+    huggingface-cli download --help
+
+    # Download file
+    huggingface-cli download gpt2 config.json
+
+    # Download entire repo
+    huggingface-cli download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78
+
+    # Download repo with filters
+    huggingface-cli download gpt2 --include="*.safetensors"
+
+    # Download with token
+    huggingface-cli download Wauplin/private-model --token=hf_***
+
+    # Download quietly (no progress bar, no warnings, only the returned path)
+    huggingface-cli download gpt2 config.json --quiet
+
+    # Download to local dir
+    huggingface-cli download gpt2 --local-dir=./models/gpt2
+"""
+
+import warnings
+from argparse import Namespace, _SubParsersAction
+from typing import List, Optional
+
+from huggingface_hub import logging
+from huggingface_hub._snapshot_download import snapshot_download
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.file_download import hf_hub_download
+from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
+
+
+logger = logging.get_logger(__name__)
+
+
+class DownloadCommand(BaseHuggingfaceCLICommand):
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        download_parser = parser.add_parser("download", help="Download files from the Hub")
+        download_parser.add_argument(
+            "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)."
+        )
+        download_parser.add_argument(
+            "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)."
+        )
+        download_parser.add_argument(
+            "--repo-type",
+            choices=["model", "dataset", "space"],
+            default="model",
+            help="Type of repo to download from (defaults to 'model').",
+        )
+        download_parser.add_argument(
+            "--revision",
+            type=str,
+            help="An optional Git revision id which can be a branch name, a tag, or a commit hash.",
+        )
+        download_parser.add_argument(
+            "--include", nargs="*", type=str, help="Glob patterns to match files to download."
+        )
+        download_parser.add_argument(
+            "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download."
+        )
+        download_parser.add_argument(
+            "--cache-dir", type=str, help="Path to the directory where to save the downloaded files."
+        )
+        download_parser.add_argument(
+            "--local-dir",
+            type=str,
+            help=(
+                "If set, the downloaded file will be placed under this directory. Check out"
+                " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more"
+                " details."
+            ),
+        )
+        download_parser.add_argument(
+            "--local-dir-use-symlinks",
+            choices=["auto", "True", "False"],
+            help=("Deprecated and ignored. Downloading to a local directory does not use symlinks anymore."),
+        )
+        download_parser.add_argument(
+            "--force-download",
+            action="store_true",
+            help="If True, the files will be downloaded even if they are already cached.",
+        )
+        download_parser.add_argument(
+            "--resume-download",
+            action="store_true",
+            help="Deprecated and ignored. Downloading a file to local dir always attempts to resume previously interrupted downloads (unless hf-transfer is enabled).",
+        )
+        download_parser.add_argument(
+            "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
+        )
+        download_parser.add_argument(
+            "--quiet",
+            action="store_true",
+            help="If True, progress bars are disabled and only the path to the download files is printed.",
+        )
+        download_parser.add_argument(
+            "--max-workers",
+            type=int,
+            default=8,
+            help="Maximum number of workers to use for downloading files. Default is 8.",
+        )
+        download_parser.set_defaults(func=DownloadCommand)
+
+    def __init__(self, args: Namespace) -> None:
+        self.token = args.token
+        self.repo_id: str = args.repo_id
+        self.filenames: List[str] = args.filenames
+        self.repo_type: str = args.repo_type
+        self.revision: Optional[str] = args.revision
+        self.include: Optional[List[str]] = args.include
+        self.exclude: Optional[List[str]] = args.exclude
+        self.cache_dir: Optional[str] = args.cache_dir
+        self.local_dir: Optional[str] = args.local_dir
+        self.force_download: bool = args.force_download
+        self.resume_download: Optional[bool] = args.resume_download or None
+        self.quiet: bool = args.quiet
+        self.max_workers: int = args.max_workers
+
+        if args.local_dir_use_symlinks is not None:
+            warnings.warn(
+                "Ignoring --local-dir-use-symlinks. Downloading to a local directory does not use symlinks anymore.",
+                FutureWarning,
+            )
+
+    def run(self) -> None:
+        if self.quiet:
+            disable_progress_bars()
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                print(self._download())  # Print path to downloaded files
+            enable_progress_bars()
+        else:
+            logging.set_verbosity_info()
+            print(self._download())  # Print path to downloaded files
+            logging.set_verbosity_warning()
+
+    def _download(self) -> str:
+        # Warn user if patterns are ignored
+        if len(self.filenames) > 0:
+            if self.include is not None and len(self.include) > 0:
+                warnings.warn("Ignoring `--include` since filenames have being explicitly set.")
+            if self.exclude is not None and len(self.exclude) > 0:
+                warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.")
+
+        # Single file to download: use `hf_hub_download`
+        if len(self.filenames) == 1:
+            return hf_hub_download(
+                repo_id=self.repo_id,
+                repo_type=self.repo_type,
+                revision=self.revision,
+                filename=self.filenames[0],
+                cache_dir=self.cache_dir,
+                resume_download=self.resume_download,
+                force_download=self.force_download,
+                token=self.token,
+                local_dir=self.local_dir,
+                library_name="huggingface-cli",
+            )
+
+        # Otherwise: use `snapshot_download` to ensure all files comes from same revision
+        elif len(self.filenames) == 0:
+            allow_patterns = self.include
+            ignore_patterns = self.exclude
+        else:
+            allow_patterns = self.filenames
+            ignore_patterns = None
+
+        return snapshot_download(
+            repo_id=self.repo_id,
+            repo_type=self.repo_type,
+            revision=self.revision,
+            allow_patterns=allow_patterns,
+            ignore_patterns=ignore_patterns,
+            resume_download=self.resume_download,
+            force_download=self.force_download,
+            cache_dir=self.cache_dir,
+            token=self.token,
+            local_dir=self.local_dir,
+            library_name="huggingface-cli",
+            max_workers=self.max_workers,
+        )

.venv/lib/python3.11/site-packages/huggingface_hub/commands/env.py

@@ -0,0 +1,36 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to print information about the environment.
+
+Usage:
+    huggingface-cli env
+"""
+
+from argparse import _SubParsersAction
+
+from ..utils import dump_environment_info
+from . import BaseHuggingfaceCLICommand
+
+
+class EnvironmentCommand(BaseHuggingfaceCLICommand):
+    def __init__(self, args):
+        self.args = args
+
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        env_parser = parser.add_parser("env", help="Print information about the environment.")
+        env_parser.set_defaults(func=EnvironmentCommand)
+
+    def run(self) -> None:
+        dump_environment_info()

.venv/lib/python3.11/site-packages/huggingface_hub/commands/huggingface_cli.py

@@ -0,0 +1,61 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from argparse import ArgumentParser
+
+from huggingface_hub.commands.delete_cache import DeleteCacheCommand
+from huggingface_hub.commands.download import DownloadCommand
+from huggingface_hub.commands.env import EnvironmentCommand
+from huggingface_hub.commands.lfs import LfsCommands
+from huggingface_hub.commands.repo_files import RepoFilesCommand
+from huggingface_hub.commands.scan_cache import ScanCacheCommand
+from huggingface_hub.commands.tag import TagCommands
+from huggingface_hub.commands.upload import UploadCommand
+from huggingface_hub.commands.upload_large_folder import UploadLargeFolderCommand
+from huggingface_hub.commands.user import UserCommands
+from huggingface_hub.commands.version import VersionCommand
+
+
+def main():
+    parser = ArgumentParser("huggingface-cli", usage="huggingface-cli <command> [<args>]")
+    commands_parser = parser.add_subparsers(help="huggingface-cli command helpers")
+
+    # Register commands
+    DownloadCommand.register_subcommand(commands_parser)
+    UploadCommand.register_subcommand(commands_parser)
+    RepoFilesCommand.register_subcommand(commands_parser)
+    EnvironmentCommand.register_subcommand(commands_parser)
+    UserCommands.register_subcommand(commands_parser)
+    LfsCommands.register_subcommand(commands_parser)
+    ScanCacheCommand.register_subcommand(commands_parser)
+    DeleteCacheCommand.register_subcommand(commands_parser)
+    TagCommands.register_subcommand(commands_parser)
+    VersionCommand.register_subcommand(commands_parser)
+
+    # Experimental
+    UploadLargeFolderCommand.register_subcommand(commands_parser)
+
+    # Let's go
+    args = parser.parse_args()
+    if not hasattr(args, "func"):
+        parser.print_help()
+        exit(1)
+
+    # Run
+    service = args.func(args)
+    service.run()
+
+
+if __name__ == "__main__":
+    main()

.venv/lib/python3.11/site-packages/huggingface_hub/commands/lfs.py

@@ -0,0 +1,200 @@
+"""
+Implementation of a custom transfer agent for the transfer type "multipart" for
+git-lfs.
+
+Inspired by:
+github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py
+
+Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
+
+
+To launch debugger while developing:
+
+``` [lfs "customtransfer.multipart"]
+path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678
+--wait-for-client
+/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
+lfs-multipart-upload ```"""
+
+import json
+import os
+import subprocess
+import sys
+from argparse import _SubParsersAction
+from typing import Dict, List, Optional
+
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND
+
+from ..utils import get_session, hf_raise_for_status, logging
+from ..utils._lfs import SliceFileObj
+
+
+logger = logging.get_logger(__name__)
+
+
+class LfsCommands(BaseHuggingfaceCLICommand):
+    """
+    Implementation of a custom transfer agent for the transfer type "multipart"
+    for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom
+    transfer agent is:
+    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
+
+    This introduces two commands to the CLI:
+
+    1. $ huggingface-cli lfs-enable-largefiles
+
+    This should be executed once for each model repo that contains a model file
+    >5GB. It's documented in the error message you get if you just try to git
+    push a 5GB file without having enabled it before.
+
+    2. $ huggingface-cli lfs-multipart-upload
+
+    This command is called by lfs directly and is not meant to be called by the
+    user.
+    """
+
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        enable_parser = parser.add_parser(
+            "lfs-enable-largefiles", help="Configure your repository to enable upload of files > 5GB."
+        )
+        enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.")
+        enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args))
+
+        # Command will get called by git-lfs, do not call it directly.
+        upload_parser = parser.add_parser(LFS_MULTIPART_UPLOAD_COMMAND, add_help=False)
+        upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args))
+
+
+class LfsEnableCommand:
+    def __init__(self, args):
+        self.args = args
+
+    def run(self):
+        local_path = os.path.abspath(self.args.path)
+        if not os.path.isdir(local_path):
+            print("This does not look like a valid git repo.")
+            exit(1)
+        subprocess.run(
+            "git config lfs.customtransfer.multipart.path huggingface-cli".split(),
+            check=True,
+            cwd=local_path,
+        )
+        subprocess.run(
+            f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
+            check=True,
+            cwd=local_path,
+        )
+        print("Local repo set up for largefiles")
+
+
+def write_msg(msg: Dict):
+    """Write out the message in Line delimited JSON."""
+    msg_str = json.dumps(msg) + "\n"
+    sys.stdout.write(msg_str)
+    sys.stdout.flush()
+
+
+def read_msg() -> Optional[Dict]:
+    """Read Line delimited JSON from stdin."""
+    msg = json.loads(sys.stdin.readline().strip())
+
+    if "terminate" in (msg.get("type"), msg.get("event")):
+        # terminate message received
+        return None
+
+    if msg.get("event") not in ("download", "upload"):
+        logger.critical("Received unexpected message")
+        sys.exit(1)
+
+    return msg
+
+
+class LfsUploadCommand:
+    def __init__(self, args) -> None:
+        self.args = args
+
+    def run(self) -> None:
+        # Immediately after invoking a custom transfer process, git-lfs
+        # sends initiation data to the process over stdin.
+        # This tells the process useful information about the configuration.
+        init_msg = json.loads(sys.stdin.readline().strip())
+        if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"):
+            write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}})
+            sys.exit(1)
+
+        # The transfer process should use the information it needs from the
+        # initiation structure, and also perform any one-off setup tasks it
+        # needs to do. It should then respond on stdout with a simple empty
+        # confirmation structure, as follows:
+        write_msg({})
+
+        # After the initiation exchange, git-lfs will send any number of
+        # transfer requests to the stdin of the transfer process, in a serial sequence.
+        while True:
+            msg = read_msg()
+            if msg is None:
+                # When all transfers have been processed, git-lfs will send
+                # a terminate event to the stdin of the transfer process.
+                # On receiving this message the transfer process should
+                # clean up and terminate. No response is expected.
+                sys.exit(0)
+
+            oid = msg["oid"]
+            filepath = msg["path"]
+            completion_url = msg["action"]["href"]
+            header = msg["action"]["header"]
+            chunk_size = int(header.pop("chunk_size"))
+            presigned_urls: List[str] = list(header.values())
+
+            # Send a "started" progress event to allow other workers to start.
+            # Otherwise they're delayed until first "progress" event is reported,
+            # i.e. after the first 5GB by default (!)
+            write_msg(
+                {
+                    "event": "progress",
+                    "oid": oid,
+                    "bytesSoFar": 1,
+                    "bytesSinceLast": 0,
+                }
+            )
+
+            parts = []
+            with open(filepath, "rb") as file:
+                for i, presigned_url in enumerate(presigned_urls):
+                    with SliceFileObj(
+                        file,
+                        seek_from=i * chunk_size,
+                        read_limit=chunk_size,
+                    ) as data:
+                        r = get_session().put(presigned_url, data=data)
+                        hf_raise_for_status(r)
+                        parts.append(
+                            {
+                                "etag": r.headers.get("etag"),
+                                "partNumber": i + 1,
+                            }
+                        )
+                        # In order to support progress reporting while data is uploading / downloading,
+                        # the transfer process should post messages to stdout
+                        write_msg(
+                            {
+                                "event": "progress",
+                                "oid": oid,
+                                "bytesSoFar": (i + 1) * chunk_size,
+                                "bytesSinceLast": chunk_size,
+                            }
+                        )
+                        # Not precise but that's ok.
+
+            r = get_session().post(
+                completion_url,
+                json={
+                    "oid": oid,
+                    "parts": parts,
+                },
+            )
+            hf_raise_for_status(r)
+
+            write_msg({"event": "complete", "oid": oid})

.venv/lib/python3.11/site-packages/huggingface_hub/commands/repo_files.py

@@ -0,0 +1,128 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to update or delete files in a repository using the CLI.
+
+Usage:
+    # delete all
+    huggingface-cli repo-files <repo_id> delete "*"
+
+    # delete single file
+    huggingface-cli repo-files <repo_id> delete file.txt
+
+    # delete single folder
+    huggingface-cli repo-files <repo_id> delete folder/
+
+    # delete multiple
+    huggingface-cli repo-files <repo_id> delete file.txt folder/ file2.txt
+
+    # delete multiple patterns
+    huggingface-cli repo-files <repo_id> delete file.txt "*.json" "folder/*.parquet"
+
+    # delete from different revision / repo-type
+    huggingface-cli repo-files <repo_id> delete file.txt --revision=refs/pr/1 --repo-type=dataset
+"""
+
+from argparse import _SubParsersAction
+from typing import List, Optional
+
+from huggingface_hub import logging
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.hf_api import HfApi
+
+
+logger = logging.get_logger(__name__)
+
+
+class DeleteFilesSubCommand:
+    def __init__(self, args) -> None:
+        self.args = args
+        self.repo_id: str = args.repo_id
+        self.repo_type: Optional[str] = args.repo_type
+        self.revision: Optional[str] = args.revision
+        self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
+        self.patterns: List[str] = args.patterns
+        self.commit_message: Optional[str] = args.commit_message
+        self.commit_description: Optional[str] = args.commit_description
+        self.create_pr: bool = args.create_pr
+        self.token: Optional[str] = args.token
+
+    def run(self) -> None:
+        logging.set_verbosity_info()
+        url = self.api.delete_files(
+            delete_patterns=self.patterns,
+            repo_id=self.repo_id,
+            repo_type=self.repo_type,
+            revision=self.revision,
+            commit_message=self.commit_message,
+            commit_description=self.commit_description,
+            create_pr=self.create_pr,
+        )
+        print(f"Files correctly deleted from repo. Commit: {url}.")
+        logging.set_verbosity_warning()
+
+
+class RepoFilesCommand(BaseHuggingfaceCLICommand):
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        repo_files_parser = parser.add_parser("repo-files", help="Manage files in a repo on the Hub")
+        repo_files_parser.add_argument(
+            "repo_id", type=str, help="The ID of the repo to manage (e.g. `username/repo-name`)."
+        )
+        repo_files_subparsers = repo_files_parser.add_subparsers(
+            help="Action to execute against the files.",
+            required=True,
+        )
+        delete_subparser = repo_files_subparsers.add_parser(
+            "delete",
+            help="Delete files from a repo on the Hub",
+        )
+        delete_subparser.set_defaults(func=lambda args: DeleteFilesSubCommand(args))
+        delete_subparser.add_argument(
+            "patterns",
+            nargs="+",
+            type=str,
+            help="Glob patterns to match files to delete.",
+        )
+        delete_subparser.add_argument(
+            "--repo-type",
+            choices=["model", "dataset", "space"],
+            default="model",
+            help="Type of the repo to upload to (e.g. `dataset`).",
+        )
+        delete_subparser.add_argument(
+            "--revision",
+            type=str,
+            help=(
+                "An optional Git revision to push to. It can be a branch name "
+                "or a PR reference. If revision does not"
+                " exist and `--create-pr` is not set, a branch will be automatically created."
+            ),
+        )
+        delete_subparser.add_argument(
+            "--commit-message", type=str, help="The summary / title / first line of the generated commit."
+        )
+        delete_subparser.add_argument(
+            "--commit-description", type=str, help="The description of the generated commit."
+        )
+        delete_subparser.add_argument(
+            "--create-pr", action="store_true", help="Whether to create a new Pull Request for these changes."
+        )
+        repo_files_parser.add_argument(
+            "--token",
+            type=str,
+            help="A User Access Token generated from https://huggingface.co/settings/tokens",
+        )
+
+        repo_files_parser.set_defaults(func=RepoFilesCommand)

.venv/lib/python3.11/site-packages/huggingface_hub/commands/scan_cache.py

@@ -0,0 +1,181 @@
+# coding=utf-8
+# Copyright 2022-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to scan the HF cache directory.
+
+Usage:
+    huggingface-cli scan-cache
+    huggingface-cli scan-cache -v
+    huggingface-cli scan-cache -vvv
+    huggingface-cli scan-cache --dir ~/.cache/huggingface/hub
+"""
+
+import time
+from argparse import Namespace, _SubParsersAction
+from typing import Optional
+
+from ..utils import CacheNotFound, HFCacheInfo, scan_cache_dir
+from . import BaseHuggingfaceCLICommand
+from ._cli_utils import ANSI, tabulate
+
+
+class ScanCacheCommand(BaseHuggingfaceCLICommand):
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        scan_cache_parser = parser.add_parser("scan-cache", help="Scan cache directory.")
+
+        scan_cache_parser.add_argument(
+            "--dir",
+            type=str,
+            default=None,
+            help="cache directory to scan (optional). Default to the default HuggingFace cache.",
+        )
+        scan_cache_parser.add_argument(
+            "-v",
+            "--verbose",
+            action="count",
+            default=0,
+            help="show a more verbose output",
+        )
+        scan_cache_parser.set_defaults(func=ScanCacheCommand)
+
+    def __init__(self, args: Namespace) -> None:
+        self.verbosity: int = args.verbose
+        self.cache_dir: Optional[str] = args.dir
+
+    def run(self):
+        try:
+            t0 = time.time()
+            hf_cache_info = scan_cache_dir(self.cache_dir)
+            t1 = time.time()
+        except CacheNotFound as exc:
+            cache_dir = exc.cache_dir
+            print(f"Cache directory not found: {cache_dir}")
+            return
+
+        self._print_hf_cache_info_as_table(hf_cache_info)
+
+        print(
+            f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)"
+            f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}."
+        )
+        if len(hf_cache_info.warnings) > 0:
+            message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning."
+            if self.verbosity >= 3:
+                print(ANSI.gray(message))
+                for warning in hf_cache_info.warnings:
+                    print(ANSI.gray(warning))
+            else:
+                print(ANSI.gray(message + " Use -vvv to print details."))
+
+    def _print_hf_cache_info_as_table(self, hf_cache_info: HFCacheInfo) -> None:
+        print(get_table(hf_cache_info, verbosity=self.verbosity))
+
+
+def get_table(hf_cache_info: HFCacheInfo, *, verbosity: int = 0) -> str:
+    """Generate a table from the [`HFCacheInfo`] object.
+
+    Pass `verbosity=0` to get a table with a single row per repo, with columns
+    "repo_id", "repo_type", "size_on_disk", "nb_files", "last_accessed", "last_modified", "refs", "local_path".
+
+    Pass `verbosity=1` to get a table with a row per repo and revision (thus multiple rows can appear for a single repo), with columns
+    "repo_id", "repo_type", "revision", "size_on_disk", "nb_files", "last_modified", "refs", "local_path".
+
+    Example:
+    ```py
+    >>> from huggingface_hub.utils import scan_cache_dir
+    >>> from huggingface_hub.commands.scan_cache import get_table
+
+    >>> hf_cache_info = scan_cache_dir()
+    HFCacheInfo(...)
+
+    >>> print(get_table(hf_cache_info, verbosity=0))
+    REPO ID                                             REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED LAST_MODIFIED REFS LOCAL PATH
+    --------------------------------------------------- --------- ------------ -------- ------------- ------------- ---- --------------------------------------------------------------------------------------------------
+    roberta-base                                        model             2.7M        5 1 day ago     1 week ago    main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--roberta-base
+    suno/bark                                           model             8.8K        1 1 week ago    1 week ago    main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--suno--bark
+    t5-base                                             model           893.8M        4 4 days ago    7 months ago  main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-base
+    t5-large                                            model             3.0G        4 5 weeks ago   5 months ago  main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-large
+
+    >>> print(get_table(hf_cache_info, verbosity=1))
+    REPO ID                                             REPO TYPE REVISION                                 SIZE ON DISK NB FILES LAST_MODIFIED REFS LOCAL PATH
+    --------------------------------------------------- --------- ---------------------------------------- ------------ -------- ------------- ---- -----------------------------------------------------------------------------------------------------------------------------------------------------
+    roberta-base                                        model     e2da8e2f811d1448a5b465c236feacd80ffbac7b         2.7M        5 1 week ago    main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--roberta-base\\snapshots\\e2da8e2f811d1448a5b465c236feacd80ffbac7b
+    suno/bark                                           model     70a8a7d34168586dc5d028fa9666aceade177992         8.8K        1 1 week ago    main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--suno--bark\\snapshots\\70a8a7d34168586dc5d028fa9666aceade177992
+    t5-base                                             model     a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1       893.8M        4 7 months ago  main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-base\\snapshots\\a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1
+    t5-large                                            model     150ebc2c4b72291e770f58e6057481c8d2ed331a         3.0G        4 5 months ago  main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-large\\snapshots\\150ebc2c4b72291e770f58e6057481c8d2ed331a                                                 ```
+    ```
+
+    Args:
+        hf_cache_info ([`HFCacheInfo`]):
+            The HFCacheInfo object to print.
+        verbosity (`int`, *optional*):
+            The verbosity level. Defaults to 0.
+
+    Returns:
+        `str`: The table as a string.
+    """
+    if verbosity == 0:
+        return tabulate(
+            rows=[
+                [
+                    repo.repo_id,
+                    repo.repo_type,
+                    "{:>12}".format(repo.size_on_disk_str),
+                    repo.nb_files,
+                    repo.last_accessed_str,
+                    repo.last_modified_str,
+                    ", ".join(sorted(repo.refs)),
+                    str(repo.repo_path),
+                ]
+                for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path)
+            ],
+            headers=[
+                "REPO ID",
+                "REPO TYPE",
+                "SIZE ON DISK",
+                "NB FILES",
+                "LAST_ACCESSED",
+                "LAST_MODIFIED",
+                "REFS",
+                "LOCAL PATH",
+            ],
+        )
+    else:
+        return tabulate(
+            rows=[
+                [
+                    repo.repo_id,
+                    repo.repo_type,
+                    revision.commit_hash,
+                    "{:>12}".format(revision.size_on_disk_str),
+                    revision.nb_files,
+                    revision.last_modified_str,
+                    ", ".join(sorted(revision.refs)),
+                    str(revision.snapshot_path),
+                ]
+                for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path)
+                for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash)
+            ],
+            headers=[
+                "REPO ID",
+                "REPO TYPE",
+                "REVISION",
+                "SIZE ON DISK",
+                "NB FILES",
+                "LAST_MODIFIED",
+                "REFS",
+                "LOCAL PATH",
+            ],
+        )

.venv/lib/python3.11/site-packages/huggingface_hub/commands/tag.py

@@ -0,0 +1,159 @@
+# coding=utf-8
+# Copyright 2024-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains commands to perform tag management with the CLI.
+
+Usage Examples:
+    - Create a tag:
+        $ huggingface-cli tag user/my-model 1.0 --message "First release"
+        $ huggingface-cli tag user/my-model 1.0 -m "First release" --revision develop
+        $ huggingface-cli tag user/my-dataset 1.0 -m "First release" --repo-type dataset
+        $ huggingface-cli tag user/my-space 1.0
+    - List all tags:
+        $ huggingface-cli tag -l user/my-model
+        $ huggingface-cli tag --list user/my-dataset --repo-type dataset
+    - Delete a tag:
+        $ huggingface-cli tag -d user/my-model 1.0
+        $ huggingface-cli tag --delete user/my-dataset 1.0 --repo-type dataset
+        $ huggingface-cli tag -d user/my-space 1.0 -y
+"""
+
+from argparse import Namespace, _SubParsersAction
+
+from requests.exceptions import HTTPError
+
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.constants import (
+    REPO_TYPES,
+)
+from huggingface_hub.hf_api import HfApi
+
+from ..errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError
+from ._cli_utils import ANSI
+
+
+class TagCommands(BaseHuggingfaceCLICommand):
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        tag_parser = parser.add_parser("tag", help="(create, list, delete) tags for a repo in the hub")
+
+        tag_parser.add_argument("repo_id", type=str, help="The ID of the repo to tag (e.g. `username/repo-name`).")
+        tag_parser.add_argument("tag", nargs="?", type=str, help="The name of the tag for creation or deletion.")
+        tag_parser.add_argument("-m", "--message", type=str, help="The description of the tag to create.")
+        tag_parser.add_argument("--revision", type=str, help="The git revision to tag.")
+        tag_parser.add_argument(
+            "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens."
+        )
+        tag_parser.add_argument(
+            "--repo-type",
+            choices=["model", "dataset", "space"],
+            default="model",
+            help="Set the type of repository (model, dataset, or space).",
+        )
+        tag_parser.add_argument("-y", "--yes", action="store_true", help="Answer Yes to prompts automatically.")
+
+        tag_parser.add_argument("-l", "--list", action="store_true", help="List tags for a repository.")
+        tag_parser.add_argument("-d", "--delete", action="store_true", help="Delete a tag for a repository.")
+
+        tag_parser.set_defaults(func=lambda args: handle_commands(args))
+
+
+def handle_commands(args: Namespace):
+    if args.list:
+        return TagListCommand(args)
+    elif args.delete:
+        return TagDeleteCommand(args)
+    else:
+        return TagCreateCommand(args)
+
+
+class TagCommand:
+    def __init__(self, args: Namespace):
+        self.args = args
+        self.api = HfApi(token=self.args.token)
+        self.repo_id = self.args.repo_id
+        self.repo_type = self.args.repo_type
+        if self.repo_type not in REPO_TYPES:
+            print("Invalid repo --repo-type")
+            exit(1)
+
+
+class TagCreateCommand(TagCommand):
+    def run(self):
+        print(f"You are about to create tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}")
+
+        try:
+            self.api.create_tag(
+                repo_id=self.repo_id,
+                tag=self.args.tag,
+                tag_message=self.args.message,
+                revision=self.args.revision,
+                repo_type=self.repo_type,
+            )
+        except RepositoryNotFoundError:
+            print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.")
+            exit(1)
+        except RevisionNotFoundError:
+            print(f"Revision {ANSI.bold(self.args.revision)} not found.")
+            exit(1)
+        except HfHubHTTPError as e:
+            if e.response.status_code == 409:
+                print(f"Tag {ANSI.bold(self.args.tag)} already exists on {ANSI.bold(self.repo_id)}")
+                exit(1)
+            raise e
+
+        print(f"Tag {ANSI.bold(self.args.tag)} created on {ANSI.bold(self.repo_id)}")
+
+
+class TagListCommand(TagCommand):
+    def run(self):
+        try:
+            refs = self.api.list_repo_refs(
+                repo_id=self.repo_id,
+                repo_type=self.repo_type,
+            )
+        except RepositoryNotFoundError:
+            print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.")
+            exit(1)
+        except HTTPError as e:
+            print(e)
+            print(ANSI.red(e.response.text))
+            exit(1)
+        if len(refs.tags) == 0:
+            print("No tags found")
+            exit(0)
+        print(f"Tags for {self.repo_type} {ANSI.bold(self.repo_id)}:")
+        for tag in refs.tags:
+            print(tag.name)
+
+
+class TagDeleteCommand(TagCommand):
+    def run(self):
+        print(f"You are about to delete tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}")
+
+        if not self.args.yes:
+            choice = input("Proceed? [Y/n] ").lower()
+            if choice not in ("", "y", "yes"):
+                print("Abort")
+                exit()
+        try:
+            self.api.delete_tag(repo_id=self.repo_id, tag=self.args.tag, repo_type=self.repo_type)
+        except RepositoryNotFoundError:
+            print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.")
+            exit(1)
+        except RevisionNotFoundError:
+            print(f"Tag {ANSI.bold(self.args.tag)} not found on {ANSI.bold(self.repo_id)}")
+            exit(1)
+        print(f"Tag {ANSI.bold(self.args.tag)} deleted on {ANSI.bold(self.repo_id)}")

.venv/lib/python3.11/site-packages/huggingface_hub/commands/upload.py

@@ -0,0 +1,299 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to upload a repo or file with the CLI.
+
+Usage:
+    # Upload file (implicit)
+    huggingface-cli upload my-cool-model ./my-cool-model.safetensors
+
+    # Upload file (explicit)
+    huggingface-cli upload my-cool-model ./my-cool-model.safetensors  model.safetensors
+
+    # Upload directory (implicit). If `my-cool-model/` is a directory it will be uploaded, otherwise an exception is raised.
+    huggingface-cli upload my-cool-model
+
+    # Upload directory (explicit)
+    huggingface-cli upload my-cool-model ./models/my-cool-model .
+
+    # Upload filtered directory (example: tensorboard logs except for the last run)
+    huggingface-cli upload my-cool-model ./model/training /logs --include "*.tfevents.*" --exclude "*20230905*"
+
+    # Upload private dataset
+    huggingface-cli upload Wauplin/my-cool-dataset ./data . --repo-type=dataset --private
+
+    # Upload with token
+    huggingface-cli upload Wauplin/my-cool-model --token=hf_****
+
+    # Sync local Space with Hub (upload new files, delete removed files)
+    huggingface-cli upload Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub"
+
+    # Schedule commits every 30 minutes
+    huggingface-cli upload Wauplin/my-cool-model --every=30
+"""
+
+import os
+import time
+import warnings
+from argparse import Namespace, _SubParsersAction
+from typing import List, Optional
+
+from huggingface_hub import logging
+from huggingface_hub._commit_scheduler import CommitScheduler
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.constants import HF_HUB_ENABLE_HF_TRANSFER
+from huggingface_hub.errors import RevisionNotFoundError
+from huggingface_hub.hf_api import HfApi
+from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
+
+
+logger = logging.get_logger(__name__)
+
+
+class UploadCommand(BaseHuggingfaceCLICommand):
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        upload_parser = parser.add_parser("upload", help="Upload a file or a folder to a repo on the Hub")
+        upload_parser.add_argument(
+            "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)."
+        )
+        upload_parser.add_argument(
+            "local_path", nargs="?", help="Local path to the file or folder to upload. Defaults to current directory."
+        )
+        upload_parser.add_argument(
+            "path_in_repo",
+            nargs="?",
+            help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.",
+        )
+        upload_parser.add_argument(
+            "--repo-type",
+            choices=["model", "dataset", "space"],
+            default="model",
+            help="Type of the repo to upload to (e.g. `dataset`).",
+        )
+        upload_parser.add_argument(
+            "--revision",
+            type=str,
+            help=(
+                "An optional Git revision to push to. It can be a branch name or a PR reference. If revision does not"
+                " exist and `--create-pr` is not set, a branch will be automatically created."
+            ),
+        )
+        upload_parser.add_argument(
+            "--private",
+            action="store_true",
+            help=(
+                "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already"
+                " exists."
+            ),
+        )
+        upload_parser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.")
+        upload_parser.add_argument(
+            "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload."
+        )
+        upload_parser.add_argument(
+            "--delete",
+            nargs="*",
+            type=str,
+            help="Glob patterns for file to be deleted from the repo while committing.",
+        )
+        upload_parser.add_argument(
+            "--commit-message", type=str, help="The summary / title / first line of the generated commit."
+        )
+        upload_parser.add_argument("--commit-description", type=str, help="The description of the generated commit.")
+        upload_parser.add_argument(
+            "--create-pr", action="store_true", help="Whether to upload content as a new Pull Request."
+        )
+        upload_parser.add_argument(
+            "--every",
+            type=float,
+            help="If set, a background job is scheduled to create commits every `every` minutes.",
+        )
+        upload_parser.add_argument(
+            "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
+        )
+        upload_parser.add_argument(
+            "--quiet",
+            action="store_true",
+            help="If True, progress bars are disabled and only the path to the uploaded files is printed.",
+        )
+        upload_parser.set_defaults(func=UploadCommand)
+
+    def __init__(self, args: Namespace) -> None:
+        self.repo_id: str = args.repo_id
+        self.repo_type: Optional[str] = args.repo_type
+        self.revision: Optional[str] = args.revision
+        self.private: bool = args.private
+
+        self.include: Optional[List[str]] = args.include
+        self.exclude: Optional[List[str]] = args.exclude
+        self.delete: Optional[List[str]] = args.delete
+
+        self.commit_message: Optional[str] = args.commit_message
+        self.commit_description: Optional[str] = args.commit_description
+        self.create_pr: bool = args.create_pr
+        self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
+        self.quiet: bool = args.quiet  # disable warnings and progress bars
+
+        # Check `--every` is valid
+        if args.every is not None and args.every <= 0:
+            raise ValueError(f"`every` must be a positive value (got '{args.every}')")
+        self.every: Optional[float] = args.every
+
+        # Resolve `local_path` and `path_in_repo`
+        repo_name: str = args.repo_id.split("/")[-1]  # e.g. "Wauplin/my-cool-model" => "my-cool-model"
+        self.local_path: str
+        self.path_in_repo: str
+        if args.local_path is None and os.path.isfile(repo_name):
+            # Implicit case 1: user provided only a repo_id which happen to be a local file as well => upload it with same name
+            self.local_path = repo_name
+            self.path_in_repo = repo_name
+        elif args.local_path is None and os.path.isdir(repo_name):
+            # Implicit case 2: user provided only a repo_id which happen to be a local folder as well => upload it at root
+            self.local_path = repo_name
+            self.path_in_repo = "."
+        elif args.local_path is None:
+            # Implicit case 3: user provided only a repo_id that does not match a local file or folder
+            # => the user must explicitly provide a local_path => raise exception
+            raise ValueError(f"'{repo_name}' is not a local file or folder. Please set `local_path` explicitly.")
+        elif args.path_in_repo is None and os.path.isfile(args.local_path):
+            # Explicit local path to file, no path in repo => upload it at root with same name
+            self.local_path = args.local_path
+            self.path_in_repo = os.path.basename(args.local_path)
+        elif args.path_in_repo is None:
+            # Explicit local path to folder, no path in repo => upload at root
+            self.local_path = args.local_path
+            self.path_in_repo = "."
+        else:
+            # Finally, if both paths are explicit
+            self.local_path = args.local_path
+            self.path_in_repo = args.path_in_repo
+
+    def run(self) -> None:
+        if self.quiet:
+            disable_progress_bars()
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                print(self._upload())
+            enable_progress_bars()
+        else:
+            logging.set_verbosity_info()
+            print(self._upload())
+            logging.set_verbosity_warning()
+
+    def _upload(self) -> str:
+        if os.path.isfile(self.local_path):
+            if self.include is not None and len(self.include) > 0:
+                warnings.warn("Ignoring `--include` since a single file is uploaded.")
+            if self.exclude is not None and len(self.exclude) > 0:
+                warnings.warn("Ignoring `--exclude` since a single file is uploaded.")
+            if self.delete is not None and len(self.delete) > 0:
+                warnings.warn("Ignoring `--delete` since a single file is uploaded.")
+
+        if not HF_HUB_ENABLE_HF_TRANSFER:
+            logger.info(
+                "Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See"
+                " https://huggingface.co/docs/huggingface_hub/hf_transfer for more details."
+            )
+
+        # Schedule commits if `every` is set
+        if self.every is not None:
+            if os.path.isfile(self.local_path):
+                # If file => watch entire folder + use allow_patterns
+                folder_path = os.path.dirname(self.local_path)
+                path_in_repo = (
+                    self.path_in_repo[: -len(self.local_path)]  # remove filename from path_in_repo
+                    if self.path_in_repo.endswith(self.local_path)
+                    else self.path_in_repo
+                )
+                allow_patterns = [self.local_path]
+                ignore_patterns = []
+            else:
+                folder_path = self.local_path
+                path_in_repo = self.path_in_repo
+                allow_patterns = self.include or []
+                ignore_patterns = self.exclude or []
+                if self.delete is not None and len(self.delete) > 0:
+                    warnings.warn("Ignoring `--delete` when uploading with scheduled commits.")
+
+            scheduler = CommitScheduler(
+                folder_path=folder_path,
+                repo_id=self.repo_id,
+                repo_type=self.repo_type,
+                revision=self.revision,
+                allow_patterns=allow_patterns,
+                ignore_patterns=ignore_patterns,
+                path_in_repo=path_in_repo,
+                private=self.private,
+                every=self.every,
+                hf_api=self.api,
+            )
+            print(f"Scheduling commits every {self.every} minutes to {scheduler.repo_id}.")
+            try:  # Block main thread until KeyboardInterrupt
+                while True:
+                    time.sleep(100)
+            except KeyboardInterrupt:
+                scheduler.stop()
+                return "Stopped scheduled commits."
+
+        # Otherwise, create repo and proceed with the upload
+        if not os.path.isfile(self.local_path) and not os.path.isdir(self.local_path):
+            raise FileNotFoundError(f"No such file or directory: '{self.local_path}'.")
+        repo_id = self.api.create_repo(
+            repo_id=self.repo_id,
+            repo_type=self.repo_type,
+            exist_ok=True,
+            private=self.private,
+            space_sdk="gradio" if self.repo_type == "space" else None,
+            # ^ We don't want it to fail when uploading to a Space => let's set Gradio by default.
+            # ^ I'd rather not add CLI args to set it explicitly as we already have `huggingface-cli repo create` for that.
+        ).repo_id
+
+        # Check if branch already exists and if not, create it
+        if self.revision is not None and not self.create_pr:
+            try:
+                self.api.repo_info(repo_id=repo_id, repo_type=self.repo_type, revision=self.revision)
+            except RevisionNotFoundError:
+                logger.info(f"Branch '{self.revision}' not found. Creating it...")
+                self.api.create_branch(repo_id=repo_id, repo_type=self.repo_type, branch=self.revision, exist_ok=True)
+                # ^ `exist_ok=True` to avoid race concurrency issues
+
+        # File-based upload
+        if os.path.isfile(self.local_path):
+            return self.api.upload_file(
+                path_or_fileobj=self.local_path,
+                path_in_repo=self.path_in_repo,
+                repo_id=repo_id,
+                repo_type=self.repo_type,
+                revision=self.revision,
+                commit_message=self.commit_message,
+                commit_description=self.commit_description,
+                create_pr=self.create_pr,
+            )
+
+        # Folder-based upload
+        else:
+            return self.api.upload_folder(
+                folder_path=self.local_path,
+                path_in_repo=self.path_in_repo,
+                repo_id=repo_id,
+                repo_type=self.repo_type,
+                revision=self.revision,
+                commit_message=self.commit_message,
+                commit_description=self.commit_description,
+                create_pr=self.create_pr,
+                allow_patterns=self.include,
+                ignore_patterns=self.exclude,
+                delete_patterns=self.delete,
+            )

.venv/lib/python3.11/site-packages/huggingface_hub/commands/upload_large_folder.py

@@ -0,0 +1,129 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to upload a large folder with the CLI."""
+
+import os
+from argparse import Namespace, _SubParsersAction
+from typing import List, Optional
+
+from huggingface_hub import logging
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.hf_api import HfApi
+from huggingface_hub.utils import disable_progress_bars
+
+from ._cli_utils import ANSI
+
+
+logger = logging.get_logger(__name__)
+
+
+class UploadLargeFolderCommand(BaseHuggingfaceCLICommand):
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        subparser = parser.add_parser("upload-large-folder", help="Upload a large folder to a repo on the Hub")
+        subparser.add_argument(
+            "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)."
+        )
+        subparser.add_argument("local_path", type=str, help="Local path to the file or folder to upload.")
+        subparser.add_argument(
+            "--repo-type",
+            choices=["model", "dataset", "space"],
+            help="Type of the repo to upload to (e.g. `dataset`).",
+        )
+        subparser.add_argument(
+            "--revision",
+            type=str,
+            help=("An optional Git revision to push to. It can be a branch name or a PR reference."),
+        )
+        subparser.add_argument(
+            "--private",
+            action="store_true",
+            help=(
+                "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists."
+            ),
+        )
+        subparser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.")
+        subparser.add_argument("--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload.")
+        subparser.add_argument(
+            "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
+        )
+        subparser.add_argument(
+            "--num-workers", type=int, help="Number of workers to use to hash, upload and commit files."
+        )
+        subparser.add_argument("--no-report", action="store_true", help="Whether to disable regular status report.")
+        subparser.add_argument("--no-bars", action="store_true", help="Whether to disable progress bars.")
+        subparser.set_defaults(func=UploadLargeFolderCommand)
+
+    def __init__(self, args: Namespace) -> None:
+        self.repo_id: str = args.repo_id
+        self.local_path: str = args.local_path
+        self.repo_type: str = args.repo_type
+        self.revision: Optional[str] = args.revision
+        self.private: bool = args.private
+
+        self.include: Optional[List[str]] = args.include
+        self.exclude: Optional[List[str]] = args.exclude
+
+        self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
+
+        self.num_workers: Optional[int] = args.num_workers
+        self.no_report: bool = args.no_report
+        self.no_bars: bool = args.no_bars
+
+        if not os.path.isdir(self.local_path):
+            raise ValueError("Large upload is only supported for folders.")
+
+    def run(self) -> None:
+        logging.set_verbosity_info()
+
+        print(
+            ANSI.yellow(
+                "You are about to upload a large folder to the Hub using `huggingface-cli upload-large-folder`. "
+                "This is a new feature so feedback is very welcome!\n"
+                "\n"
+                "A few things to keep in mind:\n"
+                "  - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n"
+                "  - Do not start several processes in parallel.\n"
+                "  - You can interrupt and resume the process at any time. "
+                "The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n"
+                "  - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n"
+                "\n"
+                f"Some temporary metadata will be stored under `{self.local_path}/.cache/huggingface`.\n"
+                "  - You must not modify those files manually.\n"
+                "  - You must not delete the `./.cache/huggingface/` folder while a process is running.\n"
+                "  - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n"
+                "\n"
+                "If the process output is too verbose, you can disable the progress bars with `--no-bars`. "
+                "You can also entirely disable the status report with `--no-report`.\n"
+                "\n"
+                "For more details, run `huggingface-cli upload-large-folder --help` or check the documentation at "
+                "https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder."
+            )
+        )
+
+        if self.no_bars:
+            disable_progress_bars()
+
+        self.api.upload_large_folder(
+            repo_id=self.repo_id,
+            folder_path=self.local_path,
+            repo_type=self.repo_type,
+            revision=self.revision,
+            private=self.private,
+            allow_patterns=self.include,
+            ignore_patterns=self.exclude,
+            num_workers=self.num_workers,
+            print_report=not self.no_report,
+        )

.venv/lib/python3.11/site-packages/huggingface_hub/commands/user.py

@@ -0,0 +1,304 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains commands to authenticate to the Hugging Face Hub and interact with your repositories.
+
+Usage:
+    # login and save token locally.
+    huggingface-cli login --token=hf_*** --add-to-git-credential
+
+    # switch between tokens
+    huggingface-cli auth switch
+
+    # list all tokens
+    huggingface-cli auth list
+
+    # logout from a specific token, if no token-name is provided, all tokens will be deleted from your machine.
+    huggingface-cli logout --token-name=your_token_name
+
+    # find out which huggingface.co account you are logged in as
+    huggingface-cli whoami
+
+    # create a new dataset repo on the Hub
+    huggingface-cli repo create mydataset --type=dataset
+
+"""
+
+import subprocess
+from argparse import _SubParsersAction
+from typing import List, Optional
+
+from requests.exceptions import HTTPError
+
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.constants import ENDPOINT, REPO_TYPES, REPO_TYPES_URL_PREFIXES, SPACES_SDK_TYPES
+from huggingface_hub.hf_api import HfApi
+
+from .._login import (  # noqa: F401 # for backward compatibility  # noqa: F401 # for backward compatibility
+    NOTEBOOK_LOGIN_PASSWORD_HTML,
+    NOTEBOOK_LOGIN_TOKEN_HTML_END,
+    NOTEBOOK_LOGIN_TOKEN_HTML_START,
+    auth_list,
+    auth_switch,
+    login,
+    logout,
+    notebook_login,
+)
+from ..utils import get_stored_tokens, get_token, logging
+from ._cli_utils import ANSI
+
+
+logger = logging.get_logger(__name__)
+
+try:
+    from InquirerPy import inquirer
+    from InquirerPy.base.control import Choice
+
+    _inquirer_py_available = True
+except ImportError:
+    _inquirer_py_available = False
+
+
+class UserCommands(BaseHuggingfaceCLICommand):
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        login_parser = parser.add_parser("login", help="Log in using a token from huggingface.co/settings/tokens")
+        login_parser.add_argument(
+            "--token",
+            type=str,
+            help="Token generated from https://huggingface.co/settings/tokens",
+        )
+        login_parser.add_argument(
+            "--add-to-git-credential",
+            action="store_true",
+            help="Optional: Save token to git credential helper.",
+        )
+        login_parser.set_defaults(func=lambda args: LoginCommand(args))
+        whoami_parser = parser.add_parser("whoami", help="Find out which huggingface.co account you are logged in as.")
+        whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args))
+
+        logout_parser = parser.add_parser("logout", help="Log out")
+        logout_parser.add_argument(
+            "--token-name",
+            type=str,
+            help="Optional: Name of the access token to log out from.",
+        )
+        logout_parser.set_defaults(func=lambda args: LogoutCommand(args))
+
+        auth_parser = parser.add_parser("auth", help="Other authentication related commands")
+        auth_subparsers = auth_parser.add_subparsers(help="Authentication subcommands")
+        auth_switch_parser = auth_subparsers.add_parser("switch", help="Switch between access tokens")
+        auth_switch_parser.add_argument(
+            "--token-name",
+            type=str,
+            help="Optional: Name of the access token to switch to.",
+        )
+        auth_switch_parser.add_argument(
+            "--add-to-git-credential",
+            action="store_true",
+            help="Optional: Save token to git credential helper.",
+        )
+        auth_switch_parser.set_defaults(func=lambda args: AuthSwitchCommand(args))
+        auth_list_parser = auth_subparsers.add_parser("list", help="List all stored access tokens")
+        auth_list_parser.set_defaults(func=lambda args: AuthListCommand(args))
+        # new system: git-based repo system
+        repo_parser = parser.add_parser("repo", help="{create} Commands to interact with your huggingface.co repos.")
+        repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands")
+        repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co")
+        repo_create_parser.add_argument(
+            "name",
+            type=str,
+            help="Name for your repo. Will be namespaced under your username to build the repo id.",
+        )
+        repo_create_parser.add_argument(
+            "--type",
+            type=str,
+            help='Optional: repo_type: set to "dataset" or "space" if creating a dataset or space, default is model.',
+        )
+        repo_create_parser.add_argument("--organization", type=str, help="Optional: organization namespace.")
+        repo_create_parser.add_argument(
+            "--space_sdk",
+            type=str,
+            help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".',
+            choices=SPACES_SDK_TYPES,
+        )
+        repo_create_parser.add_argument(
+            "-y",
+            "--yes",
+            action="store_true",
+            help="Optional: answer Yes to the prompt",
+        )
+        repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args))
+
+
+class BaseUserCommand:
+    def __init__(self, args):
+        self.args = args
+        self._api = HfApi()
+
+
+class LoginCommand(BaseUserCommand):
+    def run(self):
+        logging.set_verbosity_info()
+        login(
+            token=self.args.token,
+            add_to_git_credential=self.args.add_to_git_credential,
+        )
+
+
+class LogoutCommand(BaseUserCommand):
+    def run(self):
+        logging.set_verbosity_info()
+        logout(token_name=self.args.token_name)
+
+
+class AuthSwitchCommand(BaseUserCommand):
+    def run(self):
+        logging.set_verbosity_info()
+        token_name = self.args.token_name
+        if token_name is None:
+            token_name = self._select_token_name()
+
+        if token_name is None:
+            print("No token name provided. Aborting.")
+            exit()
+        auth_switch(token_name, add_to_git_credential=self.args.add_to_git_credential)
+
+    def _select_token_name(self) -> Optional[str]:
+        token_names = list(get_stored_tokens().keys())
+
+        if not token_names:
+            logger.error("No stored tokens found. Please login first.")
+            return None
+
+        if _inquirer_py_available:
+            return self._select_token_name_tui(token_names)
+        # if inquirer is not available, use a simpler terminal UI
+        print("Available stored tokens:")
+        for i, token_name in enumerate(token_names, 1):
+            print(f"{i}. {token_name}")
+        while True:
+            try:
+                choice = input("Enter the number of the token to switch to (or 'q' to quit): ")
+                if choice.lower() == "q":
+                    return None
+                index = int(choice) - 1
+                if 0 <= index < len(token_names):
+                    return token_names[index]
+                else:
+                    print("Invalid selection. Please try again.")
+            except ValueError:
+                print("Invalid input. Please enter a number or 'q' to quit.")
+
+    def _select_token_name_tui(self, token_names: List[str]) -> Optional[str]:
+        choices = [Choice(token_name, name=token_name) for token_name in token_names]
+        try:
+            return inquirer.select(
+                message="Select a token to switch to:",
+                choices=choices,
+                default=None,
+            ).execute()
+        except KeyboardInterrupt:
+            logger.info("Token selection cancelled.")
+            return None
+
+
+class AuthListCommand(BaseUserCommand):
+    def run(self):
+        logging.set_verbosity_info()
+        auth_list()
+
+
+class WhoamiCommand(BaseUserCommand):
+    def run(self):
+        token = get_token()
+        if token is None:
+            print("Not logged in")
+            exit()
+        try:
+            info = self._api.whoami(token)
+            print(info["name"])
+            orgs = [org["name"] for org in info["orgs"]]
+            if orgs:
+                print(ANSI.bold("orgs: "), ",".join(orgs))
+
+            if ENDPOINT != "https://huggingface.co":
+                print(f"Authenticated through private endpoint: {ENDPOINT}")
+        except HTTPError as e:
+            print(e)
+            print(ANSI.red(e.response.text))
+            exit(1)
+
+
+class RepoCreateCommand(BaseUserCommand):
+    def run(self):
+        token = get_token()
+        if token is None:
+            print("Not logged in")
+            exit(1)
+        try:
+            stdout = subprocess.check_output(["git", "--version"]).decode("utf-8")
+            print(ANSI.gray(stdout.strip()))
+        except FileNotFoundError:
+            print("Looks like you do not have git installed, please install.")
+
+        try:
+            stdout = subprocess.check_output(["git-lfs", "--version"]).decode("utf-8")
+            print(ANSI.gray(stdout.strip()))
+        except FileNotFoundError:
+            print(
+                ANSI.red(
+                    "Looks like you do not have git-lfs installed, please install."
+                    " You can install from https://git-lfs.github.com/."
+                    " Then run `git lfs install` (you only have to do this once)."
+                )
+            )
+        print("")
+
+        user = self._api.whoami(token)["name"]
+        namespace = self.args.organization if self.args.organization is not None else user
+
+        repo_id = f"{namespace}/{self.args.name}"
+
+        if self.args.type not in REPO_TYPES:
+            print("Invalid repo --type")
+            exit(1)
+
+        if self.args.type in REPO_TYPES_URL_PREFIXES:
+            prefixed_repo_id = REPO_TYPES_URL_PREFIXES[self.args.type] + repo_id
+        else:
+            prefixed_repo_id = repo_id
+
+        print(f"You are about to create {ANSI.bold(prefixed_repo_id)}")
+
+        if not self.args.yes:
+            choice = input("Proceed? [Y/n] ").lower()
+            if not (choice == "" or choice == "y" or choice == "yes"):
+                print("Abort")
+                exit()
+        try:
+            url = self._api.create_repo(
+                repo_id=repo_id,
+                token=token,
+                repo_type=self.args.type,
+                space_sdk=self.args.space_sdk,
+            )
+        except HTTPError as e:
+            print(e)
+            print(ANSI.red(e.response.text))
+            exit(1)
+        print("\nYour repo now lives at:")
+        print(f"  {ANSI.bold(url)}")
+        print("\nYou can clone it locally with the command below, and commit/push as usual.")
+        print(f"\n  git clone {url}")
+        print("")

.venv/lib/python3.11/site-packages/huggingface_hub/commands/version.py

@@ -0,0 +1,37 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to print information about the version.
+
+Usage:
+    huggingface-cli version
+"""
+
+from argparse import _SubParsersAction
+
+from huggingface_hub import __version__
+
+from . import BaseHuggingfaceCLICommand
+
+
+class VersionCommand(BaseHuggingfaceCLICommand):
+    def __init__(self, args):
+        self.args = args
+
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        version_parser = parser.add_parser("version", help="Print information about the huggingface-cli version.")
+        version_parser.set_defaults(func=VersionCommand)
+
+    def run(self) -> None:
+        print(f"huggingface_hub version: {__version__}")

.venv/lib/python3.11/site-packages/huggingface_hub/utils/__init__.py

@@ -0,0 +1,110 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+# ruff: noqa: F401
+
+from huggingface_hub.errors import (
+    BadRequestError,
+    CacheNotFound,
+    CorruptedCacheException,
+    DisabledRepoError,
+    EntryNotFoundError,
+    FileMetadataError,
+    GatedRepoError,
+    HfHubHTTPError,
+    HFValidationError,
+    LocalEntryNotFoundError,
+    LocalTokenNotFoundError,
+    NotASafetensorsRepoError,
+    OfflineModeIsEnabled,
+    RepositoryNotFoundError,
+    RevisionNotFoundError,
+    SafetensorsParsingError,
+)
+
+from . import tqdm as _tqdm  # _tqdm is the module
+from ._auth import get_stored_tokens, get_token
+from ._cache_assets import cached_assets_path
+from ._cache_manager import (
+    CachedFileInfo,
+    CachedRepoInfo,
+    CachedRevisionInfo,
+    DeleteCacheStrategy,
+    HFCacheInfo,
+    scan_cache_dir,
+)
+from ._chunk_utils import chunk_iterable
+from ._datetime import parse_datetime
+from ._experimental import experimental
+from ._fixes import SoftTemporaryDirectory, WeakFileLock, yaml_dump
+from ._git_credential import list_credential_helpers, set_git_credential, unset_git_credential
+from ._headers import build_hf_headers, get_token_to_send
+from ._hf_folder import HfFolder
+from ._http import (
+    configure_http_backend,
+    fix_hf_endpoint_in_url,
+    get_session,
+    hf_raise_for_status,
+    http_backoff,
+    reset_sessions,
+)
+from ._pagination import paginate
+from ._paths import DEFAULT_IGNORE_PATTERNS, FORBIDDEN_FOLDERS, filter_repo_objects
+from ._runtime import (
+    dump_environment_info,
+    get_aiohttp_version,
+    get_fastai_version,
+    get_fastapi_version,
+    get_fastcore_version,
+    get_gradio_version,
+    get_graphviz_version,
+    get_hf_hub_version,
+    get_hf_transfer_version,
+    get_jinja_version,
+    get_numpy_version,
+    get_pillow_version,
+    get_pydantic_version,
+    get_pydot_version,
+    get_python_version,
+    get_tensorboard_version,
+    get_tf_version,
+    get_torch_version,
+    is_aiohttp_available,
+    is_colab_enterprise,
+    is_fastai_available,
+    is_fastapi_available,
+    is_fastcore_available,
+    is_google_colab,
+    is_gradio_available,
+    is_graphviz_available,
+    is_hf_transfer_available,
+    is_jinja_available,
+    is_notebook,
+    is_numpy_available,
+    is_package_available,
+    is_pillow_available,
+    is_pydantic_available,
+    is_pydot_available,
+    is_safetensors_available,
+    is_tensorboard_available,
+    is_tf_available,
+    is_torch_available,
+)
+from ._safetensors import SafetensorsFileMetadata, SafetensorsRepoMetadata, TensorInfo
+from ._subprocess import capture_output, run_interactive_subprocess, run_subprocess
+from ._telemetry import send_telemetry
+from ._typing import is_jsonable, is_simple_optional_type, unwrap_simple_optional_type
+from ._validators import smoothly_deprecate_use_auth_token, validate_hf_hub_args, validate_repo_id
+from .tqdm import are_progress_bars_disabled, disable_progress_bars, enable_progress_bars, tqdm, tqdm_stream_file

.venv/lib/python3.11/site-packages/huggingface_hub/utils/_auth.py

@@ -0,0 +1,214 @@
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains an helper to get the token from machine (env variable, secret or config file)."""
+
+import configparser
+import logging
+import os
+import warnings
+from pathlib import Path
+from threading import Lock
+from typing import Dict, Optional
+
+from .. import constants
+from ._runtime import is_colab_enterprise, is_google_colab
+
+
+_IS_GOOGLE_COLAB_CHECKED = False
+_GOOGLE_COLAB_SECRET_LOCK = Lock()
+_GOOGLE_COLAB_SECRET: Optional[str] = None
+
+logger = logging.getLogger(__name__)
+
+
+def get_token() -> Optional[str]:
+    """
+    Get token if user is logged in.
+
+    Note: in most cases, you should use [`huggingface_hub.utils.build_hf_headers`] instead. This method is only useful
+          if you want to retrieve the token for other purposes than sending an HTTP request.
+
+    Token is retrieved in priority from the `HF_TOKEN` environment variable. Otherwise, we read the token file located
+    in the Hugging Face home folder. Returns None if user is not logged in. To log in, use [`login`] or
+    `huggingface-cli login`.
+
+    Returns:
+        `str` or `None`: The token, `None` if it doesn't exist.
+    """
+    return _get_token_from_google_colab() or _get_token_from_environment() or _get_token_from_file()
+
+
+def _get_token_from_google_colab() -> Optional[str]:
+    """Get token from Google Colab secrets vault using `google.colab.userdata.get(...)`.
+
+    Token is read from the vault only once per session and then stored in a global variable to avoid re-requesting
+    access to the vault.
+    """
+    # If it's not a Google Colab or it's Colab Enterprise, fallback to environment variable or token file authentication
+    if not is_google_colab() or is_colab_enterprise():
+        return None
+
+    # `google.colab.userdata` is not thread-safe
+    # This can lead to a deadlock if multiple threads try to access it at the same time
+    # (typically when using `snapshot_download`)
+    # => use a lock
+    # See https://github.com/huggingface/huggingface_hub/issues/1952 for more details.
+    with _GOOGLE_COLAB_SECRET_LOCK:
+        global _GOOGLE_COLAB_SECRET
+        global _IS_GOOGLE_COLAB_CHECKED
+
+        if _IS_GOOGLE_COLAB_CHECKED:  # request access only once
+            return _GOOGLE_COLAB_SECRET
+
+        try:
+            from google.colab import userdata  # type: ignore
+            from google.colab.errors import Error as ColabError  # type: ignore
+        except ImportError:
+            return None
+
+        try:
+            token = userdata.get("HF_TOKEN")
+            _GOOGLE_COLAB_SECRET = _clean_token(token)
+        except userdata.NotebookAccessError:
+            # Means the user has a secret call `HF_TOKEN` and got a popup "please grand access to HF_TOKEN" and refused it
+            # => warn user but ignore error => do not re-request access to user
+            warnings.warn(
+                "\nAccess to the secret `HF_TOKEN` has not been granted on this notebook."
+                "\nYou will not be requested again."
+                "\nPlease restart the session if you want to be prompted again."
+            )
+            _GOOGLE_COLAB_SECRET = None
+        except userdata.SecretNotFoundError:
+            # Means the user did not define a `HF_TOKEN` secret => warn
+            warnings.warn(
+                "\nThe secret `HF_TOKEN` does not exist in your Colab secrets."
+                "\nTo authenticate with the Hugging Face Hub, create a token in your settings tab "
+                "(https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session."
+                "\nYou will be able to reuse this secret in all of your notebooks."
+                "\nPlease note that authentication is recommended but still optional to access public models or datasets."
+            )
+            _GOOGLE_COLAB_SECRET = None
+        except ColabError as e:
+            # Something happen but we don't know what => recommend to open a GitHub issue
+            warnings.warn(
+                f"\nError while fetching `HF_TOKEN` secret value from your vault: '{str(e)}'."
+                "\nYou are not authenticated with the Hugging Face Hub in this notebook."
+                "\nIf the error persists, please let us know by opening an issue on GitHub "
+                "(https://github.com/huggingface/huggingface_hub/issues/new)."
+            )
+            _GOOGLE_COLAB_SECRET = None
+
+        _IS_GOOGLE_COLAB_CHECKED = True
+        return _GOOGLE_COLAB_SECRET
+
+
+def _get_token_from_environment() -> Optional[str]:
+    # `HF_TOKEN` has priority (keep `HUGGING_FACE_HUB_TOKEN` for backward compatibility)
+    return _clean_token(os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN"))
+
+
+def _get_token_from_file() -> Optional[str]:
+    try:
+        return _clean_token(Path(constants.HF_TOKEN_PATH).read_text())
+    except FileNotFoundError:
+        return None
+
+
+def get_stored_tokens() -> Dict[str, str]:
+    """
+    Returns the parsed INI file containing the access tokens.
+    The file is located at `HF_STORED_TOKENS_PATH`, defaulting to `~/.cache/huggingface/stored_tokens`.
+    If the file does not exist, an empty dictionary is returned.
+
+    Returns: `Dict[str, str]`
+        Key is the token name and value is the token.
+    """
+    tokens_path = Path(constants.HF_STORED_TOKENS_PATH)
+    if not tokens_path.exists():
+        stored_tokens = {}
+    config = configparser.ConfigParser()
+    try:
+        config.read(tokens_path)
+        stored_tokens = {token_name: config.get(token_name, "hf_token") for token_name in config.sections()}
+    except configparser.Error as e:
+        logger.error(f"Error parsing stored tokens file: {e}")
+        stored_tokens = {}
+    return stored_tokens
+
+
+def _save_stored_tokens(stored_tokens: Dict[str, str]) -> None:
+    """
+    Saves the given configuration to the stored tokens file.
+
+    Args:
+        stored_tokens (`Dict[str, str]`):
+            The stored tokens to save. Key is the token name and value is the token.
+    """
+    stored_tokens_path = Path(constants.HF_STORED_TOKENS_PATH)
+
+    # Write the stored tokens into an INI file
+    config = configparser.ConfigParser()
+    for token_name in sorted(stored_tokens.keys()):
+        config.add_section(token_name)
+        config.set(token_name, "hf_token", stored_tokens[token_name])
+
+    stored_tokens_path.parent.mkdir(parents=True, exist_ok=True)
+    with stored_tokens_path.open("w") as config_file:
+        config.write(config_file)
+
+
+def _get_token_by_name(token_name: str) -> Optional[str]:
+    """
+    Get the token by name.
+
+    Args:
+        token_name (`str`):
+            The name of the token to get.
+
+    Returns:
+        `str` or `None`: The token, `None` if it doesn't exist.
+
+    """
+    stored_tokens = get_stored_tokens()
+    if token_name not in stored_tokens:
+        return None
+    return _clean_token(stored_tokens[token_name])
+
+
+def _save_token(token: str, token_name: str) -> None:
+    """
+    Save the given token.
+
+    If the stored tokens file does not exist, it will be created.
+    Args:
+        token (`str`):
+            The token to save.
+        token_name (`str`):
+            The name of the token.
+    """
+    tokens_path = Path(constants.HF_STORED_TOKENS_PATH)
+    stored_tokens = get_stored_tokens()
+    stored_tokens[token_name] = token
+    _save_stored_tokens(stored_tokens)
+    logger.info(f"The token `{token_name}` has been saved to {tokens_path}")
+
+
+def _clean_token(token: Optional[str]) -> Optional[str]:
+    """Clean token by removing trailing and leading spaces and newlines.
+
+    If token is an empty string, return None.
+    """
+    if token is None:
+        return None
+    return token.replace("\r", "").replace("\n", "").strip() or None