| import copy |
| from dataclasses import dataclass, field |
| from pathlib import Path |
| from typing import Any, Optional, Union |
|
|
| from .. import config |
|
|
|
|
| @dataclass |
| class DownloadConfig: |
| """Configuration for our cached path manager. |
| |
| Attributes: |
| cache_dir (`str` or `Path`, *optional*): |
| Specify a cache directory to save the file to (overwrite the |
| default cache dir). |
| force_download (`bool`, defaults to `False`): |
| If `True`, re-download the file even if it's already cached in |
| the cache dir. |
| resume_download (`bool`, defaults to `False`): |
| If `True`, resume the download if an incompletely received file is |
| found. |
| proxies (`dict`, *optional*): |
| user_agent (`str`, *optional*): |
| Optional string or dict that will be appended to the user-agent on remote |
| requests. |
| extract_compressed_file (`bool`, defaults to `False`): |
| If `True` and the path point to a zip or tar file, |
| extract the compressed file in a folder along the archive. |
| force_extract (`bool`, defaults to `False`): |
| If `True` when `extract_compressed_file` is `True` and the archive |
| was already extracted, re-extract the archive and override the folder where it was extracted. |
| delete_extracted (`bool`, defaults to `False`): |
| Whether to delete (or keep) the extracted files. |
| extract_on_the_fly (`bool`, defaults to `False`): |
| If `True`, extract compressed files while they are being read. |
| use_etag (`bool`, defaults to `True`): |
| Whether to use the ETag HTTP response header to validate the cached files. |
| num_proc (`int`, *optional*): |
| The number of processes to launch to download the files in parallel. |
| max_retries (`int`, default to `1`): |
| The number of times to retry an HTTP request if it fails. |
| token (`str` or `bool`, *optional*): |
| Optional string or boolean to use as Bearer token |
| for remote files on the Datasets Hub. If `True`, or not specified, will get token from `~/.huggingface`. |
| storage_options (`dict`, *optional*): |
| Key/value pairs to be passed on to the dataset file-system backend, if any. |
| download_desc (`str`, *optional*): |
| A description to be displayed alongside with the progress bar while downloading the files. |
| disable_tqdm (`bool`, defaults to `False`): |
| Whether to disable the individual files download progress bar |
| """ |
|
|
| cache_dir: Optional[Union[str, Path]] = None |
| force_download: bool = False |
| resume_download: bool = False |
| local_files_only: bool = False |
| proxies: Optional[dict] = None |
| user_agent: Optional[str] = None |
| extract_compressed_file: bool = False |
| force_extract: bool = False |
| delete_extracted: bool = False |
| extract_on_the_fly: bool = False |
| use_etag: bool = True |
| num_proc: Optional[int] = None |
| max_retries: int = 1 |
| token: Optional[Union[str, bool]] = None |
| storage_options: dict[str, Any] = field(default_factory=dict) |
| download_desc: Optional[str] = None |
| disable_tqdm: bool = False |
|
|
| def copy(self) -> "DownloadConfig": |
| return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()}) |
|
|
| def __setattr__(self, name, value): |
| if name == "token" and getattr(self, "storage_options", None) is not None: |
| if "hf" not in self.storage_options: |
| self.storage_options["hf"] = {"token": value, "endpoint": config.HF_ENDPOINT} |
| elif getattr(self.storage_options["hf"], "token", None) is None: |
| self.storage_options["hf"]["token"] = value |
| super().__setattr__(name, value) |
|
|
