| | from __future__ import annotations |
| |
|
| | import importlib |
| | import types |
| | import warnings |
| |
|
| | __all__ = ["registry", "get_filesystem_class", "default"] |
| |
|
| | |
| | _registry: dict[str, type] = {} |
| |
|
| | |
| | registry = types.MappingProxyType(_registry) |
| | default = "file" |
| |
|
| |
|
| | def register_implementation(name, cls, clobber=False, errtxt=None): |
| | """Add implementation class to the registry |
| | |
| | Parameters |
| | ---------- |
| | name: str |
| | Protocol name to associate with the class |
| | cls: class or str |
| | if a class: fsspec-compliant implementation class (normally inherits from |
| | ``fsspec.AbstractFileSystem``, gets added straight to the registry. If a |
| | str, the full path to an implementation class like package.module.class, |
| | which gets added to known_implementations, |
| | so the import is deferred until the filesystem is actually used. |
| | clobber: bool (optional) |
| | Whether to overwrite a protocol with the same name; if False, will raise |
| | instead. |
| | errtxt: str (optional) |
| | If given, then a failure to import the given class will result in this |
| | text being given. |
| | """ |
| | if isinstance(cls, str): |
| | if name in known_implementations and clobber is False: |
| | if cls != known_implementations[name]["class"]: |
| | raise ValueError( |
| | f"Name ({name}) already in the known_implementations and clobber " |
| | f"is False" |
| | ) |
| | else: |
| | known_implementations[name] = { |
| | "class": cls, |
| | "err": errtxt or f"{cls} import failed for protocol {name}", |
| | } |
| |
|
| | else: |
| | if name in registry and clobber is False: |
| | if _registry[name] is not cls: |
| | raise ValueError( |
| | f"Name ({name}) already in the registry and clobber is False" |
| | ) |
| | else: |
| | _registry[name] = cls |
| |
|
| |
|
| | |
| | |
| | known_implementations = { |
| | "data": {"class": "fsspec.implementations.data.DataFileSystem"}, |
| | "file": {"class": "fsspec.implementations.local.LocalFileSystem"}, |
| | "local": {"class": "fsspec.implementations.local.LocalFileSystem"}, |
| | "memory": {"class": "fsspec.implementations.memory.MemoryFileSystem"}, |
| | "dropbox": { |
| | "class": "dropboxdrivefs.DropboxDriveFileSystem", |
| | "err": ( |
| | 'DropboxFileSystem requires "dropboxdrivefs",' |
| | '"requests" and "dropbox" to be installed' |
| | ), |
| | }, |
| | "http": { |
| | "class": "fsspec.implementations.http.HTTPFileSystem", |
| | "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', |
| | }, |
| | "https": { |
| | "class": "fsspec.implementations.http.HTTPFileSystem", |
| | "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', |
| | }, |
| | "zip": {"class": "fsspec.implementations.zip.ZipFileSystem"}, |
| | "tar": {"class": "fsspec.implementations.tar.TarFileSystem"}, |
| | "gcs": { |
| | "class": "gcsfs.GCSFileSystem", |
| | "err": "Please install gcsfs to access Google Storage", |
| | }, |
| | "gs": { |
| | "class": "gcsfs.GCSFileSystem", |
| | "err": "Please install gcsfs to access Google Storage", |
| | }, |
| | "gdrive": { |
| | "class": "gdrivefs.GoogleDriveFileSystem", |
| | "err": "Please install gdrivefs for access to Google Drive", |
| | }, |
| | "sftp": { |
| | "class": "fsspec.implementations.sftp.SFTPFileSystem", |
| | "err": 'SFTPFileSystem requires "paramiko" to be installed', |
| | }, |
| | "ssh": { |
| | "class": "fsspec.implementations.sftp.SFTPFileSystem", |
| | "err": 'SFTPFileSystem requires "paramiko" to be installed', |
| | }, |
| | "ftp": {"class": "fsspec.implementations.ftp.FTPFileSystem"}, |
| | "hdfs": { |
| | "class": "fsspec.implementations.arrow.HadoopFileSystem", |
| | "err": "pyarrow and local java libraries required for HDFS", |
| | }, |
| | "arrow_hdfs": { |
| | "class": "fsspec.implementations.arrow.HadoopFileSystem", |
| | "err": "pyarrow and local java libraries required for HDFS", |
| | }, |
| | "webhdfs": { |
| | "class": "fsspec.implementations.webhdfs.WebHDFS", |
| | "err": 'webHDFS access requires "requests" to be installed', |
| | }, |
| | "s3": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, |
| | "s3a": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, |
| | "wandb": {"class": "wandbfs.WandbFS", "err": "Install wandbfs to access wandb"}, |
| | "oci": { |
| | "class": "ocifs.OCIFileSystem", |
| | "err": "Install ocifs to access OCI Object Storage", |
| | }, |
| | "ocilake": { |
| | "class": "ocifs.OCIFileSystem", |
| | "err": "Install ocifs to access OCI Data Lake", |
| | }, |
| | "asynclocal": { |
| | "class": "morefs.asyn_local.AsyncLocalFileSystem", |
| | "err": "Install 'morefs[asynclocalfs]' to use AsyncLocalFileSystem", |
| | }, |
| | "adl": { |
| | "class": "adlfs.AzureDatalakeFileSystem", |
| | "err": "Install adlfs to access Azure Datalake Gen1", |
| | }, |
| | "abfs": { |
| | "class": "adlfs.AzureBlobFileSystem", |
| | "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage", |
| | }, |
| | "az": { |
| | "class": "adlfs.AzureBlobFileSystem", |
| | "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage", |
| | }, |
| | "cached": {"class": "fsspec.implementations.cached.CachingFileSystem"}, |
| | "blockcache": {"class": "fsspec.implementations.cached.CachingFileSystem"}, |
| | "filecache": {"class": "fsspec.implementations.cached.WholeFileCacheFileSystem"}, |
| | "simplecache": {"class": "fsspec.implementations.cached.SimpleCacheFileSystem"}, |
| | "dask": { |
| | "class": "fsspec.implementations.dask.DaskWorkerFileSystem", |
| | "err": "Install dask distributed to access worker file system", |
| | }, |
| | "dbfs": { |
| | "class": "fsspec.implementations.dbfs.DatabricksFileSystem", |
| | "err": "Install the requests package to use the DatabricksFileSystem", |
| | }, |
| | "github": { |
| | "class": "fsspec.implementations.github.GithubFileSystem", |
| | "err": "Install the requests package to use the github FS", |
| | }, |
| | "git": { |
| | "class": "fsspec.implementations.git.GitFileSystem", |
| | "err": "Install pygit2 to browse local git repos", |
| | }, |
| | "smb": { |
| | "class": "fsspec.implementations.smb.SMBFileSystem", |
| | "err": 'SMB requires "smbprotocol" or "smbprotocol[kerberos]" installed', |
| | }, |
| | "jupyter": { |
| | "class": "fsspec.implementations.jupyter.JupyterFileSystem", |
| | "err": "Jupyter FS requires requests to be installed", |
| | }, |
| | "jlab": { |
| | "class": "fsspec.implementations.jupyter.JupyterFileSystem", |
| | "err": "Jupyter FS requires requests to be installed", |
| | }, |
| | "libarchive": { |
| | "class": "fsspec.implementations.libarchive.LibArchiveFileSystem", |
| | "err": "LibArchive requires to be installed", |
| | }, |
| | "reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"}, |
| | "generic": {"class": "fsspec.generic.GenericFileSystem"}, |
| | "oss": { |
| | "class": "ossfs.OSSFileSystem", |
| | "err": "Install ossfs to access Alibaba Object Storage System", |
| | }, |
| | "webdav": { |
| | "class": "webdav4.fsspec.WebdavFileSystem", |
| | "err": "Install webdav4 to access WebDAV", |
| | }, |
| | "dvc": { |
| | "class": "dvc.api.DVCFileSystem", |
| | "err": "Install dvc to access DVCFileSystem", |
| | }, |
| | "hf": { |
| | "class": "huggingface_hub.HfFileSystem", |
| | "err": "Install huggingface_hub to access HfFileSystem", |
| | }, |
| | "root": { |
| | "class": "fsspec_xrootd.XRootDFileSystem", |
| | "err": "Install fsspec-xrootd to access xrootd storage system." |
| | + " Note: 'root' is the protocol name for xrootd storage systems," |
| | + " not referring to root directories", |
| | }, |
| | "dir": {"class": "fsspec.implementations.dirfs.DirFileSystem"}, |
| | "box": { |
| | "class": "boxfs.BoxFileSystem", |
| | "err": "Please install boxfs to access BoxFileSystem", |
| | }, |
| | "lakefs": { |
| | "class": "lakefs_spec.LakeFSFileSystem", |
| | "err": "Please install lakefs-spec to access LakeFSFileSystem", |
| | }, |
| | } |
| |
|
| |
|
| | def get_filesystem_class(protocol): |
| | """Fetch named protocol implementation from the registry |
| | |
| | The dict ``known_implementations`` maps protocol names to the locations |
| | of classes implementing the corresponding file-system. When used for the |
| | first time, appropriate imports will happen and the class will be placed in |
| | the registry. All subsequent calls will fetch directly from the registry. |
| | |
| | Some protocol implementations require additional dependencies, and so the |
| | import may fail. In this case, the string in the "err" field of the |
| | ``known_implementations`` will be given as the error message. |
| | """ |
| | if not protocol: |
| | protocol = default |
| |
|
| | if protocol not in registry: |
| | if protocol not in known_implementations: |
| | raise ValueError(f"Protocol not known: {protocol}") |
| | bit = known_implementations[protocol] |
| | try: |
| | register_implementation(protocol, _import_class(bit["class"])) |
| | except ImportError as e: |
| | raise ImportError(bit["err"]) from e |
| | cls = registry[protocol] |
| | if getattr(cls, "protocol", None) in ("abstract", None): |
| | cls.protocol = protocol |
| |
|
| | return cls |
| |
|
| |
|
| | s3_msg = """Your installed version of s3fs is very old and known to cause |
| | severe performance issues, see also https://github.com/dask/dask/issues/10276 |
| | |
| | To fix, you should specify a lower version bound on s3fs, or |
| | update the current installation. |
| | """ |
| |
|
| |
|
| | def _import_class(cls, minv=None): |
| | """Take a string FQP and return the imported class or identifier |
| | |
| | clas is of the form "package.module.klass" or "package.module:subobject.klass" |
| | """ |
| | if ":" in cls: |
| | mod, name = cls.rsplit(":", 1) |
| | s3 = mod == "s3fs" |
| | mod = importlib.import_module(mod) |
| | if s3 and mod.__version__.split(".") < ["0", "5"]: |
| | warnings.warn(s3_msg) |
| | for part in name.split("."): |
| | mod = getattr(mod, part) |
| | return mod |
| | else: |
| | mod, name = cls.rsplit(".", 1) |
| | s3 = mod == "s3fs" |
| | mod = importlib.import_module(mod) |
| | if s3 and mod.__version__.split(".") < ["0", "5"]: |
| | warnings.warn(s3_msg) |
| | return getattr(mod, name) |
| |
|
| |
|
| | def filesystem(protocol, **storage_options): |
| | """Instantiate filesystems for given protocol and arguments |
| | |
| | ``storage_options`` are specific to the protocol being chosen, and are |
| | passed directly to the class. |
| | """ |
| | if protocol == "arrow_hdfs": |
| | warnings.warn( |
| | "The 'arrow_hdfs' protocol has been deprecated and will be " |
| | "removed in the future. Specify it as 'hdfs'.", |
| | DeprecationWarning, |
| | ) |
| |
|
| | cls = get_filesystem_class(protocol) |
| | return cls(**storage_options) |
| |
|
| |
|
| | def available_protocols(): |
| | """Return a list of the implemented protocols. |
| | |
| | Note that any given protocol may require extra packages to be importable. |
| | """ |
| | return list(known_implementations) |
| |
|
