| import importlib |
| import importlib.util |
| import inspect |
| import os |
| import sys |
| import types |
|
|
| __all__ = ["attach", "_lazy_import"] |
|
|
|
|
| def attach(module_name, submodules=None, submod_attrs=None): |
| """Attach lazily loaded submodules, and functions or other attributes. |
| |
| Typically, modules import submodules and attributes as follows:: |
| |
| import mysubmodule |
| import anothersubmodule |
| |
| from .foo import someattr |
| |
| The idea of this function is to replace the `__init__.py` |
| module's `__getattr__`, `__dir__`, and `__all__` attributes such that |
| all imports work exactly the way they normally would, except that the |
| actual import is delayed until the resulting module object is first used. |
| |
| The typical way to call this function, replacing the above imports, is:: |
| |
| __getattr__, __lazy_dir__, __all__ = lazy.attach( |
| __name__, ["mysubmodule", "anothersubmodule"], {"foo": "someattr"} |
| ) |
| |
| This functionality requires Python 3.7 or higher. |
| |
| Parameters |
| ---------- |
| module_name : str |
| Typically use __name__. |
| submodules : set |
| List of submodules to lazily import. |
| submod_attrs : dict |
| Dictionary of submodule -> list of attributes / functions. |
| These attributes are imported as they are used. |
| |
| Returns |
| ------- |
| __getattr__, __dir__, __all__ |
| |
| """ |
| if submod_attrs is None: |
| submod_attrs = {} |
|
|
| if submodules is None: |
| submodules = set() |
| else: |
| submodules = set(submodules) |
|
|
| attr_to_modules = { |
| attr: mod for mod, attrs in submod_attrs.items() for attr in attrs |
| } |
|
|
| __all__ = list(submodules | attr_to_modules.keys()) |
|
|
| def __getattr__(name): |
| if name in submodules: |
| return importlib.import_module(f"{module_name}.{name}") |
| elif name in attr_to_modules: |
| submod = importlib.import_module(f"{module_name}.{attr_to_modules[name]}") |
| return getattr(submod, name) |
| else: |
| raise AttributeError(f"No {module_name} attribute {name}") |
|
|
| def __dir__(): |
| return __all__ |
|
|
| if os.environ.get("EAGER_IMPORT", ""): |
| for attr in set(attr_to_modules.keys()) | submodules: |
| __getattr__(attr) |
|
|
| return __getattr__, __dir__, list(__all__) |
|
|
|
|
| class DelayedImportErrorModule(types.ModuleType): |
| def __init__(self, frame_data, *args, **kwargs): |
| self.__frame_data = frame_data |
| super().__init__(*args, **kwargs) |
|
|
| def __getattr__(self, x): |
| if x in ("__class__", "__file__", "__frame_data"): |
| super().__getattr__(x) |
| else: |
| fd = self.__frame_data |
| raise ModuleNotFoundError( |
| f"No module named '{fd['spec']}'\n\n" |
| "This error is lazily reported, having originally occurred in\n" |
| f" File {fd['filename']}, line {fd['lineno']}, in {fd['function']}\n\n" |
| f"----> {''.join(fd['code_context'] or '').strip()}" |
| ) |
|
|
|
|
| def _lazy_import(fullname): |
| """Return a lazily imported proxy for a module or library. |
| |
| Warning |
| ------- |
| Importing using this function can currently cause trouble |
| when the user tries to import from a subpackage of a module before |
| the package is fully imported. In particular, this idiom may not work: |
| |
| np = lazy_import("numpy") |
| from numpy.lib import recfunctions |
| |
| This is due to a difference in the way Python's LazyLoader handles |
| subpackage imports compared to the normal import process. Hopefully |
| we will get Python's LazyLoader to fix this, or find a workaround. |
| In the meantime, this is a potential problem. |
| |
| The workaround is to import numpy before importing from the subpackage. |
| |
| Notes |
| ----- |
| We often see the following pattern:: |
| |
| def myfunc(): |
| import scipy as sp |
| sp.argmin(...) |
| .... |
| |
| This is to prevent a library, in this case `scipy`, from being |
| imported at function definition time, since that can be slow. |
| |
| This function provides a proxy module that, upon access, imports |
| the actual module. So the idiom equivalent to the above example is:: |
| |
| sp = lazy.load("scipy") |
| |
| def myfunc(): |
| sp.argmin(...) |
| .... |
| |
| The initial import time is fast because the actual import is delayed |
| until the first attribute is requested. The overall import time may |
| decrease as well for users that don't make use of large portions |
| of the library. |
| |
| Parameters |
| ---------- |
| fullname : str |
| The full name of the package or subpackage to import. For example:: |
| |
| sp = lazy.load("scipy") # import scipy as sp |
| spla = lazy.load("scipy.linalg") # import scipy.linalg as spla |
| |
| Returns |
| ------- |
| pm : importlib.util._LazyModule |
| Proxy module. Can be used like any regularly imported module. |
| Actual loading of the module occurs upon first attribute request. |
| |
| """ |
| try: |
| return sys.modules[fullname] |
| except: |
| pass |
|
|
| |
| spec = importlib.util.find_spec(fullname) |
|
|
| if spec is None: |
| try: |
| parent = inspect.stack()[1] |
| frame_data = { |
| "spec": fullname, |
| "filename": parent.filename, |
| "lineno": parent.lineno, |
| "function": parent.function, |
| "code_context": parent.code_context, |
| } |
| return DelayedImportErrorModule(frame_data, "DelayedImportErrorModule") |
| finally: |
| del parent |
|
|
| module = importlib.util.module_from_spec(spec) |
| sys.modules[fullname] = module |
|
|
| loader = importlib.util.LazyLoader(spec.loader) |
| loader.exec_module(module) |
|
|
| return module |
|
|