File size: 3,521 Bytes
aadc4ee | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | """
Shared helper for configuration_gliner_moe.py and modeling_gliner_moe.py.
transformers' trust_remote_code dynamic-module loader only auto-copies flat,
single-level relative imports (from .x import y where x.py sits directly in
the repo root) -- it does not walk real subpackages referenced via dotted
relative imports like `from .gliner_lib.config import y`, and an absolute
`import gliner_lib` would make its import-checker demand gliner_lib be
pip-installed, which it is not (it is vendored, not published). So gliner_lib
is loaded here via importlib directly from disk instead of any import/from
statement naming it, keeping it invisible to that regex-based scanner, then
registered in sys.modules like a normally-imported package.
Important: transformers copies only the single file it is resolving (e.g.
this file, or configuration_gliner_moe.py) into its own separate dynamic-
modules cache directory -- it does NOT bring the rest of the source directory
along, even when the original source was a local folder rather than a Hub
repo. So __file__ here essentially never points anywhere near an actual
gliner_lib/ directory in practice. Lookup order actually used:
1. sys.modules cache (already loaded once this process).
2. Right next to this file, in case that assumption ever does hold.
3. Any directory already on sys.path that contains a gliner_lib/ folder --
this is what actually matters for local-directory use: if you are
loading a local checkout of this repo (not yet pushed to the Hub, or
pushed but you are pointing at a local clone), do
`sys.path.insert(0, "<that local directory>")` before calling
AutoModel.from_pretrained(...) / AutoConfig.from_pretrained(...), and
this resolves without any network call.
4. Last resort: snapshot_download(repo_id) to fetch this same Hub repo's
gliner_lib/ directly. Only works once the repo actually exists on the
Hub -- by design not used for local-only testing.
"""
import importlib.util
import os
import sys
def load_gliner_lib(this_file, repo_id):
if "gliner_lib" in sys.modules:
return sys.modules["gliner_lib"]
pkg_dir = None
this_dir = os.path.dirname(os.path.abspath(this_file))
candidate = os.path.join(this_dir, "gliner_lib")
if os.path.isdir(candidate):
pkg_dir = candidate
if pkg_dir is None:
for entry in sys.path:
candidate = os.path.join(entry, "gliner_lib")
if os.path.isdir(candidate):
pkg_dir = candidate
break
if pkg_dir is None:
from huggingface_hub import snapshot_download
downloaded_dir = snapshot_download(repo_id=repo_id, allow_patterns=["gliner_lib/*", "gliner_lib/**/*"])
candidate = os.path.join(downloaded_dir, "gliner_lib")
if os.path.isdir(candidate):
pkg_dir = candidate
if pkg_dir is None:
raise ImportError(
f"Could not locate the vendored gliner_lib package (checked next to {this_file}, "
f"on sys.path, and in a fresh snapshot_download of {repo_id})."
)
parent_dir = os.path.dirname(pkg_dir)
if parent_dir not in sys.path:
sys.path.insert(0, parent_dir)
spec = importlib.util.spec_from_file_location(
"gliner_lib", os.path.join(pkg_dir, "__init__.py"), submodule_search_locations=[pkg_dir]
)
module = importlib.util.module_from_spec(spec)
sys.modules["gliner_lib"] = module
spec.loader.exec_module(module)
return module
|