fix(coins): wrap experiment.prepare() with shm and dim-expansion patches
Browse filesTwo research-code behaviours break inference inside the deployment
container; both are silenced by monkey-patching for the duration of
each experiment.prepare() call (and restored in a finally).
- torch.nn.Module.share_memory: research code calls
embedder.share_memory() to share weights across multi-process
training workers. The website is single-process and the call is
gratuitous; on Linux containers with a small /dev/shm (Docker
default 64 MB, free HF Spaces tmpfs similar) it raises Bus error
mid-prepare. No-op replacement keeps prepare() happy.
- torch.load: prepare() loads transe_model.tar to seed the embedder's
entity_embeddings_initial buffers, then KBGAT's __init__ assigns
weight.data = init, silently re-shaping the YAML-declared embedding
layer to the init's shape. For wordnet KBGAT this is fatal: the
trained checkpoint is 200d but the wordnet TransE init is 100d, so
the embedder ends up at 100d and load_state_dict fails on the
trained weights. The patch detects TransE state dicts and repeats
them along the embedding axis (e.g. 100d -> 200d via cat([init,
init])) when the YAML's embedding_dim is an integer multiple of the
init's dim — same trick _adapt_kbgat_state_dict already uses for
the GATConv multi-head expansion.
Also switches the HF_CHECKPOINTS_REPO default to the correctly-cased
Bani57/checkpoints (the HF Hub namespace check is case-sensitive).
|
@@ -64,7 +64,7 @@ def _safe_load_lightning_checkpoint(cls, ckpt_path):
|
|
| 64 |
# on-disk layout under settings.CHECKPOINTS_ROOT (RESEARCH_ROOT by default), so
|
| 65 |
# snapshot_download() drops every file into its final location and the scan
|
| 66 |
# routines below find them unchanged.
|
| 67 |
-
HF_CHECKPOINTS_REPO = os.environ.get("HF_CHECKPOINTS_REPO", "
|
| 68 |
|
| 69 |
# Per-area checkpoint subdirectories (relative to CHECKPOINTS_ROOT). Used to
|
| 70 |
# detect a fully-populated tree so we can skip the network round-trip on warm
|
|
@@ -827,6 +827,10 @@ class ModelRegistry:
|
|
| 827 |
"test": False,
|
| 828 |
"results_dir": str(Path(settings.COINS_COMPLETION_DIR) / "results"),
|
| 829 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 830 |
|
| 831 |
original_cwd = os.getcwd()
|
| 832 |
try:
|
|
@@ -850,30 +854,98 @@ class ModelRegistry:
|
|
| 850 |
# Each load_graph() reloads the full graph from disk and recomputes community
|
| 851 |
# structures; reusing a cached Loader avoids this for e.g. all four
|
| 852 |
# transe/distmult/complex/rotate variants that share the same seed on every dataset.
|
| 853 |
-
|
| 854 |
-
|
| 855 |
-
|
| 856 |
-
|
| 857 |
-
|
| 858 |
-
|
| 859 |
-
|
| 860 |
-
|
| 861 |
-
|
| 862 |
-
|
| 863 |
-
|
| 864 |
-
|
| 865 |
-
|
| 866 |
-
|
| 867 |
-
|
| 868 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 869 |
experiment.prepare()
|
| 870 |
-
|
| 871 |
-
|
| 872 |
-
|
| 873 |
-
|
| 874 |
-
|
| 875 |
-
self._coins_loaders[loader_key] = experiment.loader
|
| 876 |
-
logger.info("Cached new Loader for %s seed=%d", dataset_id, seed)
|
| 877 |
|
| 878 |
ckpt_path = (Path(settings.COINS_COMPLETION_DIR) / "checkpoints"
|
| 879 |
/ f"{dataset_id}_{algorithm}.tar")
|
|
|
|
| 64 |
# on-disk layout under settings.CHECKPOINTS_ROOT (RESEARCH_ROOT by default), so
|
| 65 |
# snapshot_download() drops every file into its final location and the scan
|
| 66 |
# routines below find them unchanged.
|
| 67 |
+
HF_CHECKPOINTS_REPO = os.environ.get("HF_CHECKPOINTS_REPO", "Bani57/checkpoints")
|
| 68 |
|
| 69 |
# Per-area checkpoint subdirectories (relative to CHECKPOINTS_ROOT). Used to
|
| 70 |
# detect a fully-populated tree so we can skip the network round-trip on warm
|
|
|
|
| 827 |
"test": False,
|
| 828 |
"results_dir": str(Path(settings.COINS_COMPLETION_DIR) / "results"),
|
| 829 |
}
|
| 830 |
+
# YAML's embedding_dim is the trained model's dim. Used below to
|
| 831 |
+
# expand the TransE init when its dim is smaller (e.g. wordnet kbgat
|
| 832 |
+
# was trained at 200d but wordnet's transe_model.tar is 100d).
|
| 833 |
+
target_embedding_dim = int(yaml_config.get("embedder_hpars", {}).get("embedding_dim", 100))
|
| 834 |
|
| 835 |
original_cwd = os.getcwd()
|
| 836 |
try:
|
|
|
|
| 854 |
# Each load_graph() reloads the full graph from disk and recomputes community
|
| 855 |
# structures; reusing a cached Loader avoids this for e.g. all four
|
| 856 |
# transe/distmult/complex/rotate variants that share the same seed on every dataset.
|
| 857 |
+
#
|
| 858 |
+
# share_memory monkey-patch: experiment.prepare() constructs the embedder
|
| 859 |
+
# and immediately calls embedder.share_memory() to share weights across
|
| 860 |
+
# multi-process training workers. We run single-process inference so the
|
| 861 |
+
# call is gratuitous, and on Linux containers with a small /dev/shm
|
| 862 |
+
# (Docker default 64 MB, free HF Spaces tmpfs similar) it raises a Bus
|
| 863 |
+
# error mid-prepare. No-op the Module.share_memory class method for the
|
| 864 |
+
# duration of prepare() and restore it after.
|
| 865 |
+
#
|
| 866 |
+
# torch.load monkey-patch: experiment.prepare() loads transe_model.tar to
|
| 867 |
+
# initialise the embedder's entity_embeddings_initial buffers. The KBGAT
|
| 868 |
+
# __init__ then assigns weight.data = init, which silently re-shapes the
|
| 869 |
+
# 200d embedding layer (built from YAML embedding_dim) to the 100d transe
|
| 870 |
+
# init's shape — and the trained checkpoint's load_state_dict afterwards
|
| 871 |
+
# blows up on the dim mismatch. Expand the transe init from 100d to
|
| 872 |
+
# target_embedding_dim by repeating along the embedding axis (same trick
|
| 873 |
+
# _adapt_kbgat_state_dict uses for GATConv multi-head expansion).
|
| 874 |
+
import torch as _pt
|
| 875 |
+
import torch.nn as _nn
|
| 876 |
+
_orig_share_memory = _nn.Module.share_memory
|
| 877 |
+
_nn.Module.share_memory = lambda self: self
|
| 878 |
+
_orig_torch_load = _pt.load
|
| 879 |
+
|
| 880 |
+
def _expand_transe_load(*args, **kwargs):
|
| 881 |
+
state_dict = _orig_torch_load(*args, **kwargs)
|
| 882 |
+
if not isinstance(state_dict, dict):
|
| 883 |
+
return state_dict
|
| 884 |
+
if not any(k.endswith("entity_embeddings.weight") for k in state_dict):
|
| 885 |
+
return state_dict
|
| 886 |
+
# Detect transe init dim and bail if it already matches.
|
| 887 |
+
sample = next(v for k, v in state_dict.items()
|
| 888 |
+
if k.endswith("entity_embeddings.weight") and hasattr(v, "shape"))
|
| 889 |
+
src_dim = int(sample.shape[-1])
|
| 890 |
+
if src_dim == target_embedding_dim or src_dim == 0:
|
| 891 |
+
return state_dict
|
| 892 |
+
if target_embedding_dim % src_dim != 0:
|
| 893 |
+
logger.warning(
|
| 894 |
+
"TransE init dim %d not a divisor of YAML embedding_dim %d; "
|
| 895 |
+
"leaving init unchanged (load_state_dict may fail).",
|
| 896 |
+
src_dim, target_embedding_dim,
|
| 897 |
+
)
|
| 898 |
+
return state_dict
|
| 899 |
+
n_repeats = target_embedding_dim // src_dim
|
| 900 |
+
expanded = {}
|
| 901 |
+
for key, value in state_dict.items():
|
| 902 |
+
if not hasattr(value, "shape") or value.ndim < 1:
|
| 903 |
+
expanded[key] = value
|
| 904 |
+
continue
|
| 905 |
+
# entity_embeddings(_initial).weight: [num_entities, dim] -> repeat dim
|
| 906 |
+
# r_embeddings_initial.weight: [num_relations, dim] -> repeat dim
|
| 907 |
+
# r_embeddings.weight: [dim, num_relations] -> repeat dim 0
|
| 908 |
+
if key.endswith(("entity_embeddings.weight",
|
| 909 |
+
"entity_embeddings_initial.weight",
|
| 910 |
+
"r_embeddings_initial.weight")) and value.shape[-1] == src_dim:
|
| 911 |
+
expanded[key] = value.repeat(*([1] * (value.ndim - 1)), n_repeats)
|
| 912 |
+
elif key.endswith("r_embeddings.weight") and value.shape[0] == src_dim:
|
| 913 |
+
expanded[key] = value.repeat(n_repeats, *([1] * (value.ndim - 1)))
|
| 914 |
+
else:
|
| 915 |
+
expanded[key] = value
|
| 916 |
+
logger.info("Expanded transe init from %dd to %dd (x%d repeat) for %s/%s",
|
| 917 |
+
src_dim, target_embedding_dim, n_repeats, dataset_id, algorithm)
|
| 918 |
+
return expanded
|
| 919 |
+
|
| 920 |
+
_pt.load = _expand_transe_load
|
| 921 |
+
try:
|
| 922 |
+
loader_key = (dataset_id, seed, leiden_resolution)
|
| 923 |
+
if loader_key in self._coins_loaders:
|
| 924 |
+
cached_loader = self._coins_loaders[loader_key]
|
| 925 |
+
# Defensive: ensure machines length matches current num_communities.
|
| 926 |
+
# _load_all_loaders already does this, but any future code path that
|
| 927 |
+
# populates _coins_loaders directly could skip it.
|
| 928 |
+
import numpy as np
|
| 929 |
+
expected_len = cached_loader.num_communities + 2
|
| 930 |
+
if len(cached_loader.machines) != expected_len:
|
| 931 |
+
cached_loader.machines = np.zeros(expected_len, dtype=int)
|
| 932 |
+
experiment.loader = cached_loader
|
| 933 |
+
# Temporarily replace load_graph with a no-op: prepare() will find all
|
| 934 |
+
# required attributes (num_nodes, communities, graph_indexes, …) already set.
|
| 935 |
+
_orig_load_graph = cached_loader.load_graph
|
| 936 |
+
cached_loader.load_graph = lambda *args, **kwargs: None
|
| 937 |
+
try:
|
| 938 |
+
experiment.prepare()
|
| 939 |
+
finally:
|
| 940 |
+
cached_loader.load_graph = _orig_load_graph
|
| 941 |
+
logger.info("Reused shared Loader for %s seed=%d", dataset_id, seed)
|
| 942 |
+
else:
|
| 943 |
experiment.prepare()
|
| 944 |
+
self._coins_loaders[loader_key] = experiment.loader
|
| 945 |
+
logger.info("Cached new Loader for %s seed=%d", dataset_id, seed)
|
| 946 |
+
finally:
|
| 947 |
+
_nn.Module.share_memory = _orig_share_memory
|
| 948 |
+
_pt.load = _orig_torch_load
|
|
|
|
|
|
|
| 949 |
|
| 950 |
ckpt_path = (Path(settings.COINS_COMPLETION_DIR) / "checkpoints"
|
| 951 |
/ f"{dataset_id}_{algorithm}.tar")
|