Andrej Janchevski commited on
Commit
15144da
·
1 Parent(s): b598a6a

fix(coins): wrap experiment.prepare() with shm and dim-expansion patches

Browse files

Two research-code behaviours break inference inside the deployment
container; both are silenced by monkey-patching for the duration of
each experiment.prepare() call (and restored in a finally).

- torch.nn.Module.share_memory: research code calls
embedder.share_memory() to share weights across multi-process
training workers. The website is single-process and the call is
gratuitous; on Linux containers with a small /dev/shm (Docker
default 64 MB, free HF Spaces tmpfs similar) it raises Bus error
mid-prepare. No-op replacement keeps prepare() happy.

- torch.load: prepare() loads transe_model.tar to seed the embedder's
entity_embeddings_initial buffers, then KBGAT's __init__ assigns
weight.data = init, silently re-shaping the YAML-declared embedding
layer to the init's shape. For wordnet KBGAT this is fatal: the
trained checkpoint is 200d but the wordnet TransE init is 100d, so
the embedder ends up at 100d and load_state_dict fails on the
trained weights. The patch detects TransE state dicts and repeats
them along the embedding axis (e.g. 100d -> 200d via cat([init,
init])) when the YAML's embedding_dim is an integer multiple of the
init's dim — same trick _adapt_kbgat_state_dict already uses for
the GATConv multi-head expansion.

Also switches the HF_CHECKPOINTS_REPO default to the correctly-cased
Bani57/checkpoints (the HF Hub namespace check is case-sensitive).

Files changed (1) hide show
  1. src/backend/api/services/registry.py +96 -24
src/backend/api/services/registry.py CHANGED
@@ -64,7 +64,7 @@ def _safe_load_lightning_checkpoint(cls, ckpt_path):
64
  # on-disk layout under settings.CHECKPOINTS_ROOT (RESEARCH_ROOT by default), so
65
  # snapshot_download() drops every file into its final location and the scan
66
  # routines below find them unchanged.
67
- HF_CHECKPOINTS_REPO = os.environ.get("HF_CHECKPOINTS_REPO", "bani57/checkpoints")
68
 
69
  # Per-area checkpoint subdirectories (relative to CHECKPOINTS_ROOT). Used to
70
  # detect a fully-populated tree so we can skip the network round-trip on warm
@@ -827,6 +827,10 @@ class ModelRegistry:
827
  "test": False,
828
  "results_dir": str(Path(settings.COINS_COMPLETION_DIR) / "results"),
829
  }
 
 
 
 
830
 
831
  original_cwd = os.getcwd()
832
  try:
@@ -850,30 +854,98 @@ class ModelRegistry:
850
  # Each load_graph() reloads the full graph from disk and recomputes community
851
  # structures; reusing a cached Loader avoids this for e.g. all four
852
  # transe/distmult/complex/rotate variants that share the same seed on every dataset.
853
- loader_key = (dataset_id, seed, leiden_resolution)
854
- if loader_key in self._coins_loaders:
855
- cached_loader = self._coins_loaders[loader_key]
856
- # Defensive: ensure machines length matches current num_communities.
857
- # _load_all_loaders already does this, but any future code path that
858
- # populates _coins_loaders directly could skip it.
859
- import numpy as np
860
- expected_len = cached_loader.num_communities + 2
861
- if len(cached_loader.machines) != expected_len:
862
- cached_loader.machines = np.zeros(expected_len, dtype=int)
863
- experiment.loader = cached_loader
864
- # Temporarily replace load_graph with a no-op: prepare() will find all
865
- # required attributes (num_nodes, communities, graph_indexes, ) already set.
866
- _orig_load_graph = cached_loader.load_graph
867
- cached_loader.load_graph = lambda *args, **kwargs: None
868
- try:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
869
  experiment.prepare()
870
- finally:
871
- cached_loader.load_graph = _orig_load_graph
872
- logger.info("Reused shared Loader for %s seed=%d", dataset_id, seed)
873
- else:
874
- experiment.prepare()
875
- self._coins_loaders[loader_key] = experiment.loader
876
- logger.info("Cached new Loader for %s seed=%d", dataset_id, seed)
877
 
878
  ckpt_path = (Path(settings.COINS_COMPLETION_DIR) / "checkpoints"
879
  / f"{dataset_id}_{algorithm}.tar")
 
64
  # on-disk layout under settings.CHECKPOINTS_ROOT (RESEARCH_ROOT by default), so
65
  # snapshot_download() drops every file into its final location and the scan
66
  # routines below find them unchanged.
67
+ HF_CHECKPOINTS_REPO = os.environ.get("HF_CHECKPOINTS_REPO", "Bani57/checkpoints")
68
 
69
  # Per-area checkpoint subdirectories (relative to CHECKPOINTS_ROOT). Used to
70
  # detect a fully-populated tree so we can skip the network round-trip on warm
 
827
  "test": False,
828
  "results_dir": str(Path(settings.COINS_COMPLETION_DIR) / "results"),
829
  }
830
+ # YAML's embedding_dim is the trained model's dim. Used below to
831
+ # expand the TransE init when its dim is smaller (e.g. wordnet kbgat
832
+ # was trained at 200d but wordnet's transe_model.tar is 100d).
833
+ target_embedding_dim = int(yaml_config.get("embedder_hpars", {}).get("embedding_dim", 100))
834
 
835
  original_cwd = os.getcwd()
836
  try:
 
854
  # Each load_graph() reloads the full graph from disk and recomputes community
855
  # structures; reusing a cached Loader avoids this for e.g. all four
856
  # transe/distmult/complex/rotate variants that share the same seed on every dataset.
857
+ #
858
+ # share_memory monkey-patch: experiment.prepare() constructs the embedder
859
+ # and immediately calls embedder.share_memory() to share weights across
860
+ # multi-process training workers. We run single-process inference so the
861
+ # call is gratuitous, and on Linux containers with a small /dev/shm
862
+ # (Docker default 64 MB, free HF Spaces tmpfs similar) it raises a Bus
863
+ # error mid-prepare. No-op the Module.share_memory class method for the
864
+ # duration of prepare() and restore it after.
865
+ #
866
+ # torch.load monkey-patch: experiment.prepare() loads transe_model.tar to
867
+ # initialise the embedder's entity_embeddings_initial buffers. The KBGAT
868
+ # __init__ then assigns weight.data = init, which silently re-shapes the
869
+ # 200d embedding layer (built from YAML embedding_dim) to the 100d transe
870
+ # init's shape — and the trained checkpoint's load_state_dict afterwards
871
+ # blows up on the dim mismatch. Expand the transe init from 100d to
872
+ # target_embedding_dim by repeating along the embedding axis (same trick
873
+ # _adapt_kbgat_state_dict uses for GATConv multi-head expansion).
874
+ import torch as _pt
875
+ import torch.nn as _nn
876
+ _orig_share_memory = _nn.Module.share_memory
877
+ _nn.Module.share_memory = lambda self: self
878
+ _orig_torch_load = _pt.load
879
+
880
+ def _expand_transe_load(*args, **kwargs):
881
+ state_dict = _orig_torch_load(*args, **kwargs)
882
+ if not isinstance(state_dict, dict):
883
+ return state_dict
884
+ if not any(k.endswith("entity_embeddings.weight") for k in state_dict):
885
+ return state_dict
886
+ # Detect transe init dim and bail if it already matches.
887
+ sample = next(v for k, v in state_dict.items()
888
+ if k.endswith("entity_embeddings.weight") and hasattr(v, "shape"))
889
+ src_dim = int(sample.shape[-1])
890
+ if src_dim == target_embedding_dim or src_dim == 0:
891
+ return state_dict
892
+ if target_embedding_dim % src_dim != 0:
893
+ logger.warning(
894
+ "TransE init dim %d not a divisor of YAML embedding_dim %d; "
895
+ "leaving init unchanged (load_state_dict may fail).",
896
+ src_dim, target_embedding_dim,
897
+ )
898
+ return state_dict
899
+ n_repeats = target_embedding_dim // src_dim
900
+ expanded = {}
901
+ for key, value in state_dict.items():
902
+ if not hasattr(value, "shape") or value.ndim < 1:
903
+ expanded[key] = value
904
+ continue
905
+ # entity_embeddings(_initial).weight: [num_entities, dim] -> repeat dim
906
+ # r_embeddings_initial.weight: [num_relations, dim] -> repeat dim
907
+ # r_embeddings.weight: [dim, num_relations] -> repeat dim 0
908
+ if key.endswith(("entity_embeddings.weight",
909
+ "entity_embeddings_initial.weight",
910
+ "r_embeddings_initial.weight")) and value.shape[-1] == src_dim:
911
+ expanded[key] = value.repeat(*([1] * (value.ndim - 1)), n_repeats)
912
+ elif key.endswith("r_embeddings.weight") and value.shape[0] == src_dim:
913
+ expanded[key] = value.repeat(n_repeats, *([1] * (value.ndim - 1)))
914
+ else:
915
+ expanded[key] = value
916
+ logger.info("Expanded transe init from %dd to %dd (x%d repeat) for %s/%s",
917
+ src_dim, target_embedding_dim, n_repeats, dataset_id, algorithm)
918
+ return expanded
919
+
920
+ _pt.load = _expand_transe_load
921
+ try:
922
+ loader_key = (dataset_id, seed, leiden_resolution)
923
+ if loader_key in self._coins_loaders:
924
+ cached_loader = self._coins_loaders[loader_key]
925
+ # Defensive: ensure machines length matches current num_communities.
926
+ # _load_all_loaders already does this, but any future code path that
927
+ # populates _coins_loaders directly could skip it.
928
+ import numpy as np
929
+ expected_len = cached_loader.num_communities + 2
930
+ if len(cached_loader.machines) != expected_len:
931
+ cached_loader.machines = np.zeros(expected_len, dtype=int)
932
+ experiment.loader = cached_loader
933
+ # Temporarily replace load_graph with a no-op: prepare() will find all
934
+ # required attributes (num_nodes, communities, graph_indexes, …) already set.
935
+ _orig_load_graph = cached_loader.load_graph
936
+ cached_loader.load_graph = lambda *args, **kwargs: None
937
+ try:
938
+ experiment.prepare()
939
+ finally:
940
+ cached_loader.load_graph = _orig_load_graph
941
+ logger.info("Reused shared Loader for %s seed=%d", dataset_id, seed)
942
+ else:
943
  experiment.prepare()
944
+ self._coins_loaders[loader_key] = experiment.loader
945
+ logger.info("Cached new Loader for %s seed=%d", dataset_id, seed)
946
+ finally:
947
+ _nn.Module.share_memory = _orig_share_memory
948
+ _pt.load = _orig_torch_load
 
 
949
 
950
  ckpt_path = (Path(settings.COINS_COMPLETION_DIR) / "checkpoints"
951
  / f"{dataset_id}_{algorithm}.tar")