Update Feather training runtime image

overlay/scripts/launch_feather_hf_job.py

@@ -94,7 +94,7 @@ def require_token() -> str:
     return TOKEN
 
 
-def wait_for_space(api: HfApi, repo_id: str, timeout_s: int = 1800) -> None:
+def wait_for_space(api: HfApi, repo_id: str, timeout_s: int = 1800) -> None:
     """Wait until the Space image has been built.
 
     We use the Space purely as a container-image builder for HF Jobs. The Space
@@ -109,24 +109,27 @@ def wait_for_space(api: HfApi, repo_id: str, timeout_s: int = 1800) -> None:
     and APP_STARTING_ERROR after a successful BUILDING→APP_STARTING transition
     are acceptable — the image exists in the registry and Jobs can use it.
     """
-    start = time.time()
-    seen_build_completion = False
-    while True:
-        runtime = api.get_space_runtime(repo_id, token=TOKEN)
-        stage = getattr(runtime, 'stage', None)
-        hardware = getattr(runtime, 'hardware', None)
-        err = getattr(runtime, 'errorMessage', None) or getattr(runtime, 'error_message', None)
-        print(f'[space] stage={stage} hardware={hardware}', flush=True)
-        if stage in {'APP_STARTING', 'RUNNING', 'PAUSED', 'SLEEPING'}:
-            seen_build_completion = True
-        if stage in {'RUNNING', 'PAUSED', 'SLEEPING'}:
-            return
-        # Image is built — Jobs can use it regardless of Space boot outcome.
-        if seen_build_completion and stage in {'RUNTIME_ERROR', 'APP_STARTING_ERROR'}:
-            print(f'[space] Space boot failed with {stage} but built image is '
-                  f'available in the Space registry and is usable by HF Jobs.',
-                  flush=True)
-            return
+    start = time.time()
+    seen_build_completion = False
+    seen_building = False
+    while True:
+        runtime = api.get_space_runtime(repo_id, token=TOKEN)
+        stage = getattr(runtime, 'stage', None)
+        hardware = getattr(runtime, 'hardware', None)
+        err = getattr(runtime, 'errorMessage', None) or getattr(runtime, 'error_message', None)
+        print(f'[space] stage={stage} hardware={hardware}', flush=True)
+        if stage == 'BUILDING':
+            seen_building = True
+        if stage in {'APP_STARTING', 'RUNNING', 'PAUSED', 'SLEEPING'}:
+            seen_build_completion = True
+        if stage in {'RUNNING', 'PAUSED', 'SLEEPING'}:
+            return
+        # Image is built — Jobs can use it regardless of Space boot outcome.
+        if (seen_build_completion or seen_building) and stage in {'RUNTIME_ERROR', 'APP_STARTING_ERROR'}:
+            print(f'[space] Space boot failed with {stage} but built image is '
+                  f'available in the Space registry and is usable by HF Jobs.',
+                  flush=True)
+            return
         # Hard build failures — no image was produced.
         if stage in {'BUILD_ERROR', 'CONFIG_ERROR', 'NO_APP_FILE'}:
             raise RuntimeError(f'Space {repo_id} build failed: stage={stage} error={err!r}')

overlay/scripts/run_domain_expanded_pretrain.sh

@@ -214,10 +214,14 @@ fi
 
 RETINA_PATH="${HYDRA_RETINA_PATH:-$CACHE_ROOT/retina.npz}"
 if [[ ! -f "$RETINA_PATH" ]]; then
-  RETINA_CMD=("${PYTHON_CMD[@]}" -c "from subsystems.sdr_retina import build_retina; build_retina()")
-  log "retina_action=build command=${RETINA_CMD[*]}"
-  if [[ "$DRY_RUN" -eq 0 ]]; then
-    "${RETINA_CMD[@]}" 2>&1 | tee -a "$LOG_FILE"
+  if [[ "${HYDRA_ALLOW_SYNTHETIC_RETINA:-0}" == "1" ]]; then
+    log "retina_action=skip reason=HYDRA_ALLOW_SYNTHETIC_RETINA=1 and retina missing"
+  else
+    RETINA_CMD=("${PYTHON_CMD[@]}" -c "from subsystems.sdr_retina import build_retina; build_retina()")
+    log "retina_action=build command=${RETINA_CMD[*]}"
+    if [[ "$DRY_RUN" -eq 0 ]]; then
+      "${RETINA_CMD[@]}" 2>&1 | tee -a "$LOG_FILE"
+    fi
   fi
 else
   log "retina_action=skip path=$RETINA_PATH"

overlay/subsystems/sdr_semantic.py

@@ -91,21 +91,39 @@ class SemanticFoldingSDR(nn.Module):
         super().__init__()
         self.vocab_size = vocab_size
         self.n_bits = n_bits
-        self.som_update_interval = int(som_update_interval)
-        self.som_warmup_steps = int(som_warmup_steps)
-        self.som_alpha = float(som_alpha)
-
-        path = retina_path or DEFAULT_RETINA_PATH
-        if not Path(path).exists():
-            raise FileNotFoundError(
-                f"Retina not found at {path}. Run subsystems/sdr_retina.py first."
-            )
-
-        with np.load(path) as f:
-            retina_sdr = f["sdr"]  # bool[V, n_bits]
-            stored_vocab = int(f["vocab_size"]) if "vocab_size" in f.files else retina_sdr.shape[0]
-            stored_nbits = int(f["n_bits"]) if "n_bits" in f.files else retina_sdr.shape[1]
-            stored_target = int(f["target_active"]) if "target_active" in f.files else int(retina_sdr[0].sum())
+        self.som_update_interval = int(som_update_interval)
+        self.som_warmup_steps = int(som_warmup_steps)
+        self.som_alpha = float(som_alpha)
+
+        path = retina_path or DEFAULT_RETINA_PATH
+        retina_path_exists = Path(path).exists()
+        allow_synthetic = os.environ.get("HYDRA_ALLOW_SYNTHETIC_RETINA", "0") == "1"
+
+        if retina_path_exists:
+            with np.load(path) as f:
+                retina_sdr = f["sdr"]  # bool[V, n_bits]
+                stored_vocab = int(f["vocab_size"]) if "vocab_size" in f.files else retina_sdr.shape[0]
+                stored_nbits = int(f["n_bits"]) if "n_bits" in f.files else retina_sdr.shape[1]
+                stored_target = int(f["target_active"]) if "target_active" in f.files else int(retina_sdr[0].sum())
+        elif allow_synthetic:
+            synth_target = int(target_active) if target_active is not None else DEFAULT_TARGET_ACTIVE
+            print(
+                f"[retina] missing {path}; HYDRA_ALLOW_SYNTHETIC_RETINA=1 so using synthetic retina "
+                f"(vocab={vocab_size}, n_bits={n_bits}, active={synth_target})",
+                flush=True,
+            )
+            base = np.arange(synth_target, dtype=np.int64)[None, :]
+            rows = np.arange(vocab_size, dtype=np.int64)[:, None]
+            cols = (rows * 2654435761 + base * 1315423911) % n_bits
+            retina_sdr = np.zeros((vocab_size, n_bits), dtype=np.bool_)
+            retina_sdr[np.arange(vocab_size)[:, None], cols] = True
+            stored_vocab = vocab_size
+            stored_nbits = n_bits
+            stored_target = synth_target
+        else:
+            raise FileNotFoundError(
+                f"Retina not found at {path}. Run subsystems/sdr_retina.py first."
+            )
 
         if retina_sdr.shape != (vocab_size, n_bits):
             raise ValueError(