Sync from GitHub: c06b3aabc1dbb351966ed76ba2b2c2896c5fbb9b

nuwave/organism.py

@@ -16,6 +16,50 @@ communication protocol (Law 1). Raw experience in, classification
 only at extraction (Law 7).
 
 # ---- Changelog ----
+# [2026-06-24] Claude Opus 4.7 (1M ctx) — Substrate-restore fix: msgpack-format checkpoint path
+#   What: Two surgical edits in organism.py — save() and _apply_state().
+#         (1) save(): graph checkpoint tmp filename changed from
+#             '_tmp_save.ckpt' → '_tmp_save.ckpt.msgpack'. Read back as
+#             BYTES (rb) instead of text (r). Field renamed
+#             'graph_checkpoint_json' → 'graph_checkpoint_msgpack' so the
+#             restore path can detect format unambiguously. Legacy field
+#             is .pop()'d from state to keep sidecars lean.
+#         (2) _apply_state(): try msgpack field first (canonical post-fix
+#             path), fall back to legacy graph_checkpoint_json for sidecars
+#             saved before this fix. The legacy fallback is known-lossy
+#             (#325 enforcer drops topology on non-.msgpack restore) but
+#             that's the broken behavior we're fixing — no regression.
+#   Why:  Run 41 boot log (2026-06-24 00:56) revealed the smoking-gun
+#         pattern: 'Organism restored from local-sidecar: 0 nodes, 0 syn,
+#         355 content, step 119, 120 amps, 0 embs' on a 128KB sidecar with
+#         all four state files intact on disk. Topology silently dropping
+#         on restore while auxiliary state (content/amplitudes/embeddings)
+#         restored fine. Earlier RuntimeWarning had named the root cause:
+#         "Restoring topology from non-'.msgpack' path '..._tmp_restore.ckpt':
+#         legacy lossy-JSON state (pre-#325). Re-checkpoint to .msgpack
+#         to stop the loss." The Mind-Not-Database re-vendor (commit
+#         `35ed177`) activated canonical NG's #325 msgpack-enforcer; the
+#         pre-existing bespoke JSON checkpoint path NuWave used since
+#         pre-BTF was suddenly the wrong side of LAW 2 / the
+#         Format-for-Purpose Principle's "topology MUST be msgpack" rule.
+#   How:  Two-edit surgical fix. Switches NuWave's save/restore tmp
+#         filenames to .msgpack so canonical Graph.checkpoint()/restore()
+#         use the proper binary format. Storage format in sidecar
+#         changes from JSON-text to msgpack-bytes; the sidecar.pt is
+#         already a torch.save dict so binary-bytes-as-value is fine.
+#         Existing sidecars without the new field fall back to the
+#         legacy (lossy) path which is the current broken behavior —
+#         so no worse than today. Once a save() runs with this fix, the
+#         resulting sidecar carries the msgpack field and round-trips
+#         topology correctly.
+#         The harvest-from-legacy path at line 721 ('_tmp_harvest.ckpt')
+#         is unchanged — that's the pre-BTF organism_state.pt migration
+#         and operates on truly-legacy JSON-text inputs by design.
+#   Side-effect: existing substrate state on /data is lost (was already
+#   loading as 0 topology). Recovery option: restore from HF dataset
+#   `backups/<ts>/` pre-Mind-Not-Database snapshot per MASTER-NuWave
+#   Operations procedure.
+# -------------------
 # [2026-06-20] Claude Opus 4.7 (1M ctx) — Mind-Not-Database: wire canonical RPC mechanisms
 #   What: Six integration points wired into organism.py to call into a new
 #         nuwave/substrate/rpc_mechanisms.py module that ports canonical NG's
@@ -838,14 +882,27 @@ class NuWaveOrganism:
     def _apply_state(self, state: dict, source: str) -> bool:
         """Apply a loaded state dict. Handles new + legacy formats."""
         try:
-            # Graph checkpoint — stored as JSON string
-            graph_json = state.get('graph_checkpoint_json')
-            if graph_json and self._graph:
-                tmp = os.path.join(self._state_path, "_tmp_restore.ckpt")
-                with open(tmp, 'w') as f:
-                    f.write(graph_json)
+            # Graph checkpoint — try canonical msgpack format first
+            # (post-#325 fix 2026-06-24), fall back to legacy JSON path
+            # for sidecars saved before this fix. The legacy path is
+            # known-lossy (#325 silently drops topology on non-.msgpack
+            # restore), so old sidecars restore as 0 topology — same as
+            # the broken state we're fixing. No regression vs current.
+            graph_mp = state.get('graph_checkpoint_msgpack')
+            if graph_mp and self._graph:
+                tmp = os.path.join(self._state_path, "_tmp_restore.ckpt.msgpack")
+                with open(tmp, 'wb') as f:
+                    f.write(graph_mp)
                 self._graph.restore(tmp)
                 os.remove(tmp)
+            else:
+                graph_json = state.get('graph_checkpoint_json')
+                if graph_json and self._graph:
+                    tmp = os.path.join(self._state_path, "_tmp_restore.ckpt")
+                    with open(tmp, 'w') as f:
+                        f.write(graph_json)
+                    self._graph.restore(tmp)
+                    os.remove(tmp)
 
                 # CES: restore node voltages with temporal decay so the
                 # substrate comes up energetically warm. Must run AFTER
@@ -2145,10 +2202,22 @@ class NuWaveOrganism:
                 for node in self._graph.nodes.values():
                     if node.metadata and 'embedding' in node.metadata:
                         del node.metadata['embedding']
-                tmp = os.path.join(self._state_path, "_tmp_save.ckpt")
+                # Canonical NG #325 enforces msgpack-only checkpoint paths —
+                # any non-'.msgpack' extension is treated as lossy-JSON and
+                # silently drops topology on restore. The Mind-Not-Database
+                # re-vendor activated this enforcer; the pre-existing
+                # '_tmp_save.ckpt' filename routed to the lossy path,
+                # producing sidecars whose graph checkpoint reloaded as
+                # 0 nodes / 0 synapses while auxiliary state (content,
+                # amplitudes, embeddings) restored fine. Diagnosed Run 41
+                # boot log 2026-06-24. Fix: use .msgpack extension, store
+                # bytes (not text). Field renamed graph_checkpoint_json →
+                # graph_checkpoint_msgpack so restore can detect format.
+                tmp = os.path.join(self._state_path, "_tmp_save.ckpt.msgpack")
                 self._graph.checkpoint(tmp)
-                with open(tmp, 'r') as f:
-                    state['graph_checkpoint_json'] = f.read()
+                with open(tmp, 'rb') as f:
+                    state['graph_checkpoint_msgpack'] = f.read()
+                state.pop('graph_checkpoint_json', None)  # drop legacy field
                 os.remove(tmp)
             except Exception as exc:
                 logger.debug("Graph checkpoint failed: %s", exc)