auto-sync 2026-07-03T15:40:37Z workspace

workspace/cil/chart_features.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import hashlib
+from pathlib import Path
 from typing import Any
 
 try:
@@ -10,7 +11,9 @@ except ImportError:  # pragma: no cover
 
 
 CONTEXT_HASH_WIDTH = 8
-CHART_FEATURE_MODES = ("base", "base_context")
+OBSERVATION_EMBED_DIM = 32
+CHART_FEATURE_MODES = ("base", "base_context", "base_context_obs")
+_EMBEDDING_CACHE: dict[str, Any] = {}
 
 
 def build_chart_feature(
@@ -23,8 +26,10 @@ def build_chart_feature(
 
     `base` preserves the original behavior: the chart token is the flattened
     base action chunk. `base_context` appends stable hashes of task/language
-    metadata that are visible at proposal time. It intentionally does not read
-    outcomes, labels, hidden chart branches, or evaluator-only fields.
+    metadata that are visible at proposal time. `base_context_obs` additionally
+    appends a precomputed observation embedding referenced by metadata. These
+    modes intentionally do not read outcomes, labels, hidden chart branches, or
+    evaluator-only fields.
     """
 
     if np is None:  # pragma: no cover
@@ -44,6 +49,12 @@ def build_chart_feature(
         ],
         dtype=np.float32,
     )
+    if mode == "base_context_obs":
+        obs = _load_observation_embedding(
+            metadata.get("observation_embedding_path"),
+            chart_root=metadata.get("_chart_root"),
+        )
+        context = np.concatenate([context, obs.astype(np.float32, copy=False)])
     return np.concatenate([base, context]).astype(np.float32, copy=False)
 
 
@@ -54,3 +65,36 @@ def chart_feature_dim(base_action: Any, *, mode: str = "base") -> int:
 def _stable_hash_features(text: str, width: int) -> list[float]:
     digest = hashlib.sha256(text.encode("utf-8")).digest()
     return [float(digest[index] / 127.5 - 1.0) for index in range(width)]
+
+
+def _load_observation_embedding(value: Any, *, chart_root: Any = None) -> Any:
+    if np is None:  # pragma: no cover
+        raise ImportError("build_chart_feature requires numpy")
+    if not value:
+        return np.zeros(OBSERVATION_EMBED_DIM, dtype=np.float32)
+    path_text, dataset, row_index = _parse_embedding_ref(str(value))
+    path = Path(path_text)
+    if not path.is_absolute() and chart_root:
+        path = Path(str(chart_root)) / path
+    cache_key = str(path.resolve())
+    if cache_key not in _EMBEDDING_CACHE:
+        with np.load(path, allow_pickle=False) as data:
+            _EMBEDDING_CACHE[cache_key] = np.asarray(data[dataset], dtype=np.float32)
+    matrix = _EMBEDDING_CACHE[cache_key]
+    vector = np.asarray(matrix[int(row_index)], dtype=np.float32).reshape(-1)
+    if vector.shape[0] == OBSERVATION_EMBED_DIM:
+        return vector
+    output = np.zeros(OBSERVATION_EMBED_DIM, dtype=np.float32)
+    width = min(OBSERVATION_EMBED_DIM, vector.shape[0])
+    output[:width] = vector[:width]
+    return output
+
+
+def _parse_embedding_ref(value: str) -> tuple[str, str, int]:
+    if "#" not in value:
+        return value, "embeddings", 0
+    path_text, ref = value.split("#", 1)
+    parts = [part for part in ref.split("/") if part]
+    if len(parts) != 2:
+        raise ValueError(f"invalid observation embedding ref: {value}")
+    return path_text, parts[0], int(parts[1])

workspace/data/cil_charts_rgb_refs/test/charts_00000.npz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1dab883a6d5ccf9fab9111d291de115a0fc4de77487bc7a740c207adb62e74e
+oid sha256:2145c0ec6fd38abb9b128e973e18f7f3174d34fb4b0f5b1cebc9161cf98d4932
 size 6138612

workspace/data/cil_charts_rgb_refs/test/index.json

@@ -15,10 +15,11 @@
     "wrong_direction": 410,
     "wrong_gripper": 410
   },
-  "content_hash": "a249a4d627b8113988660be3aecd867ba1b7b8b8d00f66eb0e3070439a395554",
+  "content_hash": "d258ed27d587e6f41ef6ffd520832087fe7626b5e5cba2bd6352f50a1f45125a",
   "dataset": "/scratch/knguy52/dovla/experiments/six_task_h16_collection",
   "deployment_candidate_excludes_expert": true,
   "deployment_clean": false,
+  "embedding_content_hash": "981d3b1b0aa9dc962e5a22fd695fa9befd21e1334f23b8c3843591e8be89ef83",
   "epsilon": 0.05,
   "format": "cil_charts_npz",
   "group_hashes": [
@@ -449,8 +450,19 @@
   "num_groups_scanned": 2873,
   "num_groups_skipped_by_split": 2463,
   "num_rows": 6560,
+  "observation_embedding_manifest": {
+    "dataset": "embeddings",
+    "dim": 32,
+    "extractor": "rgb_jpeg_stats_v1",
+    "num_embeddings": 410,
+    "path": "observation_embeddings_rgb_stats.npz",
+    "reads_outcomes": false
+  },
   "retrieval_index_allowed": false,
   "schema_version": 1,
+  "shard_content_hashes": {
+    "charts_00000.npz": "2145c0ec6fd38abb9b128e973e18f7f3174d34fb4b0f5b1cebc9161cf98d4932"
+  },
   "shards": [
     {
       "num_rows": 6560,

workspace/data/cil_charts_rgb_refs/train/charts_00000.npz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f3211ef91cac305045b540e71fa40156ee697abff9db3c8f994149a2fc1b663
+oid sha256:a68210ead1392615bf3389335c89e41178957c7973440409dc9d820ebc9b422c
 size 30561279

workspace/data/cil_charts_rgb_refs/train/index.json

@@ -15,10 +15,11 @@
     "wrong_direction": 2044,
     "wrong_gripper": 2044
   },
-  "content_hash": "2cad4dbf330d10ecaeb4ed4e86ceb162850223d2d9618170b6800b1c73e3d531",
+  "content_hash": "1d15143588697e89f7fc2f6375b2745e4d479c726c9a947ff7eb7b2705280e1a",
   "dataset": "/scratch/knguy52/dovla/experiments/six_task_h16_collection",
   "deployment_candidate_excludes_expert": true,
   "deployment_clean": true,
+  "embedding_content_hash": "00859bfac7590abb24a3e7cc4c26fbb17122ba52d9f63369b8bb71a6ad1bc302",
   "epsilon": 0.05,
   "format": "cil_charts_npz",
   "group_hashes": [
@@ -2083,8 +2084,19 @@
   "num_groups_scanned": 2873,
   "num_groups_skipped_by_split": 829,
   "num_rows": 32704,
+  "observation_embedding_manifest": {
+    "dataset": "embeddings",
+    "dim": 32,
+    "extractor": "rgb_jpeg_stats_v1",
+    "num_embeddings": 2044,
+    "path": "observation_embeddings_rgb_stats.npz",
+    "reads_outcomes": false
+  },
   "retrieval_index_allowed": true,
   "schema_version": 1,
+  "shard_content_hashes": {
+    "charts_00000.npz": "a68210ead1392615bf3389335c89e41178957c7973440409dc9d820ebc9b422c"
+  },
   "shards": [
     {
       "num_rows": 32704,

workspace/data/cil_charts_rgb_refs/val/charts_00000.npz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:538fac2468d1bde33b9e76f5233e331040a8c76ff5fd224727bd8140686f4071
+oid sha256:d4b8ec1d9d6619ef9c4563e1eb46ddedc253043d1dd9da29a82d6f773e6a0336
 size 6273000

workspace/data/cil_charts_rgb_refs/val/index.json

@@ -15,10 +15,11 @@
     "wrong_direction": 419,
     "wrong_gripper": 419
   },
-  "content_hash": "bad9b7241350f81c31e394e231d066a14138a61bb8e23a68e13e15eb5ac65d64",
+  "content_hash": "577761809f93f760fc7de6558a25d8085b2a0e51ca5d969680e99bcfa8586850",
   "dataset": "/scratch/knguy52/dovla/experiments/six_task_h16_collection",
   "deployment_candidate_excludes_expert": true,
   "deployment_clean": false,
+  "embedding_content_hash": "e4374b56912fdb6814eb0320518733d1602a90e0dd2f9268f65c8a148751f963",
   "epsilon": 0.05,
   "format": "cil_charts_npz",
   "group_hashes": [
@@ -458,8 +459,19 @@
   "num_groups_scanned": 2873,
   "num_groups_skipped_by_split": 2454,
   "num_rows": 6704,
+  "observation_embedding_manifest": {
+    "dataset": "embeddings",
+    "dim": 32,
+    "extractor": "rgb_jpeg_stats_v1",
+    "num_embeddings": 419,
+    "path": "observation_embeddings_rgb_stats.npz",
+    "reads_outcomes": false
+  },
   "retrieval_index_allowed": false,
   "schema_version": 1,
+  "shard_content_hashes": {
+    "charts_00000.npz": "d4b8ec1d9d6619ef9c4563e1eb46ddedc253043d1dd9da29a82d6f773e6a0336"
+  },
   "shards": [
     {
       "num_rows": 6704,