auto-sync 2026-07-03T16:58:17Z workspace

workspace/cil/chart_features.py

@@ -12,7 +12,14 @@ except ImportError:  # pragma: no cover
 
 CONTEXT_HASH_WIDTH = 8
 OBSERVATION_EMBED_DIM = 32
-CHART_FEATURE_MODES = ("base", "base_context", "base_context_obs")
+OBJECT_LAYOUT_EMBED_DIM = 64
+CHART_FEATURE_MODES = (
+    "base",
+    "base_context",
+    "base_context_obs",
+    "base_context_obj",
+    "base_context_obs_obj",
+)
 _EMBEDDING_CACHE: dict[str, Any] = {}
 
 
@@ -27,9 +34,10 @@ def build_chart_feature(
     `base` preserves the original behavior: the chart token is the flattened
     base action chunk. `base_context` appends stable hashes of task/language
     metadata that are visible at proposal time. `base_context_obs` additionally
-    appends a precomputed observation embedding referenced by metadata. These
-    modes intentionally do not read outcomes, labels, hidden chart branches, or
-    evaluator-only fields.
+    appends a precomputed observation embedding referenced by metadata.
+    `base_context_obj` appends a precomputed RGB object-layout embedding, and
+    `base_context_obs_obj` appends both. These modes intentionally do not read
+    outcomes, labels, hidden chart branches, or evaluator-only fields.
     """
 
     if np is None:  # pragma: no cover
@@ -49,12 +57,20 @@ def build_chart_feature(
         ],
         dtype=np.float32,
     )
-    if mode == "base_context_obs":
-        obs = _load_observation_embedding(
+    if mode in {"base_context_obs", "base_context_obs_obj"}:
+        obs = _load_embedding(
             metadata.get("observation_embedding_path"),
+            dim=OBSERVATION_EMBED_DIM,
             chart_root=metadata.get("_chart_root"),
         )
         context = np.concatenate([context, obs.astype(np.float32, copy=False)])
+    if mode in {"base_context_obj", "base_context_obs_obj"}:
+        obj = _load_embedding(
+            metadata.get("object_embedding_path"),
+            dim=OBJECT_LAYOUT_EMBED_DIM,
+            chart_root=metadata.get("_chart_root"),
+        )
+        context = np.concatenate([context, obj.astype(np.float32, copy=False)])
     return np.concatenate([base, context]).astype(np.float32, copy=False)
 
 
@@ -67,11 +83,11 @@ def _stable_hash_features(text: str, width: int) -> list[float]:
     return [float(digest[index] / 127.5 - 1.0) for index in range(width)]
 
 
-def _load_observation_embedding(value: Any, *, chart_root: Any = None) -> Any:
+def _load_embedding(value: Any, *, dim: int, chart_root: Any = None) -> Any:
     if np is None:  # pragma: no cover
         raise ImportError("build_chart_feature requires numpy")
     if not value:
-        return np.zeros(OBSERVATION_EMBED_DIM, dtype=np.float32)
+        return np.zeros(dim, dtype=np.float32)
     path_text, dataset, row_index = _parse_embedding_ref(str(value))
     path = Path(path_text)
     if not path.is_absolute() and chart_root:
@@ -82,10 +98,10 @@ def _load_observation_embedding(value: Any, *, chart_root: Any = None) -> Any:
             _EMBEDDING_CACHE[cache_key] = np.asarray(data[dataset], dtype=np.float32)
     matrix = _EMBEDDING_CACHE[cache_key]
     vector = np.asarray(matrix[int(row_index)], dtype=np.float32).reshape(-1)
-    if vector.shape[0] == OBSERVATION_EMBED_DIM:
+    if vector.shape[0] == dim:
         return vector
-    output = np.zeros(OBSERVATION_EMBED_DIM, dtype=np.float32)
-    width = min(OBSERVATION_EMBED_DIM, vector.shape[0])
+    output = np.zeros(dim, dtype=np.float32)
+    width = min(dim, vector.shape[0])
     output[:width] = vector[:width]
     return output
 

workspace/data/cil_charts_rgb_refs/test/charts_00000.npz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2145c0ec6fd38abb9b128e973e18f7f3174d34fb4b0f5b1cebc9161cf98d4932
-size 6138612
+oid sha256:42b54c3fe5a42b22a225fc233171d098875d16595a7754b59a2c4efeb49f8ca1
+size 6206980

workspace/data/cil_charts_rgb_refs/test/index.json

@@ -15,7 +15,7 @@
     "wrong_direction": 410,
     "wrong_gripper": 410
   },
-  "content_hash": "d258ed27d587e6f41ef6ffd520832087fe7626b5e5cba2bd6352f50a1f45125a",
+  "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af",
   "dataset": "/scratch/knguy52/dovla/experiments/six_task_h16_collection",
   "deployment_candidate_excludes_expert": true,
   "deployment_clean": false,
@@ -450,6 +450,15 @@
   "num_groups_scanned": 2873,
   "num_groups_skipped_by_split": 2463,
   "num_rows": 6560,
+  "object_embedding_content_hash": "4e8773e46aef035fe87b3a6b631ae0bd9c82ba2c7310c62545a6b8c8a86373d5",
+  "object_embedding_manifest": {
+    "dataset": "embeddings",
+    "dim": 64,
+    "extractor": "rgb_object_layout_v1",
+    "num_embeddings": 410,
+    "path": "object_embeddings_rgb_layout.npz",
+    "reads_outcomes": false
+  },
   "observation_embedding_manifest": {
     "dataset": "embeddings",
     "dim": 32,
@@ -461,7 +470,7 @@
   "retrieval_index_allowed": false,
   "schema_version": 1,
   "shard_content_hashes": {
-    "charts_00000.npz": "2145c0ec6fd38abb9b128e973e18f7f3174d34fb4b0f5b1cebc9161cf98d4932"
+    "charts_00000.npz": "42b54c3fe5a42b22a225fc233171d098875d16595a7754b59a2c4efeb49f8ca1"
   },
   "shards": [
     {

workspace/data/cil_charts_rgb_refs/train/charts_00000.npz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a68210ead1392615bf3389335c89e41178957c7973440409dc9d820ebc9b422c
-size 30561279
+oid sha256:43e04927549fe3aadafe63067bb82c16c1167c54d0c8ebd47d4114a814f338f1
+size 30903607

workspace/data/cil_charts_rgb_refs/train/index.json

@@ -15,7 +15,7 @@
     "wrong_direction": 2044,
     "wrong_gripper": 2044
   },
-  "content_hash": "1d15143588697e89f7fc2f6375b2745e4d479c726c9a947ff7eb7b2705280e1a",
+  "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42",
   "dataset": "/scratch/knguy52/dovla/experiments/six_task_h16_collection",
   "deployment_candidate_excludes_expert": true,
   "deployment_clean": true,
@@ -2084,6 +2084,15 @@
   "num_groups_scanned": 2873,
   "num_groups_skipped_by_split": 829,
   "num_rows": 32704,
+  "object_embedding_content_hash": "fe97ac428d6b0ed4da7060eb6f697c1392b4a8a09988a10c0f633b1e457bd568",
+  "object_embedding_manifest": {
+    "dataset": "embeddings",
+    "dim": 64,
+    "extractor": "rgb_object_layout_v1",
+    "num_embeddings": 2044,
+    "path": "object_embeddings_rgb_layout.npz",
+    "reads_outcomes": false
+  },
   "observation_embedding_manifest": {
     "dataset": "embeddings",
     "dim": 32,
@@ -2095,7 +2104,7 @@
   "retrieval_index_allowed": true,
   "schema_version": 1,
   "shard_content_hashes": {
-    "charts_00000.npz": "a68210ead1392615bf3389335c89e41178957c7973440409dc9d820ebc9b422c"
+    "charts_00000.npz": "43e04927549fe3aadafe63067bb82c16c1167c54d0c8ebd47d4114a814f338f1"
   },
   "shards": [
     {

workspace/data/cil_charts_rgb_refs/val/charts_00000.npz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4b8ec1d9d6619ef9c4563e1eb46ddedc253043d1dd9da29a82d6f773e6a0336
-size 6273000
+oid sha256:6f6ea9dd2f6809cbed119598842a1dd6baaf5db099587a2748a00264aafae4e8
+size 6344084

workspace/data/cil_charts_rgb_refs/val/index.json

@@ -15,7 +15,7 @@
     "wrong_direction": 419,
     "wrong_gripper": 419
   },
-  "content_hash": "577761809f93f760fc7de6558a25d8085b2a0e51ca5d969680e99bcfa8586850",
+  "content_hash": "1aacf9de98648fa36bef14e137eb43c7c6973170804650814dae41d09842c708",
   "dataset": "/scratch/knguy52/dovla/experiments/six_task_h16_collection",
   "deployment_candidate_excludes_expert": true,
   "deployment_clean": false,
@@ -459,6 +459,15 @@
   "num_groups_scanned": 2873,
   "num_groups_skipped_by_split": 2454,
   "num_rows": 6704,
+  "object_embedding_content_hash": "411ee7d4c96458879f14fc9460c0e10d5309d83aacd5552c6bcbe089e81838bc",
+  "object_embedding_manifest": {
+    "dataset": "embeddings",
+    "dim": 64,
+    "extractor": "rgb_object_layout_v1",
+    "num_embeddings": 419,
+    "path": "object_embeddings_rgb_layout.npz",
+    "reads_outcomes": false
+  },
   "observation_embedding_manifest": {
     "dataset": "embeddings",
     "dim": 32,
@@ -470,7 +479,7 @@
   "retrieval_index_allowed": false,
   "schema_version": 1,
   "shard_content_hashes": {
-    "charts_00000.npz": "d4b8ec1d9d6619ef9c4563e1eb46ddedc253043d1dd9da29a82d6f773e6a0336"
+    "charts_00000.npz": "6f6ea9dd2f6809cbed119598842a1dd6baaf5db099587a2748a00264aafae4e8"
   },
   "shards": [
     {